def get(self): # this should really be post, all these db-writes
# unpack the values from the query string
program = self.request.get('program')
sms = self.request.get('sms', None)
invite_code_name = self.request.get('invite_code', None)
user = self.get_user(invite_code_name)
if not user:
return
if sms != None and len(sms) == 10:
# need to check that no one has this phone number yet
# also, validate the phone number somewhat
user.phone = sms
user.query_medium = 'sms' # favour sms over email
user.put()
metrics = None
if program == 'casual':
metrics = json.loads(open('scripts/casual_program.json').read())
else:
metrics = json.loads(open('scripts/moderate_program.json').read())
for metric in metrics:
if metric == None:
continue
# add the metric
name = metric["name"]
frequency = int(metric["frequency"])
text = metric['text']
user_email = self.request.get('user_email')
format = metric['format']
template = TemplateMetric.get_by_name(name)
query = Query(
name = name,
# we should really do normalization inside the Query constructor,
# but it will take some time to figure out how, lacking a nearby
# python badass
normalized_name = Query.normalize_name(name),
text = text,
frequency = frequency,
user = user,
format = format,
template = template,
)
query.put()
self.redirect('/dashboard?first_time=true')
def crosssection_suite(query):
crosssection_list = []
user = query.user
for q in Query.get_by_user(user):
if q.format == 'number' and q.name != query.name:
for x in range(query_range(q)[0], query_range(q)[1]):
avg_name = 'Average when ' + q.name + ' = ' + str(x)
avg_value = float_str_format(avg_int_on_sliced_int(query, q, x))
crosssection_list.append((avg_name, avg_value))
percent_name = 'Change from average when ' + q.name + ' = ' + str(x)
percent_value =float_str_format(percent_from_avg_int_on_sliced_int(query, q, x))
crosssection_list.append((percent_name, percent_value))
elif q.format == 'text':
for word in common_words(DataPoint.get_by_query(q)).split(', '):
avg_name = 'Average when "' + word + '" is in ' + q.name
avg_value = float_str_format(avg_int_on_sliced_text(query, q, word))
crosssection_list.append((avg_name, avg_value))
#percent_name = 'Change from average when ' + word + ' in ' +q.name
#percent_value = float_str_format(percent_from_avg_int_on_sliced_int(query, q, x))
#crosssection_list[percent_name] = percent_value
return crosssection_list
def post(self):
user = self.get_user()
if not user:
return
user_email = self.request.get('user_email')
query_id = self.request.get('query_id')
csv_data = self.request.get('csv_data')
user = User.get_by_email(user_email)
query = Query.get_by_id(query_id)
for duple in self.parse_csv_data(csv_data):
timestamp = duple[0]
text = duple[1]
# for testing
self.response.out.write("<p>%s: %s\n</p>" % (timestamp, text))
dp = DataPoint(
timestamp = duple[0],
query = query,
text = duple[1])
dp.lt_put()
self.redirect('/data')
def generate_query_table(self, user):
queries = Query.get_by_user(user)
html = ''
for query in queries:
html += self.query_to_table_row(query)
return html
def covariance_suite(query):
covariance_list = []
user = query.user
for q in Query.get_by_user(user):
if q.format == 'number' and q.name != query.name:
covariance_list.append(
('Covariance with ' + q.name, float_str_format(covariance(query,q))))
return covariance_list
def correlation_suite(query):
correlation_list = []
user = query.user
for q in Query.get_by_user(user):
if q.format == 'number' and q.name != query.name:
correlation_list.append(
('Correlation with ' + q.name, float_str_format(correlation_coefficient(query,q))))
return correlation_list
def get(self):
user = self.get_user()
if not user:
return
query_id = self.request.get('query_id')
query = Query.get_by_id(query_id)
analytics_json = self.get_analytics(query)
self.response.out.write(analytics_json)
def generate_data_view(self, user):
# get all the queries
queries = Query.get_by_user(user)
html = ''
# for each query
# append query_to_data(query)
for query in queries:
html += self.query_to_data(query)
return html
def get(self):
start = datetime.now()
queries = Query.all().fetch(1000)
self.refresh_datapoints(queries)
self.refresh_most_recent_dp(queries)
end = datetime.now()
logging.info("Memcache refresh started: " + start.strftime('%s'))
logging.info("Memcache refresh ended: " + end.strftime('%s'))
def get(self):
user = self.get_user()
if not user:
return
query_id = self.request.get('query_id')
query = Query.get_by_id(query_id)
datapoints = DataPoint.get_by_query(query)
frequencies = common_word_frequencies(datapoints)
self.response.out.write('[' + json.dumps(frequencies) + ']')
def post(self):
ActionLog.log('ReceivedSMS')
sender_phone = self.request.get('From')
body = self.request.get('Body')
# normalize the phone number to 10 digits
# '+12268681112' => '2268681112'
sender_phone = sender_phone[len(sender_phone) - 10:]
logging.info('Got text from ' + sender_phone)
# get the user
user = User.get_by_phone(sender_phone)
if not user:
logging.error("Couldn't get the user for phone: " + sender_phone)
return
# parse the response
query_value_pairs = self.parse_body(body)
for query_name in query_value_pairs:
value = query_value_pairs[query_name]
if query_name == '' or value == '':
logging.error('Got a bad response');
return
query = Query.get_by_user_and_name(user, query_name)
if not query:
logging.error("Couldn't get query for user " + user.email + ' and query ' + query.name)
continue
timestamp = datetime.now()
dp = DataPoint(
user = user,
query = query,
text = value,
timestamp = timestamp,
)
dp.lt_put()
ActionLog.log('NewDatapoint', user, query.name)
query.refresh()
logging.info('Received datapoint ' + query_name + ': ' + value + '\n')
self.response.out.write('<Response><Sms>Got it!</Sms></Response>')
def receive(self, mail_message):
# we need to make a new data point
# get the user from the sender field
user_email = mail_message.sender[ mail_message.sender.find('<') + 1 : mail_message.sender.rfind('>') ]
user = User.get_by_email(user_email)
if is_admin(user) and mail_message.to.find('users@') != -1:
forward_to_users(mail_message)
return
if mail_message.to.find('feedback@') != -1:
forward_to_admins(mail_message)
return
# get the datapoint from the body
data = ''
query_name = mail_message.subject[ mail_message.subject.rfind("Re:") + 4: ]
query = Query.get_by_user_and_name(user, query_name)
for content_type, body in mail_message.bodies('text/html'):
# basically deprecated, since we're having users submit forms
# straight from the emails now.
logging.info('Message Body: ' + body.decode())
data = body.decode()[ : body.decode().find('<')]
if data == None:
data = ''
# get the time from now()
timestamp = datetime.now()
log_str = "Want to create a new datapoint for user %s and with value %s and query name %s and query %s and timestamp %s" % (user_email, data, query_name, query.key(), timestamp)
logging.info("Received a message from " + mail_message.sender)
logging.info(log_str)
# create and put the datapoint
# dp = DataPoint(...)
dp = DataPoint(
text = data,
query = query,
timestamp = timestamp)
dp.lt_put()
ActionLog.log('ReceivedEmail')
ActionLog.log('NewDatapoint', data=query.name, user=user)
query.refresh()
def get(self):
start = datetime.now().strftime("%s")
users = User.all().fetch(1000)
for user in users:
queries = Query.get_by_user(user)
for query in queries:
if query.is_stale() and query.is_time_to_send():
send_query(query)
break # only send one query per user every interval
end = datetime.now().strftime("%s")
logging.info("SendQueries started at " + start)
logging.info("SendQueries finished at " + end)
def get(self):
user = self.get_user()
if not user:
return
user_email = self.request.get('user_email')
user = User.get_by_email(user_email)
queries = Query.get_by_user(user)
query_ids = []
for query in queries:
query_ids.append(str(query.key()))
self.response.out.write(json.dumps(query_ids))
def get(self):
user = self.get_user()
if not user:
return
user_email = self.request.get('user_email')
user = User.get_by_email(user_email)
queries = Query.get_by_user(user)
datapoints = []
for query in queries:
for datapoint in DataPoint.get_by_query(query):
datapoints.append(datapoint.to_dict())
self.response.out.write(json.dumps(datapoints))
def get(self):
user = self.get_user()
if not user:
return
user_email = self.request.get('user_email')
query_id = self.request.get('query_id')
query = Query.get_by_id(query_id)
datapoints = DataPoint.get_by_query(query)
csv_data = ''
for dp in datapoints:
csv_data += self.dp_to_csv(dp)
self.response.out.write(csv_data)
def get(self):
user = self.get_user()
if not user:
return
query_id = self.request.get('query_id')
query = Query.get_by_id(query_id)
analytics_html = self.get_analytics(query)
params = {
'analytics_rows': analytics_html,
'user_email': user.email,
'query_name': query.name,
}
html = self.render_page('ui/html/analyze.html', params)
self.response.out.write(html)
def get(self):
user = self.get_user()
if not user:
return
if TemplateMetric.all().count() > 0:
return
f = open('scripts/templates.json')
json_templates = json.loads(f.read())
for json_template in json_templates:
template = TemplateMetric(
format = json_template['format'],
frequency = json_template['frequency'],
name = json_template['name'],
normalized_name = Query.normalize_name(json_template['name']),
text = json_template['text'],
)
template.put()
def post(self):
user = self.get_user()
if not user:
return
query_id = self.request.get('query_id')
name = self.request.get('name', None)
frequency = self.request.get('frequency', None)
text = self.request.get('text', None)
#user_email = self.request.get('user_email', None) #hmm
format = self.request.get('format', None) #hmm
ask_when = self.request.get('ask_when', None)
query = db.get(query_id)
if not query:
self.response.out.write('failure!')
return
if name:
query.name = name
query.normalized_name = Query.normalize_name(name)
if frequency:
query.frequency = int(frequency)
if text:
query.text = text
if format:
query.format = format
if ask_when:
# this has the amusing side-effect that it is impossible to have an
# empty 'ask_when' field, and hence a query will always be sent
ask_when_list = ask_when.split(',')
# there's always a trailing empty item in the list
ask_when_list = ask_when_list[:len(ask_when_list) -1]
query.ask_when = ask_when_list
query.put()
self.response.out.write("success")
def post(self):
user = self.get_user()
if not user:
return
name = self.request.get("name")
frequency = int(self.request.get("frequency"))
text = self.request.get('text')
user_email = self.request.get('user_email')
format = self.request.get('format').lower()
template_id = self.request.get('template_id')
# for now, default to always
ask_when = ['morning', 'afternoon', 'evening']
user = User.get_by_email(user_email)
query = Query(
name = name,
# we should really do normalization inside the Query constructor,
# but it will take some time to figure out how, lacking a nearby
# python badass
normalized_name = Query.normalize_name(name),
text = text,
frequency = frequency,
user = user,
format = format,
ask_when = ask_when,
)
if str(template_id) != "0":
query.template = db.get(template_id)
query.put()
ActionLog.log('NewMetric', user)
self.response.out.write(query.key());
def get(self):
user = self.get_user()
if not user:
return
logout_url = users.create_logout_url(self.request.uri)
yesterday = datetime.now() - timedelta(hours=24)
new_datapoints = ActionLog.get(
action = 'NewDatapoint',
timewindow = yesterday,
).count()
new_metrics = ActionLog.get(
action = 'NewMetric',
timewindow = yesterday,
).count()
queries_sent = ActionLog.get(
action = 'SentQuery',
timewindow = yesterday,
).count()
sms_sent = ActionLog.get(
action = 'SentSMS',
timewindow = yesterday,
).count()
emails_sent = ActionLog.get(
action = 'SentEmail',
timewindow = yesterday,
).count()
emails_received = ActionLog.get(
action = 'ReceivedEmail',
timewindow = yesterday,
).count()
sms_received = ActionLog.get(
action = 'ReceivedSMS',
timewindow = yesterday,
).count()
new_logins = ActionLog.get(
action = 'FirstTimeLogin',
timewindow = yesterday,
).count()
dashboard_avg_walltime = average_walltime('/dashboard')
data_avg_walltime = average_walltime('/data')
home_avg_walltime = average_walltime('/')
analyze_avg_walltime = average_walltime('/analyze')
analyze_json_avg_walltime = average_walltime('/analyzeJSON')
dashboard_worst_walltime = worst_walltime('/dashboard')
data_worst_walltime = worst_walltime('/data')
home_worst_walltime = worst_walltime('/')
analyze_worst_walltime = worst_walltime('/analyze')
analyze_json_worst_walltime = worst_walltime('/analyzeJSON')
# hackey high-number for now.
total_metrics = Query.all().count(100000)
total_datapoints = DataPoint.all().count(100000)
f = open('intern/html/engagement.html')
html = f.read()
params = {
'new_datapoints': new_datapoints,
'new_metrics': new_metrics,
'total_metrics': total_metrics,
'total_datapoints': total_datapoints,
'queries_sent': queries_sent,
'sms_sent': sms_sent,
'emails_sent': emails_sent,
'sms_received': sms_received,
'emails_received': emails_received,
'new_logins': new_logins,
'dashboard_walltime': dashboard_avg_walltime,
'data_walltime': data_avg_walltime,
'home_walltime': home_avg_walltime,
'analyze_json_walltime': analyze_json_avg_walltime,
'analyze_walltime': analyze_avg_walltime,
'dashboard_worst_walltime': dashboard_worst_walltime,
'data_worst_walltime': data_worst_walltime,
'home_worst_walltime': home_worst_walltime,
'analyze_json_worst_walltime': analyze_json_worst_walltime,
'analyze_worst_walltime': analyze_worst_walltime,
}
self.response.out.write(html % params)
