Пример #1
0
  def get(self): # this should really be post, all these db-writes
    # unpack the values from the query string
    program = self.request.get('program')
    sms = self.request.get('sms', None) 
    invite_code_name = self.request.get('invite_code', None)

    user = self.get_user(invite_code_name)
    if not user:
      return

    if sms != None and len(sms) == 10:
      # need to check that no one has this phone number yet
      # also, validate the phone number somewhat
      user.phone = sms
      user.query_medium = 'sms' # favour sms over email
      user.put()

    metrics = None

    if program == 'casual':
      metrics = json.loads(open('scripts/casual_program.json').read())
    else:
      metrics = json.loads(open('scripts/moderate_program.json').read())

    for metric in metrics:
      if metric == None:
        continue

      # add the metric
      name = metric["name"]
      frequency = int(metric["frequency"])
      text = metric['text']
      user_email = self.request.get('user_email')
      format = metric['format']
      template = TemplateMetric.get_by_name(name)

      query = Query(
        name = name,
        # we should really do normalization inside the Query constructor,
        # but it will take some time to figure out how, lacking a nearby
        # python badass
        normalized_name = Query.normalize_name(name), 
        text = text,
        frequency = frequency,
        user = user,
        format = format,
        template = template,
      )

      query.put()
    
    self.redirect('/dashboard?first_time=true')
Пример #2
0
def crosssection_suite(query):
  crosssection_list = []
  user = query.user

  for q in Query.get_by_user(user):
    if q.format == 'number' and q.name != query.name:
      for x in range(query_range(q)[0], query_range(q)[1]):
        avg_name = 'Average when ' + q.name + ' = ' + str(x)
        avg_value = float_str_format(avg_int_on_sliced_int(query, q, x))
  
        crosssection_list.append((avg_name, avg_value))

        percent_name = 'Change from average when ' + q.name + ' = ' + str(x)
        percent_value =float_str_format(percent_from_avg_int_on_sliced_int(query, q, x))

        crosssection_list.append((percent_name, percent_value))

    elif q.format == 'text':
      for word in common_words(DataPoint.get_by_query(q)).split(', '):
        avg_name = 'Average when "' + word + '" is in ' + q.name
        avg_value = float_str_format(avg_int_on_sliced_text(query, q, word))

        crosssection_list.append((avg_name, avg_value))

        #percent_name = 'Change from average when ' + word + ' in ' +q.name
        #percent_value = float_str_format(percent_from_avg_int_on_sliced_int(query, q, x))

        #crosssection_list[percent_name] = percent_value
        
  return crosssection_list
Пример #3
0
  def post(self):
    user = self.get_user()
    if not user:
      return

    user_email = self.request.get('user_email')
    query_id = self.request.get('query_id')
    csv_data = self.request.get('csv_data')

    user = User.get_by_email(user_email)
    query = Query.get_by_id(query_id)

    for duple in self.parse_csv_data(csv_data):
      timestamp = duple[0]
      text = duple[1]

      # for testing
      self.response.out.write("<p>%s: %s\n</p>" % (timestamp, text))

      dp = DataPoint(
        timestamp = duple[0],
        query = query,
        text = duple[1])
    
      dp.lt_put()

    self.redirect('/data')
Пример #4
0
  def generate_query_table(self, user):
    queries = Query.get_by_user(user)

    html = ''

    for query in queries:
      html += self.query_to_table_row(query)

    return html
Пример #5
0
def covariance_suite(query):
  covariance_list = []
  user = query.user 

  for q in Query.get_by_user(user):
    if q.format == 'number' and q.name != query.name:
      covariance_list.append(
        ('Covariance with ' + q.name, float_str_format(covariance(query,q))))

  return covariance_list
Пример #6
0
def correlation_suite(query):
  correlation_list = []
  user = query.user 

  for q in Query.get_by_user(user):
    if q.format == 'number' and q.name != query.name:
      correlation_list.append(
        ('Correlation with ' + q.name, float_str_format(correlation_coefficient(query,q))))

  return correlation_list
Пример #7
0
  def get(self):
    user = self.get_user()
    if not user:
      return
      
    query_id = self.request.get('query_id')
    query = Query.get_by_id(query_id)

    analytics_json = self.get_analytics(query)

    self.response.out.write(analytics_json)
Пример #8
0
  def generate_data_view(self, user):
    # get all the queries

    queries = Query.get_by_user(user)
    html = ''
    # for each query
    #   append query_to_data(query)
    for query in queries:
      html += self.query_to_data(query)
    
    return html
Пример #9
0
  def get(self):
    start = datetime.now()
    
    queries = Query.all().fetch(1000)

    self.refresh_datapoints(queries)
    self.refresh_most_recent_dp(queries)

    end = datetime.now()

    logging.info("Memcache refresh started: " + start.strftime('%s'))
    logging.info("Memcache refresh ended: " + end.strftime('%s'))
Пример #10
0
  def get(self):
    user = self.get_user()
    if not user:
      return
      
    query_id = self.request.get('query_id')
    query = Query.get_by_id(query_id)
   
    datapoints = DataPoint.get_by_query(query)
    
    frequencies = common_word_frequencies(datapoints)

    self.response.out.write('[' + json.dumps(frequencies) + ']')
Пример #11
0
  def post(self):
    ActionLog.log('ReceivedSMS')

    sender_phone = self.request.get('From')
    body = self.request.get('Body')

    # normalize the phone number to 10 digits
    # '+12268681112' => '2268681112'
    sender_phone = sender_phone[len(sender_phone) - 10:]
    logging.info('Got text from ' + sender_phone)

    # get the user
    user = User.get_by_phone(sender_phone)
    if not user:
      logging.error("Couldn't get the user for phone: " + sender_phone)
      return

    # parse the response
    query_value_pairs = self.parse_body(body)

    for query_name in query_value_pairs:
      value = query_value_pairs[query_name]

      if query_name == '' or value == '':
        logging.error('Got a bad response');
        return

      query = Query.get_by_user_and_name(user, query_name)

      if not query:
        logging.error("Couldn't get query for user " + user.email + ' and query ' + query.name)
        continue

      timestamp = datetime.now()
      
      dp = DataPoint(
        user = user,
        query = query,
        text = value,
        timestamp = timestamp,
      )

      dp.lt_put()

      ActionLog.log('NewDatapoint', user, query.name)

      query.refresh()

      logging.info('Received datapoint ' + query_name + ': ' + value + '\n')

    self.response.out.write('<Response><Sms>Got it!</Sms></Response>')
Пример #12
0
  def receive(self, mail_message):
    # we need to make a new data point

    # get the user from the sender field
    user_email = mail_message.sender[ mail_message.sender.find('<') + 1 : mail_message.sender.rfind('>') ]
    user = User.get_by_email(user_email)

    if is_admin(user) and mail_message.to.find('users@') != -1:
      forward_to_users(mail_message)
      return

    if mail_message.to.find('feedback@') != -1:
      forward_to_admins(mail_message)
      return

    # get the datapoint from the body
    data = ''
    query_name = mail_message.subject[ mail_message.subject.rfind("Re:") + 4: ]
  
    query = Query.get_by_user_and_name(user, query_name)

    for content_type, body in mail_message.bodies('text/html'):
      # basically deprecated, since we're having users submit forms
      # straight from the emails now.
      logging.info('Message Body: ' + body.decode())
      data = body.decode()[ : body.decode().find('<')]

    if data == None:
      data = ''

    # get the time from now()
    timestamp = datetime.now()

    log_str = "Want to create a new datapoint for user %s and with value %s and query name %s and query %s and timestamp %s" % (user_email, data, query_name, query.key(), timestamp)

    logging.info("Received a message from " + mail_message.sender)
    logging.info(log_str)

    # create and put the datapoint
    # dp = DataPoint(...)
    dp = DataPoint( 
      text = data,
      query = query,
      timestamp = timestamp)
      
    dp.lt_put()
    ActionLog.log('ReceivedEmail')
    ActionLog.log('NewDatapoint', data=query.name, user=user)

    query.refresh()
Пример #13
0
    def get(self):
        start = datetime.now().strftime("%s")
        users = User.all().fetch(1000)

        for user in users:
            queries = Query.get_by_user(user)
            for query in queries:
                if query.is_stale() and query.is_time_to_send():
                    send_query(query)
                    break  # only send one query per user every interval

        end = datetime.now().strftime("%s")

        logging.info("SendQueries started at " + start)
        logging.info("SendQueries finished at " + end)
Пример #14
0
  def get(self):
    user = self.get_user()
    if not user:
      return

    user_email = self.request.get('user_email')
    
    user = User.get_by_email(user_email)

    queries = Query.get_by_user(user)

    query_ids = []
    for query in queries:
      query_ids.append(str(query.key()))

    self.response.out.write(json.dumps(query_ids))
Пример #15
0
  def get(self):
    user = self.get_user()
    if not user:
      return

    user_email = self.request.get('user_email')
    
    user = User.get_by_email(user_email)
    queries = Query.get_by_user(user)

    datapoints = []

    for query in queries:
      for datapoint in DataPoint.get_by_query(query):
        datapoints.append(datapoint.to_dict())

    self.response.out.write(json.dumps(datapoints))
Пример #16
0
  def get(self):
    user = self.get_user()
    if not user:
      return

    user_email = self.request.get('user_email')
    query_id = self.request.get('query_id')

    query = Query.get_by_id(query_id)

    datapoints = DataPoint.get_by_query(query)
    
    csv_data = ''
    
    for dp in datapoints:
      csv_data += self.dp_to_csv(dp)

    self.response.out.write(csv_data)
Пример #17
0
  def get(self):
    user = self.get_user()
    if not user:
      return
      
    query_id = self.request.get('query_id')
    query = Query.get_by_id(query_id)

    analytics_html = self.get_analytics(query)

    params = {
      'analytics_rows': analytics_html, 
      'user_email': user.email, 
      'query_name': query.name,
    }

    html = self.render_page('ui/html/analyze.html', params)

    self.response.out.write(html)
Пример #18
0
  def get(self):
    user = self.get_user()
    if not user:
      return

    if TemplateMetric.all().count() > 0:
      return

    f = open('scripts/templates.json')
    json_templates = json.loads(f.read())

    for json_template in json_templates:
      template = TemplateMetric(
        format = json_template['format'],
        frequency = json_template['frequency'],
        name = json_template['name'],
        normalized_name = Query.normalize_name(json_template['name']),
        text = json_template['text'],
      )

      template.put()
Пример #19
0
  def post(self):
    user = self.get_user()
    if not user:
      return

    query_id = self.request.get('query_id')
    name = self.request.get('name', None)
    frequency = self.request.get('frequency', None)
    text = self.request.get('text', None)
    #user_email = self.request.get('user_email', None) #hmm
    format = self.request.get('format', None) #hmm
    ask_when = self.request.get('ask_when', None)

    query = db.get(query_id)
    if not query:
      self.response.out.write('failure!')
      return

    if name:
      query.name = name
      query.normalized_name = Query.normalize_name(name)
    if frequency:
      query.frequency = int(frequency)
    if text:
      query.text = text
    if format:
      query.format = format
    if ask_when: 
      # this has the amusing side-effect that it is impossible to have an 
      # empty 'ask_when' field, and hence a query will always be sent
      ask_when_list = ask_when.split(',')
      # there's always a trailing empty item in the list
      ask_when_list = ask_when_list[:len(ask_when_list) -1]
      query.ask_when = ask_when_list

    query.put()
    
    self.response.out.write("success")
Пример #20
0
  def post(self):
    user = self.get_user()
    if not user:
      return

    name = self.request.get("name")
    frequency = int(self.request.get("frequency"))
    text = self.request.get('text')
    user_email = self.request.get('user_email') 
    format = self.request.get('format').lower()
    template_id = self.request.get('template_id')
    # for now, default to always 
    ask_when = ['morning', 'afternoon', 'evening']

    user = User.get_by_email(user_email)

    query = Query(
      name = name,
      # we should really do normalization inside the Query constructor,
      # but it will take some time to figure out how, lacking a nearby
      # python badass
      normalized_name = Query.normalize_name(name), 
      text = text,
      frequency = frequency,
      user = user,
      format = format,
      ask_when = ask_when,
    )
  
    if str(template_id) != "0":
      query.template = db.get(template_id)

    query.put()

    ActionLog.log('NewMetric', user)

    self.response.out.write(query.key());
Пример #21
0
  def get(self):
    user = self.get_user()
    if not user:
      return

    logout_url = users.create_logout_url(self.request.uri)
    
    yesterday = datetime.now() - timedelta(hours=24)

    new_datapoints = ActionLog.get(
      action = 'NewDatapoint',
      timewindow = yesterday,
    ).count()

    new_metrics = ActionLog.get(
      action = 'NewMetric', 
      timewindow = yesterday,
    ).count()

    queries_sent = ActionLog.get(
      action = 'SentQuery',
      timewindow = yesterday,
    ).count()

    sms_sent = ActionLog.get(
      action = 'SentSMS',
      timewindow = yesterday,
    ).count()

    emails_sent = ActionLog.get(
      action = 'SentEmail',
      timewindow = yesterday,
    ).count()

    emails_received = ActionLog.get(
      action = 'ReceivedEmail',
      timewindow = yesterday,
    ).count()

    sms_received = ActionLog.get(
      action = 'ReceivedSMS',
      timewindow = yesterday,
    ).count()

    new_logins = ActionLog.get(
      action = 'FirstTimeLogin',
      timewindow = yesterday,
    ).count()

    dashboard_avg_walltime = average_walltime('/dashboard')
    data_avg_walltime = average_walltime('/data')
    home_avg_walltime = average_walltime('/')
    analyze_avg_walltime = average_walltime('/analyze')
    analyze_json_avg_walltime = average_walltime('/analyzeJSON')

    dashboard_worst_walltime = worst_walltime('/dashboard')
    data_worst_walltime = worst_walltime('/data')
    home_worst_walltime = worst_walltime('/')
    analyze_worst_walltime = worst_walltime('/analyze')
    analyze_json_worst_walltime = worst_walltime('/analyzeJSON')

    # hackey high-number for now.
    total_metrics = Query.all().count(100000)
    total_datapoints = DataPoint.all().count(100000)

    f = open('intern/html/engagement.html')
    html = f.read()

    params = {
      'new_datapoints': new_datapoints,
      'new_metrics': new_metrics,
      'total_metrics': total_metrics,
      'total_datapoints': total_datapoints,
      'queries_sent': queries_sent,
      'sms_sent': sms_sent,
      'emails_sent': emails_sent,
      'sms_received': sms_received,
      'emails_received': emails_received,
      'new_logins': new_logins,
      'dashboard_walltime': dashboard_avg_walltime,
      'data_walltime': data_avg_walltime,
      'home_walltime': home_avg_walltime,
      'analyze_json_walltime': analyze_json_avg_walltime,
      'analyze_walltime': analyze_avg_walltime,
      'dashboard_worst_walltime': dashboard_worst_walltime,
      'data_worst_walltime': data_worst_walltime,
      'home_worst_walltime': home_worst_walltime,
      'analyze_json_worst_walltime': analyze_json_worst_walltime,
      'analyze_worst_walltime': analyze_worst_walltime,
    }

    self.response.out.write(html % params)