Esempio n. 1
0
 def test_update_cq_stats_create(self):
   _clear_cq_stats()
   update_cq_stats({
       'test-project': [
         CountStats(
           name='test-count',
           description='test-count-description',
           count=123),
         ListStats(
           name='test-list',
           description='test-list-description',
           unit='test-unit'),
       ],
     }, 60, datetime(2000, 1, 1, 1), datetime(2000, 1, 1, 2))
   self.assertEqual(1, CQStats.query().count())
   cq_stats = CQStats.query().get()
   self.assertEqual('test-project', cq_stats.project)
   self.assertEqual(60, cq_stats.interval_minutes)
   self.assertEqual(datetime(2000, 1, 1, 1), cq_stats.begin)
   self.assertEqual(datetime(2000, 1, 1, 2), cq_stats.end)
   self.assertEqual([
       CountStats(
         name='test-count',
         description='test-count-description',
         count=123),
     ], cq_stats.count_stats)
   self.assertEqual([
       ListStats(
         name='test-list',
         description='test-list-description',
         unit='test-unit'),
     ], cq_stats.list_stats)
Esempio n. 2
0
def _clear_ndb(): # pragma: no cover
  for cq_stats in CQStats.query():
    cq_stats.key.delete()
  assert CQStats.query().count() == 0
  for record in Record.query():
    record.key.delete()
  assert Record.query().count() == 0
Esempio n. 3
0
def update_cq_stats(project_stats, minutes, begin, end):  # pragma: no cover
    """Ensure CQStats are updated or created as necessary with new stats data."""
    assert (end - begin).total_seconds() == 60 * minutes
    assert (begin - stats_start).total_seconds() % (60 * minutes) == 0, (
        'Interval must be aligned with %s.' % stats_start)
    for project, stats_list in project_stats.iteritems():
        count_stats_dict = {}
        list_stats_dict = {}
        for stats in stats_list:
            if type(stats) == CountStats:
                count_stats_dict[stats.name] = stats
            else:
                assert type(stats) == ListStats
                list_stats_dict[stats.name] = stats

        cq_stats = CQStats.query().filter(CQStats.project == project,
                                          CQStats.interval_minutes == minutes,
                                          CQStats.begin == begin).get()
        if cq_stats:
            update_stats_list(cq_stats.count_stats, count_stats_dict)
            update_stats_list(cq_stats.list_stats, list_stats_dict)
            cq_stats.put()
        else:
            CQStats(
                project=project,
                interval_minutes=minutes,
                begin=begin,
                end=end,
                count_stats=count_stats_dict.values(),
                list_stats=list_stats_dict.values(),
            ).put()
    update_last_cq_stats_change_timestamp(minutes)
Esempio n. 4
0
 def test_update_cq_stats_create(self):
     _clear_cq_stats()
     update_cq_stats(
         {
             'test-project': [
                 CountStats(name='test-count',
                            description='test-count-description',
                            count=123),
                 ListStats(name='test-list',
                           description='test-list-description',
                           unit='test-unit'),
             ],
         }, 60, datetime(2000, 1, 1, 1), datetime(2000, 1, 1, 2))
     self.assertEqual(1, CQStats.query().count())
     cq_stats = CQStats.query().get()
     self.assertEqual('test-project', cq_stats.project)
     self.assertEqual(60, cq_stats.interval_minutes)
     self.assertEqual(datetime(2000, 1, 1, 1), cq_stats.begin)
     self.assertEqual(datetime(2000, 1, 1, 2), cq_stats.end)
     self.assertEqual([
         CountStats(name='test-count',
                    description='test-count-description',
                    count=123),
     ], cq_stats.count_stats)
     self.assertEqual([
         ListStats(name='test-list',
                   description='test-list-description',
                   unit='test-unit'),
     ], cq_stats.list_stats)
Esempio n. 5
0
def post(handler):  # pragma: no cover
    if handler.request.get('all'):
        stats_list = CQStats.query()
    else:
        stats_list = []
        for key in handler.request.get('keys').split(','):
            stats = CQStats.get_by_id(int(key))
            assert stats, '%s must exist.' % key
            stats_list.append(stats)

    handler.response.write('CQStats removed: [\n')
    for stats in stats_list:
        handler.response.write('  %s,\n' % stats)
        stats.key.delete()
    handler.response.write(']\n')
Esempio n. 6
0
def post(handler): # pragma: no cover
  if handler.request.get('all'):
    stats_list = CQStats.query()
  else:
    stats_list = []
    for key in handler.request.get('keys').split(','):
      stats = CQStats.get_by_id(int(key))
      assert stats, '%s must exist.' % key
      stats_list.append(stats)

  handler.response.write('CQStats removed: [\n')
  for stats in stats_list:
    handler.response.write('  %s,\n' % stats)
    stats.key.delete()
  handler.response.write(']\n')
Esempio n. 7
0
def update_cq_stats(project_stats, minutes, begin, end): # pragma: no cover
  """Ensure CQStats are updated or created as necessary with new stats data."""
  assert (end - begin).total_seconds() == 60 * minutes
  assert (begin - stats_start).total_seconds() % (60 * minutes) == 0, (
      'Interval must be aligned with %s.' % stats_start)
  for project, stats_list in project_stats.iteritems():
    count_stats_dict = {}
    list_stats_dict = {}
    for stats in stats_list:
      if type(stats) == CountStats:
        count_stats_dict[stats.name] = stats
      else:
        assert type(stats) == ListStats
        list_stats_dict[stats.name] = stats

    cq_stats = CQStats.query().filter(
        CQStats.project == project,
        CQStats.interval_minutes == minutes,
        CQStats.begin == begin).get()
    if cq_stats:
      update_stats_list(cq_stats.count_stats, count_stats_dict)
      update_stats_list(cq_stats.list_stats, list_stats_dict)
      cq_stats.put()
    else:
      CQStats(
        project=project,
        interval_minutes=minutes,
        begin=begin,
        end=end,
        count_stats=count_stats_dict.values(),
        list_stats=list_stats_dict.values(),
      ).put()
  update_last_cq_stats_change_timestamp(minutes)
 def test_missing_intervals_matched_cq_stats(self):
     CQStats(project='',
             interval_minutes=1440,
             begin=datetime(2000, 1, 2, 8),
             end=datetime(2000, 1, 3, 8)).put()
     self.assertEqual([
         (datetime(2000, 1, 3, 8), datetime(2000, 1, 4, 8)),
     ], missing_intervals(1440, datetime(2000, 1, 4, 8)))
Esempio n. 9
0
def _reset_stats(): # pragma: no cover
  for cq_stats in CQStats.query():
    cq_stats.key.delete()
  assert CQStats.query().count() == 0
  cq_stats = CQStats(
    project='test',
    interval_minutes=1,
    begin=datetime.utcfromtimestamp(0),
    end=datetime.utcfromtimestamp(1),
    count_stats=[
      CountStats(
        name='test-count',
        description='Test count description',
        count=3,
        highest_100=[
          [2, {'data_point': 'c'}],
          [1, {'data_point': 'b'}],
          [0, {'data_point': 'a'}],
        ],
        lowest_100=[
          [0, {'data_point': 'a'}],
          [1, {'data_point': 'b'}],
          [2, {'data_point': 'c'}],
        ],
      ),
    ],
    list_stats=[
      ListStats(
        name='test-list',
        description='Test list description',
        unit='test_unit',
        highest_100=[
          [2, {'data_point': 'c'}],
          [1, {'data_point': 'b'}],
          [0, {'data_point': 'a'}],
        ],
        lowest_100=[
          [0, {'data_point': 'a'}],
          [1, {'data_point': 'b'}],
          [2, {'data_point': 'c'}],
        ],
      ),
    ],
  ).put()
  return cq_stats.id()
Esempio n. 10
0
def _reset_stats(): # pragma: no cover
  for cq_stats in CQStats.query():
    cq_stats.key.delete()
  assert CQStats.query().count() == 0
  cq_stats = CQStats(
    project='test',
    interval_minutes=1,
    begin=datetime.utcfromtimestamp(0),
    end=datetime.utcfromtimestamp(1),
    count_stats=[
      CountStats(
        name='test-count',
        description='Test count description',
        count=3,
        highest_100=[
          [2, {'data_point': 'c'}],
          [1, {'data_point': 'b'}],
          [0, {'data_point': 'a'}],
        ],
        lowest_100=[
          [0, {'data_point': 'a'}],
          [1, {'data_point': 'b'}],
          [2, {'data_point': 'c'}],
        ],
      ),
    ],
    list_stats=[
      ListStats(
        name='test-list',
        description='Test list description',
        unit='test_unit',
        highest_100=[
          [2, {'data_point': 'c'}],
          [1, {'data_point': 'b'}],
          [0, {'data_point': 'a'}],
        ],
        lowest_100=[
          [0, {'data_point': 'a'}],
          [1, {'data_point': 'b'}],
          [2, {'data_point': 'c'}],
        ],
      ),
    ],
  ).put()
  return cq_stats.id()
Esempio n. 11
0
 def get(self, ranking, name, cq_stats_key): # pylint: disable=R0201
   cq_stats = CQStats.get_by_id(int(cq_stats_key))
   assert cq_stats, '%s must match a CQStats entry.' % cq_stats_key
   for stats in cq_stats.count_stats + cq_stats.list_stats:
     if stats.name == name:
       if ranking == 'lowest':
         return stats.lowest_100
       assert ranking == 'highest'
       return stats.highest_100
   assert False, '%s must match a stat in the specified CQStats entry.' % name
Esempio n. 12
0
def missing_intervals(minutes, end):
  last_cq_stats = CQStats.query().filter(
      CQStats.interval_minutes == minutes).order(-CQStats.end).get()
  if last_cq_stats:
    return intervals_in_range(minutes, last_cq_stats.end, end)
  earliest_record = Record.query().order(Record.timestamp).get()
  if earliest_record:
    begin = earliest_record.timestamp - timedelta(minutes=minutes)
    return intervals_in_range(minutes, begin, end)
  return []
Esempio n. 13
0
def execute_query(key, project, interval_minutes, begin, end, names, count,
                  cursor):
    stats_list = []
    next_cursor = ''
    if key and count > 0:
        stats = CQStats.get_by_id(key)
        if stats and (  # pragma: no branch
            (not project or stats.project == project) and
            (not interval_minutes
             or stats.interval_minutes == interval_minutes) and
            (not begin or stats.begin >= begin) and
            (not end or stats.end <= end) and
            (not names or stats.has_any_names(names))):
            stats_list.append(stats)
        more = False
    else:
        more = True
        while more and len(stats_list) < count:
            filters = []
            if project:
                filters.append(CQStats.project == project)
            if interval_minutes:
                filters.append(CQStats.interval_minutes == interval_minutes)
            if begin:
                filters.append(CQStats.begin >= begin)
            if end:
                filters.append(CQStats.begin <= end)
            query = CQStats.query().filter(*filters).order(-CQStats.begin)
            page_stats, next_cursor, more = query.fetch_page(
                count - len(stats_list),
                start_cursor=Cursor(urlsafe=next_cursor or cursor))
            next_cursor = next_cursor.urlsafe() if next_cursor else ''
            for stats in page_stats:
                if not names or stats.has_any_names(names):
                    stats_list.append(stats)

    ensure_last_change_timestamp(interval_minutes)

    return {
        'results': [stats.to_dict(names) for stats in stats_list],
        'cursor': next_cursor,
        'more': more,
    }
Esempio n. 14
0
 def get(self, ranking, name, cq_stats_key): # pylint: disable=R0201
   cq_stats = CQStats.get_by_id(int(cq_stats_key))
   assert cq_stats, '%s must match a CQStats entry.' % cq_stats_key
   for stats in cq_stats.count_stats + cq_stats.list_stats:
     if stats.name == name:
       if ranking == 'lowest':
         return stats.lowest_100
       assert ranking == 'highest'
       return stats.highest_100
   assert False, '%s must match a stat in the specified CQStats entry.' % name
Esempio n. 15
0
def missing_intervals(minutes, end):
    last_cq_stats = CQStats.query().filter(
        CQStats.interval_minutes == minutes).order(-CQStats.end).get()
    if last_cq_stats:
        return intervals_in_range(minutes, last_cq_stats.end, end)
    earliest_record = Record.query().order(Record.timestamp).get()
    if earliest_record:
        begin = earliest_record.timestamp - timedelta(minutes=minutes)
        return intervals_in_range(minutes, begin, end)
    return []
Esempio n. 16
0
 def test_missing_intervals_mismatched_cq_stats(self):
     CQStats(project='',
             interval_minutes=60,
             begin=datetime(2000, 1, 3, 7),
             end=datetime(2000, 1, 3, 8)).put()
     self.mock_now(datetime(2000, 1, 2, 0))
     Record().put()
     self.assertEqual([
         (datetime(2000, 1, 1, 8), datetime(2000, 1, 2, 8)),
         (datetime(2000, 1, 2, 8), datetime(2000, 1, 3, 8)),
     ], missing_intervals(1440, datetime(2000, 1, 4, 0)))
Esempio n. 17
0
def execute_query(key, project, interval_minutes, begin, end, names,
    count, cursor): # pragma: no cover
  stats_list = []
  next_cursor = ''
  if key and count > 0:
    stats = CQStats.get_by_id(key)
    if stats and (
        (not project or stats.project == project) and
        (not interval_minutes or stats.interval_minutes == interval_minutes) and
        (not begin or stats.begin >= begin) and
        (not end or stats.end <= end) and
        (not names or stats.has_any_names(names))):
      stats_list.append(stats)
    more = False
  else:
    more = True
    while more and len(stats_list) < count:
      filters = []
      if project:
        filters.append(CQStats.project == project)
      if interval_minutes:
        filters.append(CQStats.interval_minutes == interval_minutes)
      if begin:
        filters.append(CQStats.begin >= begin)
      if end:
        filters.append(CQStats.begin <= end)
      query = CQStats.query().filter(*filters).order(-CQStats.begin)
      page_stats, next_cursor, more = query.fetch_page(count - len(stats_list),
          start_cursor=Cursor(urlsafe=next_cursor or cursor))
      next_cursor = next_cursor.urlsafe() if next_cursor else ''
      for stats in page_stats:
        if not names or stats.has_any_names(names):
          stats_list.append(stats)

  ensure_last_change_timestamp(interval_minutes)

  return {
    'results': [stats.to_dict(names) for stats in stats_list],
    'cursor': next_cursor,
    'more': more,
  }
Esempio n. 18
0
def _add_stats(project, days, begin, stats_list=None): # pragma: no cover
  minutes = days * minutes_per_day
  cq_stats = CQStats(
    project=project,
    interval_minutes=minutes,
    begin=datetime.utcfromtimestamp(begin),
    end=datetime.utcfromtimestamp(begin) + timedelta(minutes=minutes),
  )
  if stats_list:
    cq_stats.count_stats = [
        stats for stats in stats_list if type(stats) == CountStats]
    cq_stats.list_stats = [
        stats for stats in stats_list if type(stats) == ListStats]
  cq_stats.put()
  return cq_stats
Esempio n. 19
0
def _clear_cq_stats(): # pragma: no cover
  for cq_stats in CQStats.query():
    cq_stats.key.delete()
  assert CQStats.query().count() == 0
Esempio n. 20
0
 def test_update_cq_stats_empty(self):
     _clear_cq_stats()
     update_cq_stats({}, 60, datetime(2000, 1, 1, 1),
                     datetime(2000, 1, 1, 2))
     self.assertEqual(0, CQStats.query().count())
Esempio n. 21
0
def _clear_cq_stats():  # pragma: no cover
    for cq_stats in CQStats.query():
        cq_stats.key.delete()
    assert CQStats.query().count() == 0
Esempio n. 22
0
 def test_update_cq_stats_empty(self):
   _clear_cq_stats()
   update_cq_stats({}, 60, datetime(2000, 1, 1, 1), datetime(2000, 1, 1, 2))
   self.assertEqual(0, CQStats.query().count())
Esempio n. 23
0
def _clear_stats():  # pragma: no cover
    for cq_stats in CQStats.query():
        cq_stats.key.delete()
    assert CQStats.query().count() == 0
    memcache.flush_all()
Esempio n. 24
0
 def clear_cq_stats():
   for cq_stats in CQStats.query():
     cq_stats.key.delete()
   assert CQStats.query().count() == 0
Esempio n. 25
0
 def get_stats(name):
   cq_stats = CQStats.query(CQStats.begin == stats_start).get()
   for stats in cq_stats.count_stats + cq_stats.list_stats:
     if stats.name == name:
       return stats
   return None
Esempio n. 26
0
 def clear_cq_stats():
     for cq_stats in CQStats.query():
         cq_stats.key.delete()
     assert CQStats.query().count() == 0
Esempio n. 27
0
 def get_stats(name):
     cq_stats = CQStats.query(CQStats.begin == stats_start).get()
     for stats in cq_stats.count_stats + cq_stats.list_stats:
         if stats.name == name:
             return stats
     return None