def test_update_cq_stats_create(self):
_clear_cq_stats()
update_cq_stats({
'test-project': [
CountStats(
name='test-count',
description='test-count-description',
count=123),
ListStats(
name='test-list',
description='test-list-description',
unit='test-unit'),
],
}, 60, datetime(2000, 1, 1, 1), datetime(2000, 1, 1, 2))
self.assertEqual(1, CQStats.query().count())
cq_stats = CQStats.query().get()
self.assertEqual('test-project', cq_stats.project)
self.assertEqual(60, cq_stats.interval_minutes)
self.assertEqual(datetime(2000, 1, 1, 1), cq_stats.begin)
self.assertEqual(datetime(2000, 1, 1, 2), cq_stats.end)
self.assertEqual([
CountStats(
name='test-count',
description='test-count-description',
count=123),
], cq_stats.count_stats)
self.assertEqual([
ListStats(
name='test-list',
description='test-list-description',
unit='test-unit'),
], cq_stats.list_stats)
def _clear_ndb(): # pragma: no cover
for cq_stats in CQStats.query():
cq_stats.key.delete()
assert CQStats.query().count() == 0
for record in Record.query():
record.key.delete()
assert Record.query().count() == 0
def update_cq_stats(project_stats, minutes, begin, end): # pragma: no cover
"""Ensure CQStats are updated or created as necessary with new stats data."""
assert (end - begin).total_seconds() == 60 * minutes
assert (begin - stats_start).total_seconds() % (60 * minutes) == 0, (
'Interval must be aligned with %s.' % stats_start)
for project, stats_list in project_stats.iteritems():
count_stats_dict = {}
list_stats_dict = {}
for stats in stats_list:
if type(stats) == CountStats:
count_stats_dict[stats.name] = stats
else:
assert type(stats) == ListStats
list_stats_dict[stats.name] = stats
cq_stats = CQStats.query().filter(CQStats.project == project,
CQStats.interval_minutes == minutes,
CQStats.begin == begin).get()
if cq_stats:
update_stats_list(cq_stats.count_stats, count_stats_dict)
update_stats_list(cq_stats.list_stats, list_stats_dict)
cq_stats.put()
else:
CQStats(
project=project,
interval_minutes=minutes,
begin=begin,
end=end,
count_stats=count_stats_dict.values(),
list_stats=list_stats_dict.values(),
).put()
update_last_cq_stats_change_timestamp(minutes)
def test_update_cq_stats_create(self):
_clear_cq_stats()
update_cq_stats(
{
'test-project': [
CountStats(name='test-count',
description='test-count-description',
count=123),
ListStats(name='test-list',
description='test-list-description',
unit='test-unit'),
],
}, 60, datetime(2000, 1, 1, 1), datetime(2000, 1, 1, 2))
self.assertEqual(1, CQStats.query().count())
cq_stats = CQStats.query().get()
self.assertEqual('test-project', cq_stats.project)
self.assertEqual(60, cq_stats.interval_minutes)
self.assertEqual(datetime(2000, 1, 1, 1), cq_stats.begin)
self.assertEqual(datetime(2000, 1, 1, 2), cq_stats.end)
self.assertEqual([
CountStats(name='test-count',
description='test-count-description',
count=123),
], cq_stats.count_stats)
self.assertEqual([
ListStats(name='test-list',
description='test-list-description',
unit='test-unit'),
], cq_stats.list_stats)
def post(handler): # pragma: no cover
if handler.request.get('all'):
stats_list = CQStats.query()
else:
stats_list = []
for key in handler.request.get('keys').split(','):
stats = CQStats.get_by_id(int(key))
assert stats, '%s must exist.' % key
stats_list.append(stats)
handler.response.write('CQStats removed: [\n')
for stats in stats_list:
handler.response.write(' %s,\n' % stats)
stats.key.delete()
handler.response.write(']\n')
def post(handler): # pragma: no cover
if handler.request.get('all'):
stats_list = CQStats.query()
else:
stats_list = []
for key in handler.request.get('keys').split(','):
stats = CQStats.get_by_id(int(key))
assert stats, '%s must exist.' % key
stats_list.append(stats)
handler.response.write('CQStats removed: [\n')
for stats in stats_list:
handler.response.write(' %s,\n' % stats)
stats.key.delete()
handler.response.write(']\n')
def update_cq_stats(project_stats, minutes, begin, end): # pragma: no cover
"""Ensure CQStats are updated or created as necessary with new stats data."""
assert (end - begin).total_seconds() == 60 * minutes
assert (begin - stats_start).total_seconds() % (60 * minutes) == 0, (
'Interval must be aligned with %s.' % stats_start)
for project, stats_list in project_stats.iteritems():
count_stats_dict = {}
list_stats_dict = {}
for stats in stats_list:
if type(stats) == CountStats:
count_stats_dict[stats.name] = stats
else:
assert type(stats) == ListStats
list_stats_dict[stats.name] = stats
cq_stats = CQStats.query().filter(
CQStats.project == project,
CQStats.interval_minutes == minutes,
CQStats.begin == begin).get()
if cq_stats:
update_stats_list(cq_stats.count_stats, count_stats_dict)
update_stats_list(cq_stats.list_stats, list_stats_dict)
cq_stats.put()
else:
CQStats(
project=project,
interval_minutes=minutes,
begin=begin,
end=end,
count_stats=count_stats_dict.values(),
list_stats=list_stats_dict.values(),
).put()
update_last_cq_stats_change_timestamp(minutes)
def test_missing_intervals_matched_cq_stats(self):
CQStats(project='',
interval_minutes=1440,
begin=datetime(2000, 1, 2, 8),
end=datetime(2000, 1, 3, 8)).put()
self.assertEqual([
(datetime(2000, 1, 3, 8), datetime(2000, 1, 4, 8)),
], missing_intervals(1440, datetime(2000, 1, 4, 8)))
def _reset_stats(): # pragma: no cover
for cq_stats in CQStats.query():
cq_stats.key.delete()
assert CQStats.query().count() == 0
cq_stats = CQStats(
project='test',
interval_minutes=1,
begin=datetime.utcfromtimestamp(0),
end=datetime.utcfromtimestamp(1),
count_stats=[
CountStats(
name='test-count',
description='Test count description',
count=3,
highest_100=[
[2, {'data_point': 'c'}],
[1, {'data_point': 'b'}],
[0, {'data_point': 'a'}],
],
lowest_100=[
[0, {'data_point': 'a'}],
[1, {'data_point': 'b'}],
[2, {'data_point': 'c'}],
],
),
],
list_stats=[
ListStats(
name='test-list',
description='Test list description',
unit='test_unit',
highest_100=[
[2, {'data_point': 'c'}],
[1, {'data_point': 'b'}],
[0, {'data_point': 'a'}],
],
lowest_100=[
[0, {'data_point': 'a'}],
[1, {'data_point': 'b'}],
[2, {'data_point': 'c'}],
],
),
],
).put()
return cq_stats.id()
def _reset_stats(): # pragma: no cover
for cq_stats in CQStats.query():
cq_stats.key.delete()
assert CQStats.query().count() == 0
cq_stats = CQStats(
project='test',
interval_minutes=1,
begin=datetime.utcfromtimestamp(0),
end=datetime.utcfromtimestamp(1),
count_stats=[
CountStats(
name='test-count',
description='Test count description',
count=3,
highest_100=[
[2, {'data_point': 'c'}],
[1, {'data_point': 'b'}],
[0, {'data_point': 'a'}],
],
lowest_100=[
[0, {'data_point': 'a'}],
[1, {'data_point': 'b'}],
[2, {'data_point': 'c'}],
],
),
],
list_stats=[
ListStats(
name='test-list',
description='Test list description',
unit='test_unit',
highest_100=[
[2, {'data_point': 'c'}],
[1, {'data_point': 'b'}],
[0, {'data_point': 'a'}],
],
lowest_100=[
[0, {'data_point': 'a'}],
[1, {'data_point': 'b'}],
[2, {'data_point': 'c'}],
],
),
],
).put()
return cq_stats.id()
def get(self, ranking, name, cq_stats_key): # pylint: disable=R0201
cq_stats = CQStats.get_by_id(int(cq_stats_key))
assert cq_stats, '%s must match a CQStats entry.' % cq_stats_key
for stats in cq_stats.count_stats + cq_stats.list_stats:
if stats.name == name:
if ranking == 'lowest':
return stats.lowest_100
assert ranking == 'highest'
return stats.highest_100
assert False, '%s must match a stat in the specified CQStats entry.' % name
def missing_intervals(minutes, end):
last_cq_stats = CQStats.query().filter(
CQStats.interval_minutes == minutes).order(-CQStats.end).get()
if last_cq_stats:
return intervals_in_range(minutes, last_cq_stats.end, end)
earliest_record = Record.query().order(Record.timestamp).get()
if earliest_record:
begin = earliest_record.timestamp - timedelta(minutes=minutes)
return intervals_in_range(minutes, begin, end)
return []
def execute_query(key, project, interval_minutes, begin, end, names, count,
cursor):
stats_list = []
next_cursor = ''
if key and count > 0:
stats = CQStats.get_by_id(key)
if stats and ( # pragma: no branch
(not project or stats.project == project) and
(not interval_minutes
or stats.interval_minutes == interval_minutes) and
(not begin or stats.begin >= begin) and
(not end or stats.end <= end) and
(not names or stats.has_any_names(names))):
stats_list.append(stats)
more = False
else:
more = True
while more and len(stats_list) < count:
filters = []
if project:
filters.append(CQStats.project == project)
if interval_minutes:
filters.append(CQStats.interval_minutes == interval_minutes)
if begin:
filters.append(CQStats.begin >= begin)
if end:
filters.append(CQStats.begin <= end)
query = CQStats.query().filter(*filters).order(-CQStats.begin)
page_stats, next_cursor, more = query.fetch_page(
count - len(stats_list),
start_cursor=Cursor(urlsafe=next_cursor or cursor))
next_cursor = next_cursor.urlsafe() if next_cursor else ''
for stats in page_stats:
if not names or stats.has_any_names(names):
stats_list.append(stats)
ensure_last_change_timestamp(interval_minutes)
return {
'results': [stats.to_dict(names) for stats in stats_list],
'cursor': next_cursor,
'more': more,
}
def get(self, ranking, name, cq_stats_key): # pylint: disable=R0201
cq_stats = CQStats.get_by_id(int(cq_stats_key))
assert cq_stats, '%s must match a CQStats entry.' % cq_stats_key
for stats in cq_stats.count_stats + cq_stats.list_stats:
if stats.name == name:
if ranking == 'lowest':
return stats.lowest_100
assert ranking == 'highest'
return stats.highest_100
assert False, '%s must match a stat in the specified CQStats entry.' % name
def missing_intervals(minutes, end):
last_cq_stats = CQStats.query().filter(
CQStats.interval_minutes == minutes).order(-CQStats.end).get()
if last_cq_stats:
return intervals_in_range(minutes, last_cq_stats.end, end)
earliest_record = Record.query().order(Record.timestamp).get()
if earliest_record:
begin = earliest_record.timestamp - timedelta(minutes=minutes)
return intervals_in_range(minutes, begin, end)
return []
def test_missing_intervals_mismatched_cq_stats(self):
CQStats(project='',
interval_minutes=60,
begin=datetime(2000, 1, 3, 7),
end=datetime(2000, 1, 3, 8)).put()
self.mock_now(datetime(2000, 1, 2, 0))
Record().put()
self.assertEqual([
(datetime(2000, 1, 1, 8), datetime(2000, 1, 2, 8)),
(datetime(2000, 1, 2, 8), datetime(2000, 1, 3, 8)),
], missing_intervals(1440, datetime(2000, 1, 4, 0)))
def execute_query(key, project, interval_minutes, begin, end, names,
count, cursor): # pragma: no cover
stats_list = []
next_cursor = ''
if key and count > 0:
stats = CQStats.get_by_id(key)
if stats and (
(not project or stats.project == project) and
(not interval_minutes or stats.interval_minutes == interval_minutes) and
(not begin or stats.begin >= begin) and
(not end or stats.end <= end) and
(not names or stats.has_any_names(names))):
stats_list.append(stats)
more = False
else:
more = True
while more and len(stats_list) < count:
filters = []
if project:
filters.append(CQStats.project == project)
if interval_minutes:
filters.append(CQStats.interval_minutes == interval_minutes)
if begin:
filters.append(CQStats.begin >= begin)
if end:
filters.append(CQStats.begin <= end)
query = CQStats.query().filter(*filters).order(-CQStats.begin)
page_stats, next_cursor, more = query.fetch_page(count - len(stats_list),
start_cursor=Cursor(urlsafe=next_cursor or cursor))
next_cursor = next_cursor.urlsafe() if next_cursor else ''
for stats in page_stats:
if not names or stats.has_any_names(names):
stats_list.append(stats)
ensure_last_change_timestamp(interval_minutes)
return {
'results': [stats.to_dict(names) for stats in stats_list],
'cursor': next_cursor,
'more': more,
}
def _add_stats(project, days, begin, stats_list=None): # pragma: no cover
minutes = days * minutes_per_day
cq_stats = CQStats(
project=project,
interval_minutes=minutes,
begin=datetime.utcfromtimestamp(begin),
end=datetime.utcfromtimestamp(begin) + timedelta(minutes=minutes),
)
if stats_list:
cq_stats.count_stats = [
stats for stats in stats_list if type(stats) == CountStats]
cq_stats.list_stats = [
stats for stats in stats_list if type(stats) == ListStats]
cq_stats.put()
return cq_stats
def _clear_cq_stats(): # pragma: no cover
for cq_stats in CQStats.query():
cq_stats.key.delete()
assert CQStats.query().count() == 0
def test_update_cq_stats_empty(self):
_clear_cq_stats()
update_cq_stats({}, 60, datetime(2000, 1, 1, 1),
datetime(2000, 1, 1, 2))
self.assertEqual(0, CQStats.query().count())
def _clear_cq_stats(): # pragma: no cover
for cq_stats in CQStats.query():
cq_stats.key.delete()
assert CQStats.query().count() == 0
def test_update_cq_stats_empty(self):
_clear_cq_stats()
update_cq_stats({}, 60, datetime(2000, 1, 1, 1), datetime(2000, 1, 1, 2))
self.assertEqual(0, CQStats.query().count())
def _clear_stats(): # pragma: no cover
for cq_stats in CQStats.query():
cq_stats.key.delete()
assert CQStats.query().count() == 0
memcache.flush_all()
def clear_cq_stats():
for cq_stats in CQStats.query():
cq_stats.key.delete()
assert CQStats.query().count() == 0
def get_stats(name):
cq_stats = CQStats.query(CQStats.begin == stats_start).get()
for stats in cq_stats.count_stats + cq_stats.list_stats:
if stats.name == name:
return stats
return None
def clear_cq_stats():
for cq_stats in CQStats.query():
cq_stats.key.delete()
assert CQStats.query().count() == 0
def get_stats(name):
cq_stats = CQStats.query(CQStats.begin == stats_start).get()
for stats in cq_stats.count_stats + cq_stats.list_stats:
if stats.name == name:
return stats
return None
