def get(self):
ndb.get_context().set_cache_policy(lambda _: False)
seen = {}
now = utils.utcnow()
count = 0
q = task_result.TaskResultSummary.query(
task_result.TaskResultSummary.modified_ts > now -
datetime.timedelta(hours=1))
cursor = None
while True:
tasks, cursor = datastore_utils.fetch_page(q, 1000, cursor)
count += len(tasks)
for t in tasks:
for i in t.tags:
k, v = i.split(':', 1)
s = seen.setdefault(k, set())
if s is not None:
s.add(v)
# 128 is arbitrary large number to avoid OOM
if len(s) >= 128:
logging.info(
'Limiting tag %s because there are too many',
k)
seen[k] = None
if not cursor or len(tasks) == 0:
break
tags = [
task_result.TagValues(tag=k, values=sorted(values or []))
for k, values in sorted(seen.iteritems())
]
logging.info('From %d tasks, saw %d tags', count, len(tags))
task_result.TagAggregation(key=task_result.TagAggregation.KEY,
tags=tags,
ts=now).put()
def test_cron_monitoring_tasks_aggregate_tags(self):
self.mock_default_pool_acl([])
self.set_as_admin()
now = datetime.datetime(2011, 1, 2, 3, 4, 5)
self.mock_now(now)
self.client_create_task_raw(tags=['alpha:beta', 'gamma:delta'])
self.assertEqual(1, self.execute_tasks())
self.client_create_task_raw(tags=['alpha:epsilon', 'zeta:theta'])
self.assertEqual(0, self.execute_tasks())
self.app.get('/internal/cron/monitoring/tasks/aggregate_tags',
headers={'X-AppEngine-Cron': 'true'},
status=200)
actual = task_result.TagAggregation.KEY.get()
expected = task_result.TagAggregation(
key=task_result.TagAggregation.KEY,
tags=[
task_result.TagValues(tag='alpha', values=['beta', 'epsilon']),
task_result.TagValues(tag='gamma', values=['delta']),
task_result.TagValues(tag='os', values=['Amiga']),
task_result.TagValues(tag='pool', values=['default']),
task_result.TagValues(tag='priority', values=['20']),
task_result.TagValues(tag='realm', values=['none']),
task_result.TagValues(tag='service_account', values=['none']),
task_result.TagValues(tag='swarming.pool.template',
values=['none']),
task_result.TagValues(tag='swarming.pool.version',
values=['pools_cfg_rev']),
task_result.TagValues(tag='user', values=['joe@localhost']),
task_result.TagValues(tag='zeta', values=['theta']),
],
ts=now)
self.assertEqual(expected, actual)
def testCronTagsAggregateTask(self):
self.set_as_admin()
now = datetime.datetime(2011, 1, 2, 3, 4, 5)
self.mock_now(now)
self.client_create_task_raw(tags=['alpha:beta', 'gamma:delta'])
self.client_create_task_raw(tags=['alpha:epsilon', 'zeta:theta'])
self.app.get('/internal/cron/aggregate_tasks_tags',
headers={'X-AppEngine-Cron': 'true'},
status=200)
actual = task_result.TagAggregation.KEY.get()
expected = task_result.TagAggregation(
key=task_result.TagAggregation.KEY,
tags=[
task_result.TagValues(tag='alpha', values=['beta', 'epsilon']),
task_result.TagValues(tag='gamma', values=['delta']),
task_result.TagValues(tag='os', values=['Amiga']),
task_result.TagValues(tag='pool', values=['default']),
task_result.TagValues(tag='priority', values=['10']),
task_result.TagValues(tag='service_account', values=['none']),
task_result.TagValues(tag='user', values=['joe@localhost']),
task_result.TagValues(tag='zeta', values=['theta']),
],
ts=now)
self.assertEqual(expected, actual)