def test_cron_aggregate_dimensions(self):
_ensure_bot_info(bot_id='id1',
dimensions={
'pool': ['p1', 'p2'],
'foo1': ['bar1']
})
_ensure_bot_info(bot_id='id2',
dimensions={
'pool': ['p3'],
'foo2': ['bar2']
})
bot_management.cron_aggregate_dimensions()
# dimensions of all pools.
dims_all = bot_management.get_aggregation_key('all').get().dimensions
self.assertEqual(dims_all, [
bot_management.DimensionValues(dimension='foo1', values=['bar1']),
bot_management.DimensionValues(dimension='foo2', values=['bar2']),
bot_management.DimensionValues(dimension='pool',
values=['p1', 'p2', 'p3']),
])
# dimensions of p1.
dims_p1 = bot_management.get_aggregation_key('p1').get().dimensions
self.assertEqual(dims_p1, [
bot_management.DimensionValues(dimension='foo1', values=['bar1']),
bot_management.DimensionValues(dimension='pool',
values=['p1', 'p2']),
])
# dimensions of p2.
dims_p1 = bot_management.get_aggregation_key('p2').get().dimensions
self.assertEqual(dims_p1, [
bot_management.DimensionValues(dimension='foo1', values=['bar1']),
bot_management.DimensionValues(dimension='pool',
values=['p1', 'p2']),
])
# dimensions of p2.
dims_p1 = bot_management.get_aggregation_key('p3').get().dimensions
self.assertEqual(dims_p1, [
bot_management.DimensionValues(dimension='foo2', values=['bar2']),
bot_management.DimensionValues(dimension='pool', values=['p3']),
])
def test_cron_monitoring_bots_aggregate_dimensions(self):
# Tests that the aggregation works
now = datetime.datetime(2010, 1, 2, 3, 4, 5)
self.mock_now(now)
bot_management.bot_event(event_type='request_sleep',
bot_id='id1',
external_ip='8.8.4.4',
authenticated_as='bot:whitelisted-ip',
dimensions={
'foo': ['beta'],
'id': ['id1']
},
state={'ram': 65},
version='123456789',
quarantined=False,
maintenance_msg=None,
task_id=None,
task_name=None,
register_dimensions=True)
bot_management.bot_event(event_type='request_sleep',
bot_id='id2',
external_ip='8.8.4.4',
authenticated_as='bot:whitelisted-ip',
dimensions={
'foo': ['alpha'],
'id': ['id2']
},
state={'ram': 65},
version='123456789',
quarantined=True,
maintenance_msg=None,
task_id='987',
task_name=None,
register_dimensions=True)
self.app.get('/internal/cron/monitoring/bots/aggregate_dimensions',
headers={'X-AppEngine-Cron': 'true'},
status=200)
agg_key = bot_management.get_aggregation_key('all')
actual = agg_key.get()
expected = bot_management.DimensionAggregation(
key=agg_key,
dimensions=[
bot_management.DimensionValues(dimension='foo',
values=['alpha', 'beta'])
],
ts=now)
self.assertEqual(expected, actual)
def get(self):
seen = {}
now = utils.utcnow()
for b in bot_management.BotInfo.query():
for i in b.dimensions_flat:
k, v = i.split(':', 1)
if k != 'id':
seen.setdefault(k, set()).add(v)
dims = [
bot_management.DimensionValues(dimension=k, values=sorted(values))
for k, values in sorted(seen.iteritems())
]
logging.info('Saw dimensions %s', dims)
bot_management.DimensionAggregation(
key=bot_management.DimensionAggregation.KEY,
dimensions=dims,
ts=now).put()
def testCronBotsAggregateTask(self):
# Tests that the aggregation works
now = datetime.datetime(2010, 1, 2, 3, 4, 5)
self.mock_now(now)
bot_management.bot_event(event_type='bot_connected',
bot_id='id1',
external_ip='8.8.4.4',
authenticated_as='bot:whitelisted-ip',
dimensions={
'foo': ['beta'],
'id': ['id1']
},
state={'ram': 65},
version='123456789',
quarantined=False,
task_id=None,
task_name=None)
bot_management.bot_event(event_type='bot_connected',
bot_id='id2',
external_ip='8.8.4.4',
authenticated_as='bot:whitelisted-ip',
dimensions={
'foo': ['alpha'],
'id': ['id2']
},
state={'ram': 65},
version='123456789',
quarantined=True,
task_id='987',
task_name=None)
self.app.get('/internal/cron/aggregate_bots_dimensions',
headers={'X-AppEngine-Cron': 'true'},
status=200)
actual = bot_management.DimensionAggregation.KEY.get()
expected = bot_management.DimensionAggregation(
key=bot_management.DimensionAggregation.KEY,
dimensions=[
bot_management.DimensionValues(dimension='foo',
values=['alpha', 'beta'])
],
ts=now)
self.assertEqual(expected, actual)