def test_BotEvent_proto_maintenance(self):
# Also test a misconfigured bot not in a pool.
event_key = _bot_event(event_type=u'bot_connected',
bot_id=u'id1',
dimensions={u'id': [u'id1']},
maintenance_msg=u'Too hot')
actual = swarming_pb2.BotEvent()
event_key.get().to_proto(actual)
expected = swarming_pb2.BotEvent(
event=swarming_pb2.BOT_NEW_SESSION,
bot=swarming_pb2.Bot(
bot_id=u'id1',
dimensions=[
swarming_pb2.StringListPair(key=u'id', values=[u'id1']),
],
status=swarming_pb2.OVERHEAD_MAINTENANCE_EXTERNAL,
status_msg=u'Too hot',
info=swarming_pb2.BotInfo(
supplemental=struct_pb2.Struct(
fields={
u'ram': struct_pb2.Value(number_value=65),
}),
version=_VERSION,
external_ip=u'8.8.4.4',
authenticated_as=u'bot:id1.domain',
),
),
)
expected.event_time.FromDatetime(self.now)
self.assertEqual(unicode(expected), unicode(actual))
def test_BotEvent_proto_quarantine(self):
# Also test that a bot can belong to two pools.
event_key = bot_management.bot_event(
event_type=u'bot_connected',
bot_id=u'id1',
external_ip=u'8.8.4.4',
authenticated_as=u'bot:id1.domain',
dimensions={
u'id': [u'id1'],
u'pool': [u'next', u'previous']
},
state={
u'ram': 65.0,
u'quarantined': u'sad bot'
},
version=_VERSION,
quarantined=True,
maintenance_msg=None,
task_id=None,
task_name=None)
actual = swarming_pb2.BotEvent()
event_key.get().to_proto(actual)
expected = swarming_pb2.BotEvent(
event=swarming_pb2.BOT_NEW_SESSION,
bot=swarming_pb2.Bot(
bot_id=u'id1',
pools=[u'next', u'previous'],
dimensions=[
swarming_pb2.StringListPair(key=u'id', values=[u'id1']),
swarming_pb2.StringListPair(key=u'pool',
values=[u'next', u'previous']),
],
status=swarming_pb2.QUARANTINED_BY_BOT,
status_msg=u'sad bot',
info=swarming_pb2.BotInfo(
supplemental=struct_pb2.Struct(
fields={
u'quarantined':
struct_pb2.Value(string_value=u'sad bot'),
u'ram':
struct_pb2.Value(number_value=65.0),
}),
version=_VERSION,
external_ip=u'8.8.4.4',
authenticated_as=u'bot:id1.domain',
),
),
)
expected.event_time.FromDatetime(self.now)
self.assertEqual(unicode(expected), unicode(actual))
def _test_bot_events_simple(self, request):
self.set_as_bot()
self.do_handshake()
self.set_as_user()
raw_resp = self.app.post(
'/prpc/swarming.v1.BotAPI/Events', _encode(request), self._headers)
expected = swarming_pb2.BotEventsResponse(
events=[
swarming_pb2.BotEvent(
event_time=timestamp_pb2.Timestamp(seconds=1262401445),
bot=swarming_pb2.Bot(
bot_id='bot1',
pools=['default'],
info=swarming_pb2.BotInfo(
supplemental=struct_pb2.Struct(
fields={
'running_time': struct_pb2.Value(number_value=1234.0),
'sleep_streak': struct_pb2.Value(number_value=0),
'started_ts': struct_pb2.Value(number_value=1410990411.11),
}),
external_ip='192.168.2.2',
authenticated_as='bot:whitelisted-ip',
version='123',
),
dimensions=[
swarming_pb2.StringListPair(key='id', values=['bot1']),
swarming_pb2.StringListPair(key='os', values=['Amiga']),
swarming_pb2.StringListPair(key='pool', values=['default']),
]),
event=swarming_pb2.BOT_NEW_SESSION,
),
])
resp = swarming_pb2.BotEventsResponse()
_decode(raw_resp.body, resp)
self.assertEqual(unicode(expected), unicode(resp))
def test_BotEvent_proto_events(self):
# Ensures all bot event states can be converted to a proto.
dimensions = {
u'id': [u'id1'],
u'os': [u'Ubuntu', u'Ubuntu-16.04'],
u'pool': [u'default'],
}
for name in bot_management.BotEvent.ALLOWED_EVENTS:
event_key = bot_management.bot_event(
event_type=name,
bot_id=u'id1',
external_ip=u'8.8.4.4',
authenticated_as=u'bot:id1.domain',
dimensions=dimensions,
state={u'ram': 65},
version=_VERSION,
quarantined=False,
maintenance_msg=None,
task_id=None,
task_name=None)
if name in (u'request_sleep', u'task_update'):
# TODO(maruel): Store request_sleep IFF the state changed.
self.assertIsNone(event_key, name)
continue
# Just asserts it doesn't crash.
actual = swarming_pb2.BotEvent()
event_key.get().to_proto(actual)
def test_BotEvent_proto_events(self):
# Ensures all bot event states can be converted to a proto.
dimensions = {
u'id': [u'id1'],
u'os': [u'Ubuntu', u'Ubuntu-16.04'],
u'pool': [u'default'],
}
for name in bot_management.BotEvent.ALLOWED_EVENTS:
event_key = _bot_event(event_type=name,
bot_id=u'id1',
dimensions=dimensions)
if name in (u'request_sleep', u'task_update'):
# TODO(maruel): Store request_sleep IFF the state changed.
self.assertIsNone(event_key, name)
continue
# Just asserts it doesn't crash.
actual = swarming_pb2.BotEvent()
event_key.get().to_proto(actual)
def _convert(e):
"""Returns a tuple(bq_key, row)."""
out = swarming_pb2.BotEvent()
e.to_proto(out)
bq_key = e.id + ':' + e.ts.strftime(u'%Y-%m-%dT%H:%M:%S.%fZ')
return (bq_key, out)
def test_botevents(self):
# Run one task.
self.mock(random, 'getrandbits', lambda _: 0x88)
self.set_as_bot()
self.mock_now(self.now, 0)
params = self.do_handshake()
self.mock_now(self.now, 30)
self.bot_poll(params=params)
self.set_as_user()
now_60 = self.mock_now(self.now, 60)
self.client_create_task_raw()
self.set_as_bot()
self.mock_now(self.now, 120)
res = self.bot_poll(params=params)
now_180 = self.mock_now(self.now, 180)
response = self.bot_complete_task(task_id=res['manifest']['task_id'])
self.assertEqual({u'must_stop': False, u'ok': True}, response)
self.mock_now(self.now, 240)
params['event'] = 'bot_rebooting'
params['message'] = 'for the best'
# TODO(maruel): https://crbug.com/913953
response = self.post_json('/swarming/api/v1/bot/event', params)
self.assertEqual({}, response)
# Do not filter by time.
self.set_as_privileged_user()
msg = swarming_pb2.BotEventsRequest(bot_id=u'bot1', page_size=1001)
raw_resp = self.app.post(
'/prpc/swarming.v1.BotAPI/Events', _encode(msg), self._headers)
resp = swarming_pb2.BotEventsResponse()
_decode(raw_resp.body, resp)
dimensions = [
swarming_pb2.StringListPair(key='id', values=['bot1']),
swarming_pb2.StringListPair(key='os', values=['Amiga']),
swarming_pb2.StringListPair(key='pool', values=['default']),
]
common_info = swarming_pb2.BotInfo(
supplemental=struct_pb2.Struct(
fields={
'bot_group_cfg_version': struct_pb2.Value(string_value='default'),
'running_time': struct_pb2.Value(number_value=1234.0),
'sleep_streak': struct_pb2.Value(number_value=0),
'started_ts': struct_pb2.Value(number_value=1410990411.11),
}),
external_ip='192.168.2.2',
authenticated_as='bot:whitelisted-ip',
version=self.bot_version,
)
events = [
swarming_pb2.BotEvent(
event_time=timestamp_pb2.Timestamp(seconds=1262401685),
bot=swarming_pb2.Bot(
bot_id='bot1',
pools=[u'default'],
status=swarming_pb2.BOT_STATUS_UNSPECIFIED,
info=common_info,
dimensions=dimensions),
event=swarming_pb2.BOT_REBOOTING_HOST,
event_msg='for the best',
),
swarming_pb2.BotEvent(
event_time=timestamp_pb2.Timestamp(seconds=1262401625),
bot=swarming_pb2.Bot(
bot_id='bot1',
pools=[u'default'],
status=swarming_pb2.BUSY,
current_task_id='5cfcee8008811',
info=common_info,
dimensions=dimensions),
event=swarming_pb2.TASK_COMPLETED,
),
swarming_pb2.BotEvent(
event_time=timestamp_pb2.Timestamp(seconds=1262401565),
bot=swarming_pb2.Bot(
bot_id='bot1',
pools=[u'default'],
current_task_id='5cfcee8008811',
status=swarming_pb2.BUSY,
info=common_info,
dimensions=dimensions),
event=swarming_pb2.INSTRUCT_START_TASK,
),
swarming_pb2.BotEvent(
event_time=timestamp_pb2.Timestamp(seconds=1262401475),
bot=swarming_pb2.Bot(
bot_id='bot1',
pools=[u'default'],
info=common_info,
dimensions=dimensions),
event=swarming_pb2.INSTRUCT_IDLE,
),
swarming_pb2.BotEvent(
event_time=timestamp_pb2.Timestamp(seconds=1262401445),
bot=swarming_pb2.Bot(
bot_id='bot1',
pools=[u'default'],
status=swarming_pb2.BOT_STATUS_UNSPECIFIED,
info=swarming_pb2.BotInfo(
supplemental=struct_pb2.Struct(
fields={
'running_time': struct_pb2.Value(number_value=1234.0),
'sleep_streak': struct_pb2.Value(number_value=0),
'started_ts': struct_pb2.Value(
number_value=1410990411.11),
}),
external_ip='192.168.2.2',
authenticated_as='bot:whitelisted-ip',
version='123',
),
dimensions=dimensions),
event=swarming_pb2.BOT_NEW_SESSION,
),
]
self.assertEqual(len(events), len(resp.events))
for i, event in enumerate(events):
self.assertEqual(unicode(event), unicode(resp.events[i]))
# Now test with a subset. It will retrieve events 1 and 2.
msg = swarming_pb2.BotEventsRequest(bot_id=u'bot1')
msg.start_time.FromDatetime(now_60)
msg.end_time.FromDatetime(now_180 + datetime.timedelta(seconds=1))
raw_resp = self.app.post(
'/prpc/swarming.v1.BotAPI/Events', _encode(msg), self._headers)
resp = swarming_pb2.BotEventsResponse()
_decode(raw_resp.body, resp)
self.assertEqual(
unicode(swarming_pb2.BotEventsResponse(events=events[1:3])),
unicode(resp))
def test_BotEvent_proto_empty(self):
# Assert that it doesn't throw on empty entity.
actual = swarming_pb2.BotEvent()
bot_management.BotEvent().to_proto(actual)
self.assertEqual(swarming_pb2.BotEvent(), actual)
def test_cron_update_bot_info(self):
# Create two bots, one becomes dead, updating the cron job fixes composite.
timeout = bot_management.config.settings().bot_death_timeout_secs
def check(dead, alive):
q = bot_management.filter_availability(
bot_management.BotInfo.query(),
quarantined=None,
in_maintenance=None,
is_dead=True,
is_busy=None)
self.assertEqual(dead, [t.to_dict() for t in q])
q = bot_management.filter_availability(
bot_management.BotInfo.query(),
quarantined=None,
in_maintenance=None,
is_dead=False,
is_busy=None)
self.assertEqual(alive, [t.to_dict() for t in q])
_bot_event(event_type='request_sleep')
# One second before the timeout value.
then = self.mock_now(self.now, timeout - 1)
_bot_event(event_type='request_sleep',
bot_id='id2',
external_ip='8.8.4.4',
authenticated_as='bot:id2.domain',
dimensions={
'id': ['id2'],
'foo': ['bar']
})
bot1_alive = _gen_bot_info(first_seen_ts=self.now,
last_seen_ts=self.now)
bot1_dead = _gen_bot_info(
first_seen_ts=self.now,
last_seen_ts=self.now,
composite=[
bot_management.BotInfo.NOT_IN_MAINTENANCE,
bot_management.BotInfo.DEAD,
bot_management.BotInfo.HEALTHY,
bot_management.BotInfo.IDLE,
],
is_dead=True)
bot2_alive = _gen_bot_info(authenticated_as=u'bot:id2.domain',
dimensions={
u'foo': [u'bar'],
u'id': [u'id2']
},
first_seen_ts=then,
id='id2',
last_seen_ts=then)
check([], [bot1_alive, bot2_alive])
self.assertEqual(0, bot_management.cron_update_bot_info())
check([], [bot1_alive, bot2_alive])
# Just stale enough to trigger the dead logic.
then = self.mock_now(self.now, timeout)
# The cron job didn't run yet, so it still has ALIVE bit.
check([], [bot1_alive, bot2_alive])
self.assertEqual(1, bot_management.cron_update_bot_info())
# The cron job ran, so it's now correct.
check([bot1_dead], [bot2_alive])
# the last event should be bot_missing
events = list(bot_management.get_events_query('id1', order=True))
event = events[0]
bq_event = swarming_pb2.BotEvent()
event.to_proto(bq_event)
self.assertEqual(event.event_type, 'bot_missing')
self.assertEqual(event.last_seen_ts, bot1_dead['last_seen_ts'])
self.assertEqual(bq_event.event, swarming_pb2.BOT_MISSING)
self.assertEqual(bq_event.bot.status, swarming_pb2.MISSING)
last_seen_ts = timestamp_pb2.Timestamp()
last_seen_ts.FromDatetime(bot1_dead['last_seen_ts'])
self.assertEqual(bq_event.bot.info.last_seen_ts, last_seen_ts)