def post(self):
res = self._process()
event = res.request.get('event')
if event not in self.ALLOWED_EVENTS:
logging.error('Unexpected event type')
self.abort_with_error(400, error='Unsupported event type')
message = res.request.get('message')
# Record the event in a BotEvent entity so it can be listed on the bot's
# page.
bot_management.bot_event(
event_type=event,
bot_id=res.bot_id,
external_ip=self.request.remote_addr,
authenticated_as=auth.get_peer_identity().to_bytes(),
dimensions=res.dimensions,
state=res.state,
version=res.version,
quarantined=bool(res.quarantined_msg),
maintenance_msg=res.maintenance_msg,
task_id=None,
task_name=None,
message=message)
if event == 'bot_error':
# Also logs this to ereporter2, so it will be listed in the server's
# hourly ereporter2 report. THIS IS NOISY so it should only be done with
# issues requiring action. In this case, include again the bot's URL since
# there's no context in the report. Redundantly include the bot id so
# messages are bucketted by bot.
line = ('%s\n'
'\nhttps://%s/restricted/bot/%s') % (
message, app_identity.get_default_version_hostname(),
res.bot_id)
ereporter2.log_request(self.request, source='bot', message=line)
self.send_response({})
def test_get_ok(self):
"""Asserts that get shows the tasks a specific bot has executed."""
self.set_as_privileged_user()
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
now_str = unicode(now.strftime(self.DATETIME_FORMAT))
bot_management.bot_event(
event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4',
dimensions={'foo': ['bar'], 'id': ['id1']}, state={'ram': 65},
version='123456789', quarantined=False, task_id=None, task_name=None)
expected = {
u'bot_id': u'id1',
u'dimensions': [
{u'key': u'foo', u'value': [u'bar']},
{u'key': u'id', u'value': [u'id1']},
],
u'external_ip': u'8.8.4.4',
u'first_seen_ts': now_str,
u'is_dead': False,
u'last_seen_ts': now_str,
u'quarantined': False,
u'state': u'{"ram":65}',
u'version': u'123456789',
}
response = self.call_api('get', body={'bot_id': 'id1'})
self.assertEqual(expected, response.json)
def post(self, task_id=None):
request = self.parse_body()
bot_id = request.get('id')
task_id = request.get('task_id', '')
message = request.get('message', 'unknown')
bot_management.bot_event(event_type='task_error',
bot_id=bot_id,
external_ip=self.request.remote_addr,
dimensions=None,
state=None,
version=None,
quarantined=None,
task_id=task_id,
task_name=None,
message=message)
line = ('Bot: https://%s/restricted/bot/%s\n'
'Task failed: https://%s/user/task/%s\n'
'%s') % (app_identity.get_default_version_hostname(), bot_id,
app_identity.get_default_version_hostname(), task_id,
message)
ereporter2.log_request(self.request, source='bot', message=line)
msg = log_unexpected_keys(self.EXPECTED_KEYS, request, self.request,
'bot', 'keys')
if msg:
self.abort_with_error(400, error=msg)
msg = task_scheduler.bot_kill_task(
task_pack.unpack_run_result_key(task_id), bot_id)
if msg:
logging.error(msg)
self.abort_with_error(400, error=msg)
self.send_response({})
def post(self):
res = self._process()
bot_management.bot_event(
event_type='bot_connected',
bot_id=res.bot_id,
external_ip=self.request.remote_addr,
authenticated_as=auth.get_peer_identity().to_bytes(),
dimensions=res.dimensions,
state=res.state,
version=res.version,
quarantined=bool(res.quarantined_msg),
maintenance_msg=res.maintenance_msg,
task_id='',
task_name=None,
message=res.quarantined_msg)
data = {
'bot_version': bot_code.get_bot_version(self.request.host_url)[0],
'server_version': utils.get_app_version(),
'bot_group_cfg_version': res.bot_group_cfg.version,
'bot_group_cfg': {
# Let the bot know its server-side dimensions (from bots.cfg file).
'dimensions': res.bot_group_cfg.dimensions,
},
}
if res.bot_group_cfg.bot_config_script_content:
logging.info('Injecting %s: %d bytes',
res.bot_group_cfg.bot_config_script,
len(res.bot_group_cfg.bot_config_script_content))
data['bot_config'] = res.bot_group_cfg.bot_config_script_content
self.send_response(data)
def _bot_event(bot_id=None,
external_ip='8.8.4.4',
authenticated_as=None,
dimensions=None,
state=None,
version=_VERSION,
quarantined=False,
maintenance_msg=None,
task_id=None,
task_name=None,
**kwargs):
"""Calls bot_management.bot_event with default arguments."""
if not dimensions:
dimensions = {
u'id': [u'id1'],
u'os': [u'Ubuntu', u'Ubuntu-16.04'],
u'pool': [u'default'],
}
if not bot_id:
bot_id = dimensions[u'id'][0]
if not authenticated_as:
authenticated_as = u'bot:%s.domain' % bot_id
bot_management.bot_event(bot_id=bot_id,
external_ip=external_ip,
authenticated_as=authenticated_as,
dimensions=dimensions,
state=state or {'ram': 65},
version=version,
quarantined=quarantined,
maintenance_msg=maintenance_msg,
task_id=task_id,
task_name=task_name,
**kwargs)
def test_bot_event(self):
# connected.
d = {
u'id': [u'id1'],
u'os': [u'Ubuntu', u'Ubuntu-16.04'],
u'pool': [u'default'],
}
bot_management.bot_event(event_type='bot_connected',
bot_id='id1',
external_ip='8.8.4.4',
authenticated_as='bot:id1.domain',
dimensions=d,
state={'ram': 65},
version=_VERSION,
quarantined=False,
maintenance_msg=None,
task_id=None,
task_name=None)
expected = _gen_bot_info()
self.assertEqual(expected,
bot_management.get_info_key('id1').get().to_dict())
self.assertEqual(['bot_connected', 5],
memcache.get('id1:2010-01-02T03:04',
namespace='BotEvents'))
def post(self, task_id=None):
request = self.parse_body()
bot_id = request.get('id')
task_id = request.get('task_id', '')
message = request.get('message', 'unknown')
bot_management.bot_event(
event_type='task_error', bot_id=bot_id,
external_ip=self.request.remote_addr, dimensions=None, state=None,
version=None, quarantined=None, task_id=task_id, task_name=None,
message=message)
line = (
'Bot: https://%s/restricted/bot/%s\n'
'Task failed: https://%s/user/task/%s\n'
'%s') % (
app_identity.get_default_version_hostname(), bot_id,
app_identity.get_default_version_hostname(), task_id,
message)
ereporter2.log_request(self.request, source='bot', message=line)
msg = log_unexpected_keys(
self.EXPECTED_KEYS, request, self.request, 'bot', 'keys')
if msg:
self.abort_with_error(400, error=msg)
msg = task_scheduler.bot_kill_task(
task_pack.unpack_run_result_key(task_id), bot_id)
if msg:
logging.error(msg)
self.abort_with_error(400, error=msg)
self.send_response({})
def test_api_bot(self):
self.set_as_privileged_user()
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
now_str = unicode(now.strftime(utils.DATETIME_FORMAT))
self.mock_now(now)
bot_management.bot_event(
event_type="bot_connected",
bot_id="id1",
external_ip="8.8.4.4",
dimensions={"foo": ["bar"], "id": ["id1"]},
state={"ram": 65},
version="123456789",
quarantined=False,
task_id=None,
task_name=None,
)
actual = self.app.get("/swarming/api/v1/client/bot/id1", status=200).json
expected = {
u"dimensions": {u"foo": [u"bar"], u"id": [u"id1"]},
u"external_ip": u"8.8.4.4",
u"first_seen_ts": now_str,
u"id": u"id1",
u"is_dead": False,
u"last_seen_ts": now_str,
u"quarantined": False,
u"state": {u"ram": 65},
u"task_id": None,
u"task_name": None,
u"version": u"123456789",
}
self.assertEqual(expected, actual)
def post(self):
(request, bot_id, version, state, dimensions, quarantined_msg) = self._process()
event = request.get("event")
if event not in ("bot_error", "bot_rebooting", "bot_shutdown"):
self.abort_with_error(400, error="Unsupported event type")
message = request.get("message")
bot_management.bot_event(
event_type=event,
bot_id=bot_id,
external_ip=self.request.remote_addr,
dimensions=dimensions,
state=state,
version=version,
quarantined=bool(quarantined_msg),
task_id=None,
task_name=None,
message=message,
)
if event == "bot_error":
line = ("Bot: https://%s/restricted/bot/%s\n" "Bot error:\n" "%s") % (
app_identity.get_default_version_hostname(),
bot_id,
message,
)
ereporter2.log_request(self.request, source="bot", message=line)
self.send_response({})
def post(self):
res = self._process()
bot_management.bot_event(
event_type='bot_connected',
bot_id=res.bot_id,
external_ip=self.request.remote_addr,
authenticated_as=auth.get_peer_identity().to_bytes(),
dimensions=res.dimensions,
state=res.state,
version=res.version,
quarantined=bool(res.quarantined_msg),
task_id='',
task_name=None,
message=res.quarantined_msg)
data = {
'bot_version': bot_code.get_bot_version(self.request.host_url),
'server_version': utils.get_app_version(),
'bot_group_cfg_version': res.bot_group_cfg.version,
'bot_group_cfg': {
# Let the bot know its server-side dimensions (from bots.cfg file).
'dimensions': res.bot_group_cfg.dimensions,
},
}
self.send_response(data)
def test_bot_event_poll_sleep(self):
bot_management.bot_event(event_type='request_sleep',
bot_id='id1',
external_ip='8.8.4.4',
authenticated_as='bot:id1.domain',
dimensions={
'id': ['id1'],
'foo': ['bar']
},
state={'ram': 65},
version=hashlib.sha256().hexdigest(),
quarantined=True,
maintenance_msg=None,
task_id=None,
task_name=None)
# Assert that BotInfo was updated too.
expected = self._gen_bot_info(composite=[
bot_management.BotInfo.NOT_IN_MAINTENANCE,
bot_management.BotInfo.ALIVE,
bot_management.BotInfo.NOT_MACHINE_PROVIDER,
bot_management.BotInfo.QUARANTINED,
bot_management.BotInfo.IDLE,
],
quarantined=True)
bot_info = bot_management.get_info_key('id1').get()
self.assertEqual(expected, bot_info.to_dict())
# No BotEvent is registered for 'poll'.
self.assertEqual([],
bot_management.get_events_query('id1', True).fetch())
def test_bot_event_busy(self):
bot_management.bot_event(event_type='request_task',
bot_id='id1',
external_ip='8.8.4.4',
authenticated_as='bot:id1.domain',
dimensions={
'id': ['id1'],
'foo': ['bar']
},
state={'ram': 65},
version=hashlib.sha256().hexdigest(),
quarantined=False,
maintenance_msg=None,
task_id='12311',
task_name='yo')
expected = self._gen_bot_info(composite=[
bot_management.BotInfo.NOT_IN_MAINTENANCE,
bot_management.BotInfo.ALIVE,
bot_management.BotInfo.NOT_MACHINE_PROVIDER,
bot_management.BotInfo.HEALTHY,
bot_management.BotInfo.BUSY,
],
task_id=u'12311',
task_name=u'yo')
bot_info = bot_management.get_info_key('id1').get()
self.assertEqual(expected, bot_info.to_dict())
expected = [
self._gen_bot_event(event_type=u'request_task', task_id=u'12311'),
]
self.assertEqual(expected, [
e.to_dict() for e in bot_management.get_events_query('id1', True)
])
def post(self):
(request, bot_id, version, state, dimensions,
quarantined_msg) = self._process()
event = request.get('event')
if event not in ('bot_error', 'bot_rebooting', 'bot_shutdown'):
self.abort_with_error(400, error='Unsupported event type')
message = request.get('message')
bot_management.bot_event(event_type=event,
bot_id=bot_id,
external_ip=self.request.remote_addr,
dimensions=dimensions,
state=state,
version=version,
quarantined=bool(quarantined_msg),
task_id=None,
task_name=None,
message=message)
if event == 'bot_error':
line = ('Bot: https://%s/restricted/bot/%s\n'
'Bot error:\n'
'%s') % (app_identity.get_default_version_hostname(),
bot_id, message)
ereporter2.log_request(self.request, source='bot', message=line)
self.send_response({})
def test_list_ok(self):
"""Asserts that BotInfo is returned for the appropriate set of bots."""
self.set_as_privileged_user()
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
now_str = unicode(now.strftime(self.DATETIME_FORMAT))
self.mock_now(now)
bot_management.bot_event(
event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4',
dimensions={'foo': ['bar'], 'id': ['id1']}, state={'ram': 65},
version='123456789', quarantined=False, task_id=None, task_name=None)
expected = {
u'items': [
{
u'bot_id': u'id1',
u'dimensions': [
{u'key': u'foo', u'value': [u'bar']},
{u'key': u'id', u'value': [u'id1']},
],
u'external_ip': u'8.8.4.4',
u'first_seen_ts': now_str,
u'is_dead': False,
u'last_seen_ts': now_str,
u'quarantined': False,
u'version': u'123456789',
},
],
u'death_timeout': unicode(config.settings().bot_death_timeout_secs),
u'now': unicode(now.strftime(self.DATETIME_FORMAT)),
}
request = swarming_rpcs.BotsRequest()
response = self.call_api('list', body=message_to_dict(request))
self.assertEqual(expected, response.json)
def _bot_event(bot_id, pool, caches, oses):
"""Calls bot_management.bot_event with default arguments."""
dimensions = {
u'id': [bot_id],
u'os': oses or [u'Linux', u'Ubuntu', u'Ubuntu-16.04'],
u'pool': [pool],
}
# Format is named_caches: {name: [['shortname', size], timestamp]}.
state = {
'named_caches':
{name: [['a', size], 10]
for name, size in caches.items()}
}
bot_management.bot_event(event_type='request_sleep',
bot_id=bot_id,
external_ip='8.8.4.4',
authenticated_as=u'bot:%s.domain' % bot_id,
dimensions=dimensions,
state=state or {'ram': 65},
version=unicode(hashlib.sha256().hexdigest()),
quarantined=False,
maintenance_msg=None,
task_id=None,
task_name=None,
register_dimensions=True)
def test_api_bot_delete(self):
self.set_as_admin()
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
state = {"dict": {"random": "values"}, "float": 0.0, "list": ["of", "things"], "str": u"uni"}
bot_management.bot_event(
event_type="bot_connected",
bot_id="id1",
external_ip="8.8.4.4",
dimensions={"foo": ["bar"], "id": ["id1"]},
state=state,
version="123456789",
quarantined=False,
task_id=None,
task_name=None,
)
token = self.get_client_token()
actual = self.app.delete(
"/swarming/api/v1/client/bot/id1", status=200, headers={"X-XSRF-Token": str(token)}
).json
expected = {u"deleted": True}
self.assertEqual(expected, actual)
actual = self.app.get("/swarming/api/v1/client/bot/id1", status=404).json
expected = {u"error": u"Bot not found"}
self.assertEqual(expected, actual)
def test_api_bot_delete(self):
self.set_as_admin()
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
state = {
'dict': {'random': 'values'},
'float': 0.,
'list': ['of', 'things'],
'str': u'uni',
}
bot_management.bot_event(
event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4',
dimensions={'foo': ['bar'], 'id': ['id1']}, state=state,
version='123456789', quarantined=False, task_id=None, task_name=None)
token = self.get_client_token()
actual = self.app.delete(
'/swarming/api/v1/client/bot/id1',
status=200,
headers={'X-XSRF-Token': str(token)}).json
expected = {
u'deleted': True,
}
self.assertEqual(expected, actual)
actual = self.app.get('/swarming/api/v1/client/bot/id1', status=404).json
expected = {
u'error': u'Bot not found',
}
self.assertEqual(expected, actual)
def test_api_bot(self):
self.set_as_privileged_user()
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
now_str = unicode(now.strftime(utils.DATETIME_FORMAT))
self.mock_now(now)
bot_management.bot_event(
event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4',
dimensions={'foo': ['bar'], 'id': ['id1']}, state={'ram': 65},
version='123456789', quarantined=False, task_id=None, task_name=None)
actual = self.app.get('/swarming/api/v1/client/bot/id1', status=200).json
expected = {
u'dimensions': {u'foo': [u'bar'], u'id': [u'id1']},
u'external_ip': u'8.8.4.4',
u'first_seen_ts': now_str,
u'id': u'id1',
u'is_dead': False,
u'last_seen_ts': now_str,
u'quarantined': False,
u'state': {u'ram': 65},
u'task_id': None,
u'task_name': None,
u'version': u'123456789',
}
self.assertEqual(expected, actual)
def test_bot_event_poll_sleep(self):
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
bot_management.bot_event(
event_type='request_sleep', bot_id='id1', external_ip='8.8.4.4',
dimensions={'id': ['id1'], 'foo': ['bar']}, state={'ram': 65},
version=hashlib.sha1().hexdigest(), quarantined=True, task_id=None,
task_name=None)
# Assert that BotInfo was updated too.
expected = {
'dimensions': {u'foo': [u'bar'], u'id': [u'id1']},
'external_ip': u'8.8.4.4',
'first_seen_ts': now,
'id': 'id1',
'last_seen_ts': now,
'quarantined': True,
'state': {u'ram': 65},
'task_id': None,
'task_name': None,
'version': u'da39a3ee5e6b4b0d3255bfef95601890afd80709',
}
bot_info = bot_management.get_info_key('id1').get()
self.assertEqual(expected, bot_info.to_dict())
self.assertEqual(False, bot_info.is_busy)
# No BotEvent is registered for 'poll'.
self.assertEqual([], bot_management.get_events_query('id1', True).fetch())
def test_delete_ok(self):
"""Assert that delete finds and deletes a bot."""
self.set_as_admin()
self.mock(acl, 'is_admin', lambda *_args, **_kwargs: True)
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
state = {
'dict': {
'random': 'values'
},
'float': 0.,
'list': ['of', 'things'],
'str': u'uni',
}
bot_management.bot_event(event_type='bot_connected',
bot_id='id1',
external_ip='8.8.4.4',
dimensions={
'foo': ['bar'],
'id': ['id1']
},
state=state,
version='123456789',
quarantined=False,
task_id=None,
task_name=None)
# delete the bot
response = self.call_api('delete', body={'bot_id': 'id1'})
self.assertEqual({u'deleted': True}, response.json)
# is it gone?
with self.call_should_fail('404'):
self.call_api('delete', body={'bot_id': 'id1'})
def test_bot_event_poll_sleep(self):
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
bot_management.bot_event(
event_type='request_sleep', bot_id='id1', external_ip='8.8.4.4',
dimensions={'id': ['id1'], 'foo': ['bar']}, state={'ram': 65},
version=hashlib.sha1().hexdigest(), quarantined=True, task_id=None,
task_name=None)
# Assert that BotInfo was updated too.
expected = {
'dimensions': {u'foo': [u'bar'], u'id': [u'id1']},
'external_ip': u'8.8.4.4',
'first_seen_ts': now,
'id': 'id1',
'last_seen_ts': now,
'quarantined': True,
'state': {u'ram': 65},
'task_id': None,
'task_name': None,
'version': u'da39a3ee5e6b4b0d3255bfef95601890afd80709',
}
bot_info = bot_management.get_info_key('id1').get()
self.assertEqual(expected, bot_info.to_dict())
self.assertEqual(False, bot_info.is_busy)
# No BotEvent is registered for 'poll'.
self.assertEqual([], bot_management.get_events_query('id1', True).fetch())
def test_bots(self):
self.set_as_admin()
# Add bots to display.
state = {
'dict': {
'random': 'values'
},
'float': 0.,
'list': ['of', 'things'],
'str': u'uni',
}
bot_management.bot_event(event_type='bot_connected',
bot_id='id1',
external_ip='8.8.4.4',
dimensions={'id': ['id1']},
state=state,
version='123456789',
quarantined=False,
task_id=None,
task_name=None)
bot_management.bot_event(event_type='bot_connected',
bot_id='id2',
external_ip='8.8.8.8',
dimensions={'id': ['id2']},
state={'ram': 65},
version='123456789',
quarantined=False,
task_id=None,
task_name=None)
response = self.app.get('/restricted/bots', status=200)
self.assertGreater(len(response.body), 1000)
def test_get_ok(self):
"""Asserts that get shows the tasks a specific bot has executed."""
self.set_as_privileged_user()
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
now_str = unicode(now.strftime(self.DATETIME_FORMAT))
bot_management.bot_event(
event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4',
dimensions={'foo': ['bar'], 'id': ['id1']}, state={'ram': 65},
version='123456789', quarantined=False, task_id=None, task_name=None)
expected = {
u'bot_id': u'id1',
u'dimensions': [
{u'key': u'foo', u'value': [u'bar']},
{u'key': u'id', u'value': [u'id1']},
],
u'external_ip': u'8.8.4.4',
u'first_seen_ts': now_str,
u'is_dead': False,
u'last_seen_ts': now_str,
u'quarantined': False,
u'version': u'123456789',
}
response = self.call_api('get', body={'bot_id': 'id1'})
self.assertEqual(expected, response.json)
def test_api_bots(self):
self.set_as_privileged_user()
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
now_str = unicode(now.strftime(utils.DATETIME_FORMAT))
self.mock_now(now)
bot_management.bot_event(
event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4',
dimensions={'foo': ['bar'], 'id': ['id1']}, state={'ram': 65},
version='123456789', quarantined=False, task_id=None, task_name=None)
actual = self.app.get('/swarming/api/v1/client/bots', status=200).json
expected = {
u'items': [
{
u'dimensions': {u'foo': [u'bar'], u'id': [u'id1']},
u'external_ip': u'8.8.4.4',
u'first_seen_ts': now_str,
u'id': u'id1',
u'is_dead': False,
u'last_seen_ts': now_str,
u'quarantined': False,
u'state': {u'ram': 65},
u'task_id': None,
u'task_name': None,
u'version': u'123456789',
},
],
u'cursor': None,
u'death_timeout': config.settings().bot_death_timeout_secs,
u'limit': 1000,
u'now': unicode(now.strftime(utils.DATETIME_FORMAT)),
}
self.assertEqual(expected, actual)
# Test with limit.
actual = self.app.get(
'/swarming/api/v1/client/bots?limit=1', status=200).json
expected['limit'] = 1
self.assertEqual(expected, actual)
bot_management.bot_event(
event_type='bot_connected', bot_id='id2', external_ip='8.8.4.4',
dimensions={'foo': ['bar'], 'id': ['id2']}, state={'ram': 65},
version='123456789', quarantined=False, task_id=None, task_name=None)
actual = self.app.get(
'/swarming/api/v1/client/bots?limit=1', status=200).json
expected['cursor'] = actual['cursor']
self.assertTrue(actual['cursor'])
self.assertEqual(expected, actual)
# Test with cursor.
actual = self.app.get(
'/swarming/api/v1/client/bots?limit=1&cursor=%s' % actual['cursor'],
status=200).json
expected['cursor'] = None
expected['items'][0]['dimensions']['id'] = [u'id2']
expected['items'][0]['id'] = u'id2'
self.assertEqual(expected, actual)
def test_bot_event_busy(self):
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
bot_management.bot_event(event_type='request_task',
bot_id='id1',
external_ip='8.8.4.4',
dimensions={
'id': ['id1'],
'foo': ['bar']
},
state={'ram': 65},
version=hashlib.sha1().hexdigest(),
quarantined=False,
task_id='12311',
task_name='yo')
expected = {
'dimensions': {
u'foo': [u'bar'],
u'id': [u'id1']
},
'external_ip': u'8.8.4.4',
'first_seen_ts': now,
'id': 'id1',
'last_seen_ts': now,
'quarantined': False,
'state': {
u'ram': 65
},
'task_id': u'12311',
'task_name': u'yo',
'version': u'da39a3ee5e6b4b0d3255bfef95601890afd80709',
}
bot_info = bot_management.get_info_key('id1').get()
self.assertEqual(expected, bot_info.to_dict())
self.assertEqual(True, bot_info.is_busy)
expected = [
{
'dimensions': {
u'foo': [u'bar'],
u'id': [u'id1']
},
'event_type': u'request_task',
'external_ip': u'8.8.4.4',
'message': None,
'quarantined': False,
'state': {
u'ram': 65
},
'task_id': u'12311',
'ts': now,
'version': u'da39a3ee5e6b4b0d3255bfef95601890afd80709',
},
]
self.assertEqual(
expected,
[e.to_dict() for e in bot_management.get_events_query('id1')])
def _yield_next_available_task_to_dispatch(bot_dimensions, deadline):
bot_management.bot_event('bot_connected', bot_dimensions[u'id'][0],
'1.2.3.4', 'joe@localhost', bot_dimensions,
{'state': 'real'}, '1234', False, None, None,
None)
task_queues.assert_bot_async(bot_dimensions).get_result()
return [
to_run.to_dict() for _request, to_run in task_to_run.
yield_next_available_task_to_dispatch(bot_dimensions, deadline)
]
def test_list_ok(self):
"""Asserts that BotInfo is returned for the appropriate set of bots."""
self.set_as_privileged_user()
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
now_str = unicode(now.strftime(self.DATETIME_FORMAT))
self.mock_now(now)
bot_management.bot_event(event_type='bot_connected',
bot_id='id1',
external_ip='8.8.4.4',
dimensions={
'foo': ['bar'],
'id': ['id1']
},
state={'ram': 65},
version='123456789',
quarantined=False,
task_id=None,
task_name=None)
expected = {
u'items': [
{
u'bot_id':
u'id1',
u'dimensions': [
{
u'key': u'foo',
u'value': [u'bar']
},
{
u'key': u'id',
u'value': [u'id1']
},
],
u'external_ip':
u'8.8.4.4',
u'first_seen_ts':
now_str,
u'is_dead':
False,
u'last_seen_ts':
now_str,
u'quarantined':
False,
u'version':
u'123456789',
},
],
u'death_timeout':
unicode(config.settings().bot_death_timeout_secs),
u'now':
unicode(now.strftime(self.DATETIME_FORMAT)),
}
request = swarming_rpcs.BotsRequest()
response = self.call_api('list', body=message_to_dict(request))
self.assertEqual(expected, response.json)
def ensure_bot_info_exists(machine_lease):
"""Ensures a BotInfo entity exists and has Machine Provider-related fields.
Args:
machine_lease: MachineLease instance.
"""
if machine_lease.bot_id == machine_lease.hostname:
return
bot_info = bot_management.get_info_key(machine_lease.hostname).get()
if not (bot_info and bot_info.lease_id and bot_info.lease_expiration_ts
and bot_info.machine_type):
logging.info(
'Creating BotEvent\nKey: %s\nHostname: %s\nBotInfo: %s',
machine_lease.key,
machine_lease.hostname,
bot_info,
)
bot_management.bot_event(
event_type='bot_leased',
bot_id=machine_lease.hostname,
external_ip=None,
authenticated_as=None,
dimensions=None,
state=None,
version=None,
quarantined=False,
maintenance_msg=None,
task_id='',
task_name=None,
lease_id=machine_lease.lease_id,
lease_expiration_ts=machine_lease.lease_expiration_ts,
machine_type=machine_lease.machine_type.id(),
machine_lease=machine_lease.key.id(),
)
# Occasionally bot_management.bot_event fails to store the BotInfo so
# verify presence of Machine Provider fields. See https://crbug.com/681224.
bot_info = bot_management.get_info_key(machine_lease.hostname).get()
if not (bot_info and bot_info.lease_id and bot_info.lease_expiration_ts
and bot_info.machine_type and bot_info.machine_lease):
# If associate_bot_id isn't called, cron will try again later.
logging.error(
'Failed to put BotInfo\nKey: %s\nHostname: %s\nBotInfo: %s',
machine_lease.key,
machine_lease.hostname,
bot_info,
)
return
logging.info(
'Put BotInfo\nKey: %s\nHostname: %s\nBotInfo: %s',
machine_lease.key,
machine_lease.hostname,
bot_info,
)
associate_bot_id(machine_lease.key, machine_lease.hostname)
def bot_event(event_type, task_id=None, task_name=None):
bot_management.bot_event(event_type=event_type,
bot_id=bot_id,
external_ip=self.request.remote_addr,
dimensions=dimensions,
state=state,
version=version,
quarantined=quarantined,
task_id=task_id,
task_name=task_name,
message=quarantined_msg)
def _assert_bot(dimensions=None):
bot_dimensions = {
u'cpu': [u'x86-64', u'x64'],
u'id': [u'bot1'],
u'os': [u'Ubuntu-16.04', u'Ubuntu'],
u'pool': [u'default'],
}
bot_dimensions.update(dimensions or {})
bot_management.bot_event('bot_connected', u'bot1', '1.2.3.4', 'bot1',
bot_dimensions, {}, '1234', False, None, None,
None)
return task_queues.assert_bot_async(bot_dimensions).get_result()
def bot_event(event_type, task_id=None, task_name=None):
bot_management.bot_event(
event_type=event_type,
bot_id=res.bot_id,
external_ip=self.request.remote_addr,
authenticated_as=auth.get_peer_identity().to_bytes(),
dimensions=res.dimensions,
state=res.state,
version=res.version,
quarantined=quarantined,
task_id=task_id,
task_name=task_name,
message=res.quarantined_msg)
def bot_event(event_type, task_id=None, task_name=None):
bot_management.bot_event(
event_type=event_type,
bot_id=bot_id,
external_ip=self.request.remote_addr,
dimensions=dimensions,
state=state,
version=version,
quarantined=quarantined,
task_id=task_id,
task_name=task_name,
message=quarantined_msg,
)
def test_cron_monitoring_bots_aggregate_dimensions(self):
# Tests that the aggregation works
now = datetime.datetime(2010, 1, 2, 3, 4, 5)
self.mock_now(now)
bot_management.bot_event(event_type='request_sleep',
bot_id='id1',
external_ip='8.8.4.4',
authenticated_as='bot:whitelisted-ip',
dimensions={
'foo': ['beta'],
'id': ['id1']
},
state={'ram': 65},
version='123456789',
quarantined=False,
maintenance_msg=None,
task_id=None,
task_name=None,
register_dimensions=True)
bot_management.bot_event(event_type='request_sleep',
bot_id='id2',
external_ip='8.8.4.4',
authenticated_as='bot:whitelisted-ip',
dimensions={
'foo': ['alpha'],
'id': ['id2']
},
state={'ram': 65},
version='123456789',
quarantined=True,
maintenance_msg=None,
task_id='987',
task_name=None,
register_dimensions=True)
self.app.get('/internal/cron/monitoring/bots/aggregate_dimensions',
headers={'X-AppEngine-Cron': 'true'},
status=200)
agg_key = bot_management.get_aggregation_key('all')
actual = agg_key.get()
expected = bot_management.DimensionAggregation(
key=agg_key,
dimensions=[
bot_management.DimensionValues(dimension='foo',
values=['alpha', 'beta'])
],
ts=now)
self.assertEqual(expected, actual)
def post(self):
(_request, bot_id, version, state,
dimensions, quarantined_msg) = self._process()
bot_management.bot_event(
event_type='bot_connected', bot_id=bot_id,
external_ip=self.request.remote_addr, dimensions=dimensions,
state=state, version=version, quarantined=bool(quarantined_msg),
task_id='', task_name=None, message=quarantined_msg)
data = {
# This access token will be used to validate each subsequent request.
'bot_version': bot_code.get_bot_version(self.request.host_url),
'server_version': utils.get_app_version(),
'xsrf_token': self.generate_xsrf_token(),
}
self.send_response(data)
def post(self):
(_request, bot_id, version, state,
dimensions, quarantined_msg) = self._process()
bot_management.bot_event(
event_type='bot_connected', bot_id=bot_id,
external_ip=self.request.remote_addr, dimensions=dimensions,
state=state, version=version, quarantined=bool(quarantined_msg),
task_id='', task_name=None, message=quarantined_msg)
data = {
# This access token will be used to validate each subsequent request.
'bot_version': bot_code.get_bot_version(self.request.host_url),
'server_version': utils.get_app_version(),
'xsrf_token': self.generate_xsrf_token(),
}
self.send_response(data)
def test_BotEvent_proto_events(self):
# Ensures all bot event states can be converted to a proto.
dimensions = {
u'id': [u'id1'],
u'os': [u'Ubuntu', u'Ubuntu-16.04'],
u'pool': [u'default'],
}
for name in bot_management.BotEvent.ALLOWED_EVENTS:
event_key = bot_management.bot_event(
event_type=name,
bot_id=u'id1',
external_ip=u'8.8.4.4',
authenticated_as=u'bot:id1.domain',
dimensions=dimensions,
state={u'ram': 65},
version=_VERSION,
quarantined=False,
maintenance_msg=None,
task_id=None,
task_name=None)
if name in (u'request_sleep', u'task_update'):
# TODO(maruel): Store request_sleep IFF the state changed.
self.assertIsNone(event_key, name)
continue
# Just asserts it doesn't crash.
actual = swarming_pb2.BotEvent()
event_key.get().to_proto(actual)
def post(self, task_id=None):
request = self.parse_body()
bot_id = request.get('id')
task_id = request.get('task_id', '')
message = request.get('message', 'unknown')
machine_type = None
bot_info = bot_management.get_info_key(bot_id).get()
if bot_info:
machine_type = bot_info.machine_type
# Make sure bot self-reported ID matches the authentication token. Raises
# auth.AuthorizationError if not.
bot_auth.validate_bot_id_and_fetch_config(bot_id, machine_type)
bot_management.bot_event(
event_type='task_error',
bot_id=bot_id,
external_ip=self.request.remote_addr,
authenticated_as=auth.get_peer_identity().to_bytes(),
dimensions=None,
state=None,
version=None,
quarantined=None,
maintenance_msg=None,
task_id=task_id,
task_name=None,
message=message)
line = ('Bot: https://%s/restricted/bot/%s\n'
'Task failed: https://%s/user/task/%s\n'
'%s') % (app_identity.get_default_version_hostname(), bot_id,
app_identity.get_default_version_hostname(), task_id,
message)
ereporter2.log_request(self.request, source='bot', message=line)
msg = log_unexpected_keys(self.EXPECTED_KEYS, request, self.request,
'bot', 'keys')
if msg:
self.abort_with_error(400, error=msg)
msg = task_scheduler.bot_kill_task(
task_pack.unpack_run_result_key(task_id), bot_id)
if msg:
logging.error(msg)
self.abort_with_error(400, error=msg)
self.send_response({})
def test_delete_bot(self):
self.set_as_admin()
bot_management.bot_event(
event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4',
dimensions={'id': ['id1']}, state={'foo': 'bar'}, version='123456789',
quarantined=False, task_id=None, task_name=None)
response = self.app.get('/restricted/bots', status=200)
self.assertTrue('id1' in response.body)
response = self.app.post(
'/restricted/bot/id1/delete',
params={},
headers={'X-XSRF-Token': self.get_xsrf_token()})
self.assertFalse('id1' in response.body)
response = self.app.get('/restricted/bots', status=200)
self.assertFalse('id1' in response.body)
def testCronBotsAggregateTask(self):
# Tests that the aggregation works
now = datetime.datetime(2010, 1, 2, 3, 4, 5)
self.mock_now(now)
bot_management.bot_event(event_type='bot_connected',
bot_id='id1',
external_ip='8.8.4.4',
authenticated_as='bot:whitelisted-ip',
dimensions={
'foo': ['beta'],
'id': ['id1']
},
state={'ram': 65},
version='123456789',
quarantined=False,
task_id=None,
task_name=None)
bot_management.bot_event(event_type='bot_connected',
bot_id='id2',
external_ip='8.8.4.4',
authenticated_as='bot:whitelisted-ip',
dimensions={
'foo': ['alpha'],
'id': ['id2']
},
state={'ram': 65},
version='123456789',
quarantined=True,
task_id='987',
task_name=None)
self.app.get('/internal/cron/aggregate_bots_dimensions',
headers={'X-AppEngine-Cron': 'true'},
status=200)
actual = bot_management.DimensionAggregation.KEY.get()
expected = bot_management.DimensionAggregation(
key=bot_management.DimensionAggregation.KEY,
dimensions=[
bot_management.DimensionValues(dimension='foo',
values=['alpha', 'beta'])
],
ts=now)
self.assertEqual(expected, actual)
def test_get_events_query(self):
bot_management.bot_event(event_type='bot_connected',
bot_id='id1',
external_ip='8.8.4.4',
authenticated_as='bot:id1.domain',
dimensions={
'id': ['id1'],
'foo': ['bar']
},
state={'ram': 65},
version=hashlib.sha256().hexdigest(),
quarantined=False,
maintenance_msg=None,
task_id=None,
task_name=None)
expected = [self._gen_bot_event(event_type=u'bot_connected')]
self.assertEqual(expected, [
i.to_dict() for i in bot_management.get_events_query('id1', True)
])
def test_api_bot_delete(self):
self.set_as_admin()
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
state = {
'dict': {
'random': 'values'
},
'float': 0.,
'list': ['of', 'things'],
'str': u'uni',
}
bot_management.bot_event(event_type='bot_connected',
bot_id='id1',
external_ip='8.8.4.4',
dimensions={
'foo': ['bar'],
'id': ['id1']
},
state=state,
version='123456789',
quarantined=False,
task_id=None,
task_name=None)
token = self.get_client_token()
actual = self.app.delete('/swarming/api/v1/client/bot/id1',
status=200,
headers={
'X-XSRF-Token': str(token)
}).json
expected = {
u'deleted': True,
}
self.assertEqual(expected, actual)
actual = self.app.get('/swarming/api/v1/client/bot/id1',
status=404).json
expected = {
u'error': u'Bot not found',
}
self.assertEqual(expected, actual)
def test_bot_event_busy(self):
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
bot_management.bot_event(
event_type='request_task', bot_id='id1', external_ip='8.8.4.4',
dimensions={'id': ['id1'], 'foo': ['bar']}, state={'ram': 65},
version=hashlib.sha1().hexdigest(), quarantined=False, task_id='12311',
task_name='yo')
expected = {
'dimensions': {u'foo': [u'bar'], u'id': [u'id1']},
'external_ip': u'8.8.4.4',
'first_seen_ts': now,
'id': 'id1',
'last_seen_ts': now,
'quarantined': False,
'state': {u'ram': 65},
'task_id': u'12311',
'task_name': u'yo',
'version': u'da39a3ee5e6b4b0d3255bfef95601890afd80709',
}
bot_info = bot_management.get_info_key('id1').get()
self.assertEqual(expected, bot_info.to_dict())
self.assertEqual(True, bot_info.is_busy)
expected = [
{
'dimensions': {u'foo': [u'bar'], u'id': [u'id1']},
'event_type': u'request_task',
'external_ip': u'8.8.4.4',
'message': None,
'quarantined': False,
'state': {u'ram': 65},
'task_id': u'12311',
'ts': now,
'version': u'da39a3ee5e6b4b0d3255bfef95601890afd80709',
},
]
self.assertEqual(
expected,
[e.to_dict() for e in bot_management.get_events_query('id1', True)])
def post(self):
request = self.parse_body()
log_unexpected_keys(
self.EXPECTED_KEYS, request, self.request, 'bot', 'keys')
message = request.get('message', 'unknown')
bot_id = request.get('id')
if bot_id:
bot_management.bot_event(
event_type='bot_error', bot_id=bot_id,
external_ip=self.request.remote_addr, dimensions=None, state=None,
version=None, quarantined=None, task_id=None, task_name=None,
message=message)
# Also log inconditionally an ereporter2 event.
line = (
'Bot: https://%s/restricted/bot/%s\n'
'Old API error:\n'
'%s') % (
app_identity.get_default_version_hostname(), bot_id, message)
ereporter2.log_request(self.request, source='bot', message=line)
self.send_response({})
def test_bots(self):
self.set_as_admin()
# Add bots to display.
state = {
'dict': {'random': 'values'},
'float': 0.,
'list': ['of', 'things'],
'str': u'uni',
}
bot_management.bot_event(
event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4',
dimensions={'id': ['id1']}, state=state, version='123456789',
quarantined=False, task_id=None, task_name=None)
bot_management.bot_event(
event_type='bot_connected', bot_id='id2', external_ip='8.8.8.8',
dimensions={'id': ['id2']}, state={'ram': 65}, version='123456789',
quarantined=False, task_id=None, task_name=None)
response = self.app.get('/restricted/bots', status=200)
self.assertGreater(len(response.body), 1000)
def post(self):
(request, bot_id, version, state,
dimensions, quarantined_msg) = self._process()
event = request.get('event')
if event not in ('bot_error', 'bot_rebooting', 'bot_shutdown'):
self.abort_with_error(400, error='Unsupported event type')
message = request.get('message')
bot_management.bot_event(
event_type=event, bot_id=bot_id, external_ip=self.request.remote_addr,
dimensions=dimensions, state=state, version=version,
quarantined=bool(quarantined_msg), task_id=None, task_name=None,
message=message)
if event == 'bot_error':
line = (
'Bot: https://%s/restricted/bot/%s\n'
'Bot error:\n'
'%s') % (
app_identity.get_default_version_hostname(), bot_id, message)
ereporter2.log_request(self.request, source='bot', message=line)
self.send_response({})
def test_get_events_query(self):
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
bot_management.bot_event(event_type='bot_connected',
bot_id='id1',
external_ip='8.8.4.4',
authenticated_as='bot:id1.domain',
dimensions={
'id': ['id1'],
'foo': ['bar']
},
state={'ram': 65},
version=hashlib.sha1().hexdigest(),
quarantined=False,
task_id=None,
task_name=None)
expected = [
{
'authenticated_as': 'bot:id1.domain',
'dimensions': {
u'foo': [u'bar'],
u'id': [u'id1']
},
'event_type': u'bot_connected',
'external_ip': u'8.8.4.4',
'lease_id': None,
'lease_expiration_ts': None,
'message': None,
'quarantined': False,
'state': {
u'ram': 65
},
'task_id': None,
'ts': now,
'version': u'da39a3ee5e6b4b0d3255bfef95601890afd80709',
},
]
self.assertEqual(expected, [
i.to_dict() for i in bot_management.get_events_query('id1', True)
])
def test_api_bot(self):
self.set_as_privileged_user()
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
now_str = unicode(now.strftime(utils.DATETIME_FORMAT))
self.mock_now(now)
bot_management.bot_event(event_type='bot_connected',
bot_id='id1',
external_ip='8.8.4.4',
dimensions={
'foo': ['bar'],
'id': ['id1']
},
state={'ram': 65},
version='123456789',
quarantined=False,
task_id=None,
task_name=None)
actual = self.app.get('/swarming/api/v1/client/bot/id1',
status=200).json
expected = {
u'dimensions': {
u'foo': [u'bar'],
u'id': [u'id1']
},
u'external_ip': u'8.8.4.4',
u'first_seen_ts': now_str,
u'id': u'id1',
u'is_dead': False,
u'last_seen_ts': now_str,
u'quarantined': False,
u'state': {
u'ram': 65
},
u'task_id': None,
u'task_name': None,
u'version': u'123456789',
}
self.assertEqual(expected, actual)
def test_get_events_query(self):
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
bot_management.bot_event(
event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4',
dimensions={'id': ['id1'], 'foo': ['bar']}, state={'ram': 65},
version=hashlib.sha1().hexdigest(), quarantined=False, task_id=None,
task_name=None)
expected = [
{
'dimensions': {u'foo': [u'bar'], u'id': [u'id1']},
'event_type': u'bot_connected',
'external_ip': u'8.8.4.4',
'message': None,
'quarantined': False,
'state': {u'ram': 65},
'task_id': None,
'ts': now,
'version': u'da39a3ee5e6b4b0d3255bfef95601890afd80709',
},
]
self.assertEqual(
expected, [i.to_dict() for i in bot_management.get_events_query('id1')])
def test_delete_ok(self):
"""Assert that delete finds and deletes a bot."""
self.set_as_admin()
self.mock(acl, 'is_admin', lambda *_args, **_kwargs: True)
now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6)
self.mock_now(now)
state = {
'dict': {'random': 'values'},
'float': 0.,
'list': ['of', 'things'],
'str': u'uni',
}
bot_management.bot_event(
event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4',
dimensions={'foo': ['bar'], 'id': ['id1']}, state=state,
version='123456789', quarantined=False, task_id=None, task_name=None)
# delete the bot
response = self.call_api('delete', body={'bot_id': 'id1'})
self.assertEqual({u'deleted': True}, response.json)
# is it gone?
with self.call_should_fail('404'):
self.call_api('delete', body={'bot_id': 'id1'})
def post(self, task_id=None):
# Unlike handshake and poll, we do not accept invalid keys here. This code
# path is much more strict.
request = self.parse_body()
msg = log_unexpected_subset_keys(self.ACCEPTED_KEYS, self.REQUIRED_KEYS, request, self.request, "bot", "keys")
if msg:
self.abort_with_error(400, error=msg)
bot_id = request["id"]
cost_usd = request["cost_usd"]
task_id = request["task_id"]
duration = request.get("duration")
exit_code = request.get("exit_code")
hard_timeout = request.get("hard_timeout")
io_timeout = request.get("io_timeout")
output = request.get("output")
output_chunk_start = request.get("output_chunk_start")
outputs_ref = request.get("outputs_ref")
run_result_key = task_pack.unpack_run_result_key(task_id)
if output is not None:
try:
output = base64.b64decode(output)
except UnicodeEncodeError as e:
logging.error("Failed to decode output\n%s\n%r", e, output)
output = output.encode("ascii", "replace")
except TypeError as e:
# Save the output as-is instead. The error will be logged in ereporter2
# and returning a HTTP 500 would only force the bot to stay in a retry
# loop.
logging.error("Failed to decode output\n%s\n%r", e, output)
try:
success, completed = task_scheduler.bot_update_task(
run_result_key,
bot_id,
output,
output_chunk_start,
exit_code,
duration,
hard_timeout,
io_timeout,
cost_usd,
outputs_ref,
)
if not success:
self.abort_with_error(500, error="Failed to update, please retry")
action = "task_completed" if completed else "task_update"
bot_management.bot_event(
event_type=action,
bot_id=bot_id,
external_ip=self.request.remote_addr,
dimensions=None,
state=None,
version=None,
quarantined=None,
task_id=task_id,
task_name=None,
)
except ValueError as e:
ereporter2.log_request(
request=self.request, source="server", category="task_failure", message="Failed to update task: %s" % e
)
self.abort_with_error(400, error=str(e))
except Exception as e:
self.abort_with_error(500, error=str(e))
# TODO(maruel): When a task is canceled, reply with 'DIE' so that the bot
# reboots itself to abort the task abruptly. It is useful when a task hangs
# and the timeout was set too long or the task was superseded by a newer
# task with more recent executable (e.g. a new Try Server job on a newer
# patchset on Rietveld).
self.send_response({"ok": True})
def post(self, task_id=None):
# Unlike handshake and poll, we do not accept invalid keys here. This code
# path is much more strict.
request = self.parse_body()
msg = log_unexpected_subset_keys(
self.ACCEPTED_KEYS, self.REQUIRED_KEYS, request, self.request, 'bot',
'keys')
if msg:
self.abort_with_error(400, error=msg)
bot_id = request['id']
cost_usd = request['cost_usd']
task_id = request['task_id']
duration = request.get('duration')
exit_code = request.get('exit_code')
hard_timeout = request.get('hard_timeout')
io_timeout = request.get('io_timeout')
output = request.get('output')
output_chunk_start = request.get('output_chunk_start')
outputs_ref = request.get('outputs_ref')
run_result_key = task_pack.unpack_run_result_key(task_id)
if output is not None:
try:
output = base64.b64decode(output)
except UnicodeEncodeError as e:
logging.error('Failed to decode output\n%s\n%r', e, output)
output = output.encode('ascii', 'replace')
except TypeError as e:
# Save the output as-is instead. The error will be logged in ereporter2
# and returning a HTTP 500 would only force the bot to stay in a retry
# loop.
logging.error('Failed to decode output\n%s\n%r', e, output)
try:
success, completed = task_scheduler.bot_update_task(
run_result_key, bot_id, output, output_chunk_start,
exit_code, duration, hard_timeout, io_timeout, cost_usd, outputs_ref)
if not success:
logging.info('Failed to update, please retry')
self.abort_with_error(500, error='Failed to update, please retry')
action = 'task_completed' if completed else 'task_update'
bot_management.bot_event(
event_type=action, bot_id=bot_id,
external_ip=self.request.remote_addr, dimensions=None, state=None,
version=None, quarantined=None, task_id=task_id, task_name=None)
except ValueError as e:
ereporter2.log_request(
request=self.request,
source='server',
category='task_failure',
message='Failed to update task: %s' % e)
self.abort_with_error(400, error=str(e))
except webob.exc.HTTPException:
raise
except Exception as e:
logging.exception('Internal error: %s', e)
self.abort_with_error(500, error=str(e))
# TODO(maruel): When a task is canceled, reply with 'DIE' so that the bot
# reboots itself to abort the task abruptly. It is useful when a task hangs
# and the timeout was set too long or the task was superseded by a newer
# task with more recent executable (e.g. a new Try Server job on a newer
# patchset on Rietveld).
self.send_response({'ok': True})
def test_api_bots(self):
self.set_as_privileged_user()
self.mock_now(datetime.datetime(2010, 1, 2, 3, 4, 5, 6))
now_str = lambda: unicode(utils.utcnow().strftime(utils.DATETIME_FORMAT))
bot_management.bot_event(
event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4',
dimensions={'foo': ['bar'], 'id': ['id1']}, state={'ram': 65},
version='123456789', quarantined=False, task_id=None, task_name=None)
bot1_dict = {
u'dimensions': {u'foo': [u'bar'], u'id': [u'id1']},
u'external_ip': u'8.8.4.4',
u'first_seen_ts': now_str(),
u'id': u'id1',
u'is_dead': False,
u'last_seen_ts': now_str(),
u'quarantined': False,
u'state': {u'ram': 65},
u'task_id': None,
u'task_name': None,
u'version': u'123456789',
}
actual = self.app.get('/swarming/api/v1/client/bots', status=200).json
expected = {
u'items': [bot1_dict],
u'cursor': None,
u'death_timeout': config.settings().bot_death_timeout_secs,
u'limit': 1000,
u'now': now_str(),
}
self.assertEqual(expected, actual)
# Test with limit.
actual = self.app.get(
'/swarming/api/v1/client/bots?limit=1', status=200).json
expected['limit'] = 1
self.assertEqual(expected, actual)
# Advance time to make bot1 dead to test filtering for dead bots.
self.mock_now(datetime.datetime(2011, 1, 2, 3, 4, 5, 6))
bot1_dict['is_dead'] = True
expected['now'] = now_str()
# Use quarantined bot to check filtering by 'quarantined' flag.
bot_management.bot_event(
event_type='bot_connected', bot_id='id2', external_ip='8.8.4.4',
dimensions={'foo': ['bar'], 'id': ['id2']}, state={'ram': 65},
version='123456789', quarantined=True, task_id=None, task_name=None)
bot2_dict = {
u'dimensions': {u'foo': [u'bar'], u'id': [u'id2']},
u'external_ip': u'8.8.4.4',
u'first_seen_ts': now_str(),
u'id': u'id2',
u'is_dead': False,
u'last_seen_ts': now_str(),
u'quarantined': True,
u'state': {u'ram': 65},
u'task_id': None,
u'task_name': None,
u'version': u'123456789',
}
# Test limit + cursor: start the query.
actual = self.app.get(
'/swarming/api/v1/client/bots?limit=1', status=200).json
expected['cursor'] = actual['cursor']
expected['items'] = [bot1_dict]
self.assertTrue(actual['cursor'])
self.assertEqual(expected, actual)
# Test limit + cursor: continue the query.
actual = self.app.get(
'/swarming/api/v1/client/bots?limit=1&cursor=%s' % actual['cursor'],
status=200).json
expected['cursor'] = None
expected['items'] = [bot2_dict]
self.assertEqual(expected, actual)
# Filtering by 'quarantined'.
actual = self.app.get(
'/swarming/api/v1/client/bots?filter=quarantined',
status=200).json
expected['limit'] = 1000
expected['cursor'] = None
expected['items'] = [bot2_dict]
self.assertEqual(expected, actual)
# Filtering by 'is_dead'.
actual = self.app.get(
'/swarming/api/v1/client/bots?filter=is_dead',
status=200).json
expected['limit'] = 1000
expected['cursor'] = None
expected['items'] = [bot1_dict]
self.assertEqual(expected, actual)
def test_api_bots(self):
self.set_as_privileged_user()
self.mock_now(datetime.datetime(2010, 1, 2, 3, 4, 5, 6))
now_str = lambda: unicode(utils.utcnow().strftime(utils.DATETIME_FORMAT))
bot_management.bot_event(
event_type="bot_connected",
bot_id="id1",
external_ip="8.8.4.4",
dimensions={"foo": ["bar"], "id": ["id1"]},
state={"ram": 65},
version="123456789",
quarantined=False,
task_id=None,
task_name=None,
)
bot1_dict = {
u"dimensions": {u"foo": [u"bar"], u"id": [u"id1"]},
u"external_ip": u"8.8.4.4",
u"first_seen_ts": now_str(),
u"id": u"id1",
u"is_dead": False,
u"last_seen_ts": now_str(),
u"quarantined": False,
u"state": {u"ram": 65},
u"task_id": None,
u"task_name": None,
u"version": u"123456789",
}
actual = self.app.get("/swarming/api/v1/client/bots", status=200).json
expected = {
u"items": [bot1_dict],
u"cursor": None,
u"death_timeout": config.settings().bot_death_timeout_secs,
u"limit": 1000,
u"now": now_str(),
}
self.assertEqual(expected, actual)
# Test with limit.
actual = self.app.get("/swarming/api/v1/client/bots?limit=1", status=200).json
expected["limit"] = 1
self.assertEqual(expected, actual)
# Advance time to make bot1 dead to test filtering for dead bots.
self.mock_now(datetime.datetime(2011, 1, 2, 3, 4, 5, 6))
bot1_dict["is_dead"] = True
expected["now"] = now_str()
# Use quarantined bot to check filtering by 'quarantined' flag.
bot_management.bot_event(
event_type="bot_connected",
bot_id="id2",
external_ip="8.8.4.4",
dimensions={"foo": ["bar"], "id": ["id2"]},
state={"ram": 65},
version="123456789",
quarantined=True,
task_id=None,
task_name=None,
)
bot2_dict = {
u"dimensions": {u"foo": [u"bar"], u"id": [u"id2"]},
u"external_ip": u"8.8.4.4",
u"first_seen_ts": now_str(),
u"id": u"id2",
u"is_dead": False,
u"last_seen_ts": now_str(),
u"quarantined": True,
u"state": {u"ram": 65},
u"task_id": None,
u"task_name": None,
u"version": u"123456789",
}
# Test limit + cursor: start the query.
actual = self.app.get("/swarming/api/v1/client/bots?limit=1", status=200).json
expected["cursor"] = actual["cursor"]
expected["items"] = [bot1_dict]
self.assertTrue(actual["cursor"])
self.assertEqual(expected, actual)
# Test limit + cursor: continue the query.
actual = self.app.get("/swarming/api/v1/client/bots?limit=1&cursor=%s" % actual["cursor"], status=200).json
expected["cursor"] = None
expected["items"] = [bot2_dict]
self.assertEqual(expected, actual)
# Filtering by 'quarantined'.
actual = self.app.get("/swarming/api/v1/client/bots?filter=quarantined", status=200).json
expected["limit"] = 1000
expected["cursor"] = None
expected["items"] = [bot2_dict]
self.assertEqual(expected, actual)
# Filtering by 'is_dead'.
actual = self.app.get("/swarming/api/v1/client/bots?filter=is_dead", status=200).json
expected["limit"] = 1000
expected["cursor"] = None
expected["items"] = [bot1_dict]
self.assertEqual(expected, actual)
