def _yield_next_available_task_to_dispatch(bot_dimensions, deadline):
bot_management.bot_event('bot_connected', bot_dimensions[u'id'][0],
'1.2.3.4', 'joe@localhost', bot_dimensions,
{'state': 'real'}, '1234', False, None, None,
None)
task_queues.assert_bot_async(bot_dimensions).get_result()
return [
to_run.to_dict() for _request, to_run in task_to_run.
yield_next_available_task_to_dispatch(bot_dimensions, deadline)
]
def _assert_bot(dimensions=None):
bot_dimensions = {
u'cpu': [u'x86-64', u'x64'],
u'id': [u'bot1'],
u'os': [u'Ubuntu-16.04', u'Ubuntu'],
u'pool': [u'default'],
}
bot_dimensions.update(dimensions or {})
bot_management.bot_event('bot_connected', u'bot1', '1.2.3.4', 'bot1',
bot_dimensions, {}, '1234', False, None, None,
None)
return task_queues.assert_bot_async(bot_dimensions).get_result()
def _assert_bot(bot_id=u'bot1', dimensions=None):
bot_dimensions = {
u'cpu': [u'x86-64', u'x64'],
u'id': [bot_id],
u'os': [u'Ubuntu-16.04', u'Ubuntu'],
u'pool': [u'default'],
}
bot_dimensions.update(dimensions or {})
bot_management.bot_event('request_sleep',
bot_id,
'1.2.3.4',
bot_id,
bot_dimensions, {},
'1234',
False,
None,
None,
None,
register_dimensions=True)
bot_root_key = bot_management.get_root_key(bot_id)
return task_queues.assert_bot_async(bot_root_key,
bot_dimensions).get_result()
def _process(self):
"""Fetches bot info and settings, does authorization and quarantine checks.
Returns:
_ProcessResult instance, see its fields for more info.
Raises:
auth.AuthorizationError if bot's credentials are invalid.
"""
request = self.parse_body()
version = request.get('version', None)
dimensions = request.get('dimensions') or {}
state = request.get('state') or {}
bot_id = None
if dimensions.get('id'):
dimension_id = dimensions['id']
if (isinstance(dimension_id, list) and len(dimension_id) == 1
and isinstance(dimension_id[0], unicode)):
bot_id = dimensions['id'][0]
lease_expiration_ts = None
machine_type = None
if bot_id:
logging.debug('Fetching bot info and settings')
bot_info, bot_settings = ndb.get_multi([
bot_management.get_info_key(bot_id),
bot_management.get_settings_key(bot_id)
])
if bot_info:
lease_expiration_ts = bot_info.lease_expiration_ts
machine_type = bot_info.machine_type
# Make sure bot self-reported ID matches the authentication token. Raises
# auth.AuthorizationError if not.
logging.debug('Fetching bot group config')
bot_group_cfg = bot_auth.validate_bot_id_and_fetch_config(
bot_id, machine_type)
# The server side dimensions from bot_group_cfg override bot-provided ones.
# If both server side config and bot report some dimension, server side
# config wins. We still emit an warning if bot tries to supply the dimension
# and it disagrees with the server defined one. Note that this may happen
# on a first poll after server side config for a bot has changed. The bot
# doesn't know about new server-assigned dimensions yet in this case. Also
# don't report ['default'], bot sends it in the handshake before it knows
# anything at all.
for dim_key, from_cfg in bot_group_cfg.dimensions.iteritems():
from_bot = sorted(dimensions.get(dim_key) or [])
from_cfg = sorted(from_cfg)
if from_bot and from_bot != ['default'] and from_bot != from_cfg:
logging.warning(
'Dimensions in bots.cfg don\'t match ones provided by the bot\n'
'bot_id: "%s", key: "%s", from_bot: %s, from_cfg: %s',
bot_id, dim_key, from_bot, from_cfg)
dimensions[dim_key] = from_cfg
# Fill in all result fields except 'quarantined_msg'.
result = _ProcessResult(request=request,
bot_id=bot_id,
version=version,
state=state,
dimensions=dimensions,
bot_group_cfg=bot_group_cfg,
lease_expiration_ts=lease_expiration_ts,
maintenance_msg=state.get('maintenance'))
# The bot may decide to "self-quarantine" itself. Accept both via
# dimensions or via state. See bot_management._BotCommon.quarantined for
# more details.
if (bool(dimensions.get('quarantined'))
or bool(state.get('quarantined'))):
result.quarantined_msg = 'Bot self-quarantined'
return result
quarantined_msg = None
# Use a dummy 'for' to be able to break early from the block.
for _ in [0]:
quarantined_msg = has_unexpected_keys(self.EXPECTED_KEYS, request,
'keys')
if quarantined_msg:
break
quarantined_msg = has_missing_keys(self.REQUIRED_STATE_KEYS, state,
'state')
if quarantined_msg:
break
if not bot_id:
quarantined_msg = 'Missing bot id'
break
if not dimensions.get('pool'):
quarantined_msg = 'Missing \'pool\' dimension'
break
if not all(
config.validate_dimension_key(key)
and isinstance(values, list) and all(
config.validate_dimension_value(value)
for value in values)
for key, values in dimensions.iteritems()):
quarantined_msg = ('Invalid dimensions type:\n%s' %
json.dumps(dimensions,
sort_keys=True,
indent=2,
separators=(',', ': ')))
break
if quarantined_msg:
line = 'Quarantined Bot\nhttps://%s/restricted/bot/%s\n%s' % (
app_identity.get_default_version_hostname(), bot_id,
quarantined_msg)
ereporter2.log_request(self.request, source='bot', message=line)
result.quarantined_msg = quarantined_msg
return result
# Look for admin enforced quarantine.
if bool(bot_settings and bot_settings.quarantined):
result.quarantined_msg = 'Quarantined by admin'
return result
# TODO(maruel): Parallelise.
task_queues.assert_bot_async(dimensions).get_result()
return result