def _process(self):
"""Fetches bot info and settings, does authorization and quarantine checks.
Returns:
_ProcessResult instance, see its fields for more info.
Raises:
auth.AuthorizationError if bot's credentials are invalid.
"""
request = self.parse_body()
version = request.get('version', None)
dimensions = request.get('dimensions') or {}
state = request.get('state') or {}
bot_id = None
if dimensions.get('id'):
dimension_id = dimensions['id']
if (isinstance(dimension_id, list) and len(dimension_id) == 1
and isinstance(dimension_id[0], unicode)):
bot_id = dimensions['id'][0]
lease_expiration_ts = None
machine_type = None
if bot_id:
logging.debug('Fetching bot info and settings')
bot_info, bot_settings = ndb.get_multi([
bot_management.get_info_key(bot_id),
bot_management.get_settings_key(bot_id)
])
if bot_info:
lease_expiration_ts = bot_info.lease_expiration_ts
machine_type = bot_info.machine_type
# Make sure bot self-reported ID matches the authentication token. Raises
# auth.AuthorizationError if not.
logging.debug('Fetching bot group config')
bot_group_cfg = bot_auth.validate_bot_id_and_fetch_config(
bot_id, machine_type)
# The server side dimensions from bot_group_cfg override bot-provided ones.
# If both server side config and bot report some dimension, server side
# config wins. We still emit an warning if bot tries to supply the dimension
# and it disagrees with the server defined one. Note that this may happen
# on a first poll after server side config for a bot has changed. The bot
# doesn't know about new server-assigned dimensions yet in this case. Also
# don't report ['default'], bot sends it in the handshake before it knows
# anything at all.
for dim_key, from_cfg in bot_group_cfg.dimensions.iteritems():
from_bot = sorted(dimensions.get(dim_key) or [])
from_cfg = sorted(from_cfg)
if from_bot and from_bot != ['default'] and from_bot != from_cfg:
logging.warning(
'Dimensions in bots.cfg don\'t match ones provided by the bot\n'
'bot_id: "%s", key: "%s", from_bot: %s, from_cfg: %s',
bot_id, dim_key, from_bot, from_cfg)
dimensions[dim_key] = from_cfg
# Fill in all result fields except 'quarantined_msg'.
result = _ProcessResult(request=request,
bot_id=bot_id,
version=version,
state=state,
dimensions=dimensions,
bot_group_cfg=bot_group_cfg,
lease_expiration_ts=lease_expiration_ts,
maintenance_msg=state.get('maintenance'))
# The bot may decide to "self-quarantine" itself. Accept both via
# dimensions or via state. See bot_management._BotCommon.quarantined for
# more details.
if (bool(dimensions.get('quarantined'))
or bool(state.get('quarantined'))):
result.quarantined_msg = 'Bot self-quarantined'
return result
quarantined_msg = None
# Use a dummy 'for' to be able to break early from the block.
for _ in [0]:
quarantined_msg = has_unexpected_keys(self.EXPECTED_KEYS, request,
'keys')
if quarantined_msg:
break
quarantined_msg = has_missing_keys(self.REQUIRED_STATE_KEYS, state,
'state')
if quarantined_msg:
break
if not bot_id:
quarantined_msg = 'Missing bot id'
break
if not dimensions.get('pool'):
quarantined_msg = 'Missing \'pool\' dimension'
break
if not all(
config.validate_dimension_key(key)
and isinstance(values, list) and all(
config.validate_dimension_value(value)
for value in values)
for key, values in dimensions.iteritems()):
quarantined_msg = ('Invalid dimensions type:\n%s' %
json.dumps(dimensions,
sort_keys=True,
indent=2,
separators=(',', ': ')))
break
if quarantined_msg:
line = 'Quarantined Bot\nhttps://%s/restricted/bot/%s\n%s' % (
app_identity.get_default_version_hostname(), bot_id,
quarantined_msg)
ereporter2.log_request(self.request, source='bot', message=line)
result.quarantined_msg = quarantined_msg
return result
# Look for admin enforced quarantine.
if bool(bot_settings and bot_settings.quarantined):
result.quarantined_msg = 'Quarantined by admin'
return result
# TODO(maruel): Parallelise.
task_queues.assert_bot_async(dimensions).get_result()
return result
def _process(self):
"""Returns True if the bot has invalid parameter and should be automatically
quarantined.
Does one DB synchronous GET.
Returns:
tuple(request, bot_id, version, state, dimensions, quarantined_msg)
"""
request = self.parse_body()
version = request.get('version', None)
dimensions = request.get('dimensions', {})
state = request.get('state', {})
bot_id = None
if dimensions.get('id'):
dimension_id = dimensions['id']
if (isinstance(dimension_id, list) and len(dimension_id) == 1
and isinstance(dimension_id[0], unicode)):
bot_id = dimensions['id'][0]
# The bot may decide to "self-quarantine" itself. Accept both via
# dimensions or via state. See bot_management._BotCommon.quarantined for
# more details.
if (bool(dimensions.get('quarantined'))
or bool(state.get('quarantined'))):
return request, bot_id, version, state, dimensions, 'Bot self-quarantined'
quarantined_msg = None
# Use a dummy 'for' to be able to break early from the block.
for _ in [0]:
quarantined_msg = has_unexpected_keys(self.EXPECTED_KEYS, request,
'keys')
if quarantined_msg:
break
quarantined_msg = has_missing_keys(self.REQUIRED_STATE_KEYS, state,
'state')
if quarantined_msg:
break
if not bot_id:
quarantined_msg = 'Missing bot id'
break
if not all(
isinstance(key, unicode) and isinstance(values, list)
and all(isinstance(value, unicode) for value in values)
for key, values in dimensions.iteritems()):
quarantined_msg = ('Invalid dimensions type:\n%s' %
json.dumps(dimensions,
sort_keys=True,
indent=2,
separators=(',', ': ')))
break
dimensions_count = task_to_run.dimensions_powerset_count(
dimensions)
if dimensions_count > task_to_run.MAX_DIMENSIONS:
quarantined_msg = 'Dimensions product %d is too high' % dimensions_count
break
if quarantined_msg:
line = 'Quarantined Bot\nhttps://%s/restricted/bot/%s\n%s' % (
app_identity.get_default_version_hostname(), bot_id,
quarantined_msg)
ereporter2.log_request(self.request, source='bot', message=line)
return request, bot_id, version, state, dimensions, quarantined_msg
# Look for admin enforced quarantine.
bot_settings = bot_management.get_settings_key(bot_id).get()
if bool(bot_settings and bot_settings.quarantined):
return request, bot_id, version, state, dimensions, 'Quarantined by admin'
return request, bot_id, version, state, dimensions, None
def test_get_settings_key(self):
expected = ndb.Key(
bot_management.BotRoot, 'foo', bot_management.BotSettings, 'settings')
self.assertEqual(expected, bot_management.get_settings_key('foo'))
def test_get_settings_key(self):
expected = ndb.Key(
bot_management.BotRoot, 'foo', bot_management.BotSettings, 'settings')
self.assertEqual(expected, bot_management.get_settings_key('foo'))
def _process(self):
"""Returns True if the bot has invalid parameter and should be automatically
quarantined.
Does one DB synchronous GET.
Returns:
tuple(request, bot_id, version, state, dimensions, quarantined_msg)
"""
request = self.parse_body()
version = request.get("version", None)
dimensions = request.get("dimensions", {})
state = request.get("state", {})
bot_id = None
if dimensions.get("id"):
dimension_id = dimensions["id"]
if isinstance(dimension_id, list) and len(dimension_id) == 1 and isinstance(dimension_id[0], unicode):
bot_id = dimensions["id"][0]
# The bot may decide to "self-quarantine" itself. Accept both via
# dimensions or via state. See bot_management._BotCommon.quarantined for
# more details.
if bool(dimensions.get("quarantined")) or bool(state.get("quarantined")):
return request, bot_id, version, state, dimensions, "Bot self-quarantined"
quarantined_msg = None
# Use a dummy 'for' to be able to break early from the block.
for _ in [0]:
quarantined_msg = has_unexpected_keys(self.EXPECTED_KEYS, request, "keys")
if quarantined_msg:
break
quarantined_msg = has_missing_keys(self.REQUIRED_STATE_KEYS, state, "state")
if quarantined_msg:
break
if not bot_id:
quarantined_msg = "Missing bot id"
break
if not all(
isinstance(key, unicode)
and isinstance(values, list)
and all(isinstance(value, unicode) for value in values)
for key, values in dimensions.iteritems()
):
quarantined_msg = "Invalid dimensions type:\n%s" % json.dumps(
dimensions, sort_keys=True, indent=2, separators=(",", ": ")
)
break
dimensions_count = task_to_run.dimensions_powerset_count(dimensions)
if dimensions_count > task_to_run.MAX_DIMENSIONS:
quarantined_msg = "Dimensions product %d is too high" % dimensions_count
break
if quarantined_msg:
line = "Quarantined Bot\nhttps://%s/restricted/bot/%s\n%s" % (
app_identity.get_default_version_hostname(),
bot_id,
quarantined_msg,
)
ereporter2.log_request(self.request, source="bot", message=line)
return request, bot_id, version, state, dimensions, quarantined_msg
# Look for admin enforced quarantine.
bot_settings = bot_management.get_settings_key(bot_id).get()
if bool(bot_settings and bot_settings.quarantined):
return request, bot_id, version, state, dimensions, "Quarantined by admin"
return request, bot_id, version, state, dimensions, None
def _process(self):
"""Returns True if the bot has invalid parameter and should be automatically
quarantined.
Does one DB synchronous GET.
Returns:
_ProcessResult instance, see its fields for more info.
Raises:
auth.AuthorizationError if bot's credentials are invalid.
"""
request = self.parse_body()
version = request.get('version', None)
dimensions = request.get('dimensions') or {}
state = request.get('state') or {}
bot_id = None
if dimensions.get('id'):
dimension_id = dimensions['id']
if (isinstance(dimension_id, list) and len(dimension_id) == 1
and isinstance(dimension_id[0], unicode)):
bot_id = dimensions['id'][0]
# Make sure bot self-reported ID matches the authentication token. Raises
# auth.AuthorizationError if not.
bot_group_cfg = bot_auth.validate_bot_id_and_fetch_config(bot_id)
# The server side dimensions from bot_group_cfg override bot-provided ones.
# If both server side config and bot report some dimension, server side
# config wins. We still emit an error if bot tries to supply the dimension
# and it disagrees with the server defined one. Don't report ['default'] as
# an error, bot sends it in the handshake before it knows anything at all.
for dim_key, from_cfg in bot_group_cfg.dimensions.iteritems():
from_bot = sorted(dimensions.get(dim_key) or [])
from_cfg = sorted(from_cfg)
if from_bot and from_bot != ['default'] and from_bot != from_cfg:
logging.error(
'Dimensions in bots.cfg doesn\'t match ones provided by the bot\n'
'bot_id: "%s", key: "%s", from_bot: %s, from_cfg: %s',
bot_id, dim_key, from_bot, from_cfg)
dimensions[dim_key] = from_cfg
# Fill in all result fields except 'quarantined_msg'.
result = _ProcessResult(request=request,
bot_id=bot_id,
version=version,
state=state,
dimensions=dimensions,
bot_group_cfg=bot_group_cfg)
# The bot may decide to "self-quarantine" itself. Accept both via
# dimensions or via state. See bot_management._BotCommon.quarantined for
# more details.
if (bool(dimensions.get('quarantined'))
or bool(state.get('quarantined'))):
result.quarantined_msg = 'Bot self-quarantined'
return result
quarantined_msg = None
# Use a dummy 'for' to be able to break early from the block.
for _ in [0]:
quarantined_msg = has_unexpected_keys(self.EXPECTED_KEYS, request,
'keys')
if quarantined_msg:
break
quarantined_msg = has_missing_keys(self.REQUIRED_STATE_KEYS, state,
'state')
if quarantined_msg:
break
if not bot_id:
quarantined_msg = 'Missing bot id'
break
if not dimensions.get('pool'):
quarantined_msg = 'Missing \'pool\' dimension'
break
if not all(
isinstance(key, unicode)
and re.match(task_request.DIMENSION_KEY_RE, key)
and isinstance(values, list) and all(
isinstance(value, unicode) for value in values)
for key, values in dimensions.iteritems()):
quarantined_msg = ('Invalid dimensions type:\n%s' %
json.dumps(dimensions,
sort_keys=True,
indent=2,
separators=(',', ': ')))
break
dimensions_count = task_to_run.dimensions_powerset_count(
dimensions)
if dimensions_count > task_to_run.MAX_DIMENSIONS:
quarantined_msg = 'Dimensions product %d is too high' % dimensions_count
break
if not isinstance(state.get('lease_expiration_ts'),
(None.__class__, int)):
quarantined_msg = (
'lease_expiration_ts (%r) must be int or None' %
(state['lease_expiration_ts']))
break
if quarantined_msg:
line = 'Quarantined Bot\nhttps://%s/restricted/bot/%s\n%s' % (
app_identity.get_default_version_hostname(), bot_id,
quarantined_msg)
ereporter2.log_request(self.request, source='bot', message=line)
result.quarantined_msg = quarantined_msg
return result
# Look for admin enforced quarantine.
bot_settings = bot_management.get_settings_key(bot_id).get()
if bool(bot_settings and bot_settings.quarantined):
result.quarantined_msg = 'Quarantined by admin'
return result
return result
