def _resolve_bot_monitoring(ctx, bot_monitorings):
"""Validates and simplifies bot_monitoring entries in pools.cfg."""
out = {}
for m in bot_monitorings:
with ctx.prefix('bot_monitoring %r: ', m.name):
# Use the same rules for the name as for the dimensions for simplicity
# here.
if not local_config.validate_dimension_key(m.name):
ctx.error('invalid name')
if m.name in out:
ctx.error('duplicate name')
keys = set(m.dimension_key)
if len(keys) != len(m.dimension_key):
ctx.error('duplicate dimension_key')
for k in keys:
if not local_config.validate_dimension_key(k):
ctx.error('invalid dimension_key %r', k)
# pool is always implicit.
keys.add('pool')
out[m.name] = sorted(keys)
return out
def test_validate_dimension_key(self):
self.assertTrue(config.validate_dimension_key(u'b'))
self.assertTrue(config.validate_dimension_key(u'-'))
self.assertTrue(config.validate_dimension_key(u'b1'))
self.assertFalse(config.validate_dimension_key(u'1b'))
self.assertFalse(config.validate_dimension_key(u''))
self.assertFalse(config.validate_dimension_key(u'+'))
def _process(self):
"""Fetches bot info and settings, does authorization and quarantine checks.
Returns:
_ProcessResult instance, see its fields for more info.
Raises:
auth.AuthorizationError if bot's credentials are invalid.
"""
request = self.parse_body()
version = request.get('version', None)
dimensions = request.get('dimensions') or {}
state = request.get('state') or {}
bot_id = None
if dimensions.get('id'):
dimension_id = dimensions['id']
if (isinstance(dimension_id, list) and len(dimension_id) == 1
and isinstance(dimension_id[0], unicode)):
bot_id = dimensions['id'][0]
lease_expiration_ts = None
machine_type = None
if bot_id:
logging.debug('Fetching bot info and settings')
bot_info, bot_settings = ndb.get_multi([
bot_management.get_info_key(bot_id),
bot_management.get_settings_key(bot_id)
])
if bot_info:
lease_expiration_ts = bot_info.lease_expiration_ts
machine_type = bot_info.machine_type
# Make sure bot self-reported ID matches the authentication token. Raises
# auth.AuthorizationError if not.
logging.debug('Fetching bot group config')
bot_group_cfg = bot_auth.validate_bot_id_and_fetch_config(
bot_id, machine_type)
# The server side dimensions from bot_group_cfg override bot-provided ones.
# If both server side config and bot report some dimension, server side
# config wins. We still emit an warning if bot tries to supply the dimension
# and it disagrees with the server defined one. Note that this may happen
# on a first poll after server side config for a bot has changed. The bot
# doesn't know about new server-assigned dimensions yet in this case. Also
# don't report ['default'], bot sends it in the handshake before it knows
# anything at all.
for dim_key, from_cfg in bot_group_cfg.dimensions.iteritems():
from_bot = sorted(dimensions.get(dim_key) or [])
from_cfg = sorted(from_cfg)
if from_bot and from_bot != ['default'] and from_bot != from_cfg:
logging.warning(
'Dimensions in bots.cfg don\'t match ones provided by the bot\n'
'bot_id: "%s", key: "%s", from_bot: %s, from_cfg: %s',
bot_id, dim_key, from_bot, from_cfg)
dimensions[dim_key] = from_cfg
# Fill in all result fields except 'quarantined_msg'.
result = _ProcessResult(request=request,
bot_id=bot_id,
version=version,
state=state,
dimensions=dimensions,
bot_group_cfg=bot_group_cfg,
lease_expiration_ts=lease_expiration_ts,
maintenance_msg=state.get('maintenance'))
# The bot may decide to "self-quarantine" itself. Accept both via
# dimensions or via state. See bot_management._BotCommon.quarantined for
# more details.
if (bool(dimensions.get('quarantined'))
or bool(state.get('quarantined'))):
result.quarantined_msg = 'Bot self-quarantined'
return result
quarantined_msg = None
# Use a dummy 'for' to be able to break early from the block.
for _ in [0]:
quarantined_msg = has_unexpected_keys(self.EXPECTED_KEYS, request,
'keys')
if quarantined_msg:
break
quarantined_msg = has_missing_keys(self.REQUIRED_STATE_KEYS, state,
'state')
if quarantined_msg:
break
if not bot_id:
quarantined_msg = 'Missing bot id'
break
if not dimensions.get('pool'):
quarantined_msg = 'Missing \'pool\' dimension'
break
if not all(
config.validate_dimension_key(key)
and isinstance(values, list) and all(
config.validate_dimension_value(value)
for value in values)
for key, values in dimensions.iteritems()):
quarantined_msg = ('Invalid dimensions type:\n%s' %
json.dumps(dimensions,
sort_keys=True,
indent=2,
separators=(',', ': ')))
break
if quarantined_msg:
line = 'Quarantined Bot\nhttps://%s/restricted/bot/%s\n%s' % (
app_identity.get_default_version_hostname(), bot_id,
quarantined_msg)
ereporter2.log_request(self.request, source='bot', message=line)
result.quarantined_msg = quarantined_msg
return result
# Look for admin enforced quarantine.
if bool(bot_settings and bot_settings.quarantined):
result.quarantined_msg = 'Quarantined by admin'
return result
# TODO(maruel): Parallelise.
task_queues.assert_bot_async(dimensions).get_result()
return result
def test_validate_dimension_key_length(self):
l = config.DIMENSION_KEY_LENGTH
self.assertTrue(config.validate_dimension_key(u'b' * l))
self.assertFalse(config.validate_dimension_key(u'b' * (l + 1)))
def _validate_bots_cfg(cfg, ctx):
"""Validates bots.cfg file."""
with ctx.prefix('trusted_dimensions: '):
for dim_key in cfg.trusted_dimensions:
if not local_config.validate_dimension_key(dim_key):
ctx.error('invalid dimension key %r', dim_key)
# Explicitly mentioned bot_id => index of a group where it was mentioned.
bot_ids = {}
# bot_id_prefix => index of a group where it was defined.
bot_id_prefixes = {}
# Index of a group to use as default fallback (there can be only one).
default_group_idx = None
# machine_type names.
machine_type_names = set()
for i, entry in enumerate(cfg.bot_group):
with ctx.prefix('bot_group #%d: ', i):
# Validate bot_id field and make sure bot_id groups do not intersect.
_validate_group_bot_ids(ctx, entry.bot_id, i, bot_ids,
bot_id_prefixes)
# Validate bot_id_prefix and make sure bot_id_prefix groups do not
# intersect.
_validate_group_bot_id_prefixes(ctx, entry.bot_id_prefix, i,
bot_id_prefixes, bot_ids)
# A group without bot_id, bot_id_prefix and machine_type is applied to
# bots that don't fit any other groups. There should be at most one such
# group.
if (not entry.bot_id and not entry.bot_id_prefix
and not entry.machine_type):
if default_group_idx is not None:
ctx.error('group #%d is already set as default',
default_group_idx)
else:
default_group_idx = i
# Validate machine_type.
for i, machine_type in enumerate(entry.machine_type):
with ctx.prefix('machine_type #%d: ', i):
_validate_machine_type(ctx, machine_type,
machine_type_names)
# Validate 'auth' and 'system_service_account' fields.
_validate_group_auth_and_system_service_account(ctx, entry)
# Validate 'owners'. Just check they are emails.
for own in entry.owners:
_validate_email(ctx, own, 'owner')
# Validate 'dimensions'.
for dim in entry.dimensions:
if not local_config.validate_flat_dimension(dim):
ctx.error('bad dimension %r', dim)
# Validate 'bot_config_script': the supplemental bot_config.py.
if entry.bot_config_script:
# Another check in bot_code.py confirms that the script itself is valid
# python before it is accepted by the config service. See
# _validate_scripts validator there. We later recheck this (see below)
# when assembling the final expanded bots.cfg.
if not entry.bot_config_script.endswith('.py'):
ctx.error(
'invalid bot_config_script name: must end with .py')
if os.path.basename(
entry.bot_config_script) != entry.bot_config_script:
ctx.error(
'invalid bot_config_script name: must not contain path entry'
)
# We can't validate that the file exists here. We'll do it later in
# _fetch_and_expand_bots_cfg when assembling the final config from
# individual files.
# Validate 'bot_config_script_content': the content must be valid python.
# This validation is hit when validating the expanded bot config.
if entry.bot_config_script_content:
try:
ast.parse(entry.bot_config_script_content)
except (SyntaxError, TypeError) as e:
ctx.error('invalid bot config script "%s": %s' %
(entry.bot_config_script, e))