def _process(self):
"""Fetches bot info and settings, does authorization and quarantine checks.
Returns:
_ProcessResult instance, see its fields for more info.
Raises:
auth.AuthorizationError if bot's credentials are invalid.
"""
request = self.parse_body()
version = request.get('version', None)
dimensions = request.get('dimensions') or {}
state = request.get('state') or {}
bot_id = None
if dimensions.get('id'):
dimension_id = dimensions['id']
if (isinstance(dimension_id, list) and len(dimension_id) == 1
and isinstance(dimension_id[0], unicode)):
bot_id = dimensions['id'][0]
lease_expiration_ts = None
machine_type = None
if bot_id:
logging.debug('Fetching bot info and settings')
bot_info, bot_settings = ndb.get_multi([
bot_management.get_info_key(bot_id),
bot_management.get_settings_key(bot_id)
])
if bot_info:
lease_expiration_ts = bot_info.lease_expiration_ts
machine_type = bot_info.machine_type
# Make sure bot self-reported ID matches the authentication token. Raises
# auth.AuthorizationError if not.
logging.debug('Fetching bot group config')
bot_group_cfg = bot_auth.validate_bot_id_and_fetch_config(
bot_id, machine_type)
# The server side dimensions from bot_group_cfg override bot-provided ones.
# If both server side config and bot report some dimension, server side
# config wins. We still emit an warning if bot tries to supply the dimension
# and it disagrees with the server defined one. Note that this may happen
# on a first poll after server side config for a bot has changed. The bot
# doesn't know about new server-assigned dimensions yet in this case. Also
# don't report ['default'], bot sends it in the handshake before it knows
# anything at all.
for dim_key, from_cfg in bot_group_cfg.dimensions.iteritems():
from_bot = sorted(dimensions.get(dim_key) or [])
from_cfg = sorted(from_cfg)
if from_bot and from_bot != ['default'] and from_bot != from_cfg:
logging.warning(
'Dimensions in bots.cfg don\'t match ones provided by the bot\n'
'bot_id: "%s", key: "%s", from_bot: %s, from_cfg: %s',
bot_id, dim_key, from_bot, from_cfg)
dimensions[dim_key] = from_cfg
# Fill in all result fields except 'quarantined_msg'.
result = _ProcessResult(request=request,
bot_id=bot_id,
version=version,
state=state,
dimensions=dimensions,
bot_group_cfg=bot_group_cfg,
lease_expiration_ts=lease_expiration_ts,
maintenance_msg=state.get('maintenance'))
# The bot may decide to "self-quarantine" itself. Accept both via
# dimensions or via state. See bot_management._BotCommon.quarantined for
# more details.
if (bool(dimensions.get('quarantined'))
or bool(state.get('quarantined'))):
result.quarantined_msg = 'Bot self-quarantined'
return result
quarantined_msg = None
# Use a dummy 'for' to be able to break early from the block.
for _ in [0]:
quarantined_msg = has_unexpected_keys(self.EXPECTED_KEYS, request,
'keys')
if quarantined_msg:
break
quarantined_msg = has_missing_keys(self.REQUIRED_STATE_KEYS, state,
'state')
if quarantined_msg:
break
if not bot_id:
quarantined_msg = 'Missing bot id'
break
if not dimensions.get('pool'):
quarantined_msg = 'Missing \'pool\' dimension'
break
if not all(
config.validate_dimension_key(key)
and isinstance(values, list) and all(
config.validate_dimension_value(value)
for value in values)
for key, values in dimensions.iteritems()):
quarantined_msg = ('Invalid dimensions type:\n%s' %
json.dumps(dimensions,
sort_keys=True,
indent=2,
separators=(',', ': ')))
break
if quarantined_msg:
line = 'Quarantined Bot\nhttps://%s/restricted/bot/%s\n%s' % (
app_identity.get_default_version_hostname(), bot_id,
quarantined_msg)
ereporter2.log_request(self.request, source='bot', message=line)
result.quarantined_msg = quarantined_msg
return result
# Look for admin enforced quarantine.
if bool(bot_settings and bot_settings.quarantined):
result.quarantined_msg = 'Quarantined by admin'
return result
# TODO(maruel): Parallelise.
task_queues.assert_bot_async(dimensions).get_result()
return result
def _validate_pools_cfg(cfg, ctx):
"""Validates pools.cfg file."""
template_map = _resolve_task_template_inclusions(
ctx, cfg.task_template)
deployment_map = _resolve_task_template_deployments(
ctx, template_map, cfg.task_template_deployment)
bot_monitorings = _resolve_bot_monitoring(ctx, cfg.bot_monitoring)
bot_monitoring_unreferred = set(bot_monitorings)
# Currently optional
if cfg.HasField("default_external_services"):
_validate_external_services(ctx, cfg.default_external_services)
pools = set()
for i, msg in enumerate(cfg.pool):
with ctx.prefix('pool #%d (%s): ', i, '|'.join(msg.name) or 'unnamed'):
# Validate names.
if not msg.name:
ctx.error('at least one pool name must be given')
for name in msg.name:
if not local_config.validate_dimension_value(name):
ctx.error('bad pool name "%s", not a valid dimension value', name)
elif name in pools:
ctx.error('pool "%s" was already declared', name)
else:
pools.add(name)
# Validate schedulers.user.
for u in msg.schedulers.user:
_validate_ident(ctx, 'user', u)
# Validate schedulers.group.
for g in msg.schedulers.group:
if not auth.is_valid_group_name(g):
ctx.error('bad group name "%s"', g)
# Validate schedulers.trusted_delegation.
seen_peers = set()
for d in msg.schedulers.trusted_delegation:
with ctx.prefix('trusted_delegation #%d (%s): ', i, d.peer_id):
if not d.peer_id:
ctx.error('"peer_id" is required')
else:
peer_id = _validate_ident(ctx, 'peer_id', d.peer_id)
if peer_id in seen_peers:
ctx.error('peer "%s" was specified twice', d.peer_id)
elif peer_id:
seen_peers.add(peer_id)
for i, tag in enumerate(d.require_any_of.tag):
if ':' not in tag:
ctx.error('bad tag #%d "%s" - must be <key>:<value>', i, tag)
# Validate service accounts.
for i, account in enumerate(msg.allowed_service_account):
if not service_accounts.is_service_account(account):
ctx.error('bad allowed_service_account #%d "%s"', i, account)
# Validate service account groups.
for i, group in enumerate(msg.allowed_service_account_group):
if not auth.is_valid_group_name(group):
ctx.error('bad allowed_service_account_group #%d "%s"', i, group)
# Validate external schedulers.
for i, es in enumerate(msg.external_schedulers):
if not es.address:
ctx.error('%sth external scheduler config had no address', i)
_resolve_deployment(ctx, msg, template_map, deployment_map)
if msg.bot_monitoring:
if msg.bot_monitoring not in bot_monitorings:
ctx.error('refer to missing bot_monitoring %r', msg.bot_monitoring)
else:
bot_monitoring_unreferred.discard(msg.bot_monitoring)
if bot_monitoring_unreferred:
ctx.error(
'bot_monitoring not referred to: %s',
', '.join(sorted(bot_monitoring_unreferred)))
def test_validate_dimension_value(self):
self.assertTrue(config.validate_dimension_value(u'b'))
self.assertFalse(config.validate_dimension_value(u''))
self.assertFalse(config.validate_dimension_value(u' a'))
def test_validate_dimension_value_length(self):
l = config.DIMENSION_VALUE_LENGTH
self.assertTrue(config.validate_dimension_value(u'b' * l))
self.assertFalse(config.validate_dimension_value(u'b' * (l + 1)))
def _validate_pools_cfg(cfg, ctx):
"""Validates pools.cfg file."""
template_map = _resolve_task_template_inclusions(ctx, cfg.task_template)
deployment_map = _resolve_task_template_deployments(
ctx, template_map, cfg.task_template_deployment)
pools = set()
for i, msg in enumerate(cfg.pool):
with ctx.prefix('pool #%d (%s): ', i, '|'.join(msg.name) or 'unnamed'):
# Validate names.
if not msg.name:
ctx.error('at least one pool name must be given')
for name in msg.name:
if not local_config.validate_dimension_value(name):
ctx.error(
'bad pool name "%s", not a valid dimension value',
name)
elif name in pools:
ctx.error('pool "%s" was already declared', name)
else:
pools.add(name)
# Validate schedulers.user.
for u in msg.schedulers.user:
_validate_ident(ctx, 'user', u)
# Validate schedulers.group.
for g in msg.schedulers.group:
if not auth.is_valid_group_name(g):
ctx.error('bad group name "%s"', g)
# Validate schedulers.trusted_delegation.
seen_peers = set()
for d in msg.schedulers.trusted_delegation:
with ctx.prefix('trusted_delegation #%d (%s): ', i, d.peer_id):
if not d.peer_id:
ctx.error('"peer_id" is required')
else:
peer_id = _validate_ident(ctx, 'peer_id', d.peer_id)
if peer_id in seen_peers:
ctx.error('peer "%s" was specified twice',
d.peer_id)
elif peer_id:
seen_peers.add(peer_id)
for i, tag in enumerate(d.require_any_of.tag):
if ':' not in tag:
ctx.error(
'bad tag #%d "%s" - must be <key>:<value>', i,
tag)
# Validate service accounts.
for i, account in enumerate(msg.allowed_service_account):
if not service_accounts.is_service_account(account):
ctx.error('bad allowed_service_account #%d "%s"', i,
account)
# Validate service account groups.
for i, group in enumerate(msg.allowed_service_account_group):
if not auth.is_valid_group_name(group):
ctx.error('bad allowed_service_account_group #%d "%s"', i,
group)
_resolve_deployment(ctx, msg, template_map, deployment_map)
