def test_get_pool_config(self):
self.mock_config(TEST_CONFIG)
self.assertTrue(pools_config.forbid_unknown_pools())
self.assertEqual(None, pools_config.get_pool_config('unknown'))
expected1 = pools_config.PoolConfig(
name=u'pool_name',
rev='rev',
scheduling_users=frozenset([
auth.Identity('user', '*****@*****.**'),
auth.Identity('user', '*****@*****.**'),
]),
scheduling_groups=frozenset([u'group2', u'group1']),
trusted_delegatees={
auth.Identity('user', '*****@*****.**'):
pools_config.TrustedDelegatee(
peer_id=auth.Identity('user', '*****@*****.**'),
required_delegation_tags=frozenset([u'k:tag1', u'k:tag2']),
),
},
service_accounts=frozenset([u'*****@*****.**', u'*****@*****.**']),
service_accounts_groups=(u'accounts_group1', u'accounts_group2'),
task_template_deployment=None)
expected2 = expected1._replace(name='another_name')
self.assertEqual(expected1, pools_config.get_pool_config('pool_name'))
self.assertEqual(expected2,
pools_config.get_pool_config('another_name'))
def test_get_pool_config(self):
self.mock_config(TEST_CONFIG)
self.assertEqual(None, pools_config.get_pool_config('unknown'))
expected1 = pools_config.init_pool_config(
name=u'pool_name',
rev='rev',
scheduling_users=frozenset([
auth.Identity('user', '*****@*****.**'),
auth.Identity('user', '*****@*****.**'),
]),
scheduling_groups=frozenset([u'group2', u'group1']),
trusted_delegatees={
auth.Identity('user', '*****@*****.**'):
pools_config.TrustedDelegatee(
peer_id=auth.Identity('user', '*****@*****.**'),
required_delegation_tags=frozenset([u'k:tag1', u'k:tag2']),
),
},
service_accounts=frozenset([u'*****@*****.**', u'*****@*****.**']),
service_accounts_groups=(u'accounts_group1', u'accounts_group2'),
realm='test:pool/realm',
enforced_realm_permissions=frozenset(
[realms_pb2.REALM_PERMISSION_POOLS_CREATE_TASK]),
default_isolate=pools_config.IsolateServer(
server='https://isolate.server.example.com',
namespace='default-gzip',
),
default_cipd=pools_config.CipdServer(
server='https://cipd.server.example.com',
package_name='some-cipd-client',
client_version='latest',
),
external_schedulers=(pools_config.ExternalSchedulerConfig(
address=u'externalscheduler.google.com',
id=u'ext1',
dimensions=frozenset(['key2:value2', 'key1:value1']),
all_dimensions=frozenset(),
any_dimensions=frozenset(),
enabled=True,
allow_es_fallback=True,
), ),
)
expected2 = expected1._replace(name='another_name')
self.assertEqual(expected1, pools_config.get_pool_config('pool_name'))
self.assertEqual(expected2,
pools_config.get_pool_config('another_name'))
self.assertEqual(['another_name', 'pool_name'], pools_config.known())
def config_for_task(request):
"""Retrieves the ExternalSchedulerConfig for this task request, if any.
Arguments:
request: a task_request.TaskRequest instance.
Returns:
pools_config.ExternalSchedulerConfig for external scheduler to use for
this bot, if it exists, or None otherwise.
"""
s0 = request.task_slice(0)
pool = s0.properties.pool
if not pool:
return None
pool_cfg = pools_config.get_pool_config(pool)
if not pool_cfg or not pool_cfg.external_schedulers:
return None
# Determine the dimension intersection across all task slices.
common_dimensions = set(
task_queues.bot_dimensions_to_flat(s0.properties.dimensions))
for i in range(1, request.num_task_slices):
s = request.task_slice(i)
common_dimensions.intersection_update(
task_queues.bot_dimensions_to_flat(s.properties.dimensions))
return _config_for_dimensions(pool_cfg, common_dimensions)
def check_schedule_request_acl(request):
"""Verifies the current caller can schedule a given task request.
Arguments:
- request: TaskRequest entity with information about the new task.
Raises:
auth.AuthorizationError if the caller is not allowed to schedule this task.
"""
# Only terminate tasks don't have a pool. ACLs for them are handled through
# 'acl.can_edit_bot', see 'terminate' RPC handler. Such tasks do not end up
# hitting this function, and so we can assume there's a pool set (this is
# checked in TaskProperties's pre put hook).
#
# It is guaranteed that at most one item can be specified in 'pool' and that
# every TaskSlice property dimensions use the same pool and id dimensions.
pool = request.task_slice(0).properties.dimensions[u'pool'][0]
pool_cfg = pools_config.get_pool_config(pool)
# request.service_account can be 'bot' or 'none'. We don't care about these,
# they are always allowed. We care when the service account is a real email.
has_service_account = service_accounts.is_service_account(
request.service_account)
if not pool_cfg:
logging.info('Pool "%s" is not in pools.cfg', pool)
# Deployments that use pools.cfg may forbid unknown pools completely.
# This is a safeguard against anyone executing tasks in a new not fully
# configured pool.
if pools_config.forbid_unknown_pools():
raise auth.AuthorizationError(
'Can\'t submit tasks to pool "%s" not defined in pools.cfg' % pool)
# To be backward compatible with Swarming deployments that do not care about
# pools isolation pools without pools.cfg entry are relatively wide open
# (anyone with 'can_create_task' permission can use them). As a downside,
# they are not allowed to use service accounts, since Swarming doesn't know
# what accounts are allowed in such unknown pools.
if has_service_account:
raise auth.AuthorizationError(
'Can\'t use account "%s" in the pool "%s" not defined in pools.cfg' %
(request.service_account, pool))
return
logging.info(
'Looking at the pool "%s" in pools.cfg, rev "%s"', pool, pool_cfg.rev)
# Verify the caller can use the pool at all.
if not _is_allowed_to_schedule(pool_cfg):
raise auth.AuthorizationError(
'User "%s" is not allowed to schedule tasks in the pool "%s", '
'see pools.cfg' % (auth.get_current_identity().to_bytes(), pool))
# Verify the requested task service account is allowed in this pool.
if (has_service_account and
not _is_allowed_service_account(request.service_account, pool_cfg)):
raise auth.AuthorizationError(
'Service account "%s" is not allowed in the pool "%s", see pools.cfg' %
(request.service_account, pool))
def check_task_cancel_acl(task_request):
"""Checks if the caller is allowed to cancel the task.
Checks if the caller has global permission using acl.can_edit_task().
If the caller doesn't have any global permissions,
Checks if the caller has either of 'swarming.pools.cancelTask' or
'swarming.tasks.cancel' permission.
Args:
task_request: An instance of TaskRequest.
Returns:
None
Raises:
auth.AuthorizationError: if the caller is not allowed.
"""
# check global permission.
if acl.can_edit_task(task_request):
return
# check 'swarming.pools.cancelTask' permission of the pool in task dimensions.
if task_request.pool:
pool_cfg = pools_config.get_pool_config(task_request.pool)
if not pool_cfg:
raise endpoints.InternalServerErrorException(
'Pool cfg not found. pool: %s' % task_request.pool)
if pool_cfg.realm and auth.has_permission(
get_permission(realms_pb2.REALM_PERMISSION_POOLS_CANCEL_TASK),
[pool_cfg.realm]):
return
# check 'swarming.pools.cancelTask' permission of the pool in bot dimensions.
if task_request.bot_id:
pools = bot_management.get_pools_from_dimensions_flat(
_retrieve_bot_dimensions(task_request.bot_id))
pool_realms = [
p.realm for p in map(pools_config.get_pool_config, pools)
if p.realm
]
if pool_realms and auth.has_permission(
get_permission(realms_pb2.REALM_PERMISSION_POOLS_CANCEL_TASK),
pool_realms):
return
# check 'swarming.tasks.cancel' permission.
task_realm = task_request.realm
if task_realm and auth.has_permission(
get_permission(realms_pb2.REALM_PERMISSION_TASKS_CANCEL),
[task_realm]):
return
raise auth.AuthorizationError('Task "%s" is not accessible' %
task_request.task_id)
def apply_server_property_defaults(properties):
"""Fills ndb task properties with default values read from server settings."""
settings = config.settings()
# TODO(iannucci): This was an artifact of the existing test harnesses;
# get_pool_config raises on None, but the way it's mocked in
# ./test_env_handlers.py allows `get_pool_config` to return None in this case.
# This try/except will be cleaned up in a subsequent CL, once I remove these
# default services from `config`.
try:
pool_cfg = pools_config.get_pool_config(properties.pool)
except ValueError:
pool_cfg = None
if not settings and not pool_cfg:
return
iso_server = settings.isolate.default_server
iso_ns = settings.isolate.default_namespace
if pool_cfg and pool_cfg.default_isolate:
iso_server = pool_cfg.default_isolate.server
iso_ns = pool_cfg.default_isolate.namespace
if iso_server and iso_ns:
properties.inputs_ref = properties.inputs_ref or task_request.FilesRef(
)
properties.inputs_ref.isolatedserver = (
properties.inputs_ref.isolatedserver or iso_server)
properties.inputs_ref.namespace = (properties.inputs_ref.namespace
or iso_ns)
cipd_server = settings.cipd.default_server
cipd_vers = settings.cipd.default_client_package.version
if pool_cfg and pool_cfg.default_cipd:
cipd_server = pool_cfg.default_cipd.server
cipd_vers = pool_cfg.default_cipd.client_version
if cipd_server and properties.cipd_input:
properties.cipd_input.server = (properties.cipd_input.server
or cipd_server)
properties.cipd_input.client_package = (
properties.cipd_input.client_package or task_request.CipdPackage())
# TODO(iannucci) - finish removing 'client_package' as a task-configurable
# setting.
properties.cipd_input.client_package.package_name = (
'infra/tools/cipd/${platform}')
properties.cipd_input.client_package.version = (
properties.cipd_input.client_package.version or cipd_vers)
def _bot_pool_cfg(bot_dimensions):
"""Retrieves the PoolConfig for a bot.
Arguments:
- bot_dimensions: The dimensions of the bot as a dictionary in
{string key: list of string values} format.
Returns:
PoolConfig for the bot if it exists, or None otherwise.
"""
pools = bot_dimensions.get(u'pool')
if not pools:
return None
if len(pools) == 1:
return pools_config.get_pool_config(pools[0])
else:
logging.warning(
'Bot with dimensions %s was found to be in multiple '
'pools. Unable to determine pool config.', bot_dimensions)
return None
def _check_pools_filters_acl(perm_enum, pools):
"""Checks if the caller has the permission in the specified pools.
The caller needs to have the permission in *all* pools.
Raises:
auth.AuthorizationError: if the caller is not allowed.
"""
# Pool dimension is required if the caller doesn't have global permission.
if not pools:
raise auth.AuthorizationError('No pool is specified')
perm = get_permission(perm_enum)
# the caller needs to have all permissions of the pools.
for p in pools:
pool_cfg = pools_config.get_pool_config(p)
if not pool_cfg:
raise endpoints.BadRequestException('Pool "%s" not found' % p)
_check_permission(perm, [pool_cfg.realm])
def can_list_bots(pool):
"""Checks if the caller is allowed to list tasks of the pool.
Args:
pool: Pool name
Returns:
allowed: True if allowed, False otherwise.
"""
if acl.can_view_bot():
return True
pool_cfg = pools_config.get_pool_config(pool)
if not pool_cfg:
logging.warning('Pool "%s" not found', pool)
return False
try:
_check_permission(
get_permission(realms_pb2.REALM_PERMISSION_POOLS_LIST_BOTS),
[pool_cfg.realm])
return True
except auth.AuthorizationError:
return False
def _check_bot_acl(perm_enum, bot_id):
"""Checks if the caller is allowed to access the resource associated with
the bot.
The caller needs to have the permission in *any* pools the bot belong to.
Raises:
auth.AuthorizationError: if the caller is not allowed.
"""
# retrieve the pools from bot dimensions.
pools = bot_management.get_pools_from_dimensions_flat(
_retrieve_bot_dimensions(bot_id))
realms = []
for p in pools:
pool_cfg = pools_config.get_pool_config(p)
if not pool_cfg:
logging.warning('PoolCfg is missing. pool: %s', p)
continue
if not pool_cfg.realm:
continue
realms.append(pool_cfg.realm)
# the caller needs to have any permission of the pools.
_check_permission(get_permission(perm_enum), realms)
def get_task_account_token(task_id, bot_id, scopes):
"""Returns an access token for a service account associated with a task.
Assumes authorization checks have been made already. If the task is not
configured to use service account returns ('none', None). If the task is
configured to use whatever bot is using when calling Swarming, returns
('bot', None).
Otherwise returns (<email>, AccessToken with valid token for <email>).
If the task has realm, it calls MintServiceAccountToken rpc using the realm.
Otherwise, it calls MintOAuthTokenViaGrant with grant token. The legacy path
will be deprecated after migrating to Realm-based configurations.
Args:
task_id: ID of the task.
bot_id: ID of the bot that executes the task, for logs.
scopes: list of requested OAuth scopes.
Returns:
(<service account email> or 'bot' or 'none', AccessToken or None).
Raises:
PermissionError if the token server forbids the usage.
MisconfigurationError if the service account is misconfigured.
InternalError if the RPC fails unexpectedly.
"""
# Grab corresponding TaskRequest.
try:
result_summary_key = task_pack.run_result_key_to_result_summary_key(
task_pack.unpack_run_result_key(task_id))
task_request_key = task_pack.result_summary_key_to_request_key(
result_summary_key)
except ValueError as exc:
logging.error('Unexpectedly bad task_id: %s', exc)
raise MisconfigurationError('Bad task_id: %s' % task_id)
task_request = task_request_key.get()
if not task_request:
raise MisconfigurationError('No such task request: %s' % task_id)
# 'none' or 'bot' cases are handled by the bot locally, no token for them.
if task_request.service_account in ('none', 'bot'):
return task_request.service_account, None
# The only possible case is a service account email. Double check this.
if not service_accounts_utils.is_service_account(
task_request.service_account):
raise MisconfigurationError('Not a service account email: %s' %
task_request.service_account)
# Additional information for Token Server's logs.
audit_tags = [
'swarming:bot_id:%s' % bot_id,
'swarming:task_id:%s' % task_id,
'swarming:task_name:%s' % task_request.name,
]
# task_request.service_account_token can be empty here only when the task has
# a realm and the service account was authorized via realm ACLs. Use
# MintServiceAccountToken RPC for such tasks.
if not task_request.service_account_token:
assert task_request.realm
# Re-check if the service account is still allowed to run in the realm,
# because it may have changed since the last check.
pool_cfg = pools_config.get_pool_config(task_request.pool)
realms.check_tasks_act_as(task_request, pool_cfg, enforce=True)
access_token, expiry = _mint_service_account_token(
task_request.service_account, task_request.realm, scopes,
audit_tags)
else:
# Use grant token to grab the real OAuth token. Note that the bot caches the
# resulting OAuth token internally, so we don't bother to cache it here.
access_token, expiry = _mint_oauth_token_via_grant(
task_request.service_account_token, scopes, audit_tags)
# Log and return the token.
token = AccessToken(access_token,
int(utils.datetime_to_timestamp(expiry) / 1e6))
_check_and_log_token('task associated', task_request.service_account,
token)
return task_request.service_account, token