def resolve_group_names(self, r, target_group_ids, groups):
"""Resolve any security group names to the corresponding group ids
With the context of a given network attached resource.
"""
names = self.get_group_names(target_group_ids)
if not names:
return target_group_ids
target_group_ids = list(target_group_ids)
vpc_id = self.vpc_expr.search(r)
if not vpc_id:
raise PolicyExecutionError(
self._format_error(
"policy:{policy} non vpc attached resource used "
"with modify-security-group: {resource_id}",
resource_id=r[self.manager.resource_type.id]))
found = False
for n in names:
for g in groups:
if g['GroupName'] == n and g['VpcId'] == vpc_id:
found = g['GroupId']
if not found:
raise PolicyExecutionError(
self._format_error(
("policy:{policy} could not resolve sg:{name} for "
"resource:{resource_id} in vpc:{vpc}"),
name=n,
resource_id=r[self.manager.resource_type.id],
vpc=vpc_id))
target_group_ids.remove(n)
target_group_ids.append(found)
return target_group_ids
def process(self, resources):
related_resources = dict(
zip(jmespath.search('[].%s' % self.data['key'], resources),
resources))
related_ids = set(related_resources)
related_tag_map = self.get_resource_tag_map(self.data['resource'],
related_ids)
missing_related_tags = related_ids.difference(related_tag_map.keys())
if not self.data.get('skip_missing', True) and missing_related_tags:
raise PolicyExecutionError(
"Unable to find all %d %s related resources tags %d missing" %
(len(related_ids), self.data['resource'],
len(missing_related_tags)))
# rely on resource manager tag action implementation as it can differ between resources
tag_action = self.manager.action_registry.get('tag')({}, self.manager)
tag_action.id_key = tag_action.manager.get_model().id
stats = Counter()
for related, r in related_resources.items():
if related in missing_related_tags or not related_tag_map[related]:
stats['missing'] += 1
elif self.process_resource(r, related_tag_map[related],
self.data['tags'], tag_action):
stats['tagged'] += 1
else:
stats['unchanged'] += 1
self.log.info(
'Tagged %d resources from related, missing-skipped %d unchanged %d',
stats['tagged'], stats['missing'], stats['unchanged'])
def process(self, resources):
client = local_session(
self.manager.session_factory).client('service-quotas')
multiplier = self.data.get('multiplier', 1.2)
error = None
for r in resources:
count = math.floor(multiplier * r['Value'])
if not r['Adjustable']:
continue
try:
client.request_service_quota_increase(
ServiceCode=r['ServiceCode'],
QuotaCode=r['QuotaCode'],
DesiredValue=count)
except client.exceptions.QuotaExceededException as e:
error = e
self.log.error('Requested:%s exceeds quota limit for %s' %
(count, r['QuotaCode']))
continue
except (
client.exceptions.AccessDeniedException,
client.exceptions.DependencyAccessDeniedException,
):
raise PolicyExecutionError(
'Access Denied to increase quota: %s' % r['QuotaCode'])
except (
client.exceptions.NoSuchResourceException,
client.exceptions.InvalidResourceStateException,
client.exceptions.ResourceAlreadyExistsException,
) as e:
error = e
continue
if error:
raise PolicyExecutionError from error
def get_groups_by_names(self, names):
"""Resolve security names to security groups resources."""
if not names:
return []
client = utils.local_session(
self.manager.session_factory).client('ec2')
sgs = self.manager.retry(client.describe_security_groups,
Filters=[{
'Name': 'group-name',
'Values': names
}]).get('SecurityGroups', [])
unresolved = set(names)
for s in sgs:
if s['GroupName'] in unresolved:
unresolved.remove(s['GroupName'])
if unresolved:
raise PolicyExecutionError(
self._format_error(
"policy:{policy} security groups not found "
"requested: {names}, found: {groups}",
names=list(unresolved),
groups=[g['GroupId'] for g in sgs]))
return sgs
def _get_identity(self, session):
identity = jmespath.search('mode."provision-options".identity',
self.policy.data) or {
'type': AUTH_TYPE_EMBED
}
if identity['type'] != AUTH_TYPE_UAI:
return identity
# We need to resolve the client id of the uai, as the metadata
# service in functions is old and doesn't support newer
# metadata api versions where this would be extraneous
# (ie. versions 2018-02-01 or 2019-08-01). notably the
# official docs here are wrong
# https://docs.microsoft.com/en-us/azure/app-service/overview-managed-identity
# TODO: switch out to using uai resource manager so we get some cache
# benefits across policies using the same uai.
id_client = session.client(
'azure.mgmt.msi.ManagedServiceIdentityClient')
found = None
for uai in id_client.user_assigned_identities.list_by_subscription():
if uai.id == identity['id'] or uai.name == identity['id']:
found = uai
break
if not found:
raise PolicyExecutionError(
"policy:%s Could not find the user assigned identity %s" %
(self.policy.name, identity['id']))
identity['id'] = found.id
identity['client_id'] = found.client_id
return identity
def test_dispatch_err_handle(self, mock_collection):
self.patch(
handler, 'policy_config', {
'execution-options': {
'output_dir': 's3://xyz',
'account_id': '004'
},
'policies': [{
'resource': 'ec2',
'name': 'xyz'
}]
})
output = self.capture_logging('custodian.lambda',
level=logging.WARNING)
pmock = mock.MagicMock()
pmock.push.side_effect = PolicyExecutionError("foo")
mock_collection.from_data.return_value = [pmock]
self.assertRaises(PolicyExecutionError, handler.dispatch_event,
{'detail': {
'xyz': 'oui'
}}, None)
self.patch(handler, 'C7N_CATCH_ERR', True)
handler.dispatch_event({'detail': {'xyz': 'oui'}}, None)
self.assertEqual(output.getvalue().count('error during'), 2)
def get_resource_sets(self, event):
# return a mapping of (account_id, region): [resource_arns]
# per the finding in the event.
# Group resources by account_id, region for role assumes
resource_sets = {}
# Loop over findings and set resource set accordingly
# Lazy import to avoid aws sdk runtime dep in core
from c7n.resources.aws import Arn
for finding in event['detail']['findings']:
resource_sets.setdefault(
(finding['AwsAccountId'], finding['Resources'][0]['Region']),
[]).append(Arn.parse(finding['Resources'][0]['Id']))
# Warn if not configured for member-role and have multiple accounts resources.
if (not self.policy.data['mode'].get('member-role')
and {self.policy.options.account_id} != {
account_id
for (account_id, region), rarns in resource_sets.items()
}):
msg = ('hub-mode not configured for multi-account member-role '
'but multiple resource accounts found')
self.policy.log.warning(msg)
raise PolicyExecutionError(msg)
return resource_sets
def load_file(self, path, resource_key):
data = load_file(path)
if resource_key:
data = jmespath.search(resource_key, data)
if not isinstance(data, list):
raise PolicyExecutionError(
"found disk records at %s in non list format %s" %
(path, type(data)))
return DataFile(path, resource_key, data)
def initialize_source(self):
# Ideally we'll be given a source, but we'll attempt to auto create it
# if given an org_domain or org_id.
if self._source:
return self._source
elif 'source' in self.data:
self._source = self.data['source']
return self._source
session = local_session(self.manager.session_factory)
# Resolve Organization Id
if 'org-id' in self.data:
org_id = self.data['org-id']
else:
orgs = session.client('cloudresourcemanager', 'v1',
'organizations')
res = orgs.execute_query('search', {
'body': {
'filter': 'domain:%s' % self.data['org-domain']
}
}).get('organizations')
if not res:
raise PolicyExecutionError(
"Could not determine organization id")
org_id = res[0]['name'].rsplit('/', 1)[-1]
# Resolve Source
client = session.client(self.Service, self.ServiceVersion,
'organizations.sources')
source = None
res = [
s for s in
client.execute_query('list', {
'parent': 'organizations/{}'.format(org_id)
}).get('sources') if s['displayName'] == self.CustodianSourceName
]
if res:
source = res[0]['name']
if source is None:
source = client.execute_command(
'create', {
'parent': 'organizations/{}'.format(org_id),
'body': {
'displayName': self.CustodianSourceName,
'description': 'Cloud Management Rules Engine'
}
}).get('name')
self.log.info(
"policy:%s resolved cscc source: %s, update policy with this source value",
self.manager.ctx.policy.name, source)
self._source = source
return self._source
def get_analyzer(self, client):
if self.data.get('analyzer'):
return self.data['analyzer']
analyzers = client.list_analyzers(type='ACCOUNT').get('analyzers', ())
found = False
for a in analyzers:
if a['status'] != 'ACTIVE':
continue
found = a
if not found:
raise PolicyExecutionError(
"policy:%s no access analyzer found in account or org analyzer specified"
% (self.manager.policy.name))
return found['arn']
def get_resource_sets(self, event):
# return a mapping of (account_id, region): [resource_arns]
# per the finding in the event.
resource_arns = self.get_resource_arns(event)
# Group resources by account_id, region for role assumes
resource_sets = {}
for rarn in resource_arns:
resource_sets.setdefault((rarn.account_id, rarn.region), []).append(rarn)
# Warn if not configured for member-role and have multiple accounts resources.
if (not self.policy.data['mode'].get('member-role') and
{self.policy.options.account_id} != {
rarn.account_id for rarn in resource_arns}):
msg = ('hub-mode not configured for multi-account member-role '
'but multiple resource accounts found')
self.policy.log.warning(msg)
raise PolicyExecutionError(msg)
return resource_sets
def process(self, resources):
related_resources = []
for rrid, r in zip(
jmespath.search('[].[%s]' % self.data['key'], resources),
resources):
related_resources.append((rrid[0], r))
related_ids = {r[0] for r in related_resources}
missing = False
if None in related_ids:
missing = True
related_ids.discard(None)
related_tag_map = self.get_resource_tag_map(self.data['resource'],
related_ids)
missing_related_tags = related_ids.difference(related_tag_map.keys())
if not self.data.get('skip_missing', True) and (missing_related_tags
or missing):
raise PolicyExecutionError(
"Unable to find all %d %s related resources tags %d missing" %
(len(related_ids), self.data['resource'],
len(missing_related_tags) + int(missing)))
# rely on resource manager tag action implementation as it can differ between resources
tag_action = self.manager.action_registry.get('tag')({}, self.manager)
tag_action.id_key = tag_action.manager.get_model().id
client = tag_action.get_client()
stats = Counter()
for related, r in related_resources:
if (related is None or related in missing_related_tags
or not related_tag_map[related]):
stats['missing'] += 1
elif self.process_resource(client, r, related_tag_map[related],
self.data['tags'], tag_action):
stats['tagged'] += 1
else:
stats['unchanged'] += 1
self.log.info(
'Tagged %d resources from related, missing-skipped %d unchanged %d',
stats['tagged'], stats['missing'], stats['unchanged'])
def get_resources(self, ids, cache=True):
"""Retrieve ecs resources for serverless policies or related resources
Requires arns in new format.
https://docs.aws.amazon.com/AmazonECS/latest/userguide/ecs-resource-ids.html
"""
cluster_resources = {}
for i in ids:
_, ident = i.rsplit(':', 1)
parts = ident.split('/', 2)
if len(parts) != 3:
raise PolicyExecutionError("New format ecs arn required")
cluster_resources.setdefault(parts[1], []).append(parts[2])
results = []
client = local_session(self.manager.session_factory).client('ecs')
for cid, resource_ids in cluster_resources.items():
results.extend(
self.process_cluster_resources(client, cid, resource_ids))
return results
def get_query_params(self, query):
"""Parse config select expression from policy and parameter.
On policy config supports a full statement being given, or
a clause that will be added to the where expression.
If no query is specified, a default query is utilized.
A valid query should at minimum select fields
for configuration, supplementaryConfiguration and
must have resourceType qualifier.
"""
if query and not isinstance(query, dict):
raise PolicyExecutionError("invalid config source query %s" %
(query, ))
if query is None and 'query' in self.manager.data:
_q = [q for q in self.manager.data['query'] if 'expr' in q]
if _q:
query = _q.pop()
if query is None and 'query' in self.manager.data:
_c = [
q['clause'] for q in self.manager.data['query']
if 'clause' in q
]
if _c:
_c = _c.pop()
elif query:
return query
else:
_c = None
s = ("select resourceId, configuration, supplementaryConfiguration "
"where resourceType = '{}'").format(
self.manager.resource_type.config_type)
if _c:
s += "AND {}".format(_c)
return {'expr': s}
def test_dispatch_err_handle(self):
output, executions = self.setupLambdaEnv(
{
'execution-options': {
'output_dir': 's3://xyz',
'account_id': '004'
},
'policies': [{
'resource': 'ec2',
'name': 'xyz'
}]
},
err_execs=[PolicyExecutionError("foo")] * 2)
self.assertRaises(PolicyExecutionError, handler.dispatch_event,
{'detail': {
'xyz': 'oui'
}}, None)
self.patch(handler, 'C7N_CATCH_ERR', True)
handler.dispatch_event({'detail': {'xyz': 'oui'}}, None)
self.assertEqual(output.getvalue().count('error during'), 2)
