def resolve_group_names(self, r, target_group_ids, groups): """Resolve any security group names to the corresponding group ids With the context of a given network attached resource. """ names = self.get_group_names(target_group_ids) if not names: return target_group_ids target_group_ids = list(target_group_ids) vpc_id = self.vpc_expr.search(r) if not vpc_id: raise PolicyExecutionError( self._format_error( "policy:{policy} non vpc attached resource used " "with modify-security-group: {resource_id}", resource_id=r[self.manager.resource_type.id])) found = False for n in names: for g in groups: if g['GroupName'] == n and g['VpcId'] == vpc_id: found = g['GroupId'] if not found: raise PolicyExecutionError( self._format_error( ("policy:{policy} could not resolve sg:{name} for " "resource:{resource_id} in vpc:{vpc}"), name=n, resource_id=r[self.manager.resource_type.id], vpc=vpc_id)) target_group_ids.remove(n) target_group_ids.append(found) return target_group_ids
def process(self, resources): related_resources = dict( zip(jmespath.search('[].%s' % self.data['key'], resources), resources)) related_ids = set(related_resources) related_tag_map = self.get_resource_tag_map(self.data['resource'], related_ids) missing_related_tags = related_ids.difference(related_tag_map.keys()) if not self.data.get('skip_missing', True) and missing_related_tags: raise PolicyExecutionError( "Unable to find all %d %s related resources tags %d missing" % (len(related_ids), self.data['resource'], len(missing_related_tags))) # rely on resource manager tag action implementation as it can differ between resources tag_action = self.manager.action_registry.get('tag')({}, self.manager) tag_action.id_key = tag_action.manager.get_model().id stats = Counter() for related, r in related_resources.items(): if related in missing_related_tags or not related_tag_map[related]: stats['missing'] += 1 elif self.process_resource(r, related_tag_map[related], self.data['tags'], tag_action): stats['tagged'] += 1 else: stats['unchanged'] += 1 self.log.info( 'Tagged %d resources from related, missing-skipped %d unchanged %d', stats['tagged'], stats['missing'], stats['unchanged'])
def process(self, resources): client = local_session( self.manager.session_factory).client('service-quotas') multiplier = self.data.get('multiplier', 1.2) error = None for r in resources: count = math.floor(multiplier * r['Value']) if not r['Adjustable']: continue try: client.request_service_quota_increase( ServiceCode=r['ServiceCode'], QuotaCode=r['QuotaCode'], DesiredValue=count) except client.exceptions.QuotaExceededException as e: error = e self.log.error('Requested:%s exceeds quota limit for %s' % (count, r['QuotaCode'])) continue except ( client.exceptions.AccessDeniedException, client.exceptions.DependencyAccessDeniedException, ): raise PolicyExecutionError( 'Access Denied to increase quota: %s' % r['QuotaCode']) except ( client.exceptions.NoSuchResourceException, client.exceptions.InvalidResourceStateException, client.exceptions.ResourceAlreadyExistsException, ) as e: error = e continue if error: raise PolicyExecutionError from error
def get_groups_by_names(self, names): """Resolve security names to security groups resources.""" if not names: return [] client = utils.local_session( self.manager.session_factory).client('ec2') sgs = self.manager.retry(client.describe_security_groups, Filters=[{ 'Name': 'group-name', 'Values': names }]).get('SecurityGroups', []) unresolved = set(names) for s in sgs: if s['GroupName'] in unresolved: unresolved.remove(s['GroupName']) if unresolved: raise PolicyExecutionError( self._format_error( "policy:{policy} security groups not found " "requested: {names}, found: {groups}", names=list(unresolved), groups=[g['GroupId'] for g in sgs])) return sgs
def _get_identity(self, session): identity = jmespath.search('mode."provision-options".identity', self.policy.data) or { 'type': AUTH_TYPE_EMBED } if identity['type'] != AUTH_TYPE_UAI: return identity # We need to resolve the client id of the uai, as the metadata # service in functions is old and doesn't support newer # metadata api versions where this would be extraneous # (ie. versions 2018-02-01 or 2019-08-01). notably the # official docs here are wrong # https://docs.microsoft.com/en-us/azure/app-service/overview-managed-identity # TODO: switch out to using uai resource manager so we get some cache # benefits across policies using the same uai. id_client = session.client( 'azure.mgmt.msi.ManagedServiceIdentityClient') found = None for uai in id_client.user_assigned_identities.list_by_subscription(): if uai.id == identity['id'] or uai.name == identity['id']: found = uai break if not found: raise PolicyExecutionError( "policy:%s Could not find the user assigned identity %s" % (self.policy.name, identity['id'])) identity['id'] = found.id identity['client_id'] = found.client_id return identity
def test_dispatch_err_handle(self, mock_collection): self.patch( handler, 'policy_config', { 'execution-options': { 'output_dir': 's3://xyz', 'account_id': '004' }, 'policies': [{ 'resource': 'ec2', 'name': 'xyz' }] }) output = self.capture_logging('custodian.lambda', level=logging.WARNING) pmock = mock.MagicMock() pmock.push.side_effect = PolicyExecutionError("foo") mock_collection.from_data.return_value = [pmock] self.assertRaises(PolicyExecutionError, handler.dispatch_event, {'detail': { 'xyz': 'oui' }}, None) self.patch(handler, 'C7N_CATCH_ERR', True) handler.dispatch_event({'detail': {'xyz': 'oui'}}, None) self.assertEqual(output.getvalue().count('error during'), 2)
def get_resource_sets(self, event): # return a mapping of (account_id, region): [resource_arns] # per the finding in the event. # Group resources by account_id, region for role assumes resource_sets = {} # Loop over findings and set resource set accordingly # Lazy import to avoid aws sdk runtime dep in core from c7n.resources.aws import Arn for finding in event['detail']['findings']: resource_sets.setdefault( (finding['AwsAccountId'], finding['Resources'][0]['Region']), []).append(Arn.parse(finding['Resources'][0]['Id'])) # Warn if not configured for member-role and have multiple accounts resources. if (not self.policy.data['mode'].get('member-role') and {self.policy.options.account_id} != { account_id for (account_id, region), rarns in resource_sets.items() }): msg = ('hub-mode not configured for multi-account member-role ' 'but multiple resource accounts found') self.policy.log.warning(msg) raise PolicyExecutionError(msg) return resource_sets
def load_file(self, path, resource_key): data = load_file(path) if resource_key: data = jmespath.search(resource_key, data) if not isinstance(data, list): raise PolicyExecutionError( "found disk records at %s in non list format %s" % (path, type(data))) return DataFile(path, resource_key, data)
def initialize_source(self): # Ideally we'll be given a source, but we'll attempt to auto create it # if given an org_domain or org_id. if self._source: return self._source elif 'source' in self.data: self._source = self.data['source'] return self._source session = local_session(self.manager.session_factory) # Resolve Organization Id if 'org-id' in self.data: org_id = self.data['org-id'] else: orgs = session.client('cloudresourcemanager', 'v1', 'organizations') res = orgs.execute_query('search', { 'body': { 'filter': 'domain:%s' % self.data['org-domain'] } }).get('organizations') if not res: raise PolicyExecutionError( "Could not determine organization id") org_id = res[0]['name'].rsplit('/', 1)[-1] # Resolve Source client = session.client(self.Service, self.ServiceVersion, 'organizations.sources') source = None res = [ s for s in client.execute_query('list', { 'parent': 'organizations/{}'.format(org_id) }).get('sources') if s['displayName'] == self.CustodianSourceName ] if res: source = res[0]['name'] if source is None: source = client.execute_command( 'create', { 'parent': 'organizations/{}'.format(org_id), 'body': { 'displayName': self.CustodianSourceName, 'description': 'Cloud Management Rules Engine' } }).get('name') self.log.info( "policy:%s resolved cscc source: %s, update policy with this source value", self.manager.ctx.policy.name, source) self._source = source return self._source
def get_analyzer(self, client): if self.data.get('analyzer'): return self.data['analyzer'] analyzers = client.list_analyzers(type='ACCOUNT').get('analyzers', ()) found = False for a in analyzers: if a['status'] != 'ACTIVE': continue found = a if not found: raise PolicyExecutionError( "policy:%s no access analyzer found in account or org analyzer specified" % (self.manager.policy.name)) return found['arn']
def get_resource_sets(self, event): # return a mapping of (account_id, region): [resource_arns] # per the finding in the event. resource_arns = self.get_resource_arns(event) # Group resources by account_id, region for role assumes resource_sets = {} for rarn in resource_arns: resource_sets.setdefault((rarn.account_id, rarn.region), []).append(rarn) # Warn if not configured for member-role and have multiple accounts resources. if (not self.policy.data['mode'].get('member-role') and {self.policy.options.account_id} != { rarn.account_id for rarn in resource_arns}): msg = ('hub-mode not configured for multi-account member-role ' 'but multiple resource accounts found') self.policy.log.warning(msg) raise PolicyExecutionError(msg) return resource_sets
def process(self, resources): related_resources = [] for rrid, r in zip( jmespath.search('[].[%s]' % self.data['key'], resources), resources): related_resources.append((rrid[0], r)) related_ids = {r[0] for r in related_resources} missing = False if None in related_ids: missing = True related_ids.discard(None) related_tag_map = self.get_resource_tag_map(self.data['resource'], related_ids) missing_related_tags = related_ids.difference(related_tag_map.keys()) if not self.data.get('skip_missing', True) and (missing_related_tags or missing): raise PolicyExecutionError( "Unable to find all %d %s related resources tags %d missing" % (len(related_ids), self.data['resource'], len(missing_related_tags) + int(missing))) # rely on resource manager tag action implementation as it can differ between resources tag_action = self.manager.action_registry.get('tag')({}, self.manager) tag_action.id_key = tag_action.manager.get_model().id client = tag_action.get_client() stats = Counter() for related, r in related_resources: if (related is None or related in missing_related_tags or not related_tag_map[related]): stats['missing'] += 1 elif self.process_resource(client, r, related_tag_map[related], self.data['tags'], tag_action): stats['tagged'] += 1 else: stats['unchanged'] += 1 self.log.info( 'Tagged %d resources from related, missing-skipped %d unchanged %d', stats['tagged'], stats['missing'], stats['unchanged'])
def get_resources(self, ids, cache=True): """Retrieve ecs resources for serverless policies or related resources Requires arns in new format. https://docs.aws.amazon.com/AmazonECS/latest/userguide/ecs-resource-ids.html """ cluster_resources = {} for i in ids: _, ident = i.rsplit(':', 1) parts = ident.split('/', 2) if len(parts) != 3: raise PolicyExecutionError("New format ecs arn required") cluster_resources.setdefault(parts[1], []).append(parts[2]) results = [] client = local_session(self.manager.session_factory).client('ecs') for cid, resource_ids in cluster_resources.items(): results.extend( self.process_cluster_resources(client, cid, resource_ids)) return results
def get_query_params(self, query): """Parse config select expression from policy and parameter. On policy config supports a full statement being given, or a clause that will be added to the where expression. If no query is specified, a default query is utilized. A valid query should at minimum select fields for configuration, supplementaryConfiguration and must have resourceType qualifier. """ if query and not isinstance(query, dict): raise PolicyExecutionError("invalid config source query %s" % (query, )) if query is None and 'query' in self.manager.data: _q = [q for q in self.manager.data['query'] if 'expr' in q] if _q: query = _q.pop() if query is None and 'query' in self.manager.data: _c = [ q['clause'] for q in self.manager.data['query'] if 'clause' in q ] if _c: _c = _c.pop() elif query: return query else: _c = None s = ("select resourceId, configuration, supplementaryConfiguration " "where resourceType = '{}'").format( self.manager.resource_type.config_type) if _c: s += "AND {}".format(_c) return {'expr': s}
def test_dispatch_err_handle(self): output, executions = self.setupLambdaEnv( { 'execution-options': { 'output_dir': 's3://xyz', 'account_id': '004' }, 'policies': [{ 'resource': 'ec2', 'name': 'xyz' }] }, err_execs=[PolicyExecutionError("foo")] * 2) self.assertRaises(PolicyExecutionError, handler.dispatch_event, {'detail': { 'xyz': 'oui' }}, None) self.patch(handler, 'C7N_CATCH_ERR', True) handler.dispatch_event({'detail': {'xyz': 'oui'}}, None) self.assertEqual(output.getvalue().count('error during'), 2)