def test_type_schema(self): self.assertEqual( utils.type_schema('tester'), {'type': 'object', 'additionalProperties': False, 'required': ['type'], 'properties': { 'type': {'enum': ['tester']}}}) res = utils.type_schema('tester', inherits=['tested']) self.assertIn({'$ref': 'tested'}, res['allOf'])
def test_type_schema(self): self.assertEqual( utils.type_schema("tester"), { "type": "object", "additionalProperties": False, "required": ["type"], "properties": {"type": {"enum": ["tester"]}}, }, ) res = utils.type_schema("tester", inherits=["tested"]) self.assertIn({"$ref": "tested"}, res["allOf"])
def test_type_schema(self): self.assertEqual( utils.type_schema('tester'), {'type': 'object', 'additionalProperties': False, 'required': ['type'], 'properties': { 'type': {'enum': ['tester']}}})
class IAMSummary(ValueFilter): """Return annotated account resource if iam summary filter matches. Some use cases include, detecting root api keys or mfa usage. Example iam summary wrt to matchable fields:: { "UsersQuota": 5000, "GroupsPerUserQuota": 10, "AttachedPoliciesPerGroupQuota": 10, "PoliciesQuota": 1000, "GroupsQuota": 100, "InstanceProfiles": 0, "SigningCertificatesPerUserQuota": 2, "PolicySizeQuota": 5120, "PolicyVersionsInUseQuota": 10000, "RolePolicySizeQuota": 10240, "AccountSigningCertificatesPresent": 0, "Users": 5, "ServerCertificatesQuota": 20, "ServerCertificates": 0, "AssumeRolePolicySizeQuota": 2048, "Groups": 1, "MFADevicesInUse": 2, "RolesQuota": 250, "VersionsPerPolicyQuota": 5, "AccountAccessKeysPresent": 0, "Roles": 4, "AccountMFAEnabled": 1, "MFADevices": 3, "Policies": 3, "GroupPolicySizeQuota": 5120, "InstanceProfilesQuota": 100, "AccessKeysPerUserQuota": 2, "AttachedPoliciesPerRoleQuota": 10, "PolicyVersionsInUse": 5, "Providers": 0, "AttachedPoliciesPerUserQuota": 10, "UserPolicySizeQuota": 2048 } For example to determine if an account has either not been enabled with root mfa or has root api keys. .. code-block: yaml policies: - name: root-keys-or-no-mfa resource: account filters: - type: iam-summary key: AccountMFAEnabled value: true op: eq value_type: swap """ schema = type_schema('iam-summary', rinherit=ValueFilter.schema) permissions = ('iam:GetAccountSummary', ) def process(self, resources, event=None): if not resources[0].get('c7n:iam_summary'): client = local_session(self.manager.session_factory).client('iam') resources[0]['c7n:iam_summary'] = client.get_account_summary( )['SummaryMap'] if self.match(resources[0]['c7n:iam_summary']): return resources return []
class ConfigRuleMode(LambdaMode): """a lambda policy that executes as a config service rule. The policy is invoked on configuration changes to resources. See `AWS Config <https://aws.amazon.com/config/>`_ for more details. """ cfg_event = None schema = utils.type_schema('config-rule', rinherit=LambdaMode.schema) def validate(self): super(ConfigRuleMode, self).validate() if not self.policy.resource_manager.resource_type.config_type: raise PolicyValidationError( "policy:%s AWS Config does not support resource-type:%s" % ( self.policy.name, self.policy.resource_type)) if self.policy.data['mode'].get('pattern'): raise PolicyValidationError( "policy:%s AWS Config does not support event pattern filtering" % ( self.policy.name)) def resolve_resources(self, event): source = self.policy.resource_manager.get_source('config') return [source.load_resource(self.cfg_event['configurationItem'])] def run(self, event, lambda_context): self.cfg_event = json.loads(event['invokingEvent']) cfg_item = self.cfg_event['configurationItem'] evaluation = None resources = [] # TODO config resource type matches policy check if event.get('eventLeftScope') or cfg_item['configurationItemStatus'] in ( "ResourceDeleted", "ResourceNotRecorded", "ResourceDeletedNotRecorded"): evaluation = { 'annotation': 'The rule does not apply.', 'compliance_type': 'NOT_APPLICABLE'} if evaluation is None: resources = super(ConfigRuleMode, self).run(event, lambda_context) match = self.policy.data['mode'].get('match-compliant', False) self.policy.log.info( "found resources:%d match-compliant:%s", len(resources or ()), match) if (match and resources) or (not match and not resources): evaluation = { 'compliance_type': 'COMPLIANT', 'annotation': 'The resource is compliant with policy:%s.' % ( self.policy.name)} else: evaluation = { 'compliance_type': 'NON_COMPLIANT', 'annotation': 'Resource is not compliant with policy:%s' % ( self.policy.name) } client = utils.local_session( self.policy.session_factory).client('config') client.put_evaluations( Evaluations=[{ 'ComplianceResourceType': cfg_item['resourceType'], 'ComplianceResourceId': cfg_item['resourceId'], 'ComplianceType': evaluation['compliance_type'], 'Annotation': evaluation['annotation'], # TODO ? if not applicable use current timestamp 'OrderingTimestamp': cfg_item[ 'configurationItemCaptureTime']}], ResultToken=event.get('resultToken', 'No token found.')) return resources
class TagTrim(Action): """Automatically remove tags from an ec2 resource. EC2 Resources have a limit of 50 tags, in order to make additional tags space on a set of resources, this action can be used to remove enough tags to make the desired amount of space while preserving a given set of tags. .. code-block :: yaml policies: - name: ec2-tag-trim comment: | Any instances with 48 or more tags get tags removed until they match the target tag count, in this case 47 so we that we free up a tag slot for another usage. resource: ec2 filters: # Filter down to resources which already have 8 tags # as we need space for 3 more, this also ensures that # metrics reporting is correct for the policy. - type: value key: "length(Tags)" op: ge value: 48 actions: - type: tag-trim space: 3 preserve: - OwnerContact - ASV - CMDBEnvironment - downtime - custodian_status """ max_tag_count = 50 schema = utils.type_schema('tag-trim', space={'type': 'integer'}, preserve={ 'type': 'array', 'items': { 'type': 'string' } }) schema_alias = True permissions = ('ec2:DeleteTags', ) def process(self, resources): self.id_key = self.manager.get_model().id self.preserve = set(self.data.get('preserve')) self.space = self.data.get('space', 3) client = utils.local_session(self.manager.session_factory).client( self.manager.resource_type.service) futures = {} mid = self.manager.get_model().id with self.executor_factory(max_workers=2) as w: for r in resources: futures[w.submit(self.process_resource, client, r)] = r for f in as_completed(futures): if f.exception(): self.log.warning( "Error processing tag-trim on resource:%s", futures[f][mid]) def process_resource(self, client, i): # Can't really go in batch parallel without some heuristics # without some more complex matching wrt to grouping resources # by common tags populations. tag_map = { t['Key']: t['Value'] for t in i.get('Tags', []) if not t['Key'].startswith('aws:') } # Space == 0 means remove all but specified if self.space and len(tag_map) + self.space <= self.max_tag_count: return keys = set(tag_map) preserve = self.preserve.intersection(keys) candidates = keys - self.preserve if self.space: # Free up slots to fit remove = len(candidates) - (self.max_tag_count - (self.space + len(preserve))) candidates = list(sorted(candidates))[:remove] if not candidates: self.log.warning("Could not find any candidates to trim %s" % i[self.id_key]) return self.process_tag_removal(i, candidates) def process_tag_removal(self, client, resource, tags): self.manager.retry(client.delete_tags, Tags=[{ 'Key': c } for c in tags], Resources=[resource[self.id_key]], DryRun=self.manager.config.dryrun)
class PropagateTags(Action): """Propagate tags to an asg instances. In AWS changing an asg tag does not propagate to instances. This action exists to do that, and can also trim older tags not present on the asg anymore that are present on instances. :example: .. code-block: yaml policies: - name: asg-propagate-required resource: asg filters: - "tag:OwnerName": present actions: - type: propagate-tags tags: - OwnerName """ schema = type_schema('propagate-tags', tags={ 'type': 'array', 'items': { 'type': 'string' } }, trim={'type': 'boolean'}) permissions = ('ec2:DeleteTags', 'ec2:CreateTags') def validate(self): if not isinstance(self.data.get('tags', []), (list, tuple)): raise ValueError("No tags specified") return self def process(self, asgs): if not asgs: return if self.data.get('trim', False): self.instance_map = self.get_instance_map(asgs) with self.executor_factory(max_workers=10) as w: instance_count = sum(list(w.map(self.process_asg, asgs))) self.log.info("Applied tags to %d instances" % instance_count) def process_asg(self, asg): client = local_session(self.manager.session_factory).client('ec2') instance_ids = [i['InstanceId'] for i in asg['Instances']] tag_map = { t['Key']: t['Value'] for t in asg.get('Tags', []) if t['PropagateAtLaunch'] and not t['Key'].startswith('aws:') } if self.data.get('tags'): tag_map = { k: v for k, v in tag_map.items() if k in self.data['tags'] } tag_set = set(tag_map) if self.data.get('trim', False): instances = [self.instance_map[i] for i in instance_ids] self.prune_instance_tags(client, asg, tag_set, instances) if not self.manager.config.dryrun: client.create_tags(Resources=instance_ids, Tags=[{ 'Key': k, 'Value': v } for k, v in tag_map.items()]) return len(instance_ids) def prune_instance_tags(self, client, asg, tag_set, instances): """Remove tags present on all asg instances which are not present on the asg. """ instance_tags = Counter() instance_count = len(instances) remove_tags = [] extra_tags = [] for i in instances: instance_tags.update([ t['Key'] for t in i['Tags'] if not t['Key'].startswith('aws:') ]) for k, v in instance_tags.items(): if not v >= instance_count: extra_tags.append(k) continue if k not in tag_set: remove_tags.append(k) if remove_tags: log.debug( "Pruning asg:%s instances:%d of old tags: %s" % (asg['AutoScalingGroupName'], instance_count, remove_tags)) if extra_tags: log.debug("Asg: %s has uneven tags population: %s" % (asg['AutoScalingGroupName'], instance_tags)) # Remove orphan tags remove_tags.extend(extra_tags) if not self.manager.config.dryrun: client.delete_tags(Resources=[i['InstanceId'] for i in instances], Tags=[{ 'Key': t } for t in remove_tags]) def get_instance_map(self, asgs): instance_ids = [ i['InstanceId'] for i in list( itertools.chain( *[g['Instances'] for g in asgs if g['Instances']])) ] if not instance_ids: return {} return { i['InstanceId']: i for i in self.manager.get_resource_manager('ec2').get_resources( instance_ids) }
class PullMode(PolicyExecutionMode): """Pull mode execution of a policy. Queries resources from cloud provider for filtering and actions. """ schema = utils.type_schema('pull') def run(self, *args, **kw): if not self.policy.is_runnable(): return [] with self.policy.ctx: self.policy.log.debug( "Running policy:%s resource:%s region:%s c7n:%s", self.policy.name, self.policy.resource_type, self.policy.options.region or 'default', version) s = time.time() try: resources = self.policy.resource_manager.resources() except ResourceLimitExceeded as e: self.policy.log.error(str(e)) self.policy.ctx.metrics.put_metric( 'ResourceLimitExceeded', e.selection_count, "Count") raise rt = time.time() - s self.policy.log.info( "policy:%s resource:%s region:%s count:%d time:%0.2f" % ( self.policy.name, self.policy.resource_type, self.policy.options.region, len(resources), rt)) self.policy.ctx.metrics.put_metric( "ResourceCount", len(resources), "Count", Scope="Policy") self.policy.ctx.metrics.put_metric( "ResourceTime", rt, "Seconds", Scope="Policy") self.policy._write_file( 'resources.json', utils.dumps(resources, indent=2)) if not resources: return [] if self.policy.options.dryrun: self.policy.log.debug("dryrun: skipping actions") return resources at = time.time() for a in self.policy.resource_manager.actions: s = time.time() with self.policy.ctx.tracer.subsegment('action:%s' % a.type): results = a.process(resources) self.policy.log.info( "policy:%s action:%s" " resources:%d" " execution_time:%0.2f" % ( self.policy.name, a.name, len(resources), time.time() - s)) if results: self.policy._write_file( "action-%s" % a.name, utils.dumps(results)) self.policy.ctx.metrics.put_metric( "ActionTime", time.time() - at, "Seconds", Scope="Policy") return resources
class Suspend(Action): """Action to suspend ASG processes and instances AWS ASG suspend/resume and process docs https://goo.gl/XYtKQ8 :example: .. code-block: yaml policies: - name: asg-suspend-processes resource: asg filters: - "tag:SuspendTag": present actions: - type: suspend """ permissions = ("autoscaling:SuspendProcesses", "ec2:StopInstances") ASG_PROCESSES = [ "Launch", "Terminate", "HealthCheck", "ReplaceUnhealthy", "AZRebalance", "AlarmNotification", "ScheduledActions", "AddToLoadBalancer" ] schema = type_schema('suspend', exclude={ 'type': 'array', 'title': 'ASG Processes to not suspend', 'items': { 'enum': ASG_PROCESSES } }) ASG_PROCESSES = set(ASG_PROCESSES) def process(self, asgs): original_count = len(asgs) asgs = [a for a in asgs if a['Instances']] self.log.debug("Filtered from %d to %d asgs with instances" % (original_count, len(asgs))) with self.executor_factory(max_workers=3) as w: list(w.map(self.process_asg, asgs)) def process_asg(self, asg): """Multistep process to stop an asg aprori of setup - suspend processes - stop instances """ session = local_session(self.manager.session_factory) asg_client = session.client('autoscaling') processes = list( self.ASG_PROCESSES.difference(self.data.get('exclude', ()))) try: self.manager.retry( asg_client.suspend_processes, ScalingProcesses=processes, AutoScalingGroupName=asg['AutoScalingGroupName']) except ClientError as e: if e.response['Error']['Code'] == 'ValidationError': return raise ec2_client = session.client('ec2') try: instance_ids = [i['InstanceId'] for i in asg['Instances']] if not instance_ids: return retry = get_retry( ('RequestLimitExceeded', 'Client.RequestLimitExceeded')) retry(ec2_client.stop_instances, InstanceIds=instance_ids) except ClientError as e: if e.response['Error']['Code'] in ('InvalidInstanceID.NotFound', 'IncorrectInstanceState'): log.warning("Erroring stopping asg instances %s %s" % (asg['AutoScalingGroupName'], e)) return raise
class NotEncryptedFilter(Filter, LaunchConfigFilterBase): """Check if an ASG is configured to have unencrypted volumes. Checks both the ami snapshots and the launch configuration. :example: .. code-block: yaml policies: - name: asg-unencrypted resource: asg filters: - type: not-encrypted exclude_image: true """ schema = type_schema('not-encrypted', exclude_image={'type': 'boolean'}) permissions = ('ec2:DescribeImages', 'ec2:DescribeSnapshots', 'autoscaling:DescribeLaunchConfigurations') images = unencrypted_configs = unencrypted_images = None # TODO: resource-manager, notfound err mgr def process(self, asgs, event=None): self.initialize(asgs) return super(NotEncryptedFilter, self).process(asgs, event) def __call__(self, asg): cfg = self.configs.get(asg['LaunchConfigurationName']) if not cfg: self.log.warning("ASG %s instances: %d has missing config: %s", asg['AutoScalingGroupName'], len(asg['Instances']), asg['LaunchConfigurationName']) return False unencrypted = [] if (not self.data.get('exclude_image') and cfg['ImageId'] in self.unencrypted_images): unencrypted.append('Image') if cfg['LaunchConfigurationName'] in self.unencrypted_configs: unencrypted.append('LaunchConfig') if unencrypted: asg['Unencrypted'] = unencrypted return bool(unencrypted) def initialize(self, asgs): super(NotEncryptedFilter, self).initialize(asgs) ec2 = local_session(self.manager.session_factory).client('ec2') self.unencrypted_images = self.get_unencrypted_images(ec2) self.unencrypted_configs = self.get_unencrypted_configs(ec2) def _fetch_images(self, ec2, image_ids): while True: try: return ec2.describe_images(ImageIds=list(image_ids)) except ClientError as e: if e.response['Error']['Code'] == 'InvalidAMIID.NotFound': msg = e.response['Error']['Message'] e_ami_ids = [ e_ami_id.strip() for e_ami_id in msg[msg.find("'[") + 2:msg.rfind("]'")].split(',') ] self.log.warning( "asg:not-encrypted filter image not found %s", e_ami_ids) for e_ami_id in e_ami_ids: image_ids.remove(e_ami_id) continue raise def get_unencrypted_images(self, ec2): """retrieve images which have unencrypted snapshots referenced.""" image_ids = set() for cfg in self.configs.values(): image_ids.add(cfg['ImageId']) self.log.debug("querying %d images", len(image_ids)) results = self._fetch_images(ec2, image_ids) self.images = {i['ImageId']: i for i in results['Images']} unencrypted_images = set() for i in self.images.values(): for bd in i['BlockDeviceMappings']: if 'Ebs' in bd and not bd['Ebs'].get('Encrypted'): unencrypted_images.add(i['ImageId']) break return unencrypted_images def get_unencrypted_configs(self, ec2): """retrieve configs that have unencrypted ebs voluems referenced.""" unencrypted_configs = set() snaps = {} for cid, c in self.configs.items(): image = self.images.get(c['ImageId']) # image deregistered/unavailable if image is not None: image_block_devs = { bd['DeviceName']: bd['Ebs'] for bd in image['BlockDeviceMappings'] if 'Ebs' in bd } else: image_block_devs = {} for bd in c['BlockDeviceMappings']: if 'Ebs' not in bd: continue # Launch configs can shadow image devices, images have # precedence. if bd['DeviceName'] in image_block_devs: continue if 'SnapshotId' in bd['Ebs']: snaps.setdefault(bd['Ebs']['SnapshotId'].strip(), []).append(cid) elif not bd['Ebs'].get('Encrypted'): unencrypted_configs.add(cid) if not snaps: return unencrypted_configs self.log.debug("querying %d snapshots", len(snaps)) for s in self.get_snapshots(ec2, snaps.keys()): if not s.get('Encrypted'): unencrypted_configs.update(snaps[s['SnapshotId']]) return unencrypted_configs def get_snapshots(self, ec2, snap_ids): """get snapshots corresponding to id, but tolerant of missing.""" while True: try: result = ec2.describe_snapshots(SnapshotIds=snap_ids) except ClientError as e: if e.response['Error']['Code'] == 'InvalidSnapshot.NotFound': msg = e.response['Error']['Message'] e_snap_id = msg[msg.find("'") + 1:msg.rfind("'")] self.log.warning("Snapshot not found %s" % e_snap_id) snap_ids.remove(e_snap_id) continue raise else: return result.get('Snapshots', ())
class CopyRelatedResourceTag(Tag): """ Copy a related resource tag to its associated resource In some scenarios, resource tags from a related resource should be applied to its child resource. For example, EBS Volume tags propogating to their snapshots. To use this action, specify the resource type that contains the tags that are to be copied, which can be found by using the `custodian schema` command. Then, specify the key on the resource that references the related resource. In the case of ebs-snapshot, the VolumeId attribute would be the key that identifies the related resource, ebs. Finally, specify a list of tag keys to copy from the related resource onto the original resource. The special character "*" can be used to signify that all tags from the related resource should be copied to the original resource. To raise an error when related resources cannot be found, use the `skip_missing` option. By default, this is set to True. :example: .. code-block:: yaml policies: - name: copy-tags-from-ebs-volume-to-snapshot resource: ebs-snapshot actions: - type: copy-related-tag resource: ebs skip_missing: True key: VolumeId tags: '*' """ schema = utils.type_schema( 'copy-related-tag', resource={'type': 'string'}, skip_missing={'type': 'boolean'}, key={'type': 'string'}, tags={'oneOf': [{ 'enum': ['*'] }, { 'type': 'array' }]}, required=['tags', 'key', 'resource']) schema_alias = True def get_permissions(self): return self.manager.action_registry.get('tag').permissions def validate(self): related_resource = self.data['resource'] if related_resource not in aws_resources.keys(): raise PolicyValidationError( "Error: Invalid resource type selected: %s" % related_resource) # ideally should never raise here since we shouldn't be applying this # action to a resource if it doesn't have a tag action implemented if self.manager.action_registry.get('tag') is None: raise PolicyValidationError( "Error: Tag action missing on resource") return self def process(self, resources): related_resources = [] for rrid, r in zip( jmespath.search('[].[%s]' % self.data['key'], resources), resources): related_resources.append((rrid[0], r)) related_ids = set([r[0] for r in related_resources]) missing = False if None in related_ids: missing = True related_ids.discard(None) related_tag_map = self.get_resource_tag_map(self.data['resource'], related_ids) missing_related_tags = related_ids.difference(related_tag_map.keys()) if not self.data.get('skip_missing', True) and (missing_related_tags or missing): raise PolicyExecutionError( "Unable to find all %d %s related resources tags %d missing" % (len(related_ids), self.data['resource'], len(missing_related_tags) + int(missing))) # rely on resource manager tag action implementation as it can differ between resources tag_action = self.manager.action_registry.get('tag')({}, self.manager) tag_action.id_key = tag_action.manager.get_model().id client = tag_action.get_client() stats = Counter() for related, r in related_resources: if (related is None or related in missing_related_tags or not related_tag_map[related]): stats['missing'] += 1 elif self.process_resource(client, r, related_tag_map[related], self.data['tags'], tag_action): stats['tagged'] += 1 else: stats['unchanged'] += 1 self.log.info( 'Tagged %d resources from related, missing-skipped %d unchanged %d', stats['tagged'], stats['missing'], stats['unchanged']) def process_resource(self, client, r, related_tags, tag_keys, tag_action): tags = {} resource_tags = { t['Key']: t['Value'] for t in r.get('Tags', []) if not t['Key'].startswith('aws:') } if tag_keys == '*': tags = { k: v for k, v in related_tags.items() if resource_tags.get(k) != v } else: tags = { k: v for k, v in related_tags.items() if k in tag_keys and resource_tags.get(k) != v } if not tags: return if not isinstance(tag_action, UniversalTag): tags = [{'Key': k, 'Value': v} for k, v in tags.items()] tag_action.process_resource_set(client, resource_set=[r], tags=tags) return True def get_resource_tag_map(self, r_type, ids): """ Returns a mapping of {resource_id: {tagkey: tagvalue}} """ manager = self.manager.get_resource_manager(r_type) r_id = manager.resource_type.id return { r[r_id]: {t['Key']: t['Value'] for t in r.get('Tags', [])} for r in manager.get_resources(list(ids)) } @classmethod def register_resources(klass, registry, resource_class): if not resource_class.action_registry.get('tag'): return resource_class.action_registry.register('copy-related-tag', klass)
class IsSSLFilter(Filter): schema = type_schema('is-ssl') def process(self, balancers, event=None): return [b for b in balancers if is_ssl(b)]
class TagDelayedAction(Action): """Tag resources for future action. The optional 'tz' parameter can be used to adjust the clock to align with a given timezone. The default value is 'utc'. If neither 'days' nor 'hours' is specified, Cloud Custodian will default to marking the resource for action 4 days in the future. .. code-block :: yaml policies: - name: ec2-mark-for-stop-in-future resource: ec2 filters: - type: value key: Name value: instance-to-stop-in-four-days actions: - type: mark-for-op op: stop """ schema = utils.type_schema('mark-for-op', tag={'type': 'string'}, msg={'type': 'string'}, days={ 'type': 'integer', 'minimum': 0, 'exclusiveMinimum': False }, hours={ 'type': 'integer', 'minimum': 0, 'exclusiveMinimum': False }, tz={'type': 'string'}, op={'type': 'string'}) schema_alias = True batch_size = 200 concurrency = 2 default_template = 'Resource does not meet policy: {op}@{action_date}' def get_permissions(self): return self.manager.action_registry['tag'].permissions def validate(self): op = self.data.get('op') if self.manager and op not in self.manager.action_registry.keys(): raise PolicyValidationError( "mark-for-op specifies invalid op:%s in %s" % (op, self.manager.data)) self.tz = tzutil.gettz(Time.TZ_ALIASES.get(self.data.get('tz', 'utc'))) if not self.tz: raise PolicyValidationError("Invalid timezone specified %s in %s" % (self.tz, self.manager.data)) return self def generate_timestamp(self, days, hours): n = datetime.now(tz=self.tz) if days is None or hours is None: # maintains default value of days being 4 if nothing is provided days = 4 action_date = (n + timedelta(days=days, hours=hours)) if hours > 0: action_date_string = action_date.strftime('%Y/%m/%d %H%M %Z') else: action_date_string = action_date.strftime('%Y/%m/%d') return action_date_string def get_config_values(self): d = { 'op': self.data.get('op', 'stop'), 'tag': self.data.get('tag', DEFAULT_TAG), 'msg': self.data.get('msg', self.default_template), 'tz': self.data.get('tz', 'utc'), 'days': self.data.get('days', 0), 'hours': self.data.get('hours', 0) } d['action_date'] = self.generate_timestamp(d['days'], d['hours']) return d def process(self, resources): cfg = self.get_config_values() self.tz = tzutil.gettz(Time.TZ_ALIASES.get(cfg['tz'])) self.id_key = self.manager.get_model().id msg = cfg['msg'].format(op=cfg['op'], action_date=cfg['action_date']) self.log.info("Tagging %d resources for %s on %s" % (len(resources), cfg['op'], cfg['action_date'])) tags = [{'Key': cfg['tag'], 'Value': msg}] # if the tag implementation has a specified batch size, it's typically # due to some restraint on the api so we defer to that. batch_size = getattr(self.manager.action_registry.get('tag'), 'batch_size', self.batch_size) client = self.get_client() _common_tag_processer(self.executor_factory, batch_size, self.concurrency, client, self.process_resource_set, self.id_key, resources, tags, self.log) def process_resource_set(self, client, resource_set, tags): tagger = self.manager.action_registry['tag']({}, self.manager) tagger.process_resource_set(client, resource_set, tags) def get_client(self): return utils.local_session(self.manager.session_factory).client( self.manager.resource_type.service)
class NormalizeTag(Action): """Transform the value of a tag. Set the tag value to uppercase, title, lowercase, or strip text from a tag key. .. code-block :: yaml policies: - name: ec2-service-transform-lower resource: ec2 comment: | ec2-service-tag-value-to-lower query: - instance-state-name: running filters: - "tag:testing8882": present actions: - type: normalize-tag key: lower_key action: lower - name: ec2-service-strip resource: ec2 comment: | ec2-service-tag-strip-blah query: - instance-state-name: running filters: - "tag:testing8882": present actions: - type: normalize-tag key: strip_key action: strip value: blah """ schema_alias = True schema = utils.type_schema( 'normalize-tag', key={'type': 'string'}, action={ 'type': 'string', 'items': { 'enum': ['upper', 'lower', 'title' 'strip', 'replace'] } }, value={'type': 'string'}) permissions = ('ec2:CreateTags', ) def create_tag(self, client, ids, key, value): self.manager.retry(client.create_tags, Resources=ids, Tags=[{ 'Key': key, 'Value': value }]) def process_transform(self, tag_value, resource_set): """ Transform tag value - Collect value from tag - Transform Tag value - Assign new value for key """ self.log.info("Transforming tag value on %s instances" % (len(resource_set))) key = self.data.get('key') c = utils.local_session(self.manager.session_factory).client('ec2') self.create_tag(c, [ r[self.id_key] for r in resource_set if len(r.get('Tags', [])) < 50 ], key, tag_value) def create_set(self, instances): key = self.data.get('key', None) resource_set = {} for r in instances: tags = {t['Key']: t['Value'] for t in r.get('Tags', [])} if tags[key] not in resource_set: resource_set[tags[key]] = [] resource_set[tags[key]].append(r) return resource_set def filter_resources(self, resources): key = self.data.get('key', None) res = 0 for r in resources: tags = {t['Key']: t['Value'] for t in r.get('Tags', [])} if key not in tags.keys(): resources.pop(res) res += 1 return resources def process(self, resources): count = len(resources) resources = self.filter_resources(resources) self.log.info("Filtered from %s resources to %s" % (count, len(resources))) self.id_key = self.manager.get_model().id resource_set = self.create_set(resources) with self.executor_factory(max_workers=3) as w: futures = [] for r in resource_set: action = self.data.get('action') value = self.data.get('value') new_value = False if action == 'lower' and not r.islower(): new_value = r.lower() elif action == 'upper' and not r.isupper(): new_value = r.upper() elif action == 'title' and not r.istitle(): new_value = r.title() elif action == 'strip' and value and value in r: new_value = r.strip(value) if new_value: futures.append( w.submit(self.process_transform, new_value, resource_set[r])) for f in as_completed(futures): if f.exception(): self.log.error("Exception renaming tag set \n %s" % (f.exception())) return resources
class RenameTag(Action): """ Create a new tag with identical value & remove old tag """ schema = utils.type_schema('rename-tag', old_key={'type': 'string'}, new_key={'type': 'string'}) schema_alias = True permissions = ('ec2:CreateTags', 'ec2:DeleteTags') tag_count_max = 50 def delete_tag(self, client, ids, key, value): client.delete_tags(Resources=ids, Tags=[{'Key': key, 'Value': value}]) def create_tag(self, client, ids, key, value): client.create_tags(Resources=ids, Tags=[{'Key': key, 'Value': value}]) def process_rename(self, client, tag_value, resource_set): """ Move source tag value to destination tag value - Collect value from old tag - Delete old tag - Create new tag & assign stored value """ self.log.info("Renaming tag on %s instances" % (len(resource_set))) old_key = self.data.get('old_key') new_key = self.data.get('new_key') # We have a preference to creating the new tag when possible first resource_ids = [ r[self.id_key] for r in resource_set if len(r.get('Tags', [])) < self.tag_count_max ] if resource_ids: self.create_tag(client, resource_ids, new_key, tag_value) self.delete_tag(client, [r[self.id_key] for r in resource_set], old_key, tag_value) # For resources with 50 tags, we need to delete first and then create. resource_ids = [ r[self.id_key] for r in resource_set if len(r.get('Tags', [])) > self.tag_count_max - 1 ] if resource_ids: self.create_tag(client, resource_ids, new_key, tag_value) def create_set(self, instances): old_key = self.data.get('old_key', None) resource_set = {} for r in instances: tags = {t['Key']: t['Value'] for t in r.get('Tags', [])} if tags[old_key] not in resource_set: resource_set[tags[old_key]] = [] resource_set[tags[old_key]].append(r) return resource_set def filter_resources(self, resources): old_key = self.data.get('old_key', None) res = 0 for r in resources: tags = {t['Key']: t['Value'] for t in r.get('Tags', [])} if old_key not in tags.keys(): resources.pop(res) res += 1 return resources def process(self, resources): count = len(resources) resources = self.filter_resources(resources) self.log.info("Filtered from %s resources to %s" % (count, len(resources))) self.id_key = self.manager.get_model().id resource_set = self.create_set(resources) client = self.get_client() with self.executor_factory(max_workers=3) as w: futures = [] for r in resource_set: futures.append( w.submit(self.process_rename, client, r, resource_set[r])) for f in as_completed(futures): if f.exception(): self.log.error("Exception renaming tag set \n %s" % (f.exception())) return resources def get_client(self): return utils.local_session(self.manager.session_factory).client( self.manager.resource_type.service)
class Tag(Action): """Tag an ec2 resource. """ batch_size = 25 concurrency = 2 schema = utils.type_schema( 'tag', aliases=('mark', ), tags={'type': 'object'}, key={'type': 'string'}, value={'type': 'string'}, tag={'type': 'string'}, ) schema_alias = True permissions = ('ec2:CreateTags', ) id_key = None def validate(self): if self.data.get('key') and self.data.get('tag'): raise PolicyValidationError( "Can't specify both key and tag, choose one in %s" % (self.manager.data, )) return self def process(self, resources): # Legacy msg = self.data.get('msg') msg = self.data.get('value') or msg tag = self.data.get('tag', DEFAULT_TAG) tag = self.data.get('key') or tag # Support setting multiple tags in a single go with a mapping tags = self.data.get('tags') if tags is None: tags = [] else: tags = [{'Key': k, 'Value': v} for k, v in tags.items()] if msg: tags.append({'Key': tag, 'Value': msg}) self.interpolate_values(tags) batch_size = self.data.get('batch_size', self.batch_size) client = self.get_client() _common_tag_processer(self.executor_factory, batch_size, self.concurrency, client, self.process_resource_set, self.id_key, resources, tags, self.log) def process_resource_set(self, client, resource_set, tags): mid = self.manager.get_model().id self.manager.retry(client.create_tags, Resources=[v[mid] for v in resource_set], Tags=tags, DryRun=self.manager.config.dryrun) def interpolate_values(self, tags): params = { 'account_id': self.manager.config.account_id, 'now': utils.FormatDate.utcnow(), 'region': self.manager.config.region } for t in tags: t['Value'] = t['Value'].format(**params) def get_client(self): return utils.local_session(self.manager.session_factory).client( self.manager.resource_type.service)
class TagActionFilter(Filter): """Filter resources for tag specified future action Filters resources by a 'custodian_status' tag which specifies a future date for an action. The filter parses the tag values looking for an 'op@date' string. The date is parsed and compared to do today's date, the filter succeeds if today's date is gte to the target date. The optional 'skew' parameter provides for incrementing today's date a number of days into the future. An example use case might be sending a final notice email a few days before terminating an instance, or snapshotting a volume prior to deletion. The optional 'skew_hours' parameter provides for incrementing the current time a number of hours into the future. Optionally, the 'tz' parameter can get used to specify the timezone in which to interpret the clock (default value is 'utc') .. code-block :: yaml policies: - name: ec2-stop-marked resource: ec2 filters: - type: marked-for-op # The default tag used is custodian_status # but that is configurable tag: custodian_status op: stop # Another optional tag is skew tz: utc actions: - type: stop """ schema = utils.type_schema('marked-for-op', tag={'type': 'string'}, tz={'type': 'string'}, skew={ 'type': 'number', 'minimum': 0 }, skew_hours={ 'type': 'number', 'minimum': 0 }, op={'type': 'string'}) schema_alias = True current_date = None def validate(self): op = self.data.get('op') if self.manager and op not in self.manager.action_registry.keys(): raise PolicyValidationError("Invalid marked-for-op op:%s in %s" % (op, self.manager.data)) tz = tzutil.gettz(Time.TZ_ALIASES.get(self.data.get('tz', 'utc'))) if not tz: raise PolicyValidationError( "Invalid timezone specified '%s' in %s" % (self.data.get('tz'), self.manager.data)) return self def __call__(self, i): tag = self.data.get('tag', DEFAULT_TAG) op = self.data.get('op', 'stop') skew = self.data.get('skew', 0) skew_hours = self.data.get('skew_hours', 0) tz = tzutil.gettz(Time.TZ_ALIASES.get(self.data.get('tz', 'utc'))) v = None for n in i.get('Tags', ()): if n['Key'] == tag: v = n['Value'] break if v is None: return False if ':' not in v or '@' not in v: return False msg, tgt = v.rsplit(':', 1) action, action_date_str = tgt.strip().split('@', 1) if action != op: return False try: action_date = parse(action_date_str) except Exception: self.log.warning("could not parse tag:%s value:%s on %s" % (tag, v, i['InstanceId'])) if self.current_date is None: self.current_date = datetime.now() if action_date.tzinfo: # if action_date is timezone aware, set to timezone provided action_date = action_date.astimezone(tz) self.current_date = datetime.now(tz=tz) return self.current_date >= (action_date - timedelta(days=skew, hours=skew_hours))
class ServiceLimit(Filter): """Check if account's service limits are past a given threshold. Supported limits are per trusted advisor, which is variable based on usage in the account and support level enabled on the account. - service: AutoScaling limit: Auto Scaling groups - service: AutoScaling limit: Launch configurations - service: EBS limit: Active snapshots - service: EBS limit: Active volumes - service: EBS limit: General Purpose (SSD) volume storage (GiB) - service: EBS limit: Magnetic volume storage (GiB) - service: EBS limit: Provisioned IOPS - service: EBS limit: Provisioned IOPS (SSD) storage (GiB) - service: EC2 limit: Elastic IP addresses (EIPs) # Note this is extant for each active instance type in the account # however the total value is against sum of all instance types. # see issue https://github.com/capitalone/cloud-custodian/issues/516 - service: EC2 limit: On-Demand instances - m3.medium - service: EC2 limit: Reserved Instances - purchase limit (monthly) - service: ELB limit: Active load balancers - service: IAM limit: Groups - service: IAM limit: Instance profiles - service: IAM limit: Roles - service: IAM limit: Server certificates - service: IAM limit: Users - service: RDS limit: DB instances - service: RDS limit: DB parameter groups - service: RDS limit: DB security groups - service: RDS limit: DB snapshots per user - service: RDS limit: Storage quota (GB) - service: RDS limit: Internet gateways - service: SES limit: Daily sending quota - service: VPC limit: VPCs - service: VPC limit: VPC Elastic IP addresses (EIPs) :example: .. code-block: yaml policies: - name: account-service-limits resource: account filters: - type: service-limit services: - IAM threshold: 1.0 """ schema = type_schema('service-limit', threshold={'type': 'number'}, refresh_period={'type': 'integer'}, limits={ 'type': 'array', 'items': { 'type': 'string' } }, services={ 'type': 'array', 'items': { 'enum': [ 'EC2', 'ELB', 'VPC', 'AutoScaling', 'RDS', 'EBS', 'SES', 'IAM' ] } }) permissions = ('support:DescribeTrustedAdvisorCheckResult', ) check_id = 'eW7HH0l7J9' check_limit = ('region', 'service', 'check', 'limit', 'extant', 'color') def process(self, resources, event=None): client = local_session(self.manager.session_factory).client('support') checks = client.describe_trusted_advisor_check_result( checkId=self.check_id, language='en')['result'] resources[0]['c7n:ServiceLimits'] = checks delta = timedelta(self.data.get('refresh_period', 1)) check_date = parse_date(checks['timestamp']) if datetime.now(tz=tzutc()) - delta > check_date: client.refresh_trusted_advisor_check(checkId=self.check_id) threshold = self.data.get('threshold') services = self.data.get('services') limits = self.data.get('limits') exceeded = [] for resource in checks['flaggedResources']: if threshold is None and resource['status'] == 'ok': continue limit = dict(zip(self.check_limit, resource['metadata'])) if services and limit['service'] not in services: continue if limits and limit['check'] not in limits: continue limit['status'] = resource['status'] limit['percentage'] = float(limit['extant'] or 0) / float( limit['limit']) * 100 if threshold and limit['percentage'] < threshold: continue exceeded.append(limit) if exceeded: resources[0]['c7n:ServiceLimitsExceeded'] = exceeded return resources return []
class AutoscalerSet(MethodAction): """ `Patches <https://cloud.google.com/compute/docs/reference/rest/v1/autoscalers/patch>`_ configuration parameters for the autoscaling algorithm. The `coolDownPeriodSec` specifies the number of seconds that the autoscaler should wait before it starts collecting information from a new instance. The `cpuUtilization.utilizationTarget` specifies the target CPU utilization that the autoscaler should maintain. The `loadBalancingUtilization.utilizationTarget` specifies fraction of backend capacity utilization (set in HTTP(S) load balancing configuration) that autoscaler should maintain. The `minNumReplicas` specifies the minimum number of replicas that the autoscaler can scale down to. The `maxNumReplicas` specifies the maximum number of instances that the autoscaler can scale up to. :Example: .. code-block:: yaml policies: - name: gcp-autoscaler-set resource: gcp.autoscaler filters: - type: value key: name value: instance-group-2 actions: - type: set coolDownPeriodSec: 20 cpuUtilization: utilizationTarget: 0.7 loadBalancingUtilization: utilizationTarget: 0.7 minNumReplicas: 1 maxNumReplicas: 4 """ schema = type_schema( 'set', **{ 'coolDownPeriodSec': { 'type': 'integer', 'minimum': 15 }, 'cpuUtilization': { 'type': 'object', 'required': ['utilizationTarget'], 'properties': { 'utilizationTarget': { 'type': 'number', 'exclusiveMinimum': 0, 'maximum': 1 } }, }, 'loadBalancingUtilization': { 'type': 'object', 'required': ['utilizationTarget'], 'properties': { 'utilizationTarget': { 'type': 'number', 'exclusiveMinimum': 0, 'maximum': 1 } } }, 'maxNumReplicas': { 'type': 'integer', 'exclusiveMinimum': 0 }, 'minNumReplicas': { 'type': 'integer', 'exclusiveMinimum': 0 } }) method_spec = {'op': 'patch'} path_param_re = re.compile( '.*?/projects/(.*?)/zones/(.*?)/autoscalers/(.*)') method_perm = 'update' def get_resource_params(self, model, resource): project, zone, autoscaler = self.path_param_re.match( resource['selfLink']).groups() body = {} if 'coolDownPeriodSec' in self.data: body['coolDownPeriodSec'] = self.data['coolDownPeriodSec'] if 'cpuUtilization' in self.data: body['cpuUtilization'] = self.data['cpuUtilization'] if 'loadBalancingUtilization' in self.data: body['loadBalancingUtilization'] = self.data[ 'loadBalancingUtilization'] if 'maxNumReplicas' in self.data: body['maxNumReplicas'] = self.data['maxNumReplicas'] if 'minNumReplicas' in self.data: body['minNumReplicas'] = self.data['minNumReplicas'] result = { 'project': project, 'zone': zone, 'autoscaler': autoscaler, 'body': { 'autoscalingPolicy': body } } return result
class CloudTrailEnabled(Filter): """Verify cloud trail enabled for this account per specifications. Returns an annotated account resource if trail is not enabled. :example: .. code-block: yaml policies: - name: account-cloudtrail-enabled resource: account region: us-east-1 filters: - type: check-cloudtrail global-events: true multi-region: true running: true """ schema = type_schema( 'check-cloudtrail', **{ 'multi-region': { 'type': 'boolean' }, 'global-events': { 'type': 'boolean' }, 'running': { 'type': 'boolean' }, 'notifies': { 'type': 'boolean' }, 'file-digest': { 'type': 'boolean' }, 'kms': { 'type': 'boolean' }, 'kms-key': { 'type': 'string' } }) permissions = ('cloudtrail:DescribeTrails', 'cloudtrail:GetTrailStatus') def process(self, resources, event=None): client = local_session( self.manager.session_factory).client('cloudtrail') trails = client.describe_trails()['trailList'] resources[0]['c7n:cloudtrails'] = trails if self.data.get('global-events'): trails = [t for t in trails if t.get('IncludeGlobalServiceEvents')] if self.data.get('kms'): trails = [t for t in trails if t.get('KmsKeyId')] if self.data.get('kms-key'): trails = [ t for t in trails if t.get('KmsKeyId', '') == self.data['kms-key'] ] if self.data.get('file-digest'): trails = [t for t in trails if t.get('LogFileValidationEnabled')] if self.data.get('multi-region'): trails = [t for t in trails if t.get('IsMultiRegionTrail')] if self.data.get('notifies'): trails = [t for t in trails if t.get('SNSTopicArn')] if self.data.get('running', True): running = [] for t in list(trails): t['Status'] = status = client.get_trail_status( Name=t['TrailARN']) if status['IsLogging'] and not status.get( 'LatestDeliveryError'): running.append(t) trails = running if trails: return [] return resources
class Start(InstanceAction): schema = type_schema('start') method_spec = {'op': 'start'} attr_filter = ('status', ('TERMINATED', ))
class Resume(Action): """Resume a suspended autoscale group and its instances Parameter 'delay' is the amount of time (in seconds) to wait between resuming each instance within the ASG (default value: 30) :example: .. code-block: yaml policies: - name: asg-resume-processes resource: asg filters: - "tag:Resume": present actions: - type: resume delay: 300 """ schema = type_schema('resume', delay={'type': 'number'}) permissions = ("autoscaling:ResumeProcesses", "ec2:StartInstances") def process(self, asgs): original_count = len(asgs) asgs = [a for a in asgs if a['SuspendedProcesses']] self.delay = self.data.get('delay', 30) self.log.debug("Filtered from %d to %d suspended asgs", original_count, len(asgs)) with self.executor_factory(max_workers=3) as w: futures = {} for a in asgs: futures[w.submit(self.resume_asg_instances, a)] = a for f in as_completed(futures): if f.exception(): log.error( "Traceback resume asg:%s instances error:%s" % (futures[f]['AutoScalingGroupName'], f.exception())) continue log.debug("Sleeping for asg health check grace") time.sleep(self.delay) with self.executor_factory(max_workers=3) as w: futures = {} for a in asgs: futures[w.submit(self.resume_asg, a)] = a for f in as_completed(futures): if f.exception(): log.error( "Traceback resume asg:%s error:%s" % (futures[f]['AutoScalingGroupName'], f.exception())) def resume_asg_instances(self, asg): """Resume asg instances. """ session = local_session(self.manager.session_factory) ec2_client = session.client('ec2') instance_ids = [i['InstanceId'] for i in asg['Instances']] if not instance_ids: return ec2_client.start_instances(InstanceIds=instance_ids) def resume_asg(self, asg): """Resume asg processes. """ session = local_session(self.manager.session_factory) asg_client = session.client('autoscaling') self.manager.retry(asg_client.resume_processes, AutoScalingGroupName=asg['AutoScalingGroupName'])
class Stop(InstanceAction): schema = type_schema('stop') method_spec = {'op': 'stop'} attr_filter = ('status', ('RUNNING', ))
class Tag(Action): """Action to add a tag to an ASG The *propagate* parameter can be used to specify that the tag being added will need to be propagated down to each ASG instance associated or simply to the ASG itself. :example: .. code-block: yaml policies: - name: asg-add-owner-tag resource: asg filters: - "tag:OwnerName": absent actions: - type: tag key: OwnerName value: OwnerName propagate: true """ schema = type_schema( 'tag', key={'type': 'string'}, value={'type': 'string'}, # Backwards compatibility tag={'type': 'string'}, msg={'type': 'string'}, propagate={'type': 'boolean'}, aliases=('mark', )) permissions = ('autoscaling:CreateOrUpdateTags', ) batch_size = 1 def process(self, asgs): key = self.data.get('key', self.data.get('tag', DEFAULT_TAG)) value = self.data.get( 'value', self.data.get('msg', 'AutoScaleGroup does not meet policy guidelines')) return self.tag(asgs, key, value) def tag(self, asgs, key, value): error = None with self.executor_factory(max_workers=3) as w: futures = {} for asg_set in chunks(asgs, self.batch_size): futures[w.submit(self.process_asg_set, asg_set, key, value)] = asg_set for f in as_completed(futures): asg_set = futures[f] if f.exception(): self.log.exception( "Exception untagging tag:%s error:%s asg:%s" % (self.data.get( 'key', DEFAULT_TAG), f.exception(), ", ".join( [a['AutoScalingGroupName'] for a in asg_set]))) if error: raise error def process_asg_set(self, asgs, key, value): session = local_session(self.manager.session_factory) client = session.client('autoscaling') propagate = self.data.get('propagate_launch', True) tags = [ dict(Key=key, ResourceType='auto-scaling-group', Value=value, PropagateAtLaunch=propagate, ResourceId=a['AutoScalingGroupName']) for a in asgs ] self.manager.retry(client.create_or_update_tags, Tags=tags)
class Webhook(EventAction): """Calls a webhook with optional parameters and body populated from JMESPath queries. .. code-block:: yaml policies: - name: call-webhook resource: ec2 description: | Call webhook with list of resource groups actions: - type: webhook url: http://foo.com query-params: resource_name: resource.name policy_name: policy.name """ schema_alias = True schema = utils.type_schema( 'webhook', required=['url'], **{ 'url': { 'type': 'string' }, 'body': { 'type': 'string' }, 'batch': { 'type': 'boolean' }, 'batch-size': { 'type': 'number' }, 'method': { 'type': 'string', 'enum': ['PUT', 'POST', 'GET', 'PATCH', 'DELETE'] }, 'query-params': { "type": "object", "additionalProperties": { "type": "string", "description": "query string values" } }, 'headers': { "type": "object", "additionalProperties": { "type": "string", "description": "header values" } } }) def __init__(self, data=None, manager=None, log_dir=None): super(Webhook, self).__init__(data, manager, log_dir) self.http = None self.url = self.data.get('url') self.body = self.data.get('body') self.batch = self.data.get('batch', False) self.batch_size = self.data.get('batch-size', 500) self.query_params = self.data.get('query-params', {}) self.headers = self.data.get('headers', {}) self.method = self.data.get('method', 'POST') self.lookup_data = None def process(self, resources, event=None): self.lookup_data = { 'account_id': self.manager.config.account_id, 'region': self.manager.config.region, 'execution_id': self.manager.ctx.execution_id, 'execution_start': self.manager.ctx.start_time, 'policy': self.manager.data } self.http = self._build_http_manager() if self.batch: for chunk in utils.chunks(resources, self.batch_size): resource_data = self.lookup_data resource_data['resources'] = chunk self._process_call(resource_data) else: for r in resources: resource_data = self.lookup_data resource_data['resource'] = r self._process_call(resource_data) def _process_call(self, resource): prepared_url = self._build_url(resource) prepared_body = self._build_body(resource) prepared_headers = self._build_headers(resource) if prepared_body: prepared_headers['Content-Type'] = 'application/json' try: res = self.http.request(method=self.method, url=prepared_url, body=prepared_body, headers=prepared_headers) self.log.info("%s got response %s with URL %s" % (self.method, res.status, prepared_url)) except urllib3.exceptions.HTTPError as e: self.log.error("Error calling %s. Code: %s" % (prepared_url, e.reason)) def _build_http_manager(self): pool_kwargs = { 'cert_reqs': 'CERT_REQUIRED', 'ca_certs': certifi and certifi.where() or None } proxy_url = utils.get_proxy_url(self.url) if proxy_url: return urllib3.ProxyManager(proxy_url, **pool_kwargs) else: return urllib3.PoolManager(**pool_kwargs) def _build_headers(self, resource): return { k: jmespath.search(v, resource) for k, v in self.headers.items() } def _build_url(self, resource): """ Compose URL with query string parameters. Will not lose existing static parameters in the URL string but does not support 'duplicate' parameter entries """ if not self.query_params: return self.url evaluated_params = { k: jmespath.search(v, resource) for k, v in self.query_params.items() } url_parts = list(parse.urlparse(self.url)) query = dict(parse.parse_qsl(url_parts[4])) query.update(evaluated_params) url_parts[4] = parse.urlencode(query) return parse.urlunparse(url_parts) def _build_body(self, resource): """Create a JSON body and dump it to encoded bytes.""" if not self.body: return None return json.dumps(jmespath.search(self.body, resource)).encode('utf-8')
class RenameTag(Action): """Rename a tag on an AutoScaleGroup. :example: .. code-block: yaml policies: - name: asg-rename-owner-tag resource: asg filters: - "tag:OwnerNames": present actions: - type: rename-tag propagate: true source: OwnerNames dest: OwnerName """ schema = type_schema('rename-tag', required=['source', 'dest'], propagate={'type': 'boolean'}, source={'type': 'string'}, dest={'type': 'string'}) def get_permissions(self): permissions = ('autoscaling:CreateOrUpdateTags', 'autoscaling:DeleteTags') if self.data.get('propagate', True): permissions += ('ec2:CreateTags', 'ec2:DeleteTags') return permissions def process(self, asgs): source = self.data.get('source') dest = self.data.get('dest') count = len(asgs) filtered = [] for a in asgs: for t in a.get('Tags'): if t['Key'] == source: filtered.append(a) break asgs = filtered self.log.info("Filtered from %d asgs to %d", count, len(asgs)) self.log.info("Renaming %s to %s on %d asgs", source, dest, len(filtered)) with self.executor_factory(max_workers=3) as w: list(w.map(self.process_asg, asgs)) def process_asg(self, asg): """Move source tag to destination tag. Check tag count on asg Create new tag tag Delete old tag Check tag count on instance Create new tag Delete old tag """ source_tag = self.data.get('source') tag_map = {t['Key']: t for t in asg.get('Tags', [])} source = tag_map[source_tag] destination_tag = self.data.get('dest') propagate = self.data.get('propagate', True) client = local_session( self.manager.session_factory).client('autoscaling') # technically safer to create first, but running into # max tags constraints, otherwise. # # delete_first = len([t for t in tag_map if not t.startswith('aws:')]) client.delete_tags(Tags=[{ 'ResourceId': asg['AutoScalingGroupName'], 'ResourceType': 'auto-scaling-group', 'Key': source_tag, 'Value': source['Value'] }]) client.create_or_update_tags( Tags=[{ 'ResourceId': asg['AutoScalingGroupName'], 'ResourceType': 'auto-scaling-group', 'PropagateAtLaunch': propagate, 'Key': destination_tag, 'Value': source['Value'] }]) if propagate: self.propogate_instance_tag(source, destination_tag, asg) def propogate_instance_tag(self, source, destination_tag, asg): client = local_session(self.manager.session_factory).client('ec2') client.delete_tags( Resources=[i['InstanceId'] for i in asg['Instances']], Tags=[{ "Key": source['Key'] }]) client.create_tags( Resources=[i['InstanceId'] for i in asg['Instances']], Tags=[{ 'Key': source['Key'], 'Value': source['Value'] }])
class CrossAccountAccessFilter(Filter): """Check a resource's embedded iam policy for cross account access. """ schema = type_schema( 'cross-account', # only consider policies that grant one of the given actions. actions={ 'type': 'array', 'items': { 'type': 'string' } }, # only consider policies which grant to * everyone_only={'type': 'boolean'}, # disregard statements using these conditions. whitelist_conditions={ 'type': 'array', 'items': { 'type': 'string' } }, # white list accounts whitelist_from=ValuesFrom.schema, whitelist={ 'type': 'array', 'items': { 'type': 'string' } }, whitelist_vpce_from=ValuesFrom.schema, whitelist_vpce={ 'type': 'array', 'items': { 'type': 'string' } }, whitelist_vpc_from=ValuesFrom.schema, whitelist_vpc={ 'type': 'array', 'items': { 'type': 'string' } }) policy_attribute = 'Policy' annotation_key = 'CrossAccountViolations' checker_factory = PolicyChecker def process(self, resources, event=None): self.everyone_only = self.data.get('everyone_only', False) self.conditions = set( self.data.get('whitelist_conditions', ("aws:userid", "aws:username"))) self.actions = self.data.get('actions', ()) self.accounts = self.get_accounts() self.vpcs = self.get_vpcs() self.vpces = self.get_vpces() self.checker = self.checker_factory({ 'allowed_accounts': self.accounts, 'allowed_vpc': self.vpcs, 'allowed_vpce': self.vpces, 'check_actions': self.actions, 'everyone_only': self.everyone_only, 'whitelist_conditions': self.conditions }) return super(CrossAccountAccessFilter, self).process(resources, event) def get_accounts(self): owner_id = self.manager.config.account_id accounts = set(self.data.get('whitelist', ())) if 'whitelist_from' in self.data: values = ValuesFrom(self.data['whitelist_from'], self.manager) accounts = accounts.union(values.get_values()) accounts.add(owner_id) return accounts def get_vpcs(self): vpc = set(self.data.get('whitelist_vpc', ())) if 'whitelist_vpc_from' in self.data: values = ValuesFrom(self.data['whitelist_vpc_from'], self.manager) vpc = vpc.union(values.get_values()) return vpc def get_vpces(self): vpce = set(self.data.get('whitelist_vpce', ())) if 'whitelist_vpce_from' in self.data: values = ValuesFrom(self.data['whitelist_vpce_from'], self.manager) vpce = vpce.union(values.get_values()) return vpce def get_resource_policy(self, r): return r.get(self.policy_attribute, None) def __call__(self, r): p = self.get_resource_policy(r) if p is None: return False violations = self.checker.check(p) if violations: r[self.annotation_key] = violations return True
class ConfigPollRuleMode(LambdaMode, PullMode): """This mode represents a periodic/scheduled AWS config evaluation. The primary benefit this mode offers is to support additional resources beyond what config supports natively, as it can post evaluations for any resource which has a cloudformation type. If a resource is natively supported by config its highly recommended to use a `config-rule` mode instead. This mode effectively receives no data from config, instead its periodically executed by config and polls and evaluates all resources. It is equivalent to a periodic policy, except it also pushes resource evaluations to config. """ schema = utils.type_schema( 'config-poll-rule', schedule={'enum': [ "One_Hour", "Three_Hours", "Six_Hours", "Twelve_Hours", "TwentyFour_Hours"]}, rinherit=LambdaMode.schema) def validate(self): super().validate() if not self.policy.data['mode'].get('schedule'): raise PolicyValidationError( "policy:%s config-poll-rule schedule required" % ( self.policy.name)) if self.policy.resource_manager.resource_type.config_type: raise PolicyValidationError( "resource:%s fully supported by config and should use mode: config-rule" % ( self.policy.resource_type)) if self.policy.data['mode'].get('pattern'): raise PolicyValidationError( "policy:%s AWS Config does not support event pattern filtering" % ( self.policy.name)) if not self.policy.resource_manager.resource_type.cfn_type: raise PolicyValidationError(( 'policy:%s resource:%s does not have a cloudformation type' ' and is there-fore not supported by config-poll-rule')) def _get_client(self): return utils.local_session( self.policy.session_factory).client('config') def put_evaluations(self, client, token, evaluations): for eval_set in utils.chunks(evaluations, 100): self.policy.resource_manager.retry( client.put_evaluations, Evaluations=eval_set, ResultToken=token) def run(self, event, lambda_context): cfg_event = json.loads(event['invokingEvent']) resource_type = self.policy.resource_manager.resource_type.cfn_type resource_id = self.policy.resource_manager.resource_type.id client = self._get_client() token = event.get('resultToken') matched_resources = set() for r in PullMode.run(self): matched_resources.add(r[resource_id]) unmatched_resources = set() for r in self.policy.resource_manager.get_resource_manager( self.policy.resource_type).resources(): if r[resource_id] not in matched_resources: unmatched_resources.add(r[resource_id]) evaluations = [dict( ComplianceResourceType=resource_type, ComplianceResourceId=r, ComplianceType='NON_COMPLIANT', OrderingTimestamp=cfg_event['notificationCreationTime'], Annotation='The resource is not compliant with policy:%s.' % ( self.policy.name)) for r in matched_resources] if evaluations and token: self.put_evaluations(client, token, evaluations) evaluations = [dict( ComplianceResourceType=resource_type, ComplianceResourceId=r, ComplianceType='COMPLIANT', OrderingTimestamp=cfg_event['notificationCreationTime'], Annotation='The resource is compliant with policy:%s.' % ( self.policy.name)) for r in unmatched_resources] if evaluations and token: self.put_evaluations(client, token, evaluations) return list(matched_resources)
class ConfigEnabled(Filter): """Is config service enabled for this account :example: .. code-block: yaml policies: - name: account-check-config-services resource: account region: us-east-1 filters: - type: check-config all-resources: true global-resources: true running: true """ schema = type_schema( 'check-config', **{ 'all-resources': { 'type': 'boolean' }, 'running': { 'type': 'boolean' }, 'global-resources': { 'type': 'boolean' } }) permissions = ('config:DescribeDeliveryChannels', 'config:DescribeConfigurationRecorders', 'config:DescribeConfigurationRecorderStatus') def process(self, resources, event=None): client = local_session(self.manager.session_factory).client('config') channels = client.describe_delivery_channels()['DeliveryChannels'] recorders = client.describe_configuration_recorders( )['ConfigurationRecorders'] resources[0]['c7n:config_recorders'] = recorders resources[0]['c7n:config_channels'] = channels if self.data.get('global-resources'): recorders = [ r for r in recorders if r['recordingGroup'].get('includeGlobalResourceTypes') ] if self.data.get('all-resources'): recorders = [ r for r in recorders if r['recordingGroup'].get('allSupported') ] if self.data.get('running', True) and recorders: status = { s['name']: s for s in client.describe_configuration_recorder_status() ['ConfigurationRecordersStatus'] } resources[0]['c7n:config_status'] = status recorders = [ r for r in recorders if status[r['name']]['recording'] and status[r['name']] ['lastStatus'].lower() in ('pending', 'success') ] if channels and recorders: return [] return resources
class Delete(InstanceAction): schema = type_schema('delete') method_spec = {'op': 'delete'}
class EnableTrail(BaseAction): """Enables logging on the trail(s) named in the policy :Example: .. code-block: yaml policies: - name: trail-test description: Ensure CloudTrail logging is enabled resource: account actions: - type: enable-cloudtrail trail: mytrail bucket: trails """ permissions = ( 'cloudtrail:CreateTrail', 'cloudtrail:DescribeTrails', 'cloudtrail:GetTrailStatus', 'cloudtrail:StartLogging', 'cloudtrail:UpdateTrail', 's3:CreateBucket', 's3:GetBucketPolicy', 's3:PutBucketPolicy', ) schema = type_schema( 'enable-cloudtrail', **{ 'trail': {'type': 'string'}, 'bucket': {'type': 'string'}, 'bucket-region': {'type': 'string'}, 'multi-region': {'type': 'boolean'}, 'global-events': {'type': 'boolean'}, 'notify': {'type': 'string'}, 'file-digest': {'type': 'boolean'}, 'kms': {'type': 'boolean'}, 'kms-key': {'type': 'string'}, 'required': ('bucket',), } ) def process(self, accounts): """Create or enable CloudTrail""" session = local_session(self.manager.session_factory) client = session.client('cloudtrail') bucket_name = self.data['bucket'] bucket_region = self.data.get('bucket-region', 'us-east-1') trail_name = self.data.get('trail', 'default-trail') multi_region = self.data.get('multi-region', True) global_events = self.data.get('global-events', True) notify = self.data.get('notify', '') file_digest = self.data.get('file-digest', False) kms = self.data.get('kms', False) kms_key = self.data.get('kms-key', '') s3client = session.client('s3') try: s3client.create_bucket( Bucket=bucket_name, CreateBucketConfiguration={'LocationConstraint': bucket_region} ) except ClientError as ce: if not ('Error' in ce.response and ce.response['Error']['Code'] == 'BucketAlreadyOwnedByYou'): raise ce try: current_policy = s3client.get_bucket_policy(Bucket=bucket_name) except ClientError: current_policy = None policy_json = cloudtrail_policy( current_policy, bucket_name, self.manager.config.account_id) s3client.put_bucket_policy(Bucket=bucket_name, Policy=policy_json) trails = client.describe_trails().get('trailList', ()) if trail_name not in [t.get('Name') for t in trails]: new_trail = client.create_trail( Name=trail_name, S3BucketName=bucket_name, ) if new_trail: trails.append(new_trail) # the loop below will configure the new trail for trail in trails: if trail.get('Name') != trail_name: continue # enable arn = trail['TrailARN'] status = client.get_trail_status(Name=arn) if not status['IsLogging']: client.start_logging(Name=arn) # apply configuration changes (if any) update_args = {} if multi_region != trail.get('IsMultiRegionTrail'): update_args['IsMultiRegionTrail'] = multi_region if global_events != trail.get('IncludeGlobalServiceEvents'): update_args['IncludeGlobalServiceEvents'] = global_events if notify != trail.get('SNSTopicArn'): update_args['SnsTopicName'] = notify if file_digest != trail.get('LogFileValidationEnabled'): update_args['EnableLogFileValidation'] = file_digest if kms_key != trail.get('KmsKeyId'): if not kms and 'KmsKeyId' in trail: kms_key = '' update_args['KmsKeyId'] = kms_key if update_args: update_args['Name'] = trail_name client.update_trail(**update_args)
class SetSslListenerPolicy(BaseAction): """Action to set the ELB SSL listener policy :example: .. code-block:: yaml policies: - name: elb-set-listener-policy resource: elb actions: - type: set-ssl-listener-policy name: SSLNegotiation-Policy-01 attributes: - Protocol-SSLv3 - Protocol-TLSv1.1 - DHE-RSA-AES256-SHA256 """ schema = type_schema('set-ssl-listener-policy', name={'type': 'string'}, attributes={ 'type': 'array', 'items': { 'type': 'string' } }, required=['name', 'attributes']) permissions = ('elasticloadbalancing:CreateLoadBalancerPolicy', 'elasticloadbalancing:SetLoadBalancerPoliciesOfListener') def process(self, load_balancers): client = local_session(self.manager.session_factory).client('elb') rid = self.manager.resource_type.id error = None with self.executor_factory(max_workers=2) as w: futures = {} for lb in load_balancers: futures[w.submit(self.process_elb, client, lb)] = lb for f in as_completed(futures): if f.exception(): log.error( "set-ssl-listener-policy error on lb:%s error:%s", futures[f][rid], f.exception()) error = f.exception() if error is not None: raise error def process_elb(self, client, elb): if not is_ssl(elb): return # Create a custom policy with epoch timestamp. # to make it unique within the # set of policies for this load balancer. policy_name = self.data.get('name') + '-' + \ str(int(datetime.now(tz=tzutc()).strftime("%s")) * 1000) lb_name = elb['LoadBalancerName'] attrs = self.data.get('attributes') policy_attributes = [{ 'AttributeName': attr, 'AttributeValue': 'true' } for attr in attrs] try: client.create_load_balancer_policy( LoadBalancerName=lb_name, PolicyName=policy_name, PolicyTypeName='SSLNegotiationPolicyType', PolicyAttributes=policy_attributes) except ClientError as e: if e.response['Error']['Code'] not in ( 'DuplicatePolicyName', 'DuplicatePolicyNameException', 'DuplicationPolicyNameException'): raise # Apply it to all SSL listeners. ssl_policies = () if 'c7n.ssl-policies' in elb: ssl_policies = elb['c7n.ssl-policies'] for ld in elb['ListenerDescriptions']: if ld['Listener']['Protocol'] in ('HTTPS', 'SSL'): policy_names = [policy_name] # Preserve extant non-ssl listener policies policy_names.extend(ld.get('PolicyNames', ())) # Remove extant ssl listener policy if ssl_policies: policy_names = list( set(policy_names).difference(ssl_policies)) client.set_load_balancer_policies_of_listener( LoadBalancerName=lb_name, LoadBalancerPort=ld['Listener']['LoadBalancerPort'], PolicyNames=policy_names)
class FirehoseEncryptS3Destination(Action): """Action to set encryption key a Firehose S3 destination :example: .. code-block:: yaml policies: - name: encrypt-s3-destination resource: firehose filters: - KmsMasterKeyId: absent actions: - type: encrypt-s3-destination key_arn: <arn of KMS key/alias> """ schema = type_schema( 'encrypt-s3-destination', key_arn={'type': 'string'}, required=('key_arn',)) permissions = ("firehose:UpdateDestination",) DEST_MD = { 'SplunkDestinationDescription': { 'update': 'SplunkDestinationUpdate', 'clear': ['S3BackupMode'], 'encrypt_path': 'S3DestinationDescription.EncryptionConfiguration', 'remap': [('S3DestinationDescription', 'S3Update')] }, 'ElasticsearchDestinationDescription': { 'update': 'ElasticsearchDestinationUpdate', 'clear': ['S3BackupMode'], 'encrypt_path': 'S3DestinationDescription.EncryptionConfiguration', 'remap': [('S3DestinationDescription', 'S3Update')], }, 'ExtendedS3DestinationDescription': { 'update': 'ExtendedS3DestinationUpdate', 'clear': ['S3BackupMode'], 'encrypt_path': 'EncryptionConfiguration', 'remap': [] }, 'RedshiftDestinationDescription': { 'update': 'RedshiftDestinationUpdate', 'clear': ['S3BackupMode', "ClusterJDBCURL", "CopyCommand", "Username"], 'encrypt_path': 'S3DestinationDescription.EncryptionConfiguration', 'remap': [('S3DestinationDescription', 'S3Update')] }, } def process(self, resources): client = local_session(self.manager.session_factory).client('firehose') key = self.data.get('key_arn') for r in resources: if not r['DeliveryStreamStatus'] == 'ACTIVE': continue version = r['VersionId'] name = r['DeliveryStreamName'] d = r['Destinations'][0] destination_id = d['DestinationId'] for dtype, dmetadata in self.DEST_MD.items(): if dtype not in d: continue dinfo = d[dtype] for k in dmetadata['clear']: dinfo.pop(k, None) if dmetadata['encrypt_path']: encrypt_info = jmespath.search(dmetadata['encrypt_path'], dinfo) else: encrypt_info = dinfo encrypt_info.pop('NoEncryptionConfig', None) encrypt_info['KMSEncryptionConfig'] = {'AWSKMSKeyARN': key} for old_k, new_k in dmetadata['remap']: if old_k in dinfo: dinfo[new_k] = dinfo.pop(old_k) params = dict(DeliveryStreamName=name, DestinationId=destination_id, CurrentDeliveryStreamVersionId=version) params[dmetadata['update']] = dinfo client.update_destination(**params)