def filter_policies_on_tags(policies, tags, config): filtered_policies = PolicyCollection([], config) for p in policies: if tags: found = set() for t in tags: if t in p.tags: found.add(t) if found == set(tags): filtered_policies += PolicyCollection([p], config) return filtered_policies
def get_report_obj(valid_report_cache_file_name, report_to_policy_map): account, region, report_name = valid_report_cache_file_name.split( '/')[1:-1] dry_run_cache_dir = '/'.join(valid_report_cache_file_name.split('/')[0:-2]) options = argparse.Namespace( account_id=None, assume_role=None, cache=None, command='c7n.commands.report', config=None, days=1, debug=False, field=[], # ['HEADER=Tags'], format='grid', log_group=None, no_default_fields=False, output_dir=dry_run_cache_dir, policy_filter=None, profile=None, raw=None, region=region, regions='', resource_type=None, subparser='report', verbose=False) report_policy = PolicyCollection(report_to_policy_map[report_name], options) return [report_policy.policies[0], options, account, region, report_name]
def dryrun(self): # This is largely based off of mugc.main() logging.getLogger('botocore').setLevel(logging.ERROR) logging.getLogger('urllib3').setLevel(logging.ERROR) logging.getLogger('c7n.cache').setLevel(logging.WARNING) conf = Config.empty( config_files=['custodian_%s.yml' % self.region_name], regions=[self.region_name], prefix=self.config.function_prefix, policy_regex='^' + re.escape(self.config.function_prefix) + '.*', assume=None, policy_filter=None, log_group=None, external_id=None, cache_period=0, cache=None, present=False, dryrun=True ) # use cloud provider to initialize policies to get region expansion policies = AWS().initialize_policies( PolicyCollection( [ p for p in load_policies(conf, conf) if p.provider_name == 'aws' ], conf ), conf ) resources_gc_prefix(conf, conf, policies)
def run(event, context=None): # policies file should always be valid in functions so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): log.error('Invalid policy config') return False options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) # if output_dir specified use that, otherwise make a temp directory if 'output_dir' not in options_overrides: options_overrides['output_dir'] = get_tmp_output_dir() # merge all our options in options = Config.empty(**options_overrides) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: log.info("running policy %s", p.name) p.push(event, context) return True
def dispatch_event(event, context): error = event.get('detail', {}).get('errorCode') if error: log.debug("Skipping failed operation: %s" % error) return event['debug'] = True if event['debug']: log.info("Processing event\n %s", format_event(event)) # policies file should always be valid in lambda so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # TODO. This enshrines an assumption of a single policy per lambda. options_overrides = policy_config['policies'][0].get('mode', {}).get( 'execution-options', {}) options = Config.empty(**options_overrides) load_resources() policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def report_account(account, region, policies_config, output_path, debug): cache_path = os.path.join(output_path, "c7n.cache") output_path = os.path.join(output_path, account['name'], region) config = Config.empty( region=region, output_dir=output_path, account_id=account['account_id'], metrics_enabled=False, cache=cache_path, log_group=None, profile=None, external_id=None) if account.get('role'): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) records = [] for p in policies: log.debug( "Report policy:%s account:%s region:%s path:%s", p.name, account['name'], region, output_path) policy_records = fs_record_set(p.ctx.output_path, p.name) for r in policy_records: r['policy'] = p.name r['region'] = p.options.region r['account'] = account['name'] for t in account.get('tags', ()): if ':' in t: k, v = t.split(':', 1) r[k] = v records.extend(policy_records) return records
def dispatch_event(event, context): error = event.get('detail', {}).get('errorCode') if error and C7N_SKIP_EVTERR: log.debug("Skipping failed operation: %s" % error) return if C7N_DEBUG_EVENT: event['debug'] = True log.info("Processing event\n %s", format_event(event)) # Policies file should always be valid in lambda so do loading naively global policy_config if policy_config is None: with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False options = init_config(policy_config) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: try: # validation provides for an initialization point for # some filters/actions. p.validate() p.push(event, context) except Exception: log.exception("error during policy execution") if C7N_CATCH_ERR: continue raise return True
def load_policy(self, path, policies): """ Loads a YAML file and prompts scheduling updates :param path: Path to YAML file on disk :param policies: Dictionary of policies to update """ with open(path, "r") as stream: try: policy_config = yaml.safe_load(stream) new_policies = PolicyCollection.from_data( policy_config, self.options) if new_policies: for p in new_policies: log.info("Loading Policy %s from %s" % (p.name, path)) p.validate() policies.update({p.name: {'policy': p}}) # Update periodic and set event update flag self.update_periodic(p) if p.data.get('mode', {}).get('events'): self.require_event_update = True except Exception as exc: log.error('Invalid policy file %s %s' % (path, exc))
def process(self, resources, event=None): provider = clouds[self.manager.ctx.policy.provider_name]() # if the embedded policy only specifies one region, or only # being executed on a single region. if self.embedded_policy.region or len( self.manager.config.regions) <= 1: if (self.embedded_policy.region and self.embedded_policy.region != self.manager.config.region): return [] self.embedded_policy.expand_variables( self.embedded_policy.get_variables()) return not self.embedded_policy.poll() and resources or [] # For regional resources and multi-region execution, the policy matches if # the resource is missing in any region. found = {} for p in provider.initialize_policies( PolicyCollection([self.embedded_policy], self.manager.config), self.manager.config): p.expand_variables(p.get_variables()) p.validate() found[p.options.region] = p.poll() if not all(found.values()): return resources return []
def run(event, context): # policies file should always be valid in functions so do loading naively with open(context['config_file']) as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): log.error('Invalid policy config') return False options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) # setup our auth file location on disk options_overrides['authorization_file'] = context['auth_file'] # if output_dir specified use that, otherwise make a temp directory if 'output_dir' not in options_overrides: options_overrides['output_dir'] = get_tmp_output_dir() # merge all our options in options = Config.empty(**options_overrides) load_resources() options = Azure().initialize(options) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: try: p.push(event, context) except (CloudError, AzureHttpError) as error: log.error("Unable to process policy: %s :: %s" % (p.name, error)) return True
def load_policy(self, path, policies): """ Loads a YAML file and prompts scheduling updates :param path: Path to YAML file on disk :param policies: Dictionary of policies to update """ with open(path, "r") as stream: try: policy_config = yaml.safe_load(stream) new_policies = PolicyCollection.from_data( policy_config, self.options) if new_policies: for p in new_policies: log.info("Loading Policy %s from %s" % (p.name, path)) p.validate() policies.update({p.name: {'policy': p}}) # Update periodic policy_mode = p.data.get('mode', {}).get('type') if policy_mode == CONTAINER_TIME_TRIGGER_MODE: self.update_periodic(p) elif policy_mode != CONTAINER_EVENT_TRIGGER_MODE: log.warning( "Unsupported policy mode for Azure Container Host: {}. " "{} will not be run. " "Supported policy modes include \"{}\" and \"{}\"." .format(policy_mode, p.data['name'], CONTAINER_EVENT_TRIGGER_MODE, CONTAINER_TIME_TRIGGER_MODE)) except Exception as exc: log.error('Invalid policy file %s %s' % (path, exc))
def run(event, context): # policies file should always be valid in functions so do loading naively with open(context['config_file']) as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # Initialize output directory output_dir = os.environ.get( 'C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4())) if not os.path.exists(output_dir): try: os.mkdir(output_dir) except OSError as error: log.warning("Unable to make output directory: {}".format(error)) options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) options_overrides['authorization_file'] = context['auth_file'] if 'output_dir' not in options_overrides: options_overrides['output_dir'] = output_dir options = Config.empty(**options_overrides) load_resources() policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def dispatch_event(event, context): error = event.get('detail', {}).get('errorCode') if error and C7N_SKIP_EVTERR: log.debug("Skipping failed operation: %s" % error) return # one time initialization for cold starts. global policy_config, policy_data if policy_config is None: with open('config.json') as f: policy_data = json.load(f) policy_config = init_config(policy_data) load_resources(StructureParser().get_resource_types(policy_data)) if C7N_DEBUG_EVENT: event['debug'] = True log.info("Processing event\n %s", format_event(event)) if not policy_data or not policy_data.get('policies'): return False policies = PolicyCollection.from_data(policy_data, policy_config) for p in policies: try: # validation provides for an initialization point for # some filters/actions. p.validate() p.push(event, context) except Exception: log.exception("error during policy execution") if C7N_CATCH_ERR: continue raise return True
def dispatch_event(event, context): global account_id error = event.get('detail', {}).get('errorCode') if error: log.debug("Skipping failed operation: %s" % error) return event['debug'] = True if event['debug']: log.info("Processing event\n %s", format_event(event)) # Policies file should always be valid in lambda so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # Initialize output directory, we've seen occassional perm issues with # lambda on temp directory and changing unix execution users, so # use a per execution temp space. output_dir = os.environ.get('C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4())) if not os.path.exists(output_dir): try: os.mkdir(output_dir) except OSError as error: log.warning("Unable to make output directory: {}".format(error)) # TODO. This enshrines an assumption of a single policy per lambda. options_overrides = policy_config['policies'][0].get('mode', {}).get( 'execution-options', {}) # if using assume role in lambda ensure that the correct # execution account is captured in options. if 'assume_role' in options_overrides: account_id = options_overrides['assume_role'].split(':')[4] elif account_id is None: session = boto3.Session() account_id = get_account_id_from_sts(session) # Historical compatibility with manually set execution options # previously this was a boolean, its now a string value with the # boolean flag triggering a string value of 'aws' if 'metrics_enabled' in options_overrides and isinstance( options_overrides['metrics_enabled'], bool): options_overrides['metrics_enabled'] = 'aws' options_overrides['account_id'] = account_id if 'output_dir' not in options_overrides: options_overrides['output_dir'] = output_dir options = Config.empty(**options_overrides) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def run_account(account, region, policies_config, output_path, cache_period, dryrun, debug): """Execute a set of policies on an account. """ logging.getLogger('custodian.output').setLevel(logging.ERROR + 1) CONN_CACHE.session = None CONN_CACHE.time = None output_path = os.path.join(output_path, account['name'], region) if not os.path.exists(output_path): os.makedirs(output_path) cache_path = os.path.join(output_path, "c7n.cache") bag = Bag.empty(region=region, assume_role=account['role'], cache_period=cache_period, dryrun=dryrun, output_dir=output_path, account_id=account['account_id'], metrics_enabled=False, cache=cache_path, log_group=None, profile=None, external_id=None) policies = PolicyCollection.from_data(policies_config, bag) policy_counts = {} st = time.time() with environ(**account_tags(account)): for p in policies: log.debug("Running policy:%s account:%s region:%s", p.name, account['name'], region) try: resources = p.run() policy_counts[p.name] = resources and len(resources) or 0 if not resources: continue log.info( "Ran account:%s region:%s policy:%s matched:%d time:%0.2f", account['name'], region, p.name, len(resources), time.time() - st) except ClientError as e: if e.response['Error']['Code'] == 'AccessDenied': log.warning('Access denied account:%s region:%s', account['name'], region) return policy_counts log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) continue except Exception as e: log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) if not debug: continue import traceback, pdb, sys pdb.post_mortem(sys.exc_info()[-1]) raise return policy_counts
def get_untagged_resources(self, rtype): policy_data = { 'name': 'inventory-%s' % rtype, 'resource': rtype, 'filters': [{ 'tag:{}'.format(self.creator_tag): 'absent' }] } # Cloud Formation stacks can only be tagged in successful # steady state. if rtype == 'cfn': policy_data['filters'].insert( 0, { 'type': 'value', 'key': 'StackStatus', 'op': 'in', 'value': ['UPDATE_COMPLETE', 'CREATE_COMPLETE'] }) policy = list( PolicyCollection.from_data({'policies': [policy_data]}, self.config)).pop() if 'tag' not in policy.resource_manager.action_registry: return [], None resources = policy.run() return resources, policy.resource_manager
def dispatch_event(event, context): error = event.get('detail', {}).get('errorCode') if error: log.debug("Skipping failed operation: %s" % error) return event['debug'] = True if event['debug']: log.info("Processing event\n %s", format_event(event)) # policies file should always be valid in lambda so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # TODO. This enshrines an assumption of a single policy per lambda. options_overrides = policy_config[ 'policies'][0].get('mode', {}).get('execution-options', {}) options = Config.empty(**options_overrides) load_resources() policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def run(event, context): # policies file should always be valid in functions so do loading naively with open(context['config_file']) as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): log.error('Invalid policy config') return False options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) # setup our auth file location on disk options_overrides['authorization_file'] = context['auth_file'] # if output_dir specified use that, otherwise make a temp directory if 'output_dir' not in options_overrides: options_overrides['output_dir'] = get_tmp_output_dir() # merge all our options in options = Config.empty(**options_overrides) load_resources() policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def initialize_policies(self, policy_collection, options): """Return a set of policies targetted to the given regions. Supports symbolic regions like 'all'. This will automatically filter out policies if their being targetted to a region that does not support the service. Global services will target a single region (us-east-1 if only all specified, else first region in the list). Note for region partitions (govcloud and china) an explicit region from the partition must be passed in. """ from c7n.policy import Policy, PolicyCollection policies = [] service_region_map, resource_service_map = get_service_region_map( options.regions, policy_collection.resource_types) for p in policy_collection: available_regions = service_region_map.get( resource_service_map.get(p.resource_type), ()) # its a global service/endpoint, use user provided region # or us-east-1. if not available_regions and options.regions: candidates = [r for r in options.regions if r != 'all'] candidate = candidates and candidates[0] or 'us-east-1' svc_regions = [candidate] elif 'all' in options.regions: svc_regions = available_regions else: svc_regions = options.regions for region in svc_regions: if available_regions and region not in available_regions: level = ('all' in options.regions and logging.DEBUG or logging.WARNING) # TODO: fixme policy_collection.log.log( level, "policy:%s resources:%s not available in region:%s", p.name, p.resource_type, region) continue options_copy = copy.copy(options) options_copy.region = str(region) if len(options.regions ) > 1 or 'all' in options.regions and getattr( options, 'output_dir', None): options_copy.output_dir = (options.output_dir.rstrip('/') + '/%s' % region) policies.append( Policy( p.data, options_copy, session_factory=policy_collection.session_factory())) return PolicyCollection(policies, options)
def cli(**kwargs): policy_config = Config.empty() policies = PolicyCollection([ p for p in load_policies(kwargs['config'], policy_config) if p.provider_name == 'azure' ], policy_config) session = policies.policies[0].session_factory() web_client = session.client('azure.mgmt.web.WebSiteManagementClient') deployments = {} credentials_load_failed = 0 not_functions_policy = 0 for p in policies: if not p.is_lambda: not_functions_policy += 1 continue try: params = AzureFunctionMode(p).get_function_app_params() creds = web_client.web_apps.list_publishing_credentials( params.function_app_resource_group_name, params.function_app_name).result() deployments[p.name] = {'scm_uri': creds.scm_uri, 'status': None} log.info('Retrieved deployment credentials for %s policy', p.name) except Exception: log.error( 'Unable to retrieve deployment credentials for %s policy', p.name) credentials_load_failed += 1 wait_for_remote_builds(deployments) success = 0 fail = 0 not_found = 0 for name, params in deployments.items(): if params['status'] != DeploymentStatus.Succeeded: log.info('%s: %s', name, params['status']) if params['status'] == DeploymentStatus.Failed: log.error('Build logs can be retrieved here: %s', params['scm_uri']) fail += 1 else: not_found += 1 else: success += 1 log.info( 'Policies total: %i, unable to load credentials: %i, not Functions mode: %i', len(policies), credentials_load_failed, not_functions_policy) log.info( 'Status not found can happen if Linux Consumption function was deployed' 'more than 2 hours ago.') log.info( 'Deployments complete. Success: %i, Fail: %i, Status not found: %i', success, fail, not_found)
def report_account(account, region, policies_config, output_path, cache_path, debug): output_path = os.path.join(output_path, account['name'], region) cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['name'], region)) load_available() config = Config.empty(region=region, output_dir=output_path, account_id=account['account_id'], metrics_enabled=False, cache=cache_path, log_group=None, profile=None, external_id=None) if account.get('role'): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) records = [] for p in policies: # initializee policy execution context for output access p.ctx.initialize() log.debug("Report policy:%s account:%s region:%s path:%s", p.name, account['name'], region, output_path) if p.ctx.output.type == "s3": delta = timedelta(days=1) begin_date = datetime.now() - delta policy_records = record_set( p.session_factory, p.ctx.output.config['netloc'], strip_output_path(p.ctx.output.config['path'], p.name), begin_date) else: policy_records = fs_record_set(p.ctx.log_dir, p.name) for r in policy_records: r['policy'] = p.name r['region'] = p.options.region r['account'] = account['name'] for t in account.get('tags', ()): if ':' in t: k, v = t.split(':', 1) r[k] = v records.extend(policy_records) return records
def dispatch_event(event, context): global account_id if account_id is None: session = boto3.Session() account_id = get_account_id_from_sts(session) error = event.get('detail', {}).get('errorCode') if error: log.debug("Skipping failed operation: %s" % error) return event['debug'] = True if event['debug']: log.info("Processing event\n %s", format_event(event)) # policies file should always be valid in lambda so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # Initialize output directory, we've seen occassional perm issues with # lambda on temp directory and changing unix execution users, so # use a per execution temp space. output_dir = os.environ.get( 'C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4())) if not os.path.exists(output_dir): try: os.mkdir(output_dir) except OSError as error: log.warning("Unable to make output directory: {}".format(error)) # TODO. This enshrines an assumption of a single policy per lambda. options_overrides = policy_config[ 'policies'][0].get('mode', {}).get('execution-options', {}) options_overrides['account_id'] = account_id if 'output_dir' not in options_overrides: options_overrides['output_dir'] = output_dir options = Config.empty(**options_overrides) load_resources() policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def dispatch_event(event, context): global account_id if account_id is None: session = boto3.Session() account_id = get_account_id_from_sts(session) error = event.get('detail', {}).get('errorCode') if error: log.debug("Skipping failed operation: %s" % error) return event['debug'] = True if event['debug']: log.info("Processing event\n %s", format_event(event)) # policies file should always be valid in lambda so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # Initialize output directory, we've seen occassional perm issues with # lambda on temp directory and changing unix execution users, so # use a per execution temp space. output_dir = os.environ.get('C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4())) if not os.path.exists(output_dir): try: os.mkdir(output_dir) except OSError as error: log.warning("Unable to make output directory: {}".format(error)) # TODO. This enshrines an assumption of a single policy per lambda. options_overrides = policy_config['policies'][0].get('mode', {}).get( 'execution-options', {}) options_overrides['account_id'] = account_id if 'output_dir' not in options_overrides: options_overrides['output_dir'] = output_dir options = Config.empty(**options_overrides) load_resources() policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def main(): parser = setup_parser() options = parser.parse_args() log_level = logging.INFO if options.verbose: log_level = logging.DEBUG logging.basicConfig( level=log_level, format="%(asctime)s: %(name)s:%(levelname)s %(message)s") logging.getLogger('botocore').setLevel(logging.ERROR) logging.getLogger('urllib3').setLevel(logging.ERROR) logging.getLogger('c7n.cache').setLevel(logging.WARNING) if not options.policy_regex: options.policy_regex = f"^{options.prefix}.*" if not options.regions: options.regions = [os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')] files = [] files.extend(itertools.chain(*options.config_files)) files.extend(options.configs) options.config_files = files options.policy_tags = list(itertools.chain(*options.policy_tags)) if not files: parser.print_help() sys.exit(1) policy_config = Config.empty( regions=options.regions, profile=options.profile, assume_role=options.assume_role) # use cloud provider to initialize policies to get region expansion policies = AWS().initialize_policies( PolicyCollection([ p for p in load_policies( options, policy_config) if p.provider_name == 'aws'], policy_config), policy_config) resources_gc_prefix(options, policy_config, policies)
def run_account(account, region, policies_config, output_path, cache_period, dryrun, debug): """Execute a set of policies on an account. """ CONN_CACHE.session = None CONN_CACHE.time = None output_path = os.path.join(output_path, account['name'], region) if not os.path.exists(output_path): os.makedirs(output_path) cache_path = os.path.join(output_path, "c7n.cache") bag = Bag.empty( region=region, assume_role=account['role'], cache_period=cache_period, dryrun=dryrun, output_dir=output_path, account_id=account['account_id'], metrics_enabled=False, cache=cache_path, log_group=None, profile=None, external_id=None) policies = PolicyCollection.from_data(policies_config, bag) policy_counts = {} st = time.time() with environ(**account_tags(account)): for p in policies: log.debug( "Running policy:%s account:%s region:%s", p.name, account['name'], region) try: resources = p.run() policy_counts[p.name] = resources and len(resources) or 0 if not resources: continue log.info("Ran account:%s region:%s policy:%s matched:%d time:%0.2f", account['name'], region, p.name, len(resources), time.time()-st) except Exception as e: log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) if not debug: continue import traceback, pdb, sys pdb.post_mortem(sys.exc_info()[-1]) raise return policy_counts
def run(event, context, subscription_id=None): # policies file should always be valid in functions so do loading naively with open(context['config_file']) as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): log.error('Invalid policy config') return False options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) # setup our auth file location on disk options_overrides['authorization_file'] = context['auth_file'] # if output_dir specified use that, otherwise make a temp directory if 'output_dir' not in options_overrides: options_overrides['output_dir'] = get_tmp_output_dir() # merge all our options in options = Config.empty(**options_overrides) if subscription_id is not None: options['account_id'] = subscription_id load_resources(StructureParser().get_resource_types(policy_config)) options = Azure().initialize(options) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: try: p.push(event, context) except (CloudError, AzureHttpError) as error: log.error("Unable to process policy: %s :: %s" % (p.name, error)) reset_session_cache() return True
def _load_policies(options): validate = True if 'skip_validation' in options: validate = not options.skip_validation if not validate: log.debug('Policy validation disabled') vars = _load_vars(options) errors = 0 all_policies = PolicyCollection.from_data({}, options) # for a default region for policy loading, we'll expand regions later. options.region = "" for fp in options.configs: try: collection = policy_load(options, fp, validate=validate, vars=vars) except IOError: log.error('policy file does not exist ({})'.format(fp)) errors += 1 continue except yaml.YAMLError as e: log.error( "yaml syntax error loading policy file ({}) error:\n {}".format( fp, e)) errors += 1 continue except ValueError as e: log.error('problem loading policy file ({}) error: {}'.format( fp, str(e))) errors += 1 continue except PolicyValidationError as e: log.error('invalid policy file: {} error: {}'.format( fp, str(e))) errors += 1 continue if collection is None: log.debug('Loaded file {}. Contained no policies.'.format(fp)) else: log.debug( 'Loaded file {}. Contains {} policies'.format( fp, len(collection))) all_policies = all_policies + collection if errors > 0: log.error('Found {} errors. Exiting.'.format(errors)) sys.exit(1) # filter by name and resource type policies = all_policies.filter( getattr(options, 'policy_filters', []), getattr(options, 'resource_types', [])) # provider initialization provider_policies = {} for p in policies: provider_policies.setdefault(p.provider_name, []).append(p) policies = PolicyCollection.from_data({}, options) for provider_name in provider_policies: provider = clouds[provider_name]() p_options = provider.initialize(options) policies += provider.initialize_policies( PolicyCollection(provider_policies[provider_name], p_options), p_options) if len(policies) == 0: _print_no_policies_warning(options, all_policies) # If we filtered out all the policies we want to exit with a # non-zero status. But if the policy file is empty then continue # on to the specific command to determine the exit status. if len(all_policies) > 0: sys.exit(1) # Do not allow multiple policies in a region with the same name, # even across files policies_by_region = defaultdict(list) for p in policies: policies_by_region[p.options.region].append(p) for region in policies_by_region.keys(): counts = Counter([p.name for p in policies_by_region[region]]) for policy, count in six.iteritems(counts): if count > 1: log.error("duplicate policy name '{}'".format(policy)) sys.exit(1) # Variable expansion and non schema validation (not optional) for p in policies: p.expand_variables(p.get_variables()) p.validate() return f(options, list(policies))
def run_account(account, region, policies_config, output_path, cache_period, cache_path, metrics, dryrun, debug): """Execute a set of policies on an account. """ logging.getLogger('custodian.output').setLevel(logging.ERROR + 1) CONN_CACHE.session = None CONN_CACHE.time = None # allow users to specify interpolated output paths if '{' not in output_path: output_path = os.path.join(output_path, account['name'], region) cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['account_id'], region)) config = Config.empty( region=region, cache=cache_path, cache_period=cache_period, dryrun=dryrun, output_dir=output_path, account_id=account['account_id'], metrics_enabled=metrics, log_group=None, profile=None, external_id=None) env_vars = account_tags(account) if account.get('role'): if isinstance(account['role'], six.string_types): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') else: env_vars.update( _get_env_creds(get_session(account, 'custodian', region), region)) elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) policy_counts = {} success = True st = time.time() with environ(**env_vars): for p in policies: # Variable expansion and non schema validation (not optional) p.expand_variables(p.get_variables(account.get('vars', {}))) p.validate() log.debug( "Running policy:%s account:%s region:%s", p.name, account['name'], region) try: resources = p.run() policy_counts[p.name] = resources and len(resources) or 0 if not resources: continue log.info( "Ran account:%s region:%s policy:%s matched:%d time:%0.2f", account['name'], region, p.name, len(resources), time.time() - st) except ClientError as e: success = False if e.response['Error']['Code'] == 'AccessDenied': log.warning('Access denied account:%s region:%s', account['name'], region) return policy_counts, success log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) continue except Exception as e: success = False log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) if not debug: continue import traceback, pdb, sys traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1]) raise return policy_counts, success
def _load_policies(options): load_resources() vars = _load_vars(options) errors = 0 all_policies = PolicyCollection.from_data({}, options) # for a default region for policy loading, we'll expand regions later. options.region = options.regions[0] for fp in options.configs: try: collection = policy_load(options, fp, vars=vars) except IOError: log.error('policy file does not exist ({})'.format(fp)) errors += 1 continue except ValueError as e: log.error('problem loading policy file ({})'.format(e.message)) errors += 1 continue if collection is None: log.debug('Loaded file {}. Contained no policies.'.format(fp)) else: log.debug( 'Loaded file {}. Contains {} policies'.format( fp, len(collection))) all_policies = all_policies + collection if errors > 0: log.error('Found {} errors. Exiting.'.format(errors)) sys.exit(1) # filter by name and resource type policies = all_policies.filter( getattr(options, 'policy_filter', None), getattr(options, 'resource_type', None)) # expand by region, this results in a separate policy instance per region of execution. policies = policies.expand_regions(options.regions) if len(policies) == 0: _print_no_policies_warning(options, all_policies) # If we filtered out all the policies we want to exit with a # non-zero status. But if the policy file is empty then continue # on to the specific command to determine the exit status. if len(all_policies) > 0: sys.exit(1) # Do not allow multiple policies in a region with the same name, # even across files policies_by_region = defaultdict(list) for p in policies: policies_by_region[p.options.region].append(p) for region in policies_by_region.keys(): counts = Counter([p.name for p in policies_by_region[region]]) for policy, count in counts.iteritems(): if count > 1: log.error("duplicate policy name '{}'".format(policy)) sys.exit(1) return f(options, list(policies))
def initialize_policies(self, policy_collection, options): """Return a set of policies targetted to the given regions. Supports symbolic regions like 'all'. This will automatically filter out policies if their being targetted to a region that does not support the service. Global services will target a single region (us-east-1 if only all specified, else first region in the list). Note for region partitions (govcloud and china) an explicit region from the partition must be passed in. """ from c7n.policy import Policy, PolicyCollection policies = [] service_region_map, resource_service_map = get_service_region_map( options.regions, policy_collection.resource_types) if 'all' in options.regions: enabled_regions = set([ r['RegionName'] for r in get_profile_session(options).client( 'ec2').describe_regions( Filters=[{ 'Name': 'opt-in-status', 'Values': ['opt-in-not-required', 'opted-in'] }]).get('Regions') ]) for p in policy_collection: if 'aws.' in p.resource_type: _, resource_type = p.resource_type.split('.', 1) else: resource_type = p.resource_type available_regions = service_region_map.get( resource_service_map.get(resource_type), ()) # its a global service/endpoint, use user provided region # or us-east-1. if not available_regions and options.regions: candidates = [r for r in options.regions if r != 'all'] candidate = candidates and candidates[0] or 'us-east-1' svc_regions = [candidate] elif 'all' in options.regions: svc_regions = list( set(available_regions).intersection(enabled_regions)) else: svc_regions = options.regions for region in svc_regions: if available_regions and region not in available_regions: level = ('all' in options.regions and logging.DEBUG or logging.WARNING) # TODO: fixme policy_collection.log.log( level, "policy:%s resources:%s not available in region:%s", p.name, p.resource_type, region) continue options_copy = copy.copy(options) options_copy.region = str(region) if len(options.regions ) > 1 or 'all' in options.regions and getattr( options, 'output_dir', None): options_copy.output_dir = join_output( options.output_dir, region) policies.append( Policy( p.data, options_copy, session_factory=policy_collection.session_factory())) return PolicyCollection( # order policies by region to minimize local session invalidation. # note relative ordering of policies must be preserved, python sort # is stable. sorted(policies, key=operator.attrgetter('options.region')), options)
def _load_policies(options): load_resources() vars = _load_vars(options) errors = 0 all_policies = PolicyCollection.from_data({}, options) # for a default region for policy loading, we'll expand regions later. options.region = options.regions[0] for fp in options.configs: try: collection = policy_load(options, fp, vars=vars) except IOError: log.error('policy file does not exist ({})'.format(fp)) errors += 1 continue except ValueError as e: log.error('problem loading policy file ({})'.format(e.message)) errors += 1 continue if collection is None: log.debug('Loaded file {}. Contained no policies.'.format(fp)) else: log.debug('Loaded file {}. Contains {} policies'.format( fp, len(collection))) all_policies = all_policies + collection if errors > 0: log.error('Found {} errors. Exiting.'.format(errors)) sys.exit(1) # filter by name and resource type policies = all_policies.filter(getattr(options, 'policy_filter', None), getattr(options, 'resource_type', None)) # expand by region, this results in a separate policy instance per region of execution. policies = policies.expand_regions(options.regions) if len(policies) == 0: _print_no_policies_warning(options, all_policies) # If we filtered out all the policies we want to exit with a # non-zero status. But if the policy file is empty then continue # on to the specific command to determine the exit status. if len(all_policies) > 0: sys.exit(1) # Do not allow multiple policies in a region with the same name, # even across files policies_by_region = defaultdict(list) for p in policies: policies_by_region[p.options.region].append(p) for region in policies_by_region.keys(): counts = Counter([p.name for p in policies_by_region[region]]) for policy, count in six.iteritems(counts): if count > 1: log.error("duplicate policy name '{}'".format(policy)) sys.exit(1) return f(options, list(policies))
def load_policies(options, config): policies = PolicyCollection([], config) for f in options.config_files: policies += policy_load(config, f).filter(options.policy_filter) return policies
def run_account(account, region, policies_config, output_path, cache_period, cache_path, metrics, dryrun, debug): """Execute a set of policies on an account. """ logging.getLogger('custodian.output').setLevel(logging.ERROR + 1) CONN_CACHE.session = None CONN_CACHE.time = None load_available() # allow users to specify interpolated output paths if '{' not in output_path: output_path = os.path.join(output_path, account['name'], region) cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['account_id'], region)) config = Config.empty( region=region, cache=cache_path, cache_period=cache_period, dryrun=dryrun, output_dir=output_path, account_id=account['account_id'], metrics_enabled=metrics, log_group=None, profile=None, external_id=None) env_vars = account_tags(account) if account.get('role'): if isinstance(account['role'], six.string_types): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') else: env_vars.update( _get_env_creds(get_session(account, 'custodian', region), region)) elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) policy_counts = {} success = True st = time.time() with environ(**env_vars): for p in policies: # Variable expansion and non schema validation (not optional) p.expand_variables(p.get_variables(account.get('vars', {}))) p.validate() if p.region and p.region != region: continue log.debug( "Running policy:%s account:%s region:%s", p.name, account['name'], region) try: resources = p.run() policy_counts[p.name] = resources and len(resources) or 0 if not resources: continue if not config.dryrun and p.execution_mode != 'pull': log.info("Ran account:%s region:%s policy:%s provisioned time:%0.2f", account['name'], region, p.name, time.time() - st) continue log.info( "Ran account:%s region:%s policy:%s matched:%d time:%0.2f", account['name'], region, p.name, len(resources), time.time() - st) except ClientError as e: success = False if e.response['Error']['Code'] == 'AccessDenied': log.warning('Access denied api:%s policy:%s account:%s region:%s', e.operation_name, p.name, account['name'], region) return policy_counts, success log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) continue except Exception as e: success = False log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) if not debug: continue import traceback, pdb, sys traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1]) raise return policy_counts, success
def run_account(account, region, policies_config, output_path, cache_period, metrics, dryrun, debug): """Execute a set of policies on an account. """ logging.getLogger('custodian.output').setLevel(logging.ERROR + 1) CONN_CACHE.session = None CONN_CACHE.time = None output_path = os.path.join(output_path, account['name'], region) if not os.path.exists(output_path): os.makedirs(output_path) cache_path = os.path.join(output_path, "c7n.cache") config = Config.empty( region=region, cache_period=cache_period, dryrun=dryrun, output_dir=output_path, account_id=account['account_id'], metrics_enabled=metrics, cache=cache_path, log_group=None, profile=None, external_id=None) if account.get('role'): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) policy_counts = {} st = time.time() with environ(**account_tags(account)): for p in policies: log.debug( "Running policy:%s account:%s region:%s", p.name, account['name'], region) try: resources = p.run() policy_counts[p.name] = resources and len(resources) or 0 if not resources: continue log.info( "Ran account:%s region:%s policy:%s matched:%d time:%0.2f", account['name'], region, p.name, len(resources), time.time() - st) except ClientError as e: if e.response['Error']['Code'] == 'AccessDenied': log.warning('Access denied account:%s region:%s', account['name'], region) return policy_counts log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) continue except Exception as e: log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) if not debug: continue import traceback, pdb, sys traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1]) raise return policy_counts
def _load_policies(options): validate = True if 'skip_validation' in options: validate = not options.skip_validation if not validate: log.debug('Policy validation disabled') load_resources() vars = _load_vars(options) errors = 0 all_policies = PolicyCollection.from_data({}, options) # for a default region for policy loading, we'll expand regions later. options.region = "" for fp in options.configs: try: collection = policy_load(options, fp, validate=validate, vars=vars) except IOError: log.error('policy file does not exist ({})'.format(fp)) errors += 1 continue except ValueError as e: log.error('problem loading policy file ({})'.format(e.message)) errors += 1 continue if collection is None: log.debug('Loaded file {}. Contained no policies.'.format(fp)) else: log.debug( 'Loaded file {}. Contains {} policies'.format( fp, len(collection))) all_policies = all_policies + collection if errors > 0: log.error('Found {} errors. Exiting.'.format(errors)) sys.exit(1) # filter by name and resource type policies = all_policies.filter( getattr(options, 'policy_filter', None), getattr(options, 'resource_type', None)) # provider initialization provider_policies = {} for p in policies: provider_policies.setdefault(p.provider_name, []).append(p) policies = PolicyCollection.from_data({}, options) for provider_name in provider_policies: provider = clouds[provider_name]() p_options = provider.initialize(options) policies += provider.initialize_policies( PolicyCollection(provider_policies[provider_name], p_options), p_options) if len(policies) == 0: _print_no_policies_warning(options, all_policies) # If we filtered out all the policies we want to exit with a # non-zero status. But if the policy file is empty then continue # on to the specific command to determine the exit status. if len(all_policies) > 0: sys.exit(1) # Do not allow multiple policies in a region with the same name, # even across files policies_by_region = defaultdict(list) for p in policies: policies_by_region[p.options.region].append(p) for region in policies_by_region.keys(): counts = Counter([p.name for p in policies_by_region[region]]) for policy, count in six.iteritems(counts): if count > 1: log.error("duplicate policy name '{}'".format(policy)) sys.exit(1) # Variable expansion and non schema validation (not optional) for p in policies: p.expand_variables(p.get_variables()) p.validate() return f(options, list(policies))
def dispatch_event(event, context): global account_id error = event.get('detail', {}).get('errorCode') if error: log.debug("Skipping failed operation: %s" % error) return event['debug'] = True if event['debug']: log.info("Processing event\n %s", format_event(event)) # Policies file should always be valid in lambda so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # Initialize output directory, we've seen occassional perm issues with # lambda on temp directory and changing unix execution users, so # use a per execution temp space. output_dir = os.environ.get( 'C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4())) if not os.path.exists(output_dir): try: os.mkdir(output_dir) except OSError as error: log.warning("Unable to make output directory: {}".format(error)) # TODO. This enshrines an assumption of a single policy per lambda. options_overrides = policy_config[ 'policies'][0].get('mode', {}).get('execution-options', {}) # if using assume role in lambda ensure that the correct # execution account is captured in options. if 'assume_role' in options_overrides: account_id = options_overrides['assume_role'].split(':')[4] elif account_id is None: session = boto3.Session() account_id = get_account_id_from_sts(session) # Historical compatibility with manually set execution options # previously this was a boolean, its now a string value with the # boolean flag triggering a string value of 'aws' if 'metrics_enabled' in options_overrides and isinstance( options_overrides['metrics_enabled'], bool): options_overrides['metrics_enabled'] = 'aws' options_overrides['account_id'] = account_id if 'output_dir' not in options_overrides: options_overrides['output_dir'] = output_dir options = Config.empty(**options_overrides) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def load_policies(config_files, config): policies = PolicyCollection([], config) for f in config_files: policies += policy_load(config, f) return policies