def test_policy_region_expand_global(self): factory = self.replay_flight_data('test_aws_policy_global_expand') self.patch(aws, '_profile_session', factory()) original = self.policy_loader.load_data( {"policies": [ {"name": "foo", "resource": "s3"}, {"name": "iam", "resource": "iam-user"}]}, 'memory://', config=Config.empty(regions=["us-east-1", "us-west-2"]), ) collection = AWS().initialize_policies(original, Config.empty(regions=["all"])) self.assertEqual(len(collection.resource_types), 2) s3_regions = [p.options.region for p in collection if p.resource_type == "s3"] self.assertTrue("us-east-1" in s3_regions) self.assertTrue("us-east-2" in s3_regions) iam = [p for p in collection if p.resource_type == "iam-user"] self.assertEqual(len(iam), 1) self.assertEqual(iam[0].options.region, "us-east-1") collection = AWS().initialize_policies( original, Config.empty(regions=["eu-west-1", "eu-west-2"]) ) iam = [p for p in collection if p.resource_type == "iam-user"] self.assertEqual(len(iam), 1) self.assertEqual(iam[0].options.region, "eu-west-1") self.assertEqual(len(collection), 3)
def load_policy_set(self, data, config=None): filename = self.write_policy_file(data, format="json") if config: e = Config.empty(**config) else: e = Config.empty() return policy.load(e, filename)
def load_policy_set(self, data, config=None): filename = self.write_policy_file(data) if config: e = Config.empty(**config) else: e = Config.empty() return policy.load(e, filename)
def load_policy_set(self, data, config=None): filename = self.write_policy_file(data) if config: config['account_id'] = ACCOUNT_ID e = Config.empty(**config) else: e = Config.empty(account_id=ACCOUNT_ID) return policy.load(e, filename)
def main(): parser = setup_parser() argcomplete.autocomplete(parser) options = parser.parse_args() if options.subparser is None: parser.print_help(file=sys.stderr) return sys.exit(2) _setup_logger(options) # Support the deprecated -c option if getattr(options, 'config', None) is not None: options.configs.append(options.config) config = Config.empty(**vars(options)) try: command = options.command if not callable(command): command = getattr( importlib.import_module(command.rsplit('.', 1)[0]), command.rsplit('.', 1)[-1]) # Set the process name to something cleaner process_name = [os.path.basename(sys.argv[0])] process_name.extend(sys.argv[1:]) setproctitle(' '.join(process_name)) command(config) except Exception: if not options.debug: raise traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1])
def tag(assume, region, db, creator_tag, user_suffix, dryrun, summary=True, profile=None, type=()): """Tag resources with their creator. """ trail_db = TrailDB(db) load_resources() with temp_dir() as output_dir: config = ExecConfig.empty( output_dir=output_dir, assume=assume, region=region, profile=profile) factory = aws.AWS().get_session_factory(config) account_id = local_session(factory).client('sts').get_caller_identity().get('Account') config['account_id'] = account_id tagger = ResourceTagger(trail_db, config, creator_tag, user_suffix, dryrun, type) try: stats = tagger.process() except Exception: log.exception( "error processing account:%s region:%s config:%s env:%s", account_id, region, config, dict(os.environ)) raise if not summary: return stats log.info( "auto tag summary account:%s region:%s \n%s", config['account_id'], config['region'], "\n".join([" {}: {}".format(k, v) for k, v in stats.items() if v])) total = sum([v for k, v in stats.items() if not k.endswith('not-found')]) log.info("Total resources tagged: %d" % total)
def test_cloudtrail_policy(): collection = PolicyLoader(Config.empty()).load_data( { 'policies': [{ 'name': 'check-ec2', 'resource': 'ec2', 'mode': { 'type': 'cloudtrail', 'events': ['RunInstances'] } }] }, file_uri=":mem:") sam = {'Resources': {}} p = list(collection).pop() dispatch_render(p, sam) assert sam['Resources']['CheckEc2']['Properties']['Events'] == { 'PolicyTriggerA': { 'Properties': { 'Pattern': { 'detail': { 'eventName': ['RunInstances'], 'eventSource': ['ec2.amazonaws.com'] }, 'detail-type': ['AWS API Call via CloudTrail'] } }, 'Type': 'CloudWatchEvent' } }
def run(event, context=None): # policies file should always be valid in functions so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): log.error('Invalid policy config') return False options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) # if output_dir specified use that, otherwise make a temp directory if 'output_dir' not in options_overrides: options_overrides['output_dir'] = get_tmp_output_dir() # merge all our options in options = Config.empty(**options_overrides) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: log.info("running policy %s", p.name) p.push(event, context) return True
def get_dir_output(self, location): work_dir = self.change_cwd() return work_dir, DirectoryOutput( ExecutionContext(None, Bag(name="xyz", provider_name="ostack"), Config.empty(output_dir=location)), {'url': location}, )
def get_context(self, config=None, policy=None): if config is None: self.context_output_dir = self.get_temp_dir() config = Config.empty(output_dir=self.context_output_dir) ctx = ExecutionContext(Session, policy or Bag({'name': 'test-policy'}), config) return ctx
def dispatch_event(event, context): global account_id error = event.get('detail', {}).get('errorCode') if error: log.debug("Skipping failed operation: %s" % error) return event['debug'] = True if event['debug']: log.info("Processing event\n %s", format_event(event)) # Policies file should always be valid in lambda so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # Initialize output directory, we've seen occassional perm issues with # lambda on temp directory and changing unix execution users, so # use a per execution temp space. output_dir = os.environ.get('C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4())) if not os.path.exists(output_dir): try: os.mkdir(output_dir) except OSError as error: log.warning("Unable to make output directory: {}".format(error)) # TODO. This enshrines an assumption of a single policy per lambda. options_overrides = policy_config['policies'][0].get('mode', {}).get( 'execution-options', {}) # if using assume role in lambda ensure that the correct # execution account is captured in options. if 'assume_role' in options_overrides: account_id = options_overrides['assume_role'].split(':')[4] elif account_id is None: session = boto3.Session() account_id = get_account_id_from_sts(session) # Historical compatibility with manually set execution options # previously this was a boolean, its now a string value with the # boolean flag triggering a string value of 'aws' if 'metrics_enabled' in options_overrides and isinstance( options_overrides['metrics_enabled'], bool): options_overrides['metrics_enabled'] = 'aws' options_overrides['account_id'] = account_id if 'output_dir' not in options_overrides: options_overrides['output_dir'] = output_dir options = Config.empty(**options_overrides) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def report_account(account, region, policies_config, output_path, debug): cache_path = os.path.join(output_path, "c7n.cache") output_path = os.path.join(output_path, account['name'], region) config = Config.empty( region=region, output_dir=output_path, account_id=account['account_id'], metrics_enabled=False, cache=cache_path, log_group=None, profile=None, external_id=None) if account.get('role'): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) records = [] for p in policies: log.debug( "Report policy:%s account:%s region:%s path:%s", p.name, account['name'], region, output_path) policy_records = fs_record_set(p.ctx.output_path, p.name) for r in policy_records: r['policy'] = p.name r['region'] = p.options.region r['account'] = account['name'] for t in account.get('tags', ()): if ':' in t: k, v = t.split(':', 1) r[k] = v records.extend(policy_records) return records
def main(): parser = setup_parser() options = parser.parse_args() config = Config.empty() resources.load_resources() collection = policy_load( config, options.config_file).filter(options.policy_filter) sam = { 'AWSTemplateFormatVersion': '2010-09-09', 'Transform': 'AWS::Serverless-2016-10-31', 'Resources': {}} for p in collection: if p.provider_name != 'aws': continue exec_mode_type = p.data.get('mode', {'type': 'pull'}).get('type') if exec_mode_type == 'pull': continue sam_func = render(p) if sam_func: sam['Resources'][resource_name(p.name)] = sam_func sam_func['Properties']['CodeUri'] = './%s.zip' % p.name else: print("unable to render sam for policy:%s" % p.name) continue archive = mu.PolicyLambda(p).get_archive() with open(os.path.join(options.output_dir, "%s.zip" % p.name), 'wb') as fh: fh.write(archive.get_bytes()) with open(os.path.join(options.output_dir, 'deploy.yml'), 'w') as fh: fh.write(yaml.safe_dump(sam, default_flow_style=False))
def dryrun(self): # This is largely based off of mugc.main() logging.getLogger('botocore').setLevel(logging.ERROR) logging.getLogger('urllib3').setLevel(logging.ERROR) logging.getLogger('c7n.cache').setLevel(logging.WARNING) conf = Config.empty( config_files=['custodian_%s.yml' % self.region_name], regions=[self.region_name], prefix=self.config.function_prefix, policy_regex='^' + re.escape(self.config.function_prefix) + '.*', assume=None, policy_filter=None, log_group=None, external_id=None, cache_period=0, cache=None, present=False, dryrun=True ) # use cloud provider to initialize policies to get region expansion policies = AWS().initialize_policies( PolicyCollection( [ p for p in load_policies(conf, conf) if p.provider_name == 'aws' ], conf ), conf ) resources_gc_prefix(conf, conf, policies)
def load_policy( self, data, config=None, session_factory=None, validate=C7N_VALIDATE, output_dir=None, cache=False, ): if validate: if not self.custodian_schema: self.custodian_schema = generate() errors = schema_validate({"policies": [data]}, self.custodian_schema) if errors: raise errors[0] config = config or {} if not output_dir: temp_dir = self.get_temp_dir() config["output_dir"] = temp_dir if cache: config["cache"] = os.path.join(temp_dir, "c7n.cache") config["cache_period"] = 300 conf = Config.empty(**config) p = policy.Policy(data, conf, session_factory) p.validate() return p
def run(event, context): # policies file should always be valid in functions so do loading naively with open(context['config_file']) as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # Initialize output directory output_dir = os.environ.get( 'C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4())) if not os.path.exists(output_dir): try: os.mkdir(output_dir) except OSError as error: log.warning("Unable to make output directory: {}".format(error)) options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) options_overrides['authorization_file'] = context['auth_file'] if 'output_dir' not in options_overrides: options_overrides['output_dir'] = output_dir options = Config.empty(**options_overrides) load_resources() policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def run(event, context): # policies file should always be valid in functions so do loading naively with open(context['config_file']) as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): log.error('Invalid policy config') return False options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) # setup our auth file location on disk options_overrides['authorization_file'] = context['auth_file'] # if output_dir specified use that, otherwise make a temp directory if 'output_dir' not in options_overrides: options_overrides['output_dir'] = get_tmp_output_dir() # merge all our options in options = Config.empty(**options_overrides) load_resources() options = Azure().initialize(options) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: try: p.push(event, context) except (CloudError, AzureHttpError) as error: log.error("Unable to process policy: %s :: %s" % (p.name, error)) return True
def eperm(provider, el, r=None): if el.permissions: return el.permissions element_type = get_element_type(el) if r is None or r.type is None: # dummy resource type for policy if provider == 'aws': r = Bag({'type': 'kinesis'}) elif provider == 'gcp': r = Bag({'type': 'instance'}) elif provider == 'azure': r = Bag({'type': 'vm'}) # print(f'policy construction lookup {r.type}.{element_type}.{el.type}') loader = PolicyLoader(Config.empty()) pdata = { 'name': f'permissions-{r.type}', 'resource': f'{provider}.{r.type}' } pdata[element_type] = get_element_data(element_type, el) try: pset = loader.load_data({'policies': [pdata]}, ':mem:', validate=False) except Exception as e: print(f'error loading {el} as {element_type}:{el.type} error: {e} \n {pdata}') return [] el = get_policy_element(el, list(pset)[0]) return el.get_permissions()
def validate(options): load_resources() if len(options.configs) < 1: log.error('no config files specified') sys.exit(1) used_policy_names = set() schm = schema.generate() errors = [] for config_file in options.configs: config_file = os.path.expanduser(config_file) if not os.path.exists(config_file): raise ValueError("Invalid path for config %r" % config_file) options.dryrun = True fmt = config_file.rsplit('.', 1)[-1] with open(config_file) as fh: if fmt in ('yml', 'yaml'): data = yaml.safe_load(fh.read()) elif fmt in ('json', ): data = json.load(fh) else: log.error("The config file must end in .json, .yml or .yaml.") raise ValueError( "The config file must end in .json, .yml or .yaml.") errors += schema.validate(data, schm) conf_policy_names = { p.get('name', 'unknown') for p in data.get('policies', ()) } dupes = conf_policy_names.intersection(used_policy_names) if len(dupes) >= 1: errors.append( ValueError( "Only one policy with a given name allowed, duplicates: %s" % (", ".join(dupes)))) used_policy_names = used_policy_names.union(conf_policy_names) if not errors: null_config = Config.empty(dryrun=True, account_id='na', region='na') for p in data.get('policies', ()): try: policy = Policy(p, null_config, Bag()) policy.validate() except Exception as e: msg = "Policy: %s is invalid: %s" % (p.get( 'name', 'unknown'), e) errors.append(msg) if not errors: log.info("Configuration valid: {}".format(config_file)) continue log.error("Configuration invalid: {}".format(config_file)) for e in errors: log.error("%s" % e) if errors: sys.exit(1)
def test_policy_resource_limits_count(self): session_factory = self.replay_flight_data( "test_policy_resource_count") p = self.load_policy( { "name": "ecs-cluster-resource-count", "resource": "ecs", "max-resources": 1 }, session_factory=session_factory) self.assertRaises(ResourceLimitExceeded, p.run) policy = { "name": "ecs-cluster-resource-count", "resource": "ecs", "max-resources": 0 } config = Config.empty(validate=True) self.assertRaises( Exception, self.load_policy, policy, config=config, validate=True, session_factory=session_factory )
def run(self): """ Perform an actual run of cloud-custodian. This replicates the command: custodian run --region '${region}' --metrics -v -s \ cloud-custodian-${account_id}-${region}/logs \ --log-group=/cloud-custodian/${account_id}/${region} \ -c custodian_${region}.yml \ --cache '/tmp/.cache/cloud-custodian.cache' """ conf = Config.empty( configs=['custodian_%s.yml' % self.region_name], region=self.region_name, regions=[self.region_name], log_group=self.config.custodian_log_group, verbose=1, metrics_enabled=True, subparser='run', cache='/tmp/.cache/cloud-custodian.cache', command='c7n.commands.run', output_dir='%s/logs' % self.config.output_s3_bucket_name, vars=None, dryrun=False ) run(conf)
def test_verify_parent_filter(self): manager = KeyVaultKeys( ExecutionContext(None, Bag(name="xyz", provider_name='azure'), Config.empty()), { 'name': 'test-policy', 'resource': 'azure.keyvault-keys', 'filters': [{ 'type': 'parent', 'filter': { 'type': 'value', 'key': 'name', 'op': 'glob', 'value': 'cctestkv*' } }] }) self.assertEqual(len(manager.filters), 1) filter = manager.filters[0] self.assertTrue(isinstance(filter, ParentFilter)) self.assertTrue(isinstance(filter.parent_manager, KeyVault)) self.assertTrue(isinstance(filter.parent_filter, ValueFilter))
def run(event, context=None): # policies file should always be valid in functions so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): log.error('Invalid policy config') return False options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) # if output_dir specified use that, otherwise make a temp directory if 'output_dir' not in options_overrides: options_overrides['output_dir'] = get_tmp_output_dir() # merge all our options in options = Config.empty(**options_overrides) loader = PolicyLoader(options) policies = loader.load_data(policy_config, 'config.json', validate=False) if policies: for p in policies: log.info("running policy %s", p.name) p.validate() p.push(event, context) return True
def get_context(self, config=None, session_factory=None, policy=None): if config is None: self.context_output_dir = self.get_temp_dir() config = Config.empty(output_dir=self.context_output_dir) ctx = ExecutionContext(session_factory, policy or Bag({"name": "test-policy"}), config) return ctx
def main(): parser = setup_parser() options = parser.parse_args() log_level = logging.INFO if options.verbose: log_level = logging.DEBUG logging.basicConfig( level=log_level, format="%(asctime)s: %(name)s:%(levelname)s %(message)s") logging.getLogger('botocore').setLevel(logging.ERROR) logging.getLogger('c7n.cache').setLevel(logging.WARNING) if not options.regions: options.regions = [os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')] files = [] files.extend(itertools.chain(*options.config_files)) files.extend(options.configs) options.config_files = files if not files: parser.print_help() sys.exit(1) policy_config = Config.empty(regions=options.regions, profile=options.profile, assume_role=options.assume_role) # use cloud provider to initialize policies to get region expansion policies = AWS().initialize_policies(load_policies(options, policy_config), policy_config) resources_gc_prefix(options, policy_config, policies)
def main(): parser = setup_parser() argcomplete.autocomplete(parser) options = parser.parse_args() _setup_logger(options) # Support the deprecated -c option if getattr(options, 'config', None) is not None: options.configs.append(options.config) config = Config.empty(**vars(options)) try: command = options.command if not callable(command): command = getattr( importlib.import_module(command.rsplit('.', 1)[0]), command.rsplit('.', 1)[-1]) # Set the process name to something cleaner process_name = [os.path.basename(sys.argv[0])] process_name.extend(sys.argv[1:]) setproctitle(' '.join(process_name)) command(config) except Exception: if not options.debug: raise traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1])
def test_resource_shadow_source_augment(self): shadowed = [] bad = [] cfg = Config.empty() for k, v in manager.resources.items(): if not getattr(v.resource_type, "config_type", None): continue p = Bag({ "name": "permcheck", "resource": k, 'provider_name': 'aws' }) ctx = self.get_context(config=cfg, policy=p) mgr = v(ctx, p) source = mgr.get_source("config") if not isinstance(source, ConfigSource): bad.append(k) if v.__dict__.get("augment"): shadowed.append(k) if shadowed: self.fail("%s have resource managers shadowing source augments" % (", ".join(shadowed))) if bad: self.fail("%s have config types but no config source" % (", ".join(bad)))
def run(event, context): # policies file should always be valid in functions so do loading naively with open(context['config_file']) as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): log.error('Invalid policy config') return False options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) # setup our auth file location on disk options_overrides['authorization_file'] = context['auth_file'] # if output_dir specified use that, otherwise make a temp directory if 'output_dir' not in options_overrides: options_overrides['output_dir'] = get_tmp_output_dir() # merge all our options in options = Config.empty(**options_overrides) load_resources() policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
class HandlerTest(BaseTest): @patch('c7n_azure.provider.Azure.initialize', return_value=Config.empty()) @patch('azure.common.credentials.ServicePrincipalCredentials.__init__', return_value=None) @patch('c7n.policy.Policy.push') def test_run(self, push_mock, _1, initialize_mock): context = { 'config_file': join(dirname(__file__), 'data', 'test_config.json'), 'auth_file': join(dirname(__file__), 'data', 'test_auth_file.json') } self.assertTrue(run(None, context, CUSTOM_SUBSCRIPTION_ID)) push_mock.assert_called_once() self.assertEqual(push_mock.call_args_list[0], call(None, context)) initialize_mock.assert_called_once() self.assertEqual(initialize_mock.call_args_list[0][0][0]['account_id'], CUSTOM_SUBSCRIPTION_ID) self.assertEqual( initialize_mock.call_args_list[0][0][0]['authorization_file'], context['auth_file']) self.assertEqual( initialize_mock.call_args_list[0][0][0]['test_option'], "test_value") def test_run_empty_policy(self): context = { 'config_file': join(dirname(__file__), 'data', 'test_config_empty.json'), 'auth_file': join(dirname(__file__), 'data', 'test_auth_file.json') } self.assertFalse(run(None, context))
def init_config(policy_config): """Get policy lambda execution configuration. cli parameters are serialized into the policy lambda config, we merge those with any policy specific execution options. --assume role and -s output directory get special handling, as to disambiguate any cli context. account id is sourced from the config options or from api call and cached as a global. Todo: this should get refactored out to mu.py as part of the write out of configuration, instead of runtime processed. """ exec_options = policy_config.get('execution-options', {}) # Remove some configuration options that don't make sense to translate from # cli to lambda automatically. # - assume role on cli doesn't translate, it is the default lambda role and # used to provision the lambda. # - profile doesnt translate to lambda its `home` dir setup dependent # - dryrun doesn't translate (and shouldn't be present) # - region doesn't translate from cli (the lambda is bound to a region), and # on the cli represents the region the lambda is provisioned in. for k in ('assume_role', 'profile', 'region', 'dryrun', 'cache'): exec_options.pop(k, None) # a cli local directory doesn't translate to lambda if not exec_options.get('output_dir', '').startswith('s3'): exec_options['output_dir'] = '/tmp' account_id = None # we can source account id from the cli parameters to avoid the sts call if exec_options.get('account_id'): account_id = exec_options['account_id'] # merge with policy specific configuration exec_options.update( policy_config['policies'][0].get('mode', {}).get('execution-options', {})) # if using assume role in lambda ensure that the correct # execution account is captured in options. if 'assume_role' in exec_options: account_id = exec_options['assume_role'].split(':')[4] elif account_id is None: session = local_session(boto3.Session) account_id = get_account_id_from_sts(session) exec_options['account_id'] = account_id # Historical compatibility with manually set execution options # previously this was a boolean, its now a string value with the # boolean flag triggering a string value of 'aws' if 'metrics_enabled' in exec_options \ and isinstance(exec_options['metrics_enabled'], bool) \ and exec_options['metrics_enabled']: exec_options['metrics_enabled'] = 'aws' return Config.empty(**exec_options)
def get_related(self, resources): ctx = ExecutionContext(local_session(Session), self.data, Config.empty()) manager = self.factory(ctx, self.data) related = manager.source.get_resources(None) if self.data.get('op'): return [r['id'] for r in related if self.match(r)] else: return [r['id'] for r in related]
def get_context(self, config=None, session_factory=None, policy=None): if config is None: self.context_output_dir = self.get_temp_dir() config = Config.empty(output_dir=self.context_output_dir) ctx = ExecutionContext( session_factory, policy or Bag({ "name": "test-policy", "provider_name": "aws"}), config) return ctx
def _get_policy_config(self, **kw): config = kw if kw.get('output_dir') is None or config.get('cache'): config["output_dir"] = temp_dir = self.get_temp_dir() if config.get('cache'): config["cache"] = os.path.join(temp_dir, "c7n.cache") config["cache_period"] = 300 return Config.empty(**config)
def init_config(policy_config): """Get policy lambda execution configuration. cli parameters are serialized into the policy lambda config, we merge those with any policy specific execution options. --assume role and -s output directory get special handling, as to disambiguate any cli context. account id is sourced from the config options or from api call and cached as a global """ global account_id exec_options = policy_config.get('execution-options', {}) # Remove some configuration options that don't make sense to translate from # cli to lambda automatically. # - assume role on cli doesn't translate, it is the default lambda role and # used to provision the lambda. # - profile doesnt translate to lambda its `home` dir setup dependent # - dryrun doesn't translate (and shouldn't be present) # - region doesn't translate from cli (the lambda is bound to a region), and # on the cli represents the region the lambda is provisioned in. for k in ('assume_role', 'profile', 'region', 'dryrun', 'cache'): exec_options.pop(k, None) # a cli local directory doesn't translate to lambda if not exec_options.get('output_dir', '').startswith('s3'): exec_options['output_dir'] = get_local_output_dir() # we can source account id from the cli parameters to avoid the sts call if exec_options.get('account_id'): account_id = exec_options['account_id'] # merge with policy specific configuration exec_options.update( policy_config['policies'][0].get('mode', {}).get('execution-options', {})) # if using assume role in lambda ensure that the correct # execution account is captured in options. if 'assume_role' in exec_options: account_id = exec_options['assume_role'].split(':')[4] elif account_id is None: session = boto3.Session() account_id = get_account_id_from_sts(session) exec_options['account_id'] = account_id # Historical compatibility with manually set execution options # previously this was a boolean, its now a string value with the # boolean flag triggering a string value of 'aws' if 'metrics_enabled' in exec_options \ and isinstance(exec_options['metrics_enabled'], bool) \ and exec_options['metrics_enabled']: exec_options['metrics_enabled'] = 'aws' return Config.empty(**exec_options)
def get_session(role, session_name, profile): region = os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1') stats = ApiStats(Bag(), Config.empty()) if role: s = assumed_session(role, session_name, region=region) else: s = SessionFactory(region, profile)() stats(s) return stats, s
def get_context(self, config=None, session_factory=None, policy=None): if config is None: self.context_output_dir = self.get_temp_dir() config = Config.empty(output_dir=self.context_output_dir) ctx = ExecutionContext( session_factory, policy or Bag({'name': 'test-policy'}), config) return ctx
def validate(options): load_resources() if len(options.configs) < 1: log.error('no config files specified') sys.exit(1) used_policy_names = set() schm = schema.generate() errors = [] for config_file in options.configs: config_file = os.path.expanduser(config_file) if not os.path.exists(config_file): raise ValueError("Invalid path for config %r" % config_file) options.dryrun = True fmt = config_file.rsplit('.', 1)[-1] with open(config_file) as fh: if fmt in ('yml', 'yaml'): data = yaml.safe_load(fh.read()) elif fmt in ('json',): data = json.load(fh) else: log.error("The config file must end in .json, .yml or .yaml.") raise ValueError("The config file must end in .json, .yml or .yaml.") errors += schema.validate(data, schm) conf_policy_names = { p.get('name', 'unknown') for p in data.get('policies', ())} dupes = conf_policy_names.intersection(used_policy_names) if len(dupes) >= 1: errors.append(ValueError( "Only one policy with a given name allowed, duplicates: %s" % ( ", ".join(dupes) ) )) used_policy_names = used_policy_names.union(conf_policy_names) if not errors: null_config = Config.empty(dryrun=True, account_id='na', region='na') for p in data.get('policies', ()): try: policy = Policy(p, null_config, Bag()) policy.validate() except Exception as e: msg = "Policy: %s is invalid: %s" % ( p.get('name', 'unknown'), e) errors.append(msg) if not errors: log.info("Configuration valid: {}".format(config_file)) continue log.error("Configuration invalid: {}".format(config_file)) for e in errors: log.error("%s" % e) if errors: sys.exit(1)
def cli(**kwargs): policy_config = Config.empty() policies = PolicyCollection([ p for p in load_policies(kwargs['config'], policy_config) if p.provider_name == 'azure' ], policy_config) session = policies.policies[0].session_factory() web_client = session.client('azure.mgmt.web.WebSiteManagementClient') deployments = {} credentials_load_failed = 0 not_functions_policy = 0 for p in policies: if not p.is_lambda: not_functions_policy += 1 continue try: params = AzureFunctionMode(p).get_function_app_params() creds = web_client.web_apps.list_publishing_credentials( params.function_app_resource_group_name, params.function_app_name).result() deployments[p.name] = {'scm_uri': creds.scm_uri, 'status': None} log.info('Retrieved deployment credentials for %s policy', p.name) except Exception: log.error( 'Unable to retrieve deployment credentials for %s policy', p.name) credentials_load_failed += 1 wait_for_remote_builds(deployments) success = 0 fail = 0 not_found = 0 for name, params in deployments.items(): if params['status'] != DeploymentStatus.Succeeded: log.info('%s: %s', name, params['status']) if params['status'] == DeploymentStatus.Failed: log.error('Build logs can be retrieved here: %s', params['scm_uri']) fail += 1 else: not_found += 1 else: success += 1 log.info( 'Policies total: %i, unable to load credentials: %i, not Functions mode: %i', len(policies), credentials_load_failed, not_functions_policy) log.info( 'Status not found can happen if Linux Consumption function was deployed' 'more than 2 hours ago.') log.info( 'Deployments complete. Success: %i, Fail: %i, Status not found: %i', success, fail, not_found)
def test_initialize_default_account_id(self, get_subscription_id_mock): options = Config.empty() azure = Azure() azure.initialize(options) self.assertEqual(options['account_id'], DEFAULT_SUBSCRIPTION_ID) session = azure.get_session_factory(options)() session._initialize_session() self.assertEqual(session.subscription_id, DEFAULT_SUBSCRIPTION_ID)
def test_local_session_region(self): policies = [ self.load_policy( {'name': 'ec2', 'resource': 'ec2'}, config=Config.empty(region="us-east-1")), self.load_policy( {'name': 'ec2', 'resource': 'ec2'}, config=Config.empty(region='us-west-2'))] previous = None previous_region = None for p in policies: self.assertEqual(p.options.region, p.session_factory.region) session = utils.local_session(p.session_factory) self.assertNotEqual(session.region_name, previous_region) self.assertNotEqual(session, previous) previous = session previous_region = p.options.region self.assertEqual(utils.local_session(p.session_factory), previous)
def initialize_tree(self, tree): assert not self.policy_files for tree_ent in tree: fpath = tree_ent.name if not self.matcher(fpath): continue self.policy_files[fpath] = PolicyCollection.from_data( yaml.safe_load(self.repo.get(tree[fpath].id).data), Config.empty(), fpath)
def get_azure_output(self): output = AzureStorageOutput( ExecutionContext( None, Bag(name="xyz"), Config.empty(output_dir="azure://mystorage.blob.core.windows.net/logs"), ) ) self.addCleanup(shutil.rmtree, output.root_dir) return output
def test_initialize_custom_account_id(self, get_subscription_id_mock): sample_account_id = "00000000-5106-4743-99b0-c129bfa71a47" options = Config.empty() options['account_id'] = sample_account_id azure = Azure() azure.initialize(options) self.assertEqual(options['account_id'], sample_account_id) session = azure.get_session_factory(options)() session._initialize_session() self.assertEqual(session.subscription_id, sample_account_id)
def _policy_file_rev(self, f, commit): try: return self._validate_policies( PolicyCollection.from_data( yaml.safe_load(self.repo.get(commit.tree[f].id).data), Config.empty(), f)) except Exception as e: log.warning( "invalid policy file %s @ %s %s %s \n error:%s", f, str(commit.id)[:6], commit_date(commit).isoformat(), commit.author.name, e) return PolicyCollection()
def dispatch_event(event, context): global account_id if account_id is None: session = boto3.Session() account_id = get_account_id_from_sts(session) error = event.get('detail', {}).get('errorCode') if error: log.debug("Skipping failed operation: %s" % error) return event['debug'] = True if event['debug']: log.info("Processing event\n %s", format_event(event)) # policies file should always be valid in lambda so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # Initialize output directory, we've seen occassional perm issues with # lambda on temp directory and changing unix execution users, so # use a per execution temp space. output_dir = os.environ.get( 'C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4())) if not os.path.exists(output_dir): try: os.mkdir(output_dir) except OSError as error: log.warning("Unable to make output directory: {}".format(error)) # TODO. This enshrines an assumption of a single policy per lambda. options_overrides = policy_config[ 'policies'][0].get('mode', {}).get('execution-options', {}) options_overrides['account_id'] = account_id if 'output_dir' not in options_overrides: options_overrides['output_dir'] = output_dir options = Config.empty(**options_overrides) load_resources() policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def load_policy( self, data, config=None): errors = schema_validate({'policies': [data]}, C7N_SCHEMA) if errors: raise errors[0] config = config or {} temp_dir = self.get_temp_dir() config['output_dir'] = temp_dir conf = Config.empty(**config) p = policy.Policy(data, conf, Session) p.validate() return p
def get_azure_output(self, custom_pyformat=None): output_dir = "azure://mystorage.blob.core.windows.net/logs" if custom_pyformat: output_dir = AzureStorageOutput.join(output_dir, custom_pyformat) output = AzureStorageOutput( ExecutionContext( None, Bag(name="xyz", provider_name='azure'), Config.empty(output_dir=output_dir) ), {'url': output_dir}, ) self.addCleanup(shutil.rmtree, output.root_dir) return output
def load_policy( self, data, config=None, session_factory=None, validate=C7N_VALIDATE, output_dir=None, cache=False): if validate: errors = schema_validate({'policies': [data]}, C7N_SCHEMA) if errors: raise errors[0] config = config or {} if not output_dir: temp_dir = self.get_temp_dir() config['output_dir'] = temp_dir if cache: config['cache'] = os.path.join(temp_dir, 'c7n.cache') config['cache_period'] = 300 conf = Config.empty(**config) p = policy.Policy(data, conf, session_factory) p.validate() return p
def main(): parser = setup_parser() options = parser.parse_args() log_level = logging.INFO if options.verbose: log_level = logging.DEBUG logging.basicConfig( level=log_level, format="%(asctime)s: %(name)s:%(levelname)s %(message)s") logging.getLogger('botocore').setLevel(logging.ERROR) logging.getLogger('c7n.cache').setLevel(logging.WARNING) if not options.regions: options.regions = [os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')] files = [] files.extend(itertools.chain(*options.config_files)) files.extend(options.configs) options.config_files = files if not files: parser.print_help() sys.exit(1) policy_config = Config.empty( regions=options.regions, profile=options.profile, assume_role=options.assume_role) # use cloud provider to initialize policies to get region expansion policies = AWS().initialize_policies( PolicyCollection([ p for p in load_policies( options, policy_config) if p.provider_name == 'aws'], policy_config), policy_config) resources_gc_prefix(options, policy_config, policies)
def run_account(account, region, policies_config, output_path, cache_period, cache_path, metrics, dryrun, debug): """Execute a set of policies on an account. """ logging.getLogger('custodian.output').setLevel(logging.ERROR + 1) CONN_CACHE.session = None CONN_CACHE.time = None # allow users to specify interpolated output paths if '{' not in output_path: output_path = os.path.join(output_path, account['name'], region) cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['account_id'], region)) config = Config.empty( region=region, cache=cache_path, cache_period=cache_period, dryrun=dryrun, output_dir=output_path, account_id=account['account_id'], metrics_enabled=metrics, log_group=None, profile=None, external_id=None) env_vars = account_tags(account) if account.get('role'): if isinstance(account['role'], six.string_types): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') else: env_vars.update( _get_env_creds(get_session(account, 'custodian', region), region)) elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) policy_counts = {} success = True st = time.time() with environ(**env_vars): for p in policies: # Variable expansion and non schema validation (not optional) p.expand_variables(p.get_variables(account.get('vars', {}))) p.validate() log.debug( "Running policy:%s account:%s region:%s", p.name, account['name'], region) try: resources = p.run() policy_counts[p.name] = resources and len(resources) or 0 if not resources: continue log.info( "Ran account:%s region:%s policy:%s matched:%d time:%0.2f", account['name'], region, p.name, len(resources), time.time() - st) except ClientError as e: success = False if e.response['Error']['Code'] == 'AccessDenied': log.warning('Access denied account:%s region:%s', account['name'], region) return policy_counts, success log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) continue except Exception as e: success = False log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) if not debug: continue import traceback, pdb, sys traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1]) raise return policy_counts, success
def run_account(account, region, policies_config, output_path, cache_period, metrics, dryrun, debug): """Execute a set of policies on an account. """ logging.getLogger('custodian.output').setLevel(logging.ERROR + 1) CONN_CACHE.session = None CONN_CACHE.time = None output_path = os.path.join(output_path, account['name'], region) if not os.path.exists(output_path): os.makedirs(output_path) cache_path = os.path.join(output_path, "c7n.cache") config = Config.empty( region=region, cache_period=cache_period, dryrun=dryrun, output_dir=output_path, account_id=account['account_id'], metrics_enabled=metrics, cache=cache_path, log_group=None, profile=None, external_id=None) if account.get('role'): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) policy_counts = {} st = time.time() with environ(**account_tags(account)): for p in policies: log.debug( "Running policy:%s account:%s region:%s", p.name, account['name'], region) try: resources = p.run() policy_counts[p.name] = resources and len(resources) or 0 if not resources: continue log.info( "Ran account:%s region:%s policy:%s matched:%d time:%0.2f", account['name'], region, p.name, len(resources), time.time() - st) except ClientError as e: if e.response['Error']['Code'] == 'AccessDenied': log.warning('Access denied account:%s region:%s', account['name'], region) return policy_counts log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) continue except Exception as e: log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) if not debug: continue import traceback, pdb, sys traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1]) raise return policy_counts
def __init__(self, policies=None, options=None): self.policies = policies or [] self.options = options or Config.empty() self.pmap = {p.name: p for p in self.policies}
def dispatch_event(event, context): global account_id error = event.get('detail', {}).get('errorCode') if error: log.debug("Skipping failed operation: %s" % error) return event['debug'] = True if event['debug']: log.info("Processing event\n %s", format_event(event)) # Policies file should always be valid in lambda so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # Initialize output directory, we've seen occassional perm issues with # lambda on temp directory and changing unix execution users, so # use a per execution temp space. output_dir = os.environ.get( 'C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4())) if not os.path.exists(output_dir): try: os.mkdir(output_dir) except OSError as error: log.warning("Unable to make output directory: {}".format(error)) # TODO. This enshrines an assumption of a single policy per lambda. options_overrides = policy_config[ 'policies'][0].get('mode', {}).get('execution-options', {}) # if using assume role in lambda ensure that the correct # execution account is captured in options. if 'assume_role' in options_overrides: account_id = options_overrides['assume_role'].split(':')[4] elif account_id is None: session = boto3.Session() account_id = get_account_id_from_sts(session) # Historical compatibility with manually set execution options # previously this was a boolean, its now a string value with the # boolean flag triggering a string value of 'aws' if 'metrics_enabled' in options_overrides and isinstance( options_overrides['metrics_enabled'], bool): options_overrides['metrics_enabled'] = 'aws' options_overrides['account_id'] = account_id if 'output_dir' not in options_overrides: options_overrides['output_dir'] = output_dir options = Config.empty(**options_overrides) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def report(config, output, use, output_dir, accounts, field, no_default_fields, tags, region, debug, verbose, policy, policy_tags, format, resource): """report on a cross account policy execution.""" accounts_config, custodian_config, executor = init( config, use, debug, verbose, accounts, tags, policy, resource=resource, policy_tags=policy_tags) resource_types = set() for p in custodian_config.get('policies'): resource_types.add(p['resource']) if len(resource_types) > 1: raise ValueError("can only report on one resource type at a time") elif not len(custodian_config['policies']) > 0: raise ValueError("no matching policies found") records = [] with executor(max_workers=WORKER_COUNT) as w: futures = {} for a in accounts_config.get('accounts', ()): for r in resolve_regions(region or a.get('regions', ())): futures[w.submit( report_account, a, r, custodian_config, output_dir, debug)] = (a, r) for f in as_completed(futures): a, r = futures[f] if f.exception(): if debug: raise log.warning( "Error running policy in %s @ %s exception: %s", a['name'], r, f.exception()) records.extend(f.result()) log.debug( "Found %d records across %d accounts and %d policies", len(records), len(accounts_config['accounts']), len(custodian_config['policies'])) if format == 'json': dumps(records, output, indent=2) return prefix_fields = OrderedDict( (('Account', 'account'), ('Region', 'region'), ('Policy', 'policy'))) config = Config.empty() factory = resource_registry.get(list(resource_types)[0]) formatter = Formatter( factory.resource_type, extra_fields=field, include_default_fields=not(no_default_fields), include_region=False, include_policy=False, fields=prefix_fields) rows = formatter.to_csv(records, unique=False) writer = UnicodeWriter(output, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows)