def load_policy_set(self, data, config=None): filename = self.write_policy_file(data) if config: e = Config.empty(**config) else: e = Config.empty() return policy.load(e, filename)
def load_policy_set(self, data, config=None): filename = self.write_policy_file(data) if config: config['account_id'] = ACCOUNT_ID e = Config.empty(**config) else: e = Config.empty(account_id=ACCOUNT_ID) return policy.load(e, filename)
def load_policy( self, data, config=None, session_factory=None, validate=C7N_VALIDATE, output_dir=None, cache=False, ): if validate: if not self.custodian_schema: self.custodian_schema = generate() errors = schema_validate({"policies": [data]}, self.custodian_schema) if errors: raise errors[0] config = config or {} if not output_dir: temp_dir = self.get_temp_dir() config["output_dir"] = temp_dir if cache: config["cache"] = os.path.join(temp_dir, "c7n.cache") config["cache_period"] = 300 conf = Config.empty(**config) p = policy.Policy(data, conf, session_factory) p.validate() return p
def main(): parser = setup_parser() argcomplete.autocomplete(parser) options = parser.parse_args() _setup_logger(options) # Support the deprecated -c option if getattr(options, 'config', None) is not None: options.configs.append(options.config) config = Config.empty(**vars(options)) try: command = options.command if not callable(command): command = getattr( importlib.import_module(command.rsplit('.', 1)[0]), command.rsplit('.', 1)[-1]) # Set the process name to something cleaner process_name = [os.path.basename(sys.argv[0])] process_name.extend(sys.argv[1:]) setproctitle(' '.join(process_name)) command(config) except Exception: if not options.debug: raise traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1])
def run(event, context): # policies file should always be valid in functions so do loading naively with open(context['config_file']) as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): log.error('Invalid policy config') return False options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) # setup our auth file location on disk options_overrides['authorization_file'] = context['auth_file'] # if output_dir specified use that, otherwise make a temp directory if 'output_dir' not in options_overrides: options_overrides['output_dir'] = get_tmp_output_dir() # merge all our options in options = Config.empty(**options_overrides) load_resources() options = Azure().initialize(options) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: try: p.push(event, context) except (CloudError, AzureHttpError) as error: log.error("Unable to process policy: %s :: %s" % (p.name, error)) return True
def run(event, context=None): # policies file should always be valid in functions so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): log.error('Invalid policy config') return False options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) # if output_dir specified use that, otherwise make a temp directory if 'output_dir' not in options_overrides: options_overrides['output_dir'] = get_tmp_output_dir() # merge all our options in options = Config.empty(**options_overrides) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: log.info("running policy %s", p.name) p.push(event, context) return True
def report_account(account, region, policies_config, output_path, debug): cache_path = os.path.join(output_path, "c7n.cache") output_path = os.path.join(output_path, account['name'], region) config = Config.empty( region=region, output_dir=output_path, account_id=account['account_id'], metrics_enabled=False, cache=cache_path, log_group=None, profile=None, external_id=None) if account.get('role'): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) records = [] for p in policies: log.debug( "Report policy:%s account:%s region:%s path:%s", p.name, account['name'], region, output_path) policy_records = fs_record_set(p.ctx.output_path, p.name) for r in policy_records: r['policy'] = p.name r['region'] = p.options.region r['account'] = account['name'] for t in account.get('tags', ()): if ':' in t: k, v = t.split(':', 1) r[k] = v records.extend(policy_records) return records
def main(): parser = setup_parser() options = parser.parse_args() config = Config.empty() resources.load_resources() collection = policy_load( config, options.config_file).filter(options.policy_filter) sam = { 'AWSTemplateFormatVersion': '2010-09-09', 'Transform': 'AWS::Serverless-2016-10-31', 'Resources': {}} for p in collection: if p.provider_name != 'aws': continue exec_mode_type = p.data.get('mode', {'type': 'pull'}).get('type') if exec_mode_type == 'pull': continue sam_func = render(p) if sam_func: sam['Resources'][resource_name(p.name)] = sam_func sam_func['Properties']['CodeUri'] = './%s.zip' % p.name else: print("unable to render sam for policy:%s" % p.name) continue archive = mu.PolicyLambda(p).get_archive() with open(os.path.join(options.output_dir, "%s.zip" % p.name), 'wb') as fh: fh.write(archive.get_bytes()) with open(os.path.join(options.output_dir, 'deploy.yml'), 'w') as fh: fh.write(yaml.safe_dump(sam, default_flow_style=False))
def get_context(self, config=None, session_factory=None, policy=None): if config is None: self.context_output_dir = self.get_temp_dir() config = Config.empty(output_dir=self.context_output_dir) ctx = ExecutionContext( session_factory, policy or Bag({ "name": "test-policy", "provider_name": "aws"}), config) return ctx
def get_related(self, resources): ctx = ExecutionContext(local_session(Session), self.data, Config.empty()) manager = self.factory(ctx, self.data) related = manager.source.get_resources(None) if self.data.get('op'): return [r['id'] for r in related if self.match(r)] else: return [r['id'] for r in related]
def init_config(policy_config): """Get policy lambda execution configuration. cli parameters are serialized into the policy lambda config, we merge those with any policy specific execution options. --assume role and -s output directory get special handling, as to disambiguate any cli context. account id is sourced from the config options or from api call and cached as a global """ global account_id exec_options = policy_config.get('execution-options', {}) # Remove some configuration options that don't make sense to translate from # cli to lambda automatically. # - assume role on cli doesn't translate, it is the default lambda role and # used to provision the lambda. # - profile doesnt translate to lambda its `home` dir setup dependent # - dryrun doesn't translate (and shouldn't be present) # - region doesn't translate from cli (the lambda is bound to a region), and # on the cli represents the region the lambda is provisioned in. for k in ('assume_role', 'profile', 'region', 'dryrun', 'cache'): exec_options.pop(k, None) # a cli local directory doesn't translate to lambda if not exec_options.get('output_dir', '').startswith('s3'): exec_options['output_dir'] = get_local_output_dir() # we can source account id from the cli parameters to avoid the sts call if exec_options.get('account_id'): account_id = exec_options['account_id'] # merge with policy specific configuration exec_options.update( policy_config['policies'][0].get('mode', {}).get('execution-options', {})) # if using assume role in lambda ensure that the correct # execution account is captured in options. if 'assume_role' in exec_options: account_id = exec_options['assume_role'].split(':')[4] elif account_id is None: session = boto3.Session() account_id = get_account_id_from_sts(session) exec_options['account_id'] = account_id # Historical compatibility with manually set execution options # previously this was a boolean, its now a string value with the # boolean flag triggering a string value of 'aws' if 'metrics_enabled' in exec_options \ and isinstance(exec_options['metrics_enabled'], bool) \ and exec_options['metrics_enabled']: exec_options['metrics_enabled'] = 'aws' return Config.empty(**exec_options)
def get_context(self, config=None, session_factory=None, policy=None): if config is None: self.context_output_dir = self.get_temp_dir() config = Config.empty(output_dir=self.context_output_dir) ctx = ExecutionContext( session_factory, policy or Bag({'name': 'test-policy'}), config) return ctx
def validate(options): load_resources() if len(options.configs) < 1: log.error('no config files specified') sys.exit(1) used_policy_names = set() schm = schema.generate() errors = [] for config_file in options.configs: config_file = os.path.expanduser(config_file) if not os.path.exists(config_file): raise ValueError("Invalid path for config %r" % config_file) options.dryrun = True fmt = config_file.rsplit('.', 1)[-1] with open(config_file) as fh: if fmt in ('yml', 'yaml'): data = yaml.safe_load(fh.read()) elif fmt in ('json',): data = json.load(fh) else: log.error("The config file must end in .json, .yml or .yaml.") raise ValueError("The config file must end in .json, .yml or .yaml.") errors += schema.validate(data, schm) conf_policy_names = { p.get('name', 'unknown') for p in data.get('policies', ())} dupes = conf_policy_names.intersection(used_policy_names) if len(dupes) >= 1: errors.append(ValueError( "Only one policy with a given name allowed, duplicates: %s" % ( ", ".join(dupes) ) )) used_policy_names = used_policy_names.union(conf_policy_names) if not errors: null_config = Config.empty(dryrun=True, account_id='na', region='na') for p in data.get('policies', ()): try: policy = Policy(p, null_config, Bag()) policy.validate() except Exception as e: msg = "Policy: %s is invalid: %s" % ( p.get('name', 'unknown'), e) errors.append(msg) if not errors: log.info("Configuration valid: {}".format(config_file)) continue log.error("Configuration invalid: {}".format(config_file)) for e in errors: log.error("%s" % e) if errors: sys.exit(1)
def test_local_session_region(self): policies = [ self.load_policy( {'name': 'ec2', 'resource': 'ec2'}, config=Config.empty(region="us-east-1")), self.load_policy( {'name': 'ec2', 'resource': 'ec2'}, config=Config.empty(region='us-west-2'))] previous = None previous_region = None for p in policies: self.assertEqual(p.options.region, p.session_factory.region) session = utils.local_session(p.session_factory) self.assertNotEqual(session.region_name, previous_region) self.assertNotEqual(session, previous) previous = session previous_region = p.options.region self.assertEqual(utils.local_session(p.session_factory), previous)
def initialize_tree(self, tree): assert not self.policy_files for tree_ent in tree: fpath = tree_ent.name if not self.matcher(fpath): continue self.policy_files[fpath] = PolicyCollection.from_data( yaml.safe_load(self.repo.get(tree[fpath].id).data), Config.empty(), fpath)
def test_initialize_default_account_id(self, get_subscription_id_mock): options = Config.empty() azure = Azure() azure.initialize(options) self.assertEqual(options['account_id'], DEFAULT_SUBSCRIPTION_ID) session = azure.get_session_factory(options)() session._initialize_session() self.assertEqual(session.subscription_id, DEFAULT_SUBSCRIPTION_ID)
def _policy_file_rev(self, f, commit): try: return self._validate_policies( PolicyCollection.from_data( yaml.safe_load(self.repo.get(commit.tree[f].id).data), Config.empty(), f)) except Exception as e: log.warning("invalid policy file %s @ %s %s %s \n error:%s", f, str(commit.id)[:6], commit_date(commit).isoformat(), commit.author.name, e) return PolicyCollection()
def get_azure_output(self): output = AzureStorageOutput( ExecutionContext( None, Bag(name="xyz"), Config.empty( output_dir="azure://mystorage.blob.core.windows.net/logs"), )) self.addCleanup(shutil.rmtree, output.root_dir) return output
def test_expand_partitions(self): cfg = Config.empty(regions=["us-gov-west-1", "cn-north-1", "us-west-2"]) original = policy.PolicyCollection.from_data( {"policies": [{"name": "foo", "resource": "ec2"}]}, cfg ) collection = AWS().initialize_policies(original, cfg) self.assertEqual( sorted([p.options.region for p in collection]), ["cn-north-1", "us-gov-west-1", "us-west-2"], )
def get_azure_output(self): output = AzureStorageOutput( ExecutionContext( None, Bag(name="xyz"), Config.empty(output_dir="azure://mystorage.blob.core.windows.net/logs"), ) ) self.addCleanup(shutil.rmtree, output.root_dir) return output
def test_check_permissions(self): load_resources(('gcp.*', )) missing = [] invalid = [] iam_path = os.path.join(os.path.dirname(__file__), 'data', 'iam-permissions.json') with open(iam_path) as fh: valid_perms = set(json.load(fh).get('permissions')) cfg = Config.empty() for k, v in resources.items(): policy = Bag({ 'name': 'permcheck', 'resource': 'gcp.%s' % k, 'provider_name': 'gcp' }) ctx = self.get_context(config=cfg, policy=policy) mgr = v(ctx, policy) perms = mgr.get_permissions() if not perms: missing.append(k) for p in perms: if p not in valid_perms: invalid.append((k, p)) for n, a in list(v.action_registry.items()): if n in ALLOWED_NOPERM: continue policy['actions'] = [n] perms = a({}, mgr).get_permissions() if not perms: missing.append('%s.actions.%s' % (k, n)) for p in perms: if p not in valid_perms: invalid.append(('%s.actions.%s' % (k, n), p)) for n, f in list(v.filter_registry.items()): if n in ALLOWED_NOPERM: continue policy['filters'] = [n] perms = f({}, mgr).get_permissions() if not perms: missing.append('%s.filters.%s' % (k, n)) for p in perms: if p not in valid_perms: invalid.append(('%s.filters.%s' % (k, n), p)) if missing: self.fail('missing permissions %d on \n\t%s' % (len(missing), '\n\t'.join(sorted(missing)))) if invalid: self.fail('invalid permissions %d on \n\t%s' % (len(invalid), '\n\t'.join(map(str, sorted(invalid)))))
def test_initialize_default_account_id(self): # Patch get_subscription_id during provider initialization with patch('c7n_azure.session.Session.get_subscription_id', return_value=DEFAULT_SUBSCRIPTION_ID): options = Config.empty() azure = Azure() azure.initialize(options) self.assertEqual(options['account_id'], DEFAULT_SUBSCRIPTION_ID) session = azure.get_session_factory(options)() self.assertEqual(DEFAULT_SUBSCRIPTION_ID, session.get_subscription_id())
def test_initialize_custom_account_id(self, get_subscription_id_mock): sample_account_id = "00000000-5106-4743-99b0-c129bfa71a47" options = Config.empty() options['account_id'] = sample_account_id azure = Azure() azure.initialize(options) self.assertEqual(options['account_id'], sample_account_id) session = azure.get_session_factory(options)() session._initialize_session() self.assertEqual(session.subscription_id, sample_account_id)
def _policy_file_rev(self, f, commit): try: return self._validate_policies( PolicyCollection.from_data( yaml.safe_load(self.repo.get(commit.tree[f].id).data), Config.empty(), f)) except Exception as e: log.warning( "invalid policy file %s @ %s %s %s \n error:%s", f, str(commit.id)[:6], commit_date(commit).isoformat(), commit.author.name, e) return PolicyCollection()
def get_s3_output(self): output_dir = "s3://cloud-custodian/policies" output = S3Output( ExecutionContext( None, Bag(name="xyz", provider_name="ostack"), Config.empty(output_dir=output_dir)), {'url': output_dir}) self.addCleanup(shutil.rmtree, output.root_dir) return output
def test_iam_permissions_validity(self): cfg = Config.empty() missing = set() invalid = [] perms = load_data('iam-actions.json') resources.load_available() for k, v in manager.resources.items(): p = Bag({ 'name': 'permcheck', 'resource': k, 'provider_name': 'aws' }) ctx = self.get_context(config=cfg, policy=p) mgr = v(ctx, p) # if getattr(mgr, 'permissions', None): # print(mgr) found = False for s in (mgr.resource_type.service, getattr(mgr.resource_type, 'permission_prefix', None)): if s in perms: found = True if not found: missing.add("%s->%s" % (k, mgr.resource_type.service)) continue invalid.extend( self.check_permissions(perms, mgr.get_permissions(), k)) for n, a in v.action_registry.items(): p['actions'] = [n] invalid.extend( self.check_permissions(perms, a({}, mgr).get_permissions(), "{k}.actions.{n}".format(k=k, n=n))) for n, f in v.filter_registry.items(): if n in ('or', 'and', 'not', 'missing'): continue p['filters'] = [n] invalid.extend( self.check_permissions(perms, f({}, mgr).get_permissions(), "{k}.filters.{n}".format(k=k, n=n))) if missing: raise ValueError("resources missing service %s" % ('\n'.join(sorted(missing)))) if invalid: raise ValueError("invalid permissions \n %s" % ('\n'.join(sorted(map(str, invalid)))))
def test_initialize_default_azure_cloud(self): with patch('c7n_azure.session.Session.get_subscription_id'): options = Config.empty() azure = Azure() azure.initialize(options) self.assertEqual(AZURE_PUBLIC_CLOUD, azure.cloud_endpoints) self.assertEqual(AZURE_PUBLIC_CLOUD.name, options['region']) session = azure.get_session_factory(options)() self.assertEqual( AZURE_PUBLIC_CLOUD.endpoints.active_directory_resource_id, session.resource_endpoint)
def get_azure_output(self, custom_pyformat=None): output_dir = "azure://mystorage.blob.core.windows.net/logs" if custom_pyformat: output_dir = AzureStorageOutput.join(output_dir, custom_pyformat) output = AzureStorageOutput( ExecutionContext(None, Bag(name="xyz", provider_name='azure'), Config.empty(output_dir=output_dir)), {'url': output_dir}, ) self.addCleanup(shutil.rmtree, output.root_dir) return output
def report_account(account, region, policies_config, output_path, cache_path, debug): output_path = os.path.join(output_path, account['name'], region) cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['name'], region)) load_available() config = Config.empty(region=region, output_dir=output_path, account_id=account['account_id'], metrics_enabled=False, cache=cache_path, log_group=None, profile=None, external_id=None) if account.get('role'): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) records = [] for p in policies: # initializee policy execution context for output access p.ctx.initialize() log.debug("Report policy:%s account:%s region:%s path:%s", p.name, account['name'], region, output_path) if p.ctx.output.type == "s3": delta = timedelta(days=1) begin_date = datetime.now() - delta policy_records = record_set( p.session_factory, p.ctx.output.config['netloc'], strip_output_path(p.ctx.output.config['path'], p.name), begin_date) else: policy_records = fs_record_set(p.ctx.log_dir, p.name) for r in policy_records: r['policy'] = p.name r['region'] = p.options.region r['account'] = account['name'] for t in account.get('tags', ()): if ':' in t: k, v = t.split(':', 1) r[k] = v records.extend(policy_records) return records
def test_policy_region_expand_global(self): factory = self.replay_flight_data('test_aws_policy_global_expand') self.patch(aws, '_profile_session', factory()) original = self.policy_loader.load_data( {"policies": [ {"name": "foo", "resource": "s3"}, {"name": "iam", "resource": "iam-user"}]}, 'memory://', config=Config.empty(regions=["us-east-1", "us-west-2"]), ) collection = AWS().initialize_policies( original, Config.empty(regions=["all"], output_dir="/test/output/")) self.assertEqual(len(collection.resource_types), 2) s3_regions = [p.options.region for p in collection if p.resource_type == "s3"] self.assertTrue("us-east-1" in s3_regions) self.assertTrue("us-east-2" in s3_regions) iam = [p for p in collection if p.resource_type == "iam-user"] self.assertEqual(len(iam), 1) self.assertEqual(iam[0].options.region, "us-east-1") self.assertEqual(iam[0].options.output_dir, "/test/output/us-east-1") # Don't append region when it's already in the path. collection = AWS().initialize_policies( original, Config.empty(regions=["all"], output_dir="/test/{region}/output/")) self.assertEqual(len(collection.resource_types), 2) iam = [p for p in collection if p.resource_type == "iam-user"] self.assertEqual(iam[0].options.region, "us-east-1") self.assertEqual(iam[0].options.output_dir, "/test/{region}/output") collection = AWS().initialize_policies( original, Config.empty(regions=["eu-west-1", "eu-west-2"], output_dir="/test/output/") ) iam = [p for p in collection if p.resource_type == "iam-user"] self.assertEqual(len(iam), 1) self.assertEqual(iam[0].options.region, "eu-west-1") self.assertEqual(iam[0].options.output_dir, "/test/output/eu-west-1") self.assertEqual(len(collection), 3)
def load_policy(self, data, config=None): errors = schema_validate({'policies': [data]}, C7N_SCHEMA) if errors: raise errors[0] config = config or {} temp_dir = self.get_temp_dir() config['output_dir'] = temp_dir conf = Config.empty(**config) p = policy.Policy(data, conf, Session) p.validate() return p
def get_s3_output(self, output_url=None, cleanup=True, klass=S3Output): if output_url is None: output_url = "s3://cloud-custodian/policies" output = klass( ExecutionContext( lambda assume=False: mock.MagicMock(), Bag(name="xyz", provider_name="ostack"), Config.empty(output_dir=output_url, account_id='112233445566')), {'url': output_url, 'test': True}) if cleanup: self.addCleanup(shutil.rmtree, output.root_dir) return output
def get_blob_output(request, output_url=None, cleanup=True): if output_url is None: output_url = "gs://cloud-custodian/policies" output = GCPStorageOutput( ExecutionContext(lambda assume=False: mock.MagicMock(), Bag(name="xyz", provider_name="gcp"), Config.empty(output_dir=output_url, account_id='custodian-test')), parse_url_config(output_url)) if cleanup: request.addfinalizer(lambda: shutil.rmtree(output.root_dir)) # noqa request.addfinalizer(reset_session_cache) return output
def dispatch_event(event, context): global account_id if account_id is None: session = boto3.Session() account_id = get_account_id_from_sts(session) error = event.get('detail', {}).get('errorCode') if error: log.debug("Skipping failed operation: %s" % error) return event['debug'] = True if event['debug']: log.info("Processing event\n %s", format_event(event)) # policies file should always be valid in lambda so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # Initialize output directory, we've seen occassional perm issues with # lambda on temp directory and changing unix execution users, so # use a per execution temp space. output_dir = os.environ.get( 'C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4())) if not os.path.exists(output_dir): try: os.mkdir(output_dir) except OSError as error: log.warning("Unable to make output directory: {}".format(error)) # TODO. This enshrines an assumption of a single policy per lambda. options_overrides = policy_config[ 'policies'][0].get('mode', {}).get('execution-options', {}) options_overrides['account_id'] = account_id if 'output_dir' not in options_overrides: options_overrides['output_dir'] = output_dir options = Config.empty(**options_overrides) load_resources() policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def cli(**kwargs): policy_config = Config.empty() policies = PolicyCollection([ p for p in load_policies(kwargs['config'], policy_config) if p.provider_name == 'azure'], policy_config) session = policies.policies[0].session_factory() web_client = session.client('azure.mgmt.web.WebSiteManagementClient') deployments = {} credentials_load_failed = 0 not_functions_policy = 0 for p in policies: if not p.is_lambda: not_functions_policy += 1 continue try: params = AzureFunctionMode(p).get_function_app_params() creds = web_client.web_apps.begin_list_publishing_credentials( params.function_app_resource_group_name, params.function_app_name).result() deployments[p.name] = {'scm_uri': creds.scm_uri, 'status': None} log.info('Retrieved deployment credentials for %s policy', p.name) except Exception: log.error('Unable to retrieve deployment credentials for %s policy', p.name) credentials_load_failed += 1 wait_for_remote_builds(deployments) success = 0 fail = 0 not_found = 0 for name, params in deployments.items(): if params['status'] != DeploymentStatus.Succeeded: log.info('%s: %s', name, params['status']) if params['status'] == DeploymentStatus.Failed: log.error('Build logs can be retrieved here: %s', params['scm_uri']) fail += 1 else: not_found += 1 else: success += 1 log.info('Policies total: %i, unable to load credentials: %i, not Functions mode: %i', len(policies), credentials_load_failed, not_functions_policy) log.info('Status not found can happen if Linux Consumption function was deployed' 'more than 2 hours ago.') log.info('Deployments complete. Success: %i, Fail: %i, Status not found: %i', success, fail, not_found)
def load_policy( self, data, config=None): errors = schema_validate({'policies': [data]}, C7N_SCHEMA) if errors: raise errors[0] config = config or {} temp_dir = self.get_temp_dir() config['output_dir'] = temp_dir conf = Config.empty(**config) p = policy.Policy(data, conf, Session) p.validate() return p
def exec(command: str, config: C7nDefaults = C7nDefaults(), policies: Iterable[Path] = ()): def _new_cfg(policy): # TODO: Use account_id rather than profile name profile = config.profile if config.profile else "default" return replace( config, cache=str(Path(".cache", profile, policy.stem).with_suffix(".cache")), configs=[str(policy)], output_dir=str(Path("output", profile)), ) c7n_cmd = getattr(c7n.commands, command) cfg_gen = (Config.empty(**asdict(_new_cfg(policy_))) for policy_ in policies) with ThreadPoolExecutor() as executor: list(executor.map(c7n_cmd, cfg_gen))
def test_local_session_region(self): policies = [ self.load_policy({ 'name': 'ec2', 'resource': 'ec2' }, config=Config.empty(region="us-east-1")), self.load_policy({ 'name': 'ec2', 'resource': 'ec2' }, config=Config.empty(region='us-west-2')) ] previous = None previous_region = None for p in policies: self.assertEqual(p.options.region, p.session_factory.region) session = utils.local_session(p.session_factory) self.assertNotEqual(session.region_name, previous_region) self.assertNotEqual(session, previous) previous = session previous_region = p.options.region self.assertEqual(utils.local_session(p.session_factory), previous)
def run(self): logging.getLogger('botocore').setLevel(logging.ERROR) logging.getLogger('c7n.cache').setLevel(logging.WARNING) conf = Config.empty( config_files=['custodian_%s.yml' % self.region_name], region=self.region_name, prefix='custodian-', assume=None, policy_filter=None, log_group=None, external_id=None, cache_period=0, cache=None) resources.load_resources() policies = load_policies(conf) resources_gc_prefix(conf, policies)
def is_valid(policy_file: Union[Path, PathLike]) -> bool: """ Args: policy_file: file to be validated Returns: True if policy is valid """ try: c7n.commands.validate(Config.empty(configs={Path(policy_file).resolve()})) except SystemExit: _LOGGER.debug("Invalid policy %s", policy_file.name) else: _LOGGER.debug("Validated policy %s", policy_file.name) return True return False
def get_azure_output(self, custom_pyformat=None): output_dir = "azure://mystorage.blob.core.windows.net/logs" if custom_pyformat: output_dir = AzureStorageOutput.join(output_dir, custom_pyformat) output = AzureStorageOutput( ExecutionContext( None, Bag(name="xyz", provider_name='azure'), Config.empty(output_dir=output_dir) ), {'url': output_dir}, ) self.addCleanup(shutil.rmtree, output.root_dir) return output
def dispatch_event(event, context): global account_id if account_id is None: session = boto3.Session() account_id = get_account_id_from_sts(session) error = event.get('detail', {}).get('errorCode') if error: log.debug("Skipping failed operation: %s" % error) return event['debug'] = True if event['debug']: log.info("Processing event\n %s", format_event(event)) # policies file should always be valid in lambda so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # Initialize output directory, we've seen occassional perm issues with # lambda on temp directory and changing unix execution users, so # use a per execution temp space. output_dir = os.environ.get('C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4())) if not os.path.exists(output_dir): try: os.mkdir(output_dir) except OSError as error: log.warning("Unable to make output directory: {}".format(error)) # TODO. This enshrines an assumption of a single policy per lambda. options_overrides = policy_config['policies'][0].get('mode', {}).get( 'execution-options', {}) options_overrides['account_id'] = account_id if 'output_dir' not in options_overrides: options_overrides['output_dir'] = output_dir options = Config.empty(**options_overrides) load_resources() policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True
def test_initialize_azure_cloud(self): clouds = [ AZURE_PUBLIC_CLOUD, AZURE_CHINA_CLOUD, AZURE_GERMAN_CLOUD, AZURE_US_GOV_CLOUD ] with patch('c7n_azure.session.Session.get_subscription_id'): for cloud_endpoints in clouds: options = Config.empty(regions=[cloud_endpoints.name]) azure = Azure() azure.initialize(options) self.assertEqual(cloud_endpoints, azure.cloud_endpoints) self.assertEqual(cloud_endpoints.name, options['region']) session = azure.get_session_factory(options)() self.assertEqual( cloud_endpoints.endpoints.active_directory_resource_id, session.resource_endpoint)
def build_options(output_dir=None, log_group=None, metrics=None): """ Initialize the Azure provider to apply global config across all policy executions. """ if not output_dir: output_dir = tempfile.mkdtemp() log.warning('Output directory not specified. Using directory: %s' % output_dir) config = Config.empty( **{ 'log_group': log_group, 'metrics': metrics, 'output_dir': output_dir } ) return Azure().initialize(config)
def test_doc_examples(provider_name): load_resources() loader = PolicyLoader(Config.empty()) provider = clouds.get(provider_name) policies = get_doc_policies(provider.resources) for p in policies.values(): loader.load_data({'policies': [p]}, 'memory://') for p in policies.values(): # Note max name size here is 54 if it a lambda policy given # our default prefix custodian- to stay under 64 char limit on # lambda function names. This applies to AWS and GCP, and # afaict Azure. if len(p['name']) >= 54 and 'mode' in p: raise ValueError( "doc policy exceeds name limit policy:%s" % (p['name']))
def main(): parser = setup_parser() options = parser.parse_args() log_level = logging.INFO if options.verbose: log_level = logging.DEBUG logging.basicConfig( level=log_level, format="%(asctime)s: %(name)s:%(levelname)s %(message)s") logging.getLogger('botocore').setLevel(logging.ERROR) logging.getLogger('urllib3').setLevel(logging.ERROR) logging.getLogger('c7n.cache').setLevel(logging.WARNING) if not options.policy_regex: options.policy_regex = f"^{options.prefix}.*" if not options.regions: options.regions = [os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')] files = [] files.extend(itertools.chain(*options.config_files)) files.extend(options.configs) options.config_files = files options.policy_tags = list(itertools.chain(*options.policy_tags)) if not files: parser.print_help() sys.exit(1) policy_config = Config.empty( regions=options.regions, profile=options.profile, assume_role=options.assume_role) # use cloud provider to initialize policies to get region expansion policies = AWS().initialize_policies( PolicyCollection([ p for p in load_policies( options, policy_config) if p.provider_name == 'aws'], policy_config), policy_config) resources_gc_prefix(options, policy_config, policies)
def load_policy( self, data, config=None, session_factory=None, validate=C7N_VALIDATE, output_dir=None, cache=False): if validate: errors = schema_validate({'policies': [data]}, C7N_SCHEMA) if errors: raise errors[0] config = config or {} if not output_dir: temp_dir = self.get_temp_dir() config['output_dir'] = temp_dir if cache: config['cache'] = os.path.join(temp_dir, 'c7n.cache') config['cache_period'] = 300 conf = Config.empty(**config) p = policy.Policy(data, conf, session_factory) p.validate() return p
def tag(assume, region, db, creator_tag, user_suffix, dryrun, summary=True, profile=None, type=()): """Tag resources with their creator. """ trail_db = TrailDB(db) load_resources(resource_types=('aws.*', )) with temp_dir() as output_dir: config = ExecConfig.empty(output_dir=output_dir, assume=assume, region=region, profile=profile) factory = aws.AWS().get_session_factory(config) account_id = local_session(factory).client( 'sts').get_caller_identity().get('Account') config['account_id'] = account_id tagger = ResourceTagger(trail_db, config, creator_tag, user_suffix, dryrun, type) try: stats = tagger.process() except Exception: log.exception( "error processing account:%s region:%s config:%s env:%s", account_id, region, config, dict(os.environ)) raise if not summary: return stats log.info( "auto tag summary account:%s region:%s \n%s", config['account_id'], config['region'], "\n".join([" {}: {}".format(k, v) for k, v in stats.items() if v])) total = sum([v for k, v in stats.items() if not k.endswith('not-found')]) log.info("Total resources tagged: %d" % total)
def main(): parser = setup_parser() options = parser.parse_args() config = Config.empty() resources.load_resources() collection = policy_load( config, options.config_file).filter(options.policy_filter) sam = { 'AWSTemplateFormatVersion': '2010-09-09', 'Transform': 'AWS::Serverless-2016-10-31', 'Resources': {}} for p in collection: if p.provider_name != 'aws': continue exec_mode_type = p.data.get('mode', {'type': 'pull'}).get('type') if exec_mode_type == 'pull': continue sam_func = render(p) if sam_func: sam['Resources'][resource_name(p.name)] = sam_func if exec_mode_type == 'config-rule' or exec_mode_type == 'config-poll-rule': configrule_resource = render_config_rule(p) invoke_resource = render_invoke(resource_name(p.name)) sam['Resources'][resource_name(p.name) + "Invoke"] = invoke_resource sam['Resources'][resource_name(p.name) + "ConfigRule"] = configrule_resource sam_func['Properties']['CodeUri'] = './%s.zip' % p.name else: print("unable to render sam for policy:%s" % p.name) continue archive = mu.PolicyLambda(p).get_archive() with open(os.path.join(options.output_dir, "%s.zip" % p.name), 'wb') as fh: fh.write(archive.get_bytes()) with open(os.path.join(options.output_dir, 'deploy.yml'), 'w') as fh: fh.write(yaml.safe_dump(sam, default_flow_style=False))
def test_policy_expand_group_region(self): cfg = Config.empty(regions=["us-east-1", "us-east-2", "us-west-2"]) original = policy.PolicyCollection.from_data( {"policies": [ {"name": "bar", "resource": "lambda"}, {"name": "middle", "resource": "security-group"}, {"name": "foo", "resource": "ec2"}]}, cfg) collection = AWS().initialize_policies(original, cfg) self.assertEqual( [(p.name, p.options.region) for p in collection], [('bar', 'us-east-1'), ('middle', 'us-east-1'), ('foo', 'us-east-1'), ('bar', 'us-east-2'), ('middle', 'us-east-2'), ('foo', 'us-east-2'), ('bar', 'us-west-2'), ('middle', 'us-west-2'), ('foo', 'us-west-2')])
def run(event, context, subscription_id=None): # policies file should always be valid in functions so do loading naively with open(context['config_file']) as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): log.error('Invalid policy config') return False options_overrides = \ policy_config['policies'][0].get('mode', {}).get('execution-options', {}) # setup our auth file location on disk options_overrides['authorization_file'] = context['auth_file'] # if output_dir specified use that, otherwise make a temp directory if 'output_dir' not in options_overrides: options_overrides['output_dir'] = get_tmp_output_dir() # merge all our options in options = Config.empty(**options_overrides) if subscription_id is not None: options['account_id'] = subscription_id load_resources(StructureParser().get_resource_types(policy_config)) options = Azure().initialize(options) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: try: p.push(event, context) except (CloudError, AzureHttpError) as error: log.error("Unable to process policy: %s :: %s" % (p.name, error)) reset_session_cache() return True
def main(): parser = setup_parser() options = parser.parse_args() log_level = logging.INFO if options.verbose: log_level = logging.DEBUG logging.basicConfig( level=log_level, format="%(asctime)s: %(name)s:%(levelname)s %(message)s") logging.getLogger('botocore').setLevel(logging.ERROR) logging.getLogger('c7n.cache').setLevel(logging.WARNING) if not options.regions: options.regions = [os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')] files = [] files.extend(itertools.chain(*options.config_files)) files.extend(options.configs) options.config_files = files if not files: parser.print_help() sys.exit(1) policy_config = Config.empty( regions=options.regions, profile=options.profile, assume_role=options.assume_role) # use cloud provider to initialize policies to get region expansion policies = AWS().initialize_policies( PolicyCollection([ p for p in load_policies( options, policy_config) if p.provider_name == 'aws'], policy_config), policy_config) resources_gc_prefix(options, policy_config, policies)
def run_account(account, region, policies_config, output_path, cache_period, cache_path, metrics, dryrun, debug): """Execute a set of policies on an account. """ logging.getLogger('custodian.output').setLevel(logging.ERROR + 1) CONN_CACHE.session = None CONN_CACHE.time = None # allow users to specify interpolated output paths if '{' not in output_path: output_path = os.path.join(output_path, account['name'], region) cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['account_id'], region)) config = Config.empty( region=region, cache=cache_path, cache_period=cache_period, dryrun=dryrun, output_dir=output_path, account_id=account['account_id'], metrics_enabled=metrics, log_group=None, profile=None, external_id=None) env_vars = account_tags(account) if account.get('role'): if isinstance(account['role'], six.string_types): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') else: env_vars.update( _get_env_creds(get_session(account, 'custodian', region), region)) elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) policy_counts = {} success = True st = time.time() with environ(**env_vars): for p in policies: # Variable expansion and non schema validation (not optional) p.expand_variables(p.get_variables(account.get('vars', {}))) p.validate() log.debug( "Running policy:%s account:%s region:%s", p.name, account['name'], region) try: resources = p.run() policy_counts[p.name] = resources and len(resources) or 0 if not resources: continue log.info( "Ran account:%s region:%s policy:%s matched:%d time:%0.2f", account['name'], region, p.name, len(resources), time.time() - st) except ClientError as e: success = False if e.response['Error']['Code'] == 'AccessDenied': log.warning('Access denied account:%s region:%s', account['name'], region) return policy_counts, success log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) continue except Exception as e: success = False log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) if not debug: continue import traceback, pdb, sys traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1]) raise return policy_counts, success
def _load_policies(options): config = Config() config.from_cli(options) collection = policy.load(options, options.config) return f(config, collection)
def __init__(self, policies=None, options=None): self.policies = policies or [] self.options = options or Config.empty() self.pmap = {p.name: p for p in self.policies}
def report(config, output, use, output_dir, accounts, field, no_default_fields, tags, region, debug, verbose, policy, policy_tags, format, resource): """report on a cross account policy execution.""" accounts_config, custodian_config, executor = init( config, use, debug, verbose, accounts, tags, policy, resource=resource, policy_tags=policy_tags) resource_types = set() for p in custodian_config.get('policies'): resource_types.add(p['resource']) if len(resource_types) > 1: raise ValueError("can only report on one resource type at a time") elif not len(custodian_config['policies']) > 0: raise ValueError("no matching policies found") records = [] with executor(max_workers=WORKER_COUNT) as w: futures = {} for a in accounts_config.get('accounts', ()): for r in resolve_regions(region or a.get('regions', ())): futures[w.submit( report_account, a, r, custodian_config, output_dir, debug)] = (a, r) for f in as_completed(futures): a, r = futures[f] if f.exception(): if debug: raise log.warning( "Error running policy in %s @ %s exception: %s", a['name'], r, f.exception()) records.extend(f.result()) log.debug( "Found %d records across %d accounts and %d policies", len(records), len(accounts_config['accounts']), len(custodian_config['policies'])) if format == 'json': dumps(records, output, indent=2) return prefix_fields = OrderedDict( (('Account', 'account'), ('Region', 'region'), ('Policy', 'policy'))) config = Config.empty() factory = resource_registry.get(list(resource_types)[0]) formatter = Formatter( factory.resource_type, extra_fields=field, include_default_fields=not(no_default_fields), include_region=False, include_policy=False, fields=prefix_fields) rows = formatter.to_csv(records, unique=False) writer = UnicodeWriter(output, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows)
def run_account(account, region, policies_config, output_path, cache_period, metrics, dryrun, debug): """Execute a set of policies on an account. """ logging.getLogger('custodian.output').setLevel(logging.ERROR + 1) CONN_CACHE.session = None CONN_CACHE.time = None output_path = os.path.join(output_path, account['name'], region) if not os.path.exists(output_path): os.makedirs(output_path) cache_path = os.path.join(output_path, "c7n.cache") config = Config.empty( region=region, cache_period=cache_period, dryrun=dryrun, output_dir=output_path, account_id=account['account_id'], metrics_enabled=metrics, cache=cache_path, log_group=None, profile=None, external_id=None) if account.get('role'): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) policy_counts = {} st = time.time() with environ(**account_tags(account)): for p in policies: log.debug( "Running policy:%s account:%s region:%s", p.name, account['name'], region) try: resources = p.run() policy_counts[p.name] = resources and len(resources) or 0 if not resources: continue log.info( "Ran account:%s region:%s policy:%s matched:%d time:%0.2f", account['name'], region, p.name, len(resources), time.time() - st) except ClientError as e: if e.response['Error']['Code'] == 'AccessDenied': log.warning('Access denied account:%s region:%s', account['name'], region) return policy_counts log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) continue except Exception as e: log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) if not debug: continue import traceback, pdb, sys traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1]) raise return policy_counts
def dispatch_event(event, context): global account_id error = event.get('detail', {}).get('errorCode') if error: log.debug("Skipping failed operation: %s" % error) return event['debug'] = True if event['debug']: log.info("Processing event\n %s", format_event(event)) # Policies file should always be valid in lambda so do loading naively with open('config.json') as f: policy_config = json.load(f) if not policy_config or not policy_config.get('policies'): return False # Initialize output directory, we've seen occassional perm issues with # lambda on temp directory and changing unix execution users, so # use a per execution temp space. output_dir = os.environ.get( 'C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4())) if not os.path.exists(output_dir): try: os.mkdir(output_dir) except OSError as error: log.warning("Unable to make output directory: {}".format(error)) # TODO. This enshrines an assumption of a single policy per lambda. options_overrides = policy_config[ 'policies'][0].get('mode', {}).get('execution-options', {}) # if using assume role in lambda ensure that the correct # execution account is captured in options. if 'assume_role' in options_overrides: account_id = options_overrides['assume_role'].split(':')[4] elif account_id is None: session = boto3.Session() account_id = get_account_id_from_sts(session) # Historical compatibility with manually set execution options # previously this was a boolean, its now a string value with the # boolean flag triggering a string value of 'aws' if 'metrics_enabled' in options_overrides and isinstance( options_overrides['metrics_enabled'], bool): options_overrides['metrics_enabled'] = 'aws' options_overrides['account_id'] = account_id if 'output_dir' not in options_overrides: options_overrides['output_dir'] = output_dir options = Config.empty(**options_overrides) policies = PolicyCollection.from_data(policy_config, options) if policies: for p in policies: p.push(event, context) return True