def load_policy_set(self, data, config=None):
filename = self.write_policy_file(data)
if config:
e = Config.empty(**config)
else:
e = Config.empty()
return policy.load(e, filename)
def load_policy_set(self, data, config=None):
filename = self.write_policy_file(data)
if config:
config['account_id'] = ACCOUNT_ID
e = Config.empty(**config)
else:
e = Config.empty(account_id=ACCOUNT_ID)
return policy.load(e, filename)
def load_policy(
self,
data,
config=None,
session_factory=None,
validate=C7N_VALIDATE,
output_dir=None,
cache=False,
):
if validate:
if not self.custodian_schema:
self.custodian_schema = generate()
errors = schema_validate({"policies": [data]}, self.custodian_schema)
if errors:
raise errors[0]
config = config or {}
if not output_dir:
temp_dir = self.get_temp_dir()
config["output_dir"] = temp_dir
if cache:
config["cache"] = os.path.join(temp_dir, "c7n.cache")
config["cache_period"] = 300
conf = Config.empty(**config)
p = policy.Policy(data, conf, session_factory)
p.validate()
return p
def main():
parser = setup_parser()
argcomplete.autocomplete(parser)
options = parser.parse_args()
_setup_logger(options)
# Support the deprecated -c option
if getattr(options, 'config', None) is not None:
options.configs.append(options.config)
config = Config.empty(**vars(options))
try:
command = options.command
if not callable(command):
command = getattr(
importlib.import_module(command.rsplit('.', 1)[0]),
command.rsplit('.', 1)[-1])
# Set the process name to something cleaner
process_name = [os.path.basename(sys.argv[0])]
process_name.extend(sys.argv[1:])
setproctitle(' '.join(process_name))
command(config)
except Exception:
if not options.debug:
raise
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
def run(event, context):
# policies file should always be valid in functions so do loading naively
with open(context['config_file']) as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
log.error('Invalid policy config')
return False
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
# setup our auth file location on disk
options_overrides['authorization_file'] = context['auth_file']
# if output_dir specified use that, otherwise make a temp directory
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = get_tmp_output_dir()
# merge all our options in
options = Config.empty(**options_overrides)
load_resources()
options = Azure().initialize(options)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
try:
p.push(event, context)
except (CloudError, AzureHttpError) as error:
log.error("Unable to process policy: %s :: %s" % (p.name, error))
return True
def run(event, context=None):
# policies file should always be valid in functions so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
log.error('Invalid policy config')
return False
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
# if output_dir specified use that, otherwise make a temp directory
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = get_tmp_output_dir()
# merge all our options in
options = Config.empty(**options_overrides)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
log.info("running policy %s", p.name)
p.push(event, context)
return True
def report_account(account, region, policies_config, output_path, debug):
cache_path = os.path.join(output_path, "c7n.cache")
output_path = os.path.join(output_path, account['name'], region)
config = Config.empty(
region=region,
output_dir=output_path,
account_id=account['account_id'], metrics_enabled=False,
cache=cache_path, log_group=None, profile=None, external_id=None)
if account.get('role'):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
records = []
for p in policies:
log.debug(
"Report policy:%s account:%s region:%s path:%s",
p.name, account['name'], region, output_path)
policy_records = fs_record_set(p.ctx.output_path, p.name)
for r in policy_records:
r['policy'] = p.name
r['region'] = p.options.region
r['account'] = account['name']
for t in account.get('tags', ()):
if ':' in t:
k, v = t.split(':', 1)
r[k] = v
records.extend(policy_records)
return records
def main():
parser = setup_parser()
options = parser.parse_args()
config = Config.empty()
resources.load_resources()
collection = policy_load(
config, options.config_file).filter(options.policy_filter)
sam = {
'AWSTemplateFormatVersion': '2010-09-09',
'Transform': 'AWS::Serverless-2016-10-31',
'Resources': {}}
for p in collection:
if p.provider_name != 'aws':
continue
exec_mode_type = p.data.get('mode', {'type': 'pull'}).get('type')
if exec_mode_type == 'pull':
continue
sam_func = render(p)
if sam_func:
sam['Resources'][resource_name(p.name)] = sam_func
sam_func['Properties']['CodeUri'] = './%s.zip' % p.name
else:
print("unable to render sam for policy:%s" % p.name)
continue
archive = mu.PolicyLambda(p).get_archive()
with open(os.path.join(options.output_dir, "%s.zip" % p.name), 'wb') as fh:
fh.write(archive.get_bytes())
with open(os.path.join(options.output_dir, 'deploy.yml'), 'w') as fh:
fh.write(yaml.safe_dump(sam, default_flow_style=False))
def get_context(self, config=None, session_factory=None, policy=None):
if config is None:
self.context_output_dir = self.get_temp_dir()
config = Config.empty(output_dir=self.context_output_dir)
ctx = ExecutionContext(
session_factory, policy or Bag({
"name": "test-policy", "provider_name": "aws"}), config)
return ctx
def get_related(self, resources):
ctx = ExecutionContext(local_session(Session), self.data, Config.empty())
manager = self.factory(ctx, self.data)
related = manager.source.get_resources(None)
if self.data.get('op'):
return [r['id'] for r in related if self.match(r)]
else:
return [r['id'] for r in related]
def init_config(policy_config):
"""Get policy lambda execution configuration.
cli parameters are serialized into the policy lambda config,
we merge those with any policy specific execution options.
--assume role and -s output directory get special handling, as
to disambiguate any cli context.
account id is sourced from the config options or from api call
and cached as a global
"""
global account_id
exec_options = policy_config.get('execution-options', {})
# Remove some configuration options that don't make sense to translate from
# cli to lambda automatically.
# - assume role on cli doesn't translate, it is the default lambda role and
# used to provision the lambda.
# - profile doesnt translate to lambda its `home` dir setup dependent
# - dryrun doesn't translate (and shouldn't be present)
# - region doesn't translate from cli (the lambda is bound to a region), and
# on the cli represents the region the lambda is provisioned in.
for k in ('assume_role', 'profile', 'region', 'dryrun', 'cache'):
exec_options.pop(k, None)
# a cli local directory doesn't translate to lambda
if not exec_options.get('output_dir', '').startswith('s3'):
exec_options['output_dir'] = get_local_output_dir()
# we can source account id from the cli parameters to avoid the sts call
if exec_options.get('account_id'):
account_id = exec_options['account_id']
# merge with policy specific configuration
exec_options.update(
policy_config['policies'][0].get('mode', {}).get('execution-options', {}))
# if using assume role in lambda ensure that the correct
# execution account is captured in options.
if 'assume_role' in exec_options:
account_id = exec_options['assume_role'].split(':')[4]
elif account_id is None:
session = boto3.Session()
account_id = get_account_id_from_sts(session)
exec_options['account_id'] = account_id
# Historical compatibility with manually set execution options
# previously this was a boolean, its now a string value with the
# boolean flag triggering a string value of 'aws'
if 'metrics_enabled' in exec_options \
and isinstance(exec_options['metrics_enabled'], bool) \
and exec_options['metrics_enabled']:
exec_options['metrics_enabled'] = 'aws'
return Config.empty(**exec_options)
def get_context(self, config=None, session_factory=None, policy=None):
if config is None:
self.context_output_dir = self.get_temp_dir()
config = Config.empty(output_dir=self.context_output_dir)
ctx = ExecutionContext(
session_factory,
policy or Bag({'name': 'test-policy'}),
config)
return ctx
def validate(options):
load_resources()
if len(options.configs) < 1:
log.error('no config files specified')
sys.exit(1)
used_policy_names = set()
schm = schema.generate()
errors = []
for config_file in options.configs:
config_file = os.path.expanduser(config_file)
if not os.path.exists(config_file):
raise ValueError("Invalid path for config %r" % config_file)
options.dryrun = True
fmt = config_file.rsplit('.', 1)[-1]
with open(config_file) as fh:
if fmt in ('yml', 'yaml'):
data = yaml.safe_load(fh.read())
elif fmt in ('json',):
data = json.load(fh)
else:
log.error("The config file must end in .json, .yml or .yaml.")
raise ValueError("The config file must end in .json, .yml or .yaml.")
errors += schema.validate(data, schm)
conf_policy_names = {
p.get('name', 'unknown') for p in data.get('policies', ())}
dupes = conf_policy_names.intersection(used_policy_names)
if len(dupes) >= 1:
errors.append(ValueError(
"Only one policy with a given name allowed, duplicates: %s" % (
", ".join(dupes)
)
))
used_policy_names = used_policy_names.union(conf_policy_names)
if not errors:
null_config = Config.empty(dryrun=True, account_id='na', region='na')
for p in data.get('policies', ()):
try:
policy = Policy(p, null_config, Bag())
policy.validate()
except Exception as e:
msg = "Policy: %s is invalid: %s" % (
p.get('name', 'unknown'), e)
errors.append(msg)
if not errors:
log.info("Configuration valid: {}".format(config_file))
continue
log.error("Configuration invalid: {}".format(config_file))
for e in errors:
log.error("%s" % e)
if errors:
sys.exit(1)
def test_local_session_region(self):
policies = [
self.load_policy(
{'name': 'ec2', 'resource': 'ec2'},
config=Config.empty(region="us-east-1")),
self.load_policy(
{'name': 'ec2', 'resource': 'ec2'},
config=Config.empty(region='us-west-2'))]
previous = None
previous_region = None
for p in policies:
self.assertEqual(p.options.region, p.session_factory.region)
session = utils.local_session(p.session_factory)
self.assertNotEqual(session.region_name, previous_region)
self.assertNotEqual(session, previous)
previous = session
previous_region = p.options.region
self.assertEqual(utils.local_session(p.session_factory), previous)
def initialize_tree(self, tree):
assert not self.policy_files
for tree_ent in tree:
fpath = tree_ent.name
if not self.matcher(fpath):
continue
self.policy_files[fpath] = PolicyCollection.from_data(
yaml.safe_load(self.repo.get(tree[fpath].id).data),
Config.empty(), fpath)
def test_initialize_default_account_id(self, get_subscription_id_mock):
options = Config.empty()
azure = Azure()
azure.initialize(options)
self.assertEqual(options['account_id'], DEFAULT_SUBSCRIPTION_ID)
session = azure.get_session_factory(options)()
session._initialize_session()
self.assertEqual(session.subscription_id, DEFAULT_SUBSCRIPTION_ID)
def _policy_file_rev(self, f, commit):
try:
return self._validate_policies(
PolicyCollection.from_data(
yaml.safe_load(self.repo.get(commit.tree[f].id).data),
Config.empty(), f))
except Exception as e:
log.warning("invalid policy file %s @ %s %s %s \n error:%s", f,
str(commit.id)[:6],
commit_date(commit).isoformat(), commit.author.name, e)
return PolicyCollection()
def get_azure_output(self):
output = AzureStorageOutput(
ExecutionContext(
None,
Bag(name="xyz"),
Config.empty(
output_dir="azure://mystorage.blob.core.windows.net/logs"),
))
self.addCleanup(shutil.rmtree, output.root_dir)
return output
def test_expand_partitions(self):
cfg = Config.empty(regions=["us-gov-west-1", "cn-north-1", "us-west-2"])
original = policy.PolicyCollection.from_data(
{"policies": [{"name": "foo", "resource": "ec2"}]}, cfg
)
collection = AWS().initialize_policies(original, cfg)
self.assertEqual(
sorted([p.options.region for p in collection]),
["cn-north-1", "us-gov-west-1", "us-west-2"],
)
def get_azure_output(self):
output = AzureStorageOutput(
ExecutionContext(
None,
Bag(name="xyz"),
Config.empty(output_dir="azure://mystorage.blob.core.windows.net/logs"),
)
)
self.addCleanup(shutil.rmtree, output.root_dir)
return output
def test_check_permissions(self):
load_resources(('gcp.*', ))
missing = []
invalid = []
iam_path = os.path.join(os.path.dirname(__file__), 'data',
'iam-permissions.json')
with open(iam_path) as fh:
valid_perms = set(json.load(fh).get('permissions'))
cfg = Config.empty()
for k, v in resources.items():
policy = Bag({
'name': 'permcheck',
'resource': 'gcp.%s' % k,
'provider_name': 'gcp'
})
ctx = self.get_context(config=cfg, policy=policy)
mgr = v(ctx, policy)
perms = mgr.get_permissions()
if not perms:
missing.append(k)
for p in perms:
if p not in valid_perms:
invalid.append((k, p))
for n, a in list(v.action_registry.items()):
if n in ALLOWED_NOPERM:
continue
policy['actions'] = [n]
perms = a({}, mgr).get_permissions()
if not perms:
missing.append('%s.actions.%s' % (k, n))
for p in perms:
if p not in valid_perms:
invalid.append(('%s.actions.%s' % (k, n), p))
for n, f in list(v.filter_registry.items()):
if n in ALLOWED_NOPERM:
continue
policy['filters'] = [n]
perms = f({}, mgr).get_permissions()
if not perms:
missing.append('%s.filters.%s' % (k, n))
for p in perms:
if p not in valid_perms:
invalid.append(('%s.filters.%s' % (k, n), p))
if missing:
self.fail('missing permissions %d on \n\t%s' %
(len(missing), '\n\t'.join(sorted(missing))))
if invalid:
self.fail('invalid permissions %d on \n\t%s' %
(len(invalid), '\n\t'.join(map(str, sorted(invalid)))))
def test_initialize_default_account_id(self):
# Patch get_subscription_id during provider initialization
with patch('c7n_azure.session.Session.get_subscription_id',
return_value=DEFAULT_SUBSCRIPTION_ID):
options = Config.empty()
azure = Azure()
azure.initialize(options)
self.assertEqual(options['account_id'], DEFAULT_SUBSCRIPTION_ID)
session = azure.get_session_factory(options)()
self.assertEqual(DEFAULT_SUBSCRIPTION_ID, session.get_subscription_id())
def test_initialize_custom_account_id(self, get_subscription_id_mock):
sample_account_id = "00000000-5106-4743-99b0-c129bfa71a47"
options = Config.empty()
options['account_id'] = sample_account_id
azure = Azure()
azure.initialize(options)
self.assertEqual(options['account_id'], sample_account_id)
session = azure.get_session_factory(options)()
session._initialize_session()
self.assertEqual(session.subscription_id, sample_account_id)
def _policy_file_rev(self, f, commit):
try:
return self._validate_policies(
PolicyCollection.from_data(
yaml.safe_load(self.repo.get(commit.tree[f].id).data),
Config.empty(), f))
except Exception as e:
log.warning(
"invalid policy file %s @ %s %s %s \n error:%s",
f, str(commit.id)[:6], commit_date(commit).isoformat(),
commit.author.name, e)
return PolicyCollection()
def get_s3_output(self):
output_dir = "s3://cloud-custodian/policies"
output = S3Output(
ExecutionContext(
None,
Bag(name="xyz", provider_name="ostack"),
Config.empty(output_dir=output_dir)),
{'url': output_dir})
self.addCleanup(shutil.rmtree, output.root_dir)
return output
def test_iam_permissions_validity(self):
cfg = Config.empty()
missing = set()
invalid = []
perms = load_data('iam-actions.json')
resources.load_available()
for k, v in manager.resources.items():
p = Bag({
'name': 'permcheck',
'resource': k,
'provider_name': 'aws'
})
ctx = self.get_context(config=cfg, policy=p)
mgr = v(ctx, p)
# if getattr(mgr, 'permissions', None):
# print(mgr)
found = False
for s in (mgr.resource_type.service,
getattr(mgr.resource_type, 'permission_prefix', None)):
if s in perms:
found = True
if not found:
missing.add("%s->%s" % (k, mgr.resource_type.service))
continue
invalid.extend(
self.check_permissions(perms, mgr.get_permissions(), k))
for n, a in v.action_registry.items():
p['actions'] = [n]
invalid.extend(
self.check_permissions(perms,
a({}, mgr).get_permissions(),
"{k}.actions.{n}".format(k=k, n=n)))
for n, f in v.filter_registry.items():
if n in ('or', 'and', 'not', 'missing'):
continue
p['filters'] = [n]
invalid.extend(
self.check_permissions(perms,
f({}, mgr).get_permissions(),
"{k}.filters.{n}".format(k=k, n=n)))
if missing:
raise ValueError("resources missing service %s" %
('\n'.join(sorted(missing))))
if invalid:
raise ValueError("invalid permissions \n %s" %
('\n'.join(sorted(map(str, invalid)))))
def test_initialize_default_azure_cloud(self):
with patch('c7n_azure.session.Session.get_subscription_id'):
options = Config.empty()
azure = Azure()
azure.initialize(options)
self.assertEqual(AZURE_PUBLIC_CLOUD, azure.cloud_endpoints)
self.assertEqual(AZURE_PUBLIC_CLOUD.name, options['region'])
session = azure.get_session_factory(options)()
self.assertEqual(
AZURE_PUBLIC_CLOUD.endpoints.active_directory_resource_id,
session.resource_endpoint)
def get_azure_output(self, custom_pyformat=None):
output_dir = "azure://mystorage.blob.core.windows.net/logs"
if custom_pyformat:
output_dir = AzureStorageOutput.join(output_dir, custom_pyformat)
output = AzureStorageOutput(
ExecutionContext(None, Bag(name="xyz", provider_name='azure'),
Config.empty(output_dir=output_dir)),
{'url': output_dir},
)
self.addCleanup(shutil.rmtree, output.root_dir)
return output
def report_account(account, region, policies_config, output_path, cache_path,
debug):
output_path = os.path.join(output_path, account['name'], region)
cache_path = os.path.join(cache_path,
"%s-%s.cache" % (account['name'], region))
load_available()
config = Config.empty(region=region,
output_dir=output_path,
account_id=account['account_id'],
metrics_enabled=False,
cache=cache_path,
log_group=None,
profile=None,
external_id=None)
if account.get('role'):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
records = []
for p in policies:
# initializee policy execution context for output access
p.ctx.initialize()
log.debug("Report policy:%s account:%s region:%s path:%s", p.name,
account['name'], region, output_path)
if p.ctx.output.type == "s3":
delta = timedelta(days=1)
begin_date = datetime.now() - delta
policy_records = record_set(
p.session_factory, p.ctx.output.config['netloc'],
strip_output_path(p.ctx.output.config['path'], p.name),
begin_date)
else:
policy_records = fs_record_set(p.ctx.log_dir, p.name)
for r in policy_records:
r['policy'] = p.name
r['region'] = p.options.region
r['account'] = account['name']
for t in account.get('tags', ()):
if ':' in t:
k, v = t.split(':', 1)
r[k] = v
records.extend(policy_records)
return records
def test_policy_region_expand_global(self):
factory = self.replay_flight_data('test_aws_policy_global_expand')
self.patch(aws, '_profile_session', factory())
original = self.policy_loader.load_data(
{"policies": [
{"name": "foo", "resource": "s3"},
{"name": "iam", "resource": "iam-user"}]},
'memory://',
config=Config.empty(regions=["us-east-1", "us-west-2"]),
)
collection = AWS().initialize_policies(
original, Config.empty(regions=["all"], output_dir="/test/output/"))
self.assertEqual(len(collection.resource_types), 2)
s3_regions = [p.options.region for p in collection if p.resource_type == "s3"]
self.assertTrue("us-east-1" in s3_regions)
self.assertTrue("us-east-2" in s3_regions)
iam = [p for p in collection if p.resource_type == "iam-user"]
self.assertEqual(len(iam), 1)
self.assertEqual(iam[0].options.region, "us-east-1")
self.assertEqual(iam[0].options.output_dir, "/test/output/us-east-1")
# Don't append region when it's already in the path.
collection = AWS().initialize_policies(
original, Config.empty(regions=["all"], output_dir="/test/{region}/output/"))
self.assertEqual(len(collection.resource_types), 2)
iam = [p for p in collection if p.resource_type == "iam-user"]
self.assertEqual(iam[0].options.region, "us-east-1")
self.assertEqual(iam[0].options.output_dir, "/test/{region}/output")
collection = AWS().initialize_policies(
original, Config.empty(regions=["eu-west-1", "eu-west-2"], output_dir="/test/output/")
)
iam = [p for p in collection if p.resource_type == "iam-user"]
self.assertEqual(len(iam), 1)
self.assertEqual(iam[0].options.region, "eu-west-1")
self.assertEqual(iam[0].options.output_dir, "/test/output/eu-west-1")
self.assertEqual(len(collection), 3)
def load_policy(self, data, config=None):
errors = schema_validate({'policies': [data]}, C7N_SCHEMA)
if errors:
raise errors[0]
config = config or {}
temp_dir = self.get_temp_dir()
config['output_dir'] = temp_dir
conf = Config.empty(**config)
p = policy.Policy(data, conf, Session)
p.validate()
return p
def get_s3_output(self, output_url=None, cleanup=True, klass=S3Output):
if output_url is None:
output_url = "s3://cloud-custodian/policies"
output = klass(
ExecutionContext(
lambda assume=False: mock.MagicMock(),
Bag(name="xyz", provider_name="ostack"),
Config.empty(output_dir=output_url, account_id='112233445566')),
{'url': output_url, 'test': True})
if cleanup:
self.addCleanup(shutil.rmtree, output.root_dir)
return output
def get_blob_output(request, output_url=None, cleanup=True):
if output_url is None:
output_url = "gs://cloud-custodian/policies"
output = GCPStorageOutput(
ExecutionContext(lambda assume=False: mock.MagicMock(),
Bag(name="xyz", provider_name="gcp"),
Config.empty(output_dir=output_url,
account_id='custodian-test')),
parse_url_config(output_url))
if cleanup:
request.addfinalizer(lambda: shutil.rmtree(output.root_dir)) # noqa
request.addfinalizer(reset_session_cache)
return output
def dispatch_event(event, context):
global account_id
if account_id is None:
session = boto3.Session()
account_id = get_account_id_from_sts(session)
error = event.get('detail', {}).get('errorCode')
if error:
log.debug("Skipping failed operation: %s" % error)
return
event['debug'] = True
if event['debug']:
log.info("Processing event\n %s", format_event(event))
# policies file should always be valid in lambda so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# Initialize output directory, we've seen occassional perm issues with
# lambda on temp directory and changing unix execution users, so
# use a per execution temp space.
output_dir = os.environ.get(
'C7N_OUTPUT_DIR',
'/tmp/' + str(uuid.uuid4()))
if not os.path.exists(output_dir):
try:
os.mkdir(output_dir)
except OSError as error:
log.warning("Unable to make output directory: {}".format(error))
# TODO. This enshrines an assumption of a single policy per lambda.
options_overrides = policy_config[
'policies'][0].get('mode', {}).get('execution-options', {})
options_overrides['account_id'] = account_id
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = output_dir
options = Config.empty(**options_overrides)
load_resources()
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def cli(**kwargs):
policy_config = Config.empty()
policies = PolicyCollection([
p for p in load_policies(kwargs['config'], policy_config)
if p.provider_name == 'azure'], policy_config)
session = policies.policies[0].session_factory()
web_client = session.client('azure.mgmt.web.WebSiteManagementClient')
deployments = {}
credentials_load_failed = 0
not_functions_policy = 0
for p in policies:
if not p.is_lambda:
not_functions_policy += 1
continue
try:
params = AzureFunctionMode(p).get_function_app_params()
creds = web_client.web_apps.begin_list_publishing_credentials(
params.function_app_resource_group_name,
params.function_app_name).result()
deployments[p.name] = {'scm_uri': creds.scm_uri, 'status': None}
log.info('Retrieved deployment credentials for %s policy', p.name)
except Exception:
log.error('Unable to retrieve deployment credentials for %s policy', p.name)
credentials_load_failed += 1
wait_for_remote_builds(deployments)
success = 0
fail = 0
not_found = 0
for name, params in deployments.items():
if params['status'] != DeploymentStatus.Succeeded:
log.info('%s: %s', name, params['status'])
if params['status'] == DeploymentStatus.Failed:
log.error('Build logs can be retrieved here: %s', params['scm_uri'])
fail += 1
else:
not_found += 1
else:
success += 1
log.info('Policies total: %i, unable to load credentials: %i, not Functions mode: %i',
len(policies), credentials_load_failed, not_functions_policy)
log.info('Status not found can happen if Linux Consumption function was deployed'
'more than 2 hours ago.')
log.info('Deployments complete. Success: %i, Fail: %i, Status not found: %i',
success, fail, not_found)
def load_policy(
self, data, config=None):
errors = schema_validate({'policies': [data]}, C7N_SCHEMA)
if errors:
raise errors[0]
config = config or {}
temp_dir = self.get_temp_dir()
config['output_dir'] = temp_dir
conf = Config.empty(**config)
p = policy.Policy(data, conf, Session)
p.validate()
return p
def exec(command: str, config: C7nDefaults = C7nDefaults(), policies: Iterable[Path] = ()):
def _new_cfg(policy):
# TODO: Use account_id rather than profile name
profile = config.profile if config.profile else "default"
return replace(
config,
cache=str(Path(".cache", profile, policy.stem).with_suffix(".cache")),
configs=[str(policy)],
output_dir=str(Path("output", profile)),
)
c7n_cmd = getattr(c7n.commands, command)
cfg_gen = (Config.empty(**asdict(_new_cfg(policy_))) for policy_ in policies)
with ThreadPoolExecutor() as executor:
list(executor.map(c7n_cmd, cfg_gen))
def test_local_session_region(self):
policies = [
self.load_policy({
'name': 'ec2',
'resource': 'ec2'
},
config=Config.empty(region="us-east-1")),
self.load_policy({
'name': 'ec2',
'resource': 'ec2'
},
config=Config.empty(region='us-west-2'))
]
previous = None
previous_region = None
for p in policies:
self.assertEqual(p.options.region, p.session_factory.region)
session = utils.local_session(p.session_factory)
self.assertNotEqual(session.region_name, previous_region)
self.assertNotEqual(session, previous)
previous = session
previous_region = p.options.region
self.assertEqual(utils.local_session(p.session_factory), previous)
def run(self):
logging.getLogger('botocore').setLevel(logging.ERROR)
logging.getLogger('c7n.cache').setLevel(logging.WARNING)
conf = Config.empty(
config_files=['custodian_%s.yml' % self.region_name],
region=self.region_name,
prefix='custodian-',
assume=None,
policy_filter=None,
log_group=None,
external_id=None,
cache_period=0,
cache=None)
resources.load_resources()
policies = load_policies(conf)
resources_gc_prefix(conf, policies)
def is_valid(policy_file: Union[Path, PathLike]) -> bool:
"""
Args:
policy_file: file to be validated
Returns:
True if policy is valid
"""
try:
c7n.commands.validate(Config.empty(configs={Path(policy_file).resolve()}))
except SystemExit:
_LOGGER.debug("Invalid policy %s", policy_file.name)
else:
_LOGGER.debug("Validated policy %s", policy_file.name)
return True
return False
def get_azure_output(self, custom_pyformat=None):
output_dir = "azure://mystorage.blob.core.windows.net/logs"
if custom_pyformat:
output_dir = AzureStorageOutput.join(output_dir, custom_pyformat)
output = AzureStorageOutput(
ExecutionContext(
None,
Bag(name="xyz", provider_name='azure'),
Config.empty(output_dir=output_dir)
),
{'url': output_dir},
)
self.addCleanup(shutil.rmtree, output.root_dir)
return output
def dispatch_event(event, context):
global account_id
if account_id is None:
session = boto3.Session()
account_id = get_account_id_from_sts(session)
error = event.get('detail', {}).get('errorCode')
if error:
log.debug("Skipping failed operation: %s" % error)
return
event['debug'] = True
if event['debug']:
log.info("Processing event\n %s", format_event(event))
# policies file should always be valid in lambda so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# Initialize output directory, we've seen occassional perm issues with
# lambda on temp directory and changing unix execution users, so
# use a per execution temp space.
output_dir = os.environ.get('C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4()))
if not os.path.exists(output_dir):
try:
os.mkdir(output_dir)
except OSError as error:
log.warning("Unable to make output directory: {}".format(error))
# TODO. This enshrines an assumption of a single policy per lambda.
options_overrides = policy_config['policies'][0].get('mode', {}).get(
'execution-options', {})
options_overrides['account_id'] = account_id
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = output_dir
options = Config.empty(**options_overrides)
load_resources()
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def test_initialize_azure_cloud(self):
clouds = [
AZURE_PUBLIC_CLOUD, AZURE_CHINA_CLOUD, AZURE_GERMAN_CLOUD,
AZURE_US_GOV_CLOUD
]
with patch('c7n_azure.session.Session.get_subscription_id'):
for cloud_endpoints in clouds:
options = Config.empty(regions=[cloud_endpoints.name])
azure = Azure()
azure.initialize(options)
self.assertEqual(cloud_endpoints, azure.cloud_endpoints)
self.assertEqual(cloud_endpoints.name, options['region'])
session = azure.get_session_factory(options)()
self.assertEqual(
cloud_endpoints.endpoints.active_directory_resource_id,
session.resource_endpoint)
def build_options(output_dir=None, log_group=None, metrics=None):
"""
Initialize the Azure provider to apply global config across all policy executions.
"""
if not output_dir:
output_dir = tempfile.mkdtemp()
log.warning('Output directory not specified. Using directory: %s' % output_dir)
config = Config.empty(
**{
'log_group': log_group,
'metrics': metrics,
'output_dir': output_dir
}
)
return Azure().initialize(config)
def test_doc_examples(provider_name):
load_resources()
loader = PolicyLoader(Config.empty())
provider = clouds.get(provider_name)
policies = get_doc_policies(provider.resources)
for p in policies.values():
loader.load_data({'policies': [p]}, 'memory://')
for p in policies.values():
# Note max name size here is 54 if it a lambda policy given
# our default prefix custodian- to stay under 64 char limit on
# lambda function names. This applies to AWS and GCP, and
# afaict Azure.
if len(p['name']) >= 54 and 'mode' in p:
raise ValueError(
"doc policy exceeds name limit policy:%s" % (p['name']))
def main():
parser = setup_parser()
options = parser.parse_args()
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(
level=log_level,
format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
logging.getLogger('botocore').setLevel(logging.ERROR)
logging.getLogger('urllib3').setLevel(logging.ERROR)
logging.getLogger('c7n.cache').setLevel(logging.WARNING)
if not options.policy_regex:
options.policy_regex = f"^{options.prefix}.*"
if not options.regions:
options.regions = [os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')]
files = []
files.extend(itertools.chain(*options.config_files))
files.extend(options.configs)
options.config_files = files
options.policy_tags = list(itertools.chain(*options.policy_tags))
if not files:
parser.print_help()
sys.exit(1)
policy_config = Config.empty(
regions=options.regions,
profile=options.profile,
assume_role=options.assume_role)
# use cloud provider to initialize policies to get region expansion
policies = AWS().initialize_policies(
PolicyCollection([
p for p in load_policies(
options, policy_config)
if p.provider_name == 'aws'],
policy_config),
policy_config)
resources_gc_prefix(options, policy_config, policies)
def load_policy(
self, data, config=None, session_factory=None,
validate=C7N_VALIDATE, output_dir=None, cache=False):
if validate:
errors = schema_validate({'policies': [data]}, C7N_SCHEMA)
if errors:
raise errors[0]
config = config or {}
if not output_dir:
temp_dir = self.get_temp_dir()
config['output_dir'] = temp_dir
if cache:
config['cache'] = os.path.join(temp_dir, 'c7n.cache')
config['cache_period'] = 300
conf = Config.empty(**config)
p = policy.Policy(data, conf, session_factory)
p.validate()
return p
def tag(assume,
region,
db,
creator_tag,
user_suffix,
dryrun,
summary=True,
profile=None,
type=()):
"""Tag resources with their creator.
"""
trail_db = TrailDB(db)
load_resources(resource_types=('aws.*', ))
with temp_dir() as output_dir:
config = ExecConfig.empty(output_dir=output_dir,
assume=assume,
region=region,
profile=profile)
factory = aws.AWS().get_session_factory(config)
account_id = local_session(factory).client(
'sts').get_caller_identity().get('Account')
config['account_id'] = account_id
tagger = ResourceTagger(trail_db, config, creator_tag, user_suffix,
dryrun, type)
try:
stats = tagger.process()
except Exception:
log.exception(
"error processing account:%s region:%s config:%s env:%s",
account_id, region, config, dict(os.environ))
raise
if not summary:
return stats
log.info(
"auto tag summary account:%s region:%s \n%s", config['account_id'],
config['region'],
"\n".join([" {}: {}".format(k, v) for k, v in stats.items() if v]))
total = sum([v for k, v in stats.items() if not k.endswith('not-found')])
log.info("Total resources tagged: %d" % total)
def main():
parser = setup_parser()
options = parser.parse_args()
config = Config.empty()
resources.load_resources()
collection = policy_load(
config, options.config_file).filter(options.policy_filter)
sam = {
'AWSTemplateFormatVersion': '2010-09-09',
'Transform': 'AWS::Serverless-2016-10-31',
'Resources': {}}
for p in collection:
if p.provider_name != 'aws':
continue
exec_mode_type = p.data.get('mode', {'type': 'pull'}).get('type')
if exec_mode_type == 'pull':
continue
sam_func = render(p)
if sam_func:
sam['Resources'][resource_name(p.name)] = sam_func
if exec_mode_type == 'config-rule' or exec_mode_type == 'config-poll-rule':
configrule_resource = render_config_rule(p)
invoke_resource = render_invoke(resource_name(p.name))
sam['Resources'][resource_name(p.name) + "Invoke"] = invoke_resource
sam['Resources'][resource_name(p.name) + "ConfigRule"] = configrule_resource
sam_func['Properties']['CodeUri'] = './%s.zip' % p.name
else:
print("unable to render sam for policy:%s" % p.name)
continue
archive = mu.PolicyLambda(p).get_archive()
with open(os.path.join(options.output_dir, "%s.zip" % p.name), 'wb') as fh:
fh.write(archive.get_bytes())
with open(os.path.join(options.output_dir, 'deploy.yml'), 'w') as fh:
fh.write(yaml.safe_dump(sam, default_flow_style=False))
def test_policy_expand_group_region(self):
cfg = Config.empty(regions=["us-east-1", "us-east-2", "us-west-2"])
original = policy.PolicyCollection.from_data(
{"policies": [
{"name": "bar", "resource": "lambda"},
{"name": "middle", "resource": "security-group"},
{"name": "foo", "resource": "ec2"}]},
cfg)
collection = AWS().initialize_policies(original, cfg)
self.assertEqual(
[(p.name, p.options.region) for p in collection],
[('bar', 'us-east-1'),
('middle', 'us-east-1'),
('foo', 'us-east-1'),
('bar', 'us-east-2'),
('middle', 'us-east-2'),
('foo', 'us-east-2'),
('bar', 'us-west-2'),
('middle', 'us-west-2'),
('foo', 'us-west-2')])
def run(event, context, subscription_id=None):
# policies file should always be valid in functions so do loading naively
with open(context['config_file']) as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
log.error('Invalid policy config')
return False
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
# setup our auth file location on disk
options_overrides['authorization_file'] = context['auth_file']
# if output_dir specified use that, otherwise make a temp directory
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = get_tmp_output_dir()
# merge all our options in
options = Config.empty(**options_overrides)
if subscription_id is not None:
options['account_id'] = subscription_id
load_resources(StructureParser().get_resource_types(policy_config))
options = Azure().initialize(options)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
try:
p.push(event, context)
except (CloudError, AzureHttpError) as error:
log.error("Unable to process policy: %s :: %s" %
(p.name, error))
reset_session_cache()
return True
def main():
parser = setup_parser()
options = parser.parse_args()
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(
level=log_level,
format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
logging.getLogger('botocore').setLevel(logging.ERROR)
logging.getLogger('c7n.cache').setLevel(logging.WARNING)
if not options.regions:
options.regions = [os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')]
files = []
files.extend(itertools.chain(*options.config_files))
files.extend(options.configs)
options.config_files = files
if not files:
parser.print_help()
sys.exit(1)
policy_config = Config.empty(
regions=options.regions,
profile=options.profile,
assume_role=options.assume_role)
# use cloud provider to initialize policies to get region expansion
policies = AWS().initialize_policies(
PolicyCollection([
p for p in load_policies(
options, policy_config)
if p.provider_name == 'aws'],
policy_config),
policy_config)
resources_gc_prefix(options, policy_config, policies)
def run_account(account, region, policies_config, output_path,
cache_period, cache_path, metrics, dryrun, debug):
"""Execute a set of policies on an account.
"""
logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
CONN_CACHE.session = None
CONN_CACHE.time = None
# allow users to specify interpolated output paths
if '{' not in output_path:
output_path = os.path.join(output_path, account['name'], region)
cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['account_id'], region))
config = Config.empty(
region=region, cache=cache_path,
cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
account_id=account['account_id'], metrics_enabled=metrics,
log_group=None, profile=None, external_id=None)
env_vars = account_tags(account)
if account.get('role'):
if isinstance(account['role'], six.string_types):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
else:
env_vars.update(
_get_env_creds(get_session(account, 'custodian', region), region))
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
policy_counts = {}
success = True
st = time.time()
with environ(**env_vars):
for p in policies:
# Variable expansion and non schema validation (not optional)
p.expand_variables(p.get_variables(account.get('vars', {})))
p.validate()
log.debug(
"Running policy:%s account:%s region:%s",
p.name, account['name'], region)
try:
resources = p.run()
policy_counts[p.name] = resources and len(resources) or 0
if not resources:
continue
log.info(
"Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
account['name'], region, p.name, len(resources),
time.time() - st)
except ClientError as e:
success = False
if e.response['Error']['Code'] == 'AccessDenied':
log.warning('Access denied account:%s region:%s',
account['name'], region)
return policy_counts, success
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
continue
except Exception as e:
success = False
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
if not debug:
continue
import traceback, pdb, sys
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
raise
return policy_counts, success
def _load_policies(options):
config = Config()
config.from_cli(options)
collection = policy.load(options, options.config)
return f(config, collection)
def __init__(self, policies=None, options=None):
self.policies = policies or []
self.options = options or Config.empty()
self.pmap = {p.name: p for p in self.policies}
def report(config, output, use, output_dir, accounts,
field, no_default_fields, tags, region, debug, verbose,
policy, policy_tags, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(
config, use, debug, verbose, accounts, tags, policy,
resource=resource, policy_tags=policy_tags)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
elif not len(custodian_config['policies']) > 0:
raise ValueError("no matching policies found")
records = []
with executor(max_workers=WORKER_COUNT) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
for r in resolve_regions(region or a.get('regions', ())):
futures[w.submit(
report_account,
a, r,
custodian_config,
output_dir,
debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning(
"Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug(
"Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Config.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(
factory.resource_type,
extra_fields=field,
include_default_fields=not(no_default_fields),
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = UnicodeWriter(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
def run_account(account, region, policies_config, output_path,
cache_period, metrics, dryrun, debug):
"""Execute a set of policies on an account.
"""
logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
CONN_CACHE.session = None
CONN_CACHE.time = None
output_path = os.path.join(output_path, account['name'], region)
if not os.path.exists(output_path):
os.makedirs(output_path)
cache_path = os.path.join(output_path, "c7n.cache")
config = Config.empty(
region=region,
cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
account_id=account['account_id'], metrics_enabled=metrics,
cache=cache_path, log_group=None, profile=None, external_id=None)
if account.get('role'):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
policy_counts = {}
st = time.time()
with environ(**account_tags(account)):
for p in policies:
log.debug(
"Running policy:%s account:%s region:%s",
p.name, account['name'], region)
try:
resources = p.run()
policy_counts[p.name] = resources and len(resources) or 0
if not resources:
continue
log.info(
"Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
account['name'], region, p.name, len(resources),
time.time() - st)
except ClientError as e:
if e.response['Error']['Code'] == 'AccessDenied':
log.warning('Access denied account:%s region:%s',
account['name'], region)
return policy_counts
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
continue
except Exception as e:
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
if not debug:
continue
import traceback, pdb, sys
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
raise
return policy_counts
def dispatch_event(event, context):
global account_id
error = event.get('detail', {}).get('errorCode')
if error:
log.debug("Skipping failed operation: %s" % error)
return
event['debug'] = True
if event['debug']:
log.info("Processing event\n %s", format_event(event))
# Policies file should always be valid in lambda so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# Initialize output directory, we've seen occassional perm issues with
# lambda on temp directory and changing unix execution users, so
# use a per execution temp space.
output_dir = os.environ.get(
'C7N_OUTPUT_DIR',
'/tmp/' + str(uuid.uuid4()))
if not os.path.exists(output_dir):
try:
os.mkdir(output_dir)
except OSError as error:
log.warning("Unable to make output directory: {}".format(error))
# TODO. This enshrines an assumption of a single policy per lambda.
options_overrides = policy_config[
'policies'][0].get('mode', {}).get('execution-options', {})
# if using assume role in lambda ensure that the correct
# execution account is captured in options.
if 'assume_role' in options_overrides:
account_id = options_overrides['assume_role'].split(':')[4]
elif account_id is None:
session = boto3.Session()
account_id = get_account_id_from_sts(session)
# Historical compatibility with manually set execution options
# previously this was a boolean, its now a string value with the
# boolean flag triggering a string value of 'aws'
if 'metrics_enabled' in options_overrides and isinstance(
options_overrides['metrics_enabled'], bool):
options_overrides['metrics_enabled'] = 'aws'
options_overrides['account_id'] = account_id
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = output_dir
options = Config.empty(**options_overrides)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
