def test_policy_region_expand_global(self):
factory = self.replay_flight_data('test_aws_policy_global_expand')
self.patch(aws, '_profile_session', factory())
original = self.policy_loader.load_data(
{"policies": [
{"name": "foo", "resource": "s3"},
{"name": "iam", "resource": "iam-user"}]},
'memory://',
config=Config.empty(regions=["us-east-1", "us-west-2"]),
)
collection = AWS().initialize_policies(original, Config.empty(regions=["all"]))
self.assertEqual(len(collection.resource_types), 2)
s3_regions = [p.options.region for p in collection if p.resource_type == "s3"]
self.assertTrue("us-east-1" in s3_regions)
self.assertTrue("us-east-2" in s3_regions)
iam = [p for p in collection if p.resource_type == "iam-user"]
self.assertEqual(len(iam), 1)
self.assertEqual(iam[0].options.region, "us-east-1")
collection = AWS().initialize_policies(
original, Config.empty(regions=["eu-west-1", "eu-west-2"])
)
iam = [p for p in collection if p.resource_type == "iam-user"]
self.assertEqual(len(iam), 1)
self.assertEqual(iam[0].options.region, "eu-west-1")
self.assertEqual(len(collection), 3)
def load_policy_set(self, data, config=None):
filename = self.write_policy_file(data, format="json")
if config:
e = Config.empty(**config)
else:
e = Config.empty()
return policy.load(e, filename)
def load_policy_set(self, data, config=None):
filename = self.write_policy_file(data)
if config:
e = Config.empty(**config)
else:
e = Config.empty()
return policy.load(e, filename)
def load_policy_set(self, data, config=None):
filename = self.write_policy_file(data)
if config:
config['account_id'] = ACCOUNT_ID
e = Config.empty(**config)
else:
e = Config.empty(account_id=ACCOUNT_ID)
return policy.load(e, filename)
def main():
parser = setup_parser()
argcomplete.autocomplete(parser)
options = parser.parse_args()
if options.subparser is None:
parser.print_help(file=sys.stderr)
return sys.exit(2)
_setup_logger(options)
# Support the deprecated -c option
if getattr(options, 'config', None) is not None:
options.configs.append(options.config)
config = Config.empty(**vars(options))
try:
command = options.command
if not callable(command):
command = getattr(
importlib.import_module(command.rsplit('.', 1)[0]),
command.rsplit('.', 1)[-1])
# Set the process name to something cleaner
process_name = [os.path.basename(sys.argv[0])]
process_name.extend(sys.argv[1:])
setproctitle(' '.join(process_name))
command(config)
except Exception:
if not options.debug:
raise
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
def tag(assume, region, db, creator_tag, user_suffix, dryrun,
summary=True, profile=None, type=()):
"""Tag resources with their creator.
"""
trail_db = TrailDB(db)
load_resources()
with temp_dir() as output_dir:
config = ExecConfig.empty(
output_dir=output_dir, assume=assume,
region=region, profile=profile)
factory = aws.AWS().get_session_factory(config)
account_id = local_session(factory).client('sts').get_caller_identity().get('Account')
config['account_id'] = account_id
tagger = ResourceTagger(trail_db, config, creator_tag, user_suffix, dryrun, type)
try:
stats = tagger.process()
except Exception:
log.exception(
"error processing account:%s region:%s config:%s env:%s",
account_id, region, config, dict(os.environ))
raise
if not summary:
return stats
log.info(
"auto tag summary account:%s region:%s \n%s",
config['account_id'],
config['region'],
"\n".join([" {}: {}".format(k, v) for k, v in stats.items() if v]))
total = sum([v for k, v in stats.items() if not k.endswith('not-found')])
log.info("Total resources tagged: %d" % total)
def test_cloudtrail_policy():
collection = PolicyLoader(Config.empty()).load_data(
{
'policies': [{
'name': 'check-ec2',
'resource': 'ec2',
'mode': {
'type': 'cloudtrail',
'events': ['RunInstances']
}
}]
},
file_uri=":mem:")
sam = {'Resources': {}}
p = list(collection).pop()
dispatch_render(p, sam)
assert sam['Resources']['CheckEc2']['Properties']['Events'] == {
'PolicyTriggerA': {
'Properties': {
'Pattern': {
'detail': {
'eventName': ['RunInstances'],
'eventSource': ['ec2.amazonaws.com']
},
'detail-type': ['AWS API Call via CloudTrail']
}
},
'Type': 'CloudWatchEvent'
}
}
def run(event, context=None):
# policies file should always be valid in functions so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
log.error('Invalid policy config')
return False
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
# if output_dir specified use that, otherwise make a temp directory
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = get_tmp_output_dir()
# merge all our options in
options = Config.empty(**options_overrides)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
log.info("running policy %s", p.name)
p.push(event, context)
return True
def get_dir_output(self, location):
work_dir = self.change_cwd()
return work_dir, DirectoryOutput(
ExecutionContext(None, Bag(name="xyz", provider_name="ostack"),
Config.empty(output_dir=location)),
{'url': location},
)
def get_context(self, config=None, policy=None):
if config is None:
self.context_output_dir = self.get_temp_dir()
config = Config.empty(output_dir=self.context_output_dir)
ctx = ExecutionContext(Session, policy or Bag({'name': 'test-policy'}),
config)
return ctx
def dispatch_event(event, context):
global account_id
error = event.get('detail', {}).get('errorCode')
if error:
log.debug("Skipping failed operation: %s" % error)
return
event['debug'] = True
if event['debug']:
log.info("Processing event\n %s", format_event(event))
# Policies file should always be valid in lambda so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# Initialize output directory, we've seen occassional perm issues with
# lambda on temp directory and changing unix execution users, so
# use a per execution temp space.
output_dir = os.environ.get('C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4()))
if not os.path.exists(output_dir):
try:
os.mkdir(output_dir)
except OSError as error:
log.warning("Unable to make output directory: {}".format(error))
# TODO. This enshrines an assumption of a single policy per lambda.
options_overrides = policy_config['policies'][0].get('mode', {}).get(
'execution-options', {})
# if using assume role in lambda ensure that the correct
# execution account is captured in options.
if 'assume_role' in options_overrides:
account_id = options_overrides['assume_role'].split(':')[4]
elif account_id is None:
session = boto3.Session()
account_id = get_account_id_from_sts(session)
# Historical compatibility with manually set execution options
# previously this was a boolean, its now a string value with the
# boolean flag triggering a string value of 'aws'
if 'metrics_enabled' in options_overrides and isinstance(
options_overrides['metrics_enabled'], bool):
options_overrides['metrics_enabled'] = 'aws'
options_overrides['account_id'] = account_id
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = output_dir
options = Config.empty(**options_overrides)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def report_account(account, region, policies_config, output_path, debug):
cache_path = os.path.join(output_path, "c7n.cache")
output_path = os.path.join(output_path, account['name'], region)
config = Config.empty(
region=region,
output_dir=output_path,
account_id=account['account_id'], metrics_enabled=False,
cache=cache_path, log_group=None, profile=None, external_id=None)
if account.get('role'):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
records = []
for p in policies:
log.debug(
"Report policy:%s account:%s region:%s path:%s",
p.name, account['name'], region, output_path)
policy_records = fs_record_set(p.ctx.output_path, p.name)
for r in policy_records:
r['policy'] = p.name
r['region'] = p.options.region
r['account'] = account['name']
for t in account.get('tags', ()):
if ':' in t:
k, v = t.split(':', 1)
r[k] = v
records.extend(policy_records)
return records
def main():
parser = setup_parser()
options = parser.parse_args()
config = Config.empty()
resources.load_resources()
collection = policy_load(
config, options.config_file).filter(options.policy_filter)
sam = {
'AWSTemplateFormatVersion': '2010-09-09',
'Transform': 'AWS::Serverless-2016-10-31',
'Resources': {}}
for p in collection:
if p.provider_name != 'aws':
continue
exec_mode_type = p.data.get('mode', {'type': 'pull'}).get('type')
if exec_mode_type == 'pull':
continue
sam_func = render(p)
if sam_func:
sam['Resources'][resource_name(p.name)] = sam_func
sam_func['Properties']['CodeUri'] = './%s.zip' % p.name
else:
print("unable to render sam for policy:%s" % p.name)
continue
archive = mu.PolicyLambda(p).get_archive()
with open(os.path.join(options.output_dir, "%s.zip" % p.name), 'wb') as fh:
fh.write(archive.get_bytes())
with open(os.path.join(options.output_dir, 'deploy.yml'), 'w') as fh:
fh.write(yaml.safe_dump(sam, default_flow_style=False))
def dryrun(self):
# This is largely based off of mugc.main()
logging.getLogger('botocore').setLevel(logging.ERROR)
logging.getLogger('urllib3').setLevel(logging.ERROR)
logging.getLogger('c7n.cache').setLevel(logging.WARNING)
conf = Config.empty(
config_files=['custodian_%s.yml' % self.region_name],
regions=[self.region_name],
prefix=self.config.function_prefix,
policy_regex='^' + re.escape(self.config.function_prefix) + '.*',
assume=None,
policy_filter=None,
log_group=None,
external_id=None,
cache_period=0,
cache=None,
present=False,
dryrun=True
)
# use cloud provider to initialize policies to get region expansion
policies = AWS().initialize_policies(
PolicyCollection(
[
p for p in load_policies(conf, conf)
if p.provider_name == 'aws'
],
conf
),
conf
)
resources_gc_prefix(conf, conf, policies)
def main():
parser = setup_parser()
options = parser.parse_args()
config = Config.empty()
resources.load_resources()
collection = policy_load(
config, options.config_file).filter(options.policy_filter)
sam = {
'AWSTemplateFormatVersion': '2010-09-09',
'Transform': 'AWS::Serverless-2016-10-31',
'Resources': {}}
for p in collection:
if p.provider_name != 'aws':
continue
exec_mode_type = p.data.get('mode', {'type': 'pull'}).get('type')
if exec_mode_type == 'pull':
continue
sam_func = render(p)
if sam_func:
sam['Resources'][resource_name(p.name)] = sam_func
sam_func['Properties']['CodeUri'] = './%s.zip' % p.name
else:
print("unable to render sam for policy:%s" % p.name)
continue
archive = mu.PolicyLambda(p).get_archive()
with open(os.path.join(options.output_dir, "%s.zip" % p.name), 'wb') as fh:
fh.write(archive.get_bytes())
with open(os.path.join(options.output_dir, 'deploy.yml'), 'w') as fh:
fh.write(yaml.safe_dump(sam, default_flow_style=False))
def load_policy(
self,
data,
config=None,
session_factory=None,
validate=C7N_VALIDATE,
output_dir=None,
cache=False,
):
if validate:
if not self.custodian_schema:
self.custodian_schema = generate()
errors = schema_validate({"policies": [data]}, self.custodian_schema)
if errors:
raise errors[0]
config = config or {}
if not output_dir:
temp_dir = self.get_temp_dir()
config["output_dir"] = temp_dir
if cache:
config["cache"] = os.path.join(temp_dir, "c7n.cache")
config["cache_period"] = 300
conf = Config.empty(**config)
p = policy.Policy(data, conf, session_factory)
p.validate()
return p
def report_account(account, region, policies_config, output_path, debug):
cache_path = os.path.join(output_path, "c7n.cache")
output_path = os.path.join(output_path, account['name'], region)
config = Config.empty(
region=region,
output_dir=output_path,
account_id=account['account_id'], metrics_enabled=False,
cache=cache_path, log_group=None, profile=None, external_id=None)
if account.get('role'):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
records = []
for p in policies:
log.debug(
"Report policy:%s account:%s region:%s path:%s",
p.name, account['name'], region, output_path)
policy_records = fs_record_set(p.ctx.output_path, p.name)
for r in policy_records:
r['policy'] = p.name
r['region'] = p.options.region
r['account'] = account['name']
for t in account.get('tags', ()):
if ':' in t:
k, v = t.split(':', 1)
r[k] = v
records.extend(policy_records)
return records
def run(event, context):
# policies file should always be valid in functions so do loading naively
with open(context['config_file']) as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# Initialize output directory
output_dir = os.environ.get(
'C7N_OUTPUT_DIR',
'/tmp/' + str(uuid.uuid4()))
if not os.path.exists(output_dir):
try:
os.mkdir(output_dir)
except OSError as error:
log.warning("Unable to make output directory: {}".format(error))
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
options_overrides['authorization_file'] = context['auth_file']
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = output_dir
options = Config.empty(**options_overrides)
load_resources()
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def run(event, context):
# policies file should always be valid in functions so do loading naively
with open(context['config_file']) as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
log.error('Invalid policy config')
return False
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
# setup our auth file location on disk
options_overrides['authorization_file'] = context['auth_file']
# if output_dir specified use that, otherwise make a temp directory
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = get_tmp_output_dir()
# merge all our options in
options = Config.empty(**options_overrides)
load_resources()
options = Azure().initialize(options)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
try:
p.push(event, context)
except (CloudError, AzureHttpError) as error:
log.error("Unable to process policy: %s :: %s" % (p.name, error))
return True
def eperm(provider, el, r=None):
if el.permissions:
return el.permissions
element_type = get_element_type(el)
if r is None or r.type is None:
# dummy resource type for policy
if provider == 'aws':
r = Bag({'type': 'kinesis'})
elif provider == 'gcp':
r = Bag({'type': 'instance'})
elif provider == 'azure':
r = Bag({'type': 'vm'})
# print(f'policy construction lookup {r.type}.{element_type}.{el.type}')
loader = PolicyLoader(Config.empty())
pdata = {
'name': f'permissions-{r.type}',
'resource': f'{provider}.{r.type}'
}
pdata[element_type] = get_element_data(element_type, el)
try:
pset = loader.load_data({'policies': [pdata]}, ':mem:', validate=False)
except Exception as e:
print(f'error loading {el} as {element_type}:{el.type} error: {e} \n {pdata}')
return []
el = get_policy_element(el, list(pset)[0])
return el.get_permissions()
def validate(options):
load_resources()
if len(options.configs) < 1:
log.error('no config files specified')
sys.exit(1)
used_policy_names = set()
schm = schema.generate()
errors = []
for config_file in options.configs:
config_file = os.path.expanduser(config_file)
if not os.path.exists(config_file):
raise ValueError("Invalid path for config %r" % config_file)
options.dryrun = True
fmt = config_file.rsplit('.', 1)[-1]
with open(config_file) as fh:
if fmt in ('yml', 'yaml'):
data = yaml.safe_load(fh.read())
elif fmt in ('json', ):
data = json.load(fh)
else:
log.error("The config file must end in .json, .yml or .yaml.")
raise ValueError(
"The config file must end in .json, .yml or .yaml.")
errors += schema.validate(data, schm)
conf_policy_names = {
p.get('name', 'unknown')
for p in data.get('policies', ())
}
dupes = conf_policy_names.intersection(used_policy_names)
if len(dupes) >= 1:
errors.append(
ValueError(
"Only one policy with a given name allowed, duplicates: %s"
% (", ".join(dupes))))
used_policy_names = used_policy_names.union(conf_policy_names)
if not errors:
null_config = Config.empty(dryrun=True,
account_id='na',
region='na')
for p in data.get('policies', ()):
try:
policy = Policy(p, null_config, Bag())
policy.validate()
except Exception as e:
msg = "Policy: %s is invalid: %s" % (p.get(
'name', 'unknown'), e)
errors.append(msg)
if not errors:
log.info("Configuration valid: {}".format(config_file))
continue
log.error("Configuration invalid: {}".format(config_file))
for e in errors:
log.error("%s" % e)
if errors:
sys.exit(1)
def test_policy_resource_limits_count(self):
session_factory = self.replay_flight_data(
"test_policy_resource_count")
p = self.load_policy(
{
"name": "ecs-cluster-resource-count",
"resource": "ecs",
"max-resources": 1
},
session_factory=session_factory)
self.assertRaises(ResourceLimitExceeded, p.run)
policy = {
"name": "ecs-cluster-resource-count",
"resource": "ecs",
"max-resources": 0
}
config = Config.empty(validate=True)
self.assertRaises(
Exception,
self.load_policy,
policy,
config=config,
validate=True,
session_factory=session_factory
)
def run(self):
"""
Perform an actual run of cloud-custodian.
This replicates the command:
custodian run --region '${region}' --metrics -v -s \
cloud-custodian-${account_id}-${region}/logs \
--log-group=/cloud-custodian/${account_id}/${region} \
-c custodian_${region}.yml \
--cache '/tmp/.cache/cloud-custodian.cache'
"""
conf = Config.empty(
configs=['custodian_%s.yml' % self.region_name],
region=self.region_name,
regions=[self.region_name],
log_group=self.config.custodian_log_group,
verbose=1,
metrics_enabled=True,
subparser='run',
cache='/tmp/.cache/cloud-custodian.cache',
command='c7n.commands.run',
output_dir='%s/logs' % self.config.output_s3_bucket_name,
vars=None,
dryrun=False
)
run(conf)
def test_verify_parent_filter(self):
manager = KeyVaultKeys(
ExecutionContext(None, Bag(name="xyz", provider_name='azure'),
Config.empty()), {
'name':
'test-policy',
'resource':
'azure.keyvault-keys',
'filters': [{
'type': 'parent',
'filter': {
'type': 'value',
'key': 'name',
'op': 'glob',
'value': 'cctestkv*'
}
}]
})
self.assertEqual(len(manager.filters), 1)
filter = manager.filters[0]
self.assertTrue(isinstance(filter, ParentFilter))
self.assertTrue(isinstance(filter.parent_manager, KeyVault))
self.assertTrue(isinstance(filter.parent_filter, ValueFilter))
def run(event, context=None):
# policies file should always be valid in functions so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
log.error('Invalid policy config')
return False
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
# if output_dir specified use that, otherwise make a temp directory
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = get_tmp_output_dir()
# merge all our options in
options = Config.empty(**options_overrides)
loader = PolicyLoader(options)
policies = loader.load_data(policy_config, 'config.json', validate=False)
if policies:
for p in policies:
log.info("running policy %s", p.name)
p.validate()
p.push(event, context)
return True
def get_context(self, config=None, session_factory=None, policy=None):
if config is None:
self.context_output_dir = self.get_temp_dir()
config = Config.empty(output_dir=self.context_output_dir)
ctx = ExecutionContext(session_factory, policy
or Bag({"name": "test-policy"}), config)
return ctx
def main():
parser = setup_parser()
options = parser.parse_args()
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(
level=log_level,
format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
logging.getLogger('botocore').setLevel(logging.ERROR)
logging.getLogger('c7n.cache').setLevel(logging.WARNING)
if not options.regions:
options.regions = [os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')]
files = []
files.extend(itertools.chain(*options.config_files))
files.extend(options.configs)
options.config_files = files
if not files:
parser.print_help()
sys.exit(1)
policy_config = Config.empty(regions=options.regions,
profile=options.profile,
assume_role=options.assume_role)
# use cloud provider to initialize policies to get region expansion
policies = AWS().initialize_policies(load_policies(options, policy_config),
policy_config)
resources_gc_prefix(options, policy_config, policies)
def load_policy(
self,
data,
config=None,
session_factory=None,
validate=C7N_VALIDATE,
output_dir=None,
cache=False,
):
if validate:
if not self.custodian_schema:
self.custodian_schema = generate()
errors = schema_validate({"policies": [data]},
self.custodian_schema)
if errors:
raise errors[0]
config = config or {}
if not output_dir:
temp_dir = self.get_temp_dir()
config["output_dir"] = temp_dir
if cache:
config["cache"] = os.path.join(temp_dir, "c7n.cache")
config["cache_period"] = 300
conf = Config.empty(**config)
p = policy.Policy(data, conf, session_factory)
p.validate()
return p
def main():
parser = setup_parser()
argcomplete.autocomplete(parser)
options = parser.parse_args()
_setup_logger(options)
# Support the deprecated -c option
if getattr(options, 'config', None) is not None:
options.configs.append(options.config)
config = Config.empty(**vars(options))
try:
command = options.command
if not callable(command):
command = getattr(
importlib.import_module(command.rsplit('.', 1)[0]),
command.rsplit('.', 1)[-1])
# Set the process name to something cleaner
process_name = [os.path.basename(sys.argv[0])]
process_name.extend(sys.argv[1:])
setproctitle(' '.join(process_name))
command(config)
except Exception:
if not options.debug:
raise
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
def test_resource_shadow_source_augment(self):
shadowed = []
bad = []
cfg = Config.empty()
for k, v in manager.resources.items():
if not getattr(v.resource_type, "config_type", None):
continue
p = Bag({
"name": "permcheck",
"resource": k,
'provider_name': 'aws'
})
ctx = self.get_context(config=cfg, policy=p)
mgr = v(ctx, p)
source = mgr.get_source("config")
if not isinstance(source, ConfigSource):
bad.append(k)
if v.__dict__.get("augment"):
shadowed.append(k)
if shadowed:
self.fail("%s have resource managers shadowing source augments" %
(", ".join(shadowed)))
if bad:
self.fail("%s have config types but no config source" %
(", ".join(bad)))
def run(event, context):
# policies file should always be valid in functions so do loading naively
with open(context['config_file']) as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
log.error('Invalid policy config')
return False
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
# setup our auth file location on disk
options_overrides['authorization_file'] = context['auth_file']
# if output_dir specified use that, otherwise make a temp directory
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = get_tmp_output_dir()
# merge all our options in
options = Config.empty(**options_overrides)
load_resources()
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
class HandlerTest(BaseTest):
@patch('c7n_azure.provider.Azure.initialize', return_value=Config.empty())
@patch('azure.common.credentials.ServicePrincipalCredentials.__init__',
return_value=None)
@patch('c7n.policy.Policy.push')
def test_run(self, push_mock, _1, initialize_mock):
context = {
'config_file': join(dirname(__file__), 'data', 'test_config.json'),
'auth_file': join(dirname(__file__), 'data', 'test_auth_file.json')
}
self.assertTrue(run(None, context, CUSTOM_SUBSCRIPTION_ID))
push_mock.assert_called_once()
self.assertEqual(push_mock.call_args_list[0], call(None, context))
initialize_mock.assert_called_once()
self.assertEqual(initialize_mock.call_args_list[0][0][0]['account_id'],
CUSTOM_SUBSCRIPTION_ID)
self.assertEqual(
initialize_mock.call_args_list[0][0][0]['authorization_file'],
context['auth_file'])
self.assertEqual(
initialize_mock.call_args_list[0][0][0]['test_option'],
"test_value")
def test_run_empty_policy(self):
context = {
'config_file':
join(dirname(__file__), 'data', 'test_config_empty.json'),
'auth_file':
join(dirname(__file__), 'data', 'test_auth_file.json')
}
self.assertFalse(run(None, context))
def init_config(policy_config):
"""Get policy lambda execution configuration.
cli parameters are serialized into the policy lambda config,
we merge those with any policy specific execution options.
--assume role and -s output directory get special handling, as
to disambiguate any cli context.
account id is sourced from the config options or from api call
and cached as a global.
Todo: this should get refactored out to mu.py as part of the
write out of configuration, instead of runtime processed.
"""
exec_options = policy_config.get('execution-options', {})
# Remove some configuration options that don't make sense to translate from
# cli to lambda automatically.
# - assume role on cli doesn't translate, it is the default lambda role and
# used to provision the lambda.
# - profile doesnt translate to lambda its `home` dir setup dependent
# - dryrun doesn't translate (and shouldn't be present)
# - region doesn't translate from cli (the lambda is bound to a region), and
# on the cli represents the region the lambda is provisioned in.
for k in ('assume_role', 'profile', 'region', 'dryrun', 'cache'):
exec_options.pop(k, None)
# a cli local directory doesn't translate to lambda
if not exec_options.get('output_dir', '').startswith('s3'):
exec_options['output_dir'] = '/tmp'
account_id = None
# we can source account id from the cli parameters to avoid the sts call
if exec_options.get('account_id'):
account_id = exec_options['account_id']
# merge with policy specific configuration
exec_options.update(
policy_config['policies'][0].get('mode', {}).get('execution-options', {}))
# if using assume role in lambda ensure that the correct
# execution account is captured in options.
if 'assume_role' in exec_options:
account_id = exec_options['assume_role'].split(':')[4]
elif account_id is None:
session = local_session(boto3.Session)
account_id = get_account_id_from_sts(session)
exec_options['account_id'] = account_id
# Historical compatibility with manually set execution options
# previously this was a boolean, its now a string value with the
# boolean flag triggering a string value of 'aws'
if 'metrics_enabled' in exec_options \
and isinstance(exec_options['metrics_enabled'], bool) \
and exec_options['metrics_enabled']:
exec_options['metrics_enabled'] = 'aws'
return Config.empty(**exec_options)
def get_related(self, resources):
ctx = ExecutionContext(local_session(Session), self.data, Config.empty())
manager = self.factory(ctx, self.data)
related = manager.source.get_resources(None)
if self.data.get('op'):
return [r['id'] for r in related if self.match(r)]
else:
return [r['id'] for r in related]
def get_related(self, resources):
ctx = ExecutionContext(local_session(Session), self.data, Config.empty())
manager = self.factory(ctx, self.data)
related = manager.source.get_resources(None)
if self.data.get('op'):
return [r['id'] for r in related if self.match(r)]
else:
return [r['id'] for r in related]
def get_context(self, config=None, session_factory=None, policy=None):
if config is None:
self.context_output_dir = self.get_temp_dir()
config = Config.empty(output_dir=self.context_output_dir)
ctx = ExecutionContext(
session_factory, policy or Bag({
"name": "test-policy", "provider_name": "aws"}), config)
return ctx
def _get_policy_config(self, **kw):
config = kw
if kw.get('output_dir') is None or config.get('cache'):
config["output_dir"] = temp_dir = self.get_temp_dir()
if config.get('cache'):
config["cache"] = os.path.join(temp_dir, "c7n.cache")
config["cache_period"] = 300
return Config.empty(**config)
def init_config(policy_config):
"""Get policy lambda execution configuration.
cli parameters are serialized into the policy lambda config,
we merge those with any policy specific execution options.
--assume role and -s output directory get special handling, as
to disambiguate any cli context.
account id is sourced from the config options or from api call
and cached as a global
"""
global account_id
exec_options = policy_config.get('execution-options', {})
# Remove some configuration options that don't make sense to translate from
# cli to lambda automatically.
# - assume role on cli doesn't translate, it is the default lambda role and
# used to provision the lambda.
# - profile doesnt translate to lambda its `home` dir setup dependent
# - dryrun doesn't translate (and shouldn't be present)
# - region doesn't translate from cli (the lambda is bound to a region), and
# on the cli represents the region the lambda is provisioned in.
for k in ('assume_role', 'profile', 'region', 'dryrun', 'cache'):
exec_options.pop(k, None)
# a cli local directory doesn't translate to lambda
if not exec_options.get('output_dir', '').startswith('s3'):
exec_options['output_dir'] = get_local_output_dir()
# we can source account id from the cli parameters to avoid the sts call
if exec_options.get('account_id'):
account_id = exec_options['account_id']
# merge with policy specific configuration
exec_options.update(
policy_config['policies'][0].get('mode', {}).get('execution-options', {}))
# if using assume role in lambda ensure that the correct
# execution account is captured in options.
if 'assume_role' in exec_options:
account_id = exec_options['assume_role'].split(':')[4]
elif account_id is None:
session = boto3.Session()
account_id = get_account_id_from_sts(session)
exec_options['account_id'] = account_id
# Historical compatibility with manually set execution options
# previously this was a boolean, its now a string value with the
# boolean flag triggering a string value of 'aws'
if 'metrics_enabled' in exec_options \
and isinstance(exec_options['metrics_enabled'], bool) \
and exec_options['metrics_enabled']:
exec_options['metrics_enabled'] = 'aws'
return Config.empty(**exec_options)
def get_session(role, session_name, profile):
region = os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1')
stats = ApiStats(Bag(), Config.empty())
if role:
s = assumed_session(role, session_name, region=region)
else:
s = SessionFactory(region, profile)()
stats(s)
return stats, s
def get_context(self, config=None, session_factory=None, policy=None):
if config is None:
self.context_output_dir = self.get_temp_dir()
config = Config.empty(output_dir=self.context_output_dir)
ctx = ExecutionContext(
session_factory,
policy or Bag({'name': 'test-policy'}),
config)
return ctx
def validate(options):
load_resources()
if len(options.configs) < 1:
log.error('no config files specified')
sys.exit(1)
used_policy_names = set()
schm = schema.generate()
errors = []
for config_file in options.configs:
config_file = os.path.expanduser(config_file)
if not os.path.exists(config_file):
raise ValueError("Invalid path for config %r" % config_file)
options.dryrun = True
fmt = config_file.rsplit('.', 1)[-1]
with open(config_file) as fh:
if fmt in ('yml', 'yaml'):
data = yaml.safe_load(fh.read())
elif fmt in ('json',):
data = json.load(fh)
else:
log.error("The config file must end in .json, .yml or .yaml.")
raise ValueError("The config file must end in .json, .yml or .yaml.")
errors += schema.validate(data, schm)
conf_policy_names = {
p.get('name', 'unknown') for p in data.get('policies', ())}
dupes = conf_policy_names.intersection(used_policy_names)
if len(dupes) >= 1:
errors.append(ValueError(
"Only one policy with a given name allowed, duplicates: %s" % (
", ".join(dupes)
)
))
used_policy_names = used_policy_names.union(conf_policy_names)
if not errors:
null_config = Config.empty(dryrun=True, account_id='na', region='na')
for p in data.get('policies', ()):
try:
policy = Policy(p, null_config, Bag())
policy.validate()
except Exception as e:
msg = "Policy: %s is invalid: %s" % (
p.get('name', 'unknown'), e)
errors.append(msg)
if not errors:
log.info("Configuration valid: {}".format(config_file))
continue
log.error("Configuration invalid: {}".format(config_file))
for e in errors:
log.error("%s" % e)
if errors:
sys.exit(1)
def cli(**kwargs):
policy_config = Config.empty()
policies = PolicyCollection([
p for p in load_policies(kwargs['config'], policy_config)
if p.provider_name == 'azure'
], policy_config)
session = policies.policies[0].session_factory()
web_client = session.client('azure.mgmt.web.WebSiteManagementClient')
deployments = {}
credentials_load_failed = 0
not_functions_policy = 0
for p in policies:
if not p.is_lambda:
not_functions_policy += 1
continue
try:
params = AzureFunctionMode(p).get_function_app_params()
creds = web_client.web_apps.list_publishing_credentials(
params.function_app_resource_group_name,
params.function_app_name).result()
deployments[p.name] = {'scm_uri': creds.scm_uri, 'status': None}
log.info('Retrieved deployment credentials for %s policy', p.name)
except Exception:
log.error(
'Unable to retrieve deployment credentials for %s policy',
p.name)
credentials_load_failed += 1
wait_for_remote_builds(deployments)
success = 0
fail = 0
not_found = 0
for name, params in deployments.items():
if params['status'] != DeploymentStatus.Succeeded:
log.info('%s: %s', name, params['status'])
if params['status'] == DeploymentStatus.Failed:
log.error('Build logs can be retrieved here: %s',
params['scm_uri'])
fail += 1
else:
not_found += 1
else:
success += 1
log.info(
'Policies total: %i, unable to load credentials: %i, not Functions mode: %i',
len(policies), credentials_load_failed, not_functions_policy)
log.info(
'Status not found can happen if Linux Consumption function was deployed'
'more than 2 hours ago.')
log.info(
'Deployments complete. Success: %i, Fail: %i, Status not found: %i',
success, fail, not_found)
def test_initialize_default_account_id(self, get_subscription_id_mock):
options = Config.empty()
azure = Azure()
azure.initialize(options)
self.assertEqual(options['account_id'], DEFAULT_SUBSCRIPTION_ID)
session = azure.get_session_factory(options)()
session._initialize_session()
self.assertEqual(session.subscription_id, DEFAULT_SUBSCRIPTION_ID)
def test_local_session_region(self):
policies = [
self.load_policy(
{'name': 'ec2', 'resource': 'ec2'},
config=Config.empty(region="us-east-1")),
self.load_policy(
{'name': 'ec2', 'resource': 'ec2'},
config=Config.empty(region='us-west-2'))]
previous = None
previous_region = None
for p in policies:
self.assertEqual(p.options.region, p.session_factory.region)
session = utils.local_session(p.session_factory)
self.assertNotEqual(session.region_name, previous_region)
self.assertNotEqual(session, previous)
previous = session
previous_region = p.options.region
self.assertEqual(utils.local_session(p.session_factory), previous)
def initialize_tree(self, tree):
assert not self.policy_files
for tree_ent in tree:
fpath = tree_ent.name
if not self.matcher(fpath):
continue
self.policy_files[fpath] = PolicyCollection.from_data(
yaml.safe_load(self.repo.get(tree[fpath].id).data),
Config.empty(), fpath)
def get_azure_output(self):
output = AzureStorageOutput(
ExecutionContext(
None,
Bag(name="xyz"),
Config.empty(output_dir="azure://mystorage.blob.core.windows.net/logs"),
)
)
self.addCleanup(shutil.rmtree, output.root_dir)
return output
def test_initialize_custom_account_id(self, get_subscription_id_mock):
sample_account_id = "00000000-5106-4743-99b0-c129bfa71a47"
options = Config.empty()
options['account_id'] = sample_account_id
azure = Azure()
azure.initialize(options)
self.assertEqual(options['account_id'], sample_account_id)
session = azure.get_session_factory(options)()
session._initialize_session()
self.assertEqual(session.subscription_id, sample_account_id)
def _policy_file_rev(self, f, commit):
try:
return self._validate_policies(
PolicyCollection.from_data(
yaml.safe_load(self.repo.get(commit.tree[f].id).data),
Config.empty(), f))
except Exception as e:
log.warning(
"invalid policy file %s @ %s %s %s \n error:%s",
f, str(commit.id)[:6], commit_date(commit).isoformat(),
commit.author.name, e)
return PolicyCollection()
def dispatch_event(event, context):
global account_id
if account_id is None:
session = boto3.Session()
account_id = get_account_id_from_sts(session)
error = event.get('detail', {}).get('errorCode')
if error:
log.debug("Skipping failed operation: %s" % error)
return
event['debug'] = True
if event['debug']:
log.info("Processing event\n %s", format_event(event))
# policies file should always be valid in lambda so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# Initialize output directory, we've seen occassional perm issues with
# lambda on temp directory and changing unix execution users, so
# use a per execution temp space.
output_dir = os.environ.get(
'C7N_OUTPUT_DIR',
'/tmp/' + str(uuid.uuid4()))
if not os.path.exists(output_dir):
try:
os.mkdir(output_dir)
except OSError as error:
log.warning("Unable to make output directory: {}".format(error))
# TODO. This enshrines an assumption of a single policy per lambda.
options_overrides = policy_config[
'policies'][0].get('mode', {}).get('execution-options', {})
options_overrides['account_id'] = account_id
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = output_dir
options = Config.empty(**options_overrides)
load_resources()
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def load_policy(
self, data, config=None):
errors = schema_validate({'policies': [data]}, C7N_SCHEMA)
if errors:
raise errors[0]
config = config or {}
temp_dir = self.get_temp_dir()
config['output_dir'] = temp_dir
conf = Config.empty(**config)
p = policy.Policy(data, conf, Session)
p.validate()
return p
def get_azure_output(self, custom_pyformat=None):
output_dir = "azure://mystorage.blob.core.windows.net/logs"
if custom_pyformat:
output_dir = AzureStorageOutput.join(output_dir, custom_pyformat)
output = AzureStorageOutput(
ExecutionContext(
None,
Bag(name="xyz", provider_name='azure'),
Config.empty(output_dir=output_dir)
),
{'url': output_dir},
)
self.addCleanup(shutil.rmtree, output.root_dir)
return output
def load_policy(
self, data, config=None, session_factory=None,
validate=C7N_VALIDATE, output_dir=None, cache=False):
if validate:
errors = schema_validate({'policies': [data]}, C7N_SCHEMA)
if errors:
raise errors[0]
config = config or {}
if not output_dir:
temp_dir = self.get_temp_dir()
config['output_dir'] = temp_dir
if cache:
config['cache'] = os.path.join(temp_dir, 'c7n.cache')
config['cache_period'] = 300
conf = Config.empty(**config)
p = policy.Policy(data, conf, session_factory)
p.validate()
return p
def main():
parser = setup_parser()
options = parser.parse_args()
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(
level=log_level,
format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
logging.getLogger('botocore').setLevel(logging.ERROR)
logging.getLogger('c7n.cache').setLevel(logging.WARNING)
if not options.regions:
options.regions = [os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')]
files = []
files.extend(itertools.chain(*options.config_files))
files.extend(options.configs)
options.config_files = files
if not files:
parser.print_help()
sys.exit(1)
policy_config = Config.empty(
regions=options.regions,
profile=options.profile,
assume_role=options.assume_role)
# use cloud provider to initialize policies to get region expansion
policies = AWS().initialize_policies(
PolicyCollection([
p for p in load_policies(
options, policy_config)
if p.provider_name == 'aws'],
policy_config),
policy_config)
resources_gc_prefix(options, policy_config, policies)
def run_account(account, region, policies_config, output_path,
cache_period, cache_path, metrics, dryrun, debug):
"""Execute a set of policies on an account.
"""
logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
CONN_CACHE.session = None
CONN_CACHE.time = None
# allow users to specify interpolated output paths
if '{' not in output_path:
output_path = os.path.join(output_path, account['name'], region)
cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['account_id'], region))
config = Config.empty(
region=region, cache=cache_path,
cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
account_id=account['account_id'], metrics_enabled=metrics,
log_group=None, profile=None, external_id=None)
env_vars = account_tags(account)
if account.get('role'):
if isinstance(account['role'], six.string_types):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
else:
env_vars.update(
_get_env_creds(get_session(account, 'custodian', region), region))
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
policy_counts = {}
success = True
st = time.time()
with environ(**env_vars):
for p in policies:
# Variable expansion and non schema validation (not optional)
p.expand_variables(p.get_variables(account.get('vars', {})))
p.validate()
log.debug(
"Running policy:%s account:%s region:%s",
p.name, account['name'], region)
try:
resources = p.run()
policy_counts[p.name] = resources and len(resources) or 0
if not resources:
continue
log.info(
"Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
account['name'], region, p.name, len(resources),
time.time() - st)
except ClientError as e:
success = False
if e.response['Error']['Code'] == 'AccessDenied':
log.warning('Access denied account:%s region:%s',
account['name'], region)
return policy_counts, success
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
continue
except Exception as e:
success = False
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
if not debug:
continue
import traceback, pdb, sys
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
raise
return policy_counts, success
def run_account(account, region, policies_config, output_path,
cache_period, metrics, dryrun, debug):
"""Execute a set of policies on an account.
"""
logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
CONN_CACHE.session = None
CONN_CACHE.time = None
output_path = os.path.join(output_path, account['name'], region)
if not os.path.exists(output_path):
os.makedirs(output_path)
cache_path = os.path.join(output_path, "c7n.cache")
config = Config.empty(
region=region,
cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
account_id=account['account_id'], metrics_enabled=metrics,
cache=cache_path, log_group=None, profile=None, external_id=None)
if account.get('role'):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
policy_counts = {}
st = time.time()
with environ(**account_tags(account)):
for p in policies:
log.debug(
"Running policy:%s account:%s region:%s",
p.name, account['name'], region)
try:
resources = p.run()
policy_counts[p.name] = resources and len(resources) or 0
if not resources:
continue
log.info(
"Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
account['name'], region, p.name, len(resources),
time.time() - st)
except ClientError as e:
if e.response['Error']['Code'] == 'AccessDenied':
log.warning('Access denied account:%s region:%s',
account['name'], region)
return policy_counts
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
continue
except Exception as e:
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
if not debug:
continue
import traceback, pdb, sys
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
raise
return policy_counts
def __init__(self, policies=None, options=None):
self.policies = policies or []
self.options = options or Config.empty()
self.pmap = {p.name: p for p in self.policies}
def dispatch_event(event, context):
global account_id
error = event.get('detail', {}).get('errorCode')
if error:
log.debug("Skipping failed operation: %s" % error)
return
event['debug'] = True
if event['debug']:
log.info("Processing event\n %s", format_event(event))
# Policies file should always be valid in lambda so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# Initialize output directory, we've seen occassional perm issues with
# lambda on temp directory and changing unix execution users, so
# use a per execution temp space.
output_dir = os.environ.get(
'C7N_OUTPUT_DIR',
'/tmp/' + str(uuid.uuid4()))
if not os.path.exists(output_dir):
try:
os.mkdir(output_dir)
except OSError as error:
log.warning("Unable to make output directory: {}".format(error))
# TODO. This enshrines an assumption of a single policy per lambda.
options_overrides = policy_config[
'policies'][0].get('mode', {}).get('execution-options', {})
# if using assume role in lambda ensure that the correct
# execution account is captured in options.
if 'assume_role' in options_overrides:
account_id = options_overrides['assume_role'].split(':')[4]
elif account_id is None:
session = boto3.Session()
account_id = get_account_id_from_sts(session)
# Historical compatibility with manually set execution options
# previously this was a boolean, its now a string value with the
# boolean flag triggering a string value of 'aws'
if 'metrics_enabled' in options_overrides and isinstance(
options_overrides['metrics_enabled'], bool):
options_overrides['metrics_enabled'] = 'aws'
options_overrides['account_id'] = account_id
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = output_dir
options = Config.empty(**options_overrides)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def report(config, output, use, output_dir, accounts,
field, no_default_fields, tags, region, debug, verbose,
policy, policy_tags, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(
config, use, debug, verbose, accounts, tags, policy,
resource=resource, policy_tags=policy_tags)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
elif not len(custodian_config['policies']) > 0:
raise ValueError("no matching policies found")
records = []
with executor(max_workers=WORKER_COUNT) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
for r in resolve_regions(region or a.get('regions', ())):
futures[w.submit(
report_account,
a, r,
custodian_config,
output_dir,
debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning(
"Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug(
"Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Config.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(
factory.resource_type,
extra_fields=field,
include_default_fields=not(no_default_fields),
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = UnicodeWriter(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
