def filter_policies_on_tags(policies, tags, config):
filtered_policies = PolicyCollection([], config)
for p in policies:
if tags:
found = set()
for t in tags:
if t in p.tags:
found.add(t)
if found == set(tags):
filtered_policies += PolicyCollection([p], config)
return filtered_policies
def get_report_obj(valid_report_cache_file_name, report_to_policy_map):
account, region, report_name = valid_report_cache_file_name.split(
'/')[1:-1]
dry_run_cache_dir = '/'.join(valid_report_cache_file_name.split('/')[0:-2])
options = argparse.Namespace(
account_id=None,
assume_role=None,
cache=None,
command='c7n.commands.report',
config=None,
days=1,
debug=False,
field=[], # ['HEADER=Tags'],
format='grid',
log_group=None,
no_default_fields=False,
output_dir=dry_run_cache_dir,
policy_filter=None,
profile=None,
raw=None,
region=region,
regions='',
resource_type=None,
subparser='report',
verbose=False)
report_policy = PolicyCollection(report_to_policy_map[report_name],
options)
return [report_policy.policies[0], options, account, region, report_name]
def dryrun(self):
# This is largely based off of mugc.main()
logging.getLogger('botocore').setLevel(logging.ERROR)
logging.getLogger('urllib3').setLevel(logging.ERROR)
logging.getLogger('c7n.cache').setLevel(logging.WARNING)
conf = Config.empty(
config_files=['custodian_%s.yml' % self.region_name],
regions=[self.region_name],
prefix=self.config.function_prefix,
policy_regex='^' + re.escape(self.config.function_prefix) + '.*',
assume=None,
policy_filter=None,
log_group=None,
external_id=None,
cache_period=0,
cache=None,
present=False,
dryrun=True
)
# use cloud provider to initialize policies to get region expansion
policies = AWS().initialize_policies(
PolicyCollection(
[
p for p in load_policies(conf, conf)
if p.provider_name == 'aws'
],
conf
),
conf
)
resources_gc_prefix(conf, conf, policies)
def run(event, context=None):
# policies file should always be valid in functions so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
log.error('Invalid policy config')
return False
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
# if output_dir specified use that, otherwise make a temp directory
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = get_tmp_output_dir()
# merge all our options in
options = Config.empty(**options_overrides)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
log.info("running policy %s", p.name)
p.push(event, context)
return True
def dispatch_event(event, context):
error = event.get('detail', {}).get('errorCode')
if error:
log.debug("Skipping failed operation: %s" % error)
return
event['debug'] = True
if event['debug']:
log.info("Processing event\n %s", format_event(event))
# policies file should always be valid in lambda so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# TODO. This enshrines an assumption of a single policy per lambda.
options_overrides = policy_config['policies'][0].get('mode', {}).get(
'execution-options', {})
options = Config.empty(**options_overrides)
load_resources()
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def report_account(account, region, policies_config, output_path, debug):
cache_path = os.path.join(output_path, "c7n.cache")
output_path = os.path.join(output_path, account['name'], region)
config = Config.empty(
region=region,
output_dir=output_path,
account_id=account['account_id'], metrics_enabled=False,
cache=cache_path, log_group=None, profile=None, external_id=None)
if account.get('role'):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
records = []
for p in policies:
log.debug(
"Report policy:%s account:%s region:%s path:%s",
p.name, account['name'], region, output_path)
policy_records = fs_record_set(p.ctx.output_path, p.name)
for r in policy_records:
r['policy'] = p.name
r['region'] = p.options.region
r['account'] = account['name']
for t in account.get('tags', ()):
if ':' in t:
k, v = t.split(':', 1)
r[k] = v
records.extend(policy_records)
return records
def dispatch_event(event, context):
error = event.get('detail', {}).get('errorCode')
if error and C7N_SKIP_EVTERR:
log.debug("Skipping failed operation: %s" % error)
return
if C7N_DEBUG_EVENT:
event['debug'] = True
log.info("Processing event\n %s", format_event(event))
# Policies file should always be valid in lambda so do loading naively
global policy_config
if policy_config is None:
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
options = init_config(policy_config)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
try:
# validation provides for an initialization point for
# some filters/actions.
p.validate()
p.push(event, context)
except Exception:
log.exception("error during policy execution")
if C7N_CATCH_ERR:
continue
raise
return True
def load_policy(self, path, policies):
"""
Loads a YAML file and prompts scheduling updates
:param path: Path to YAML file on disk
:param policies: Dictionary of policies to update
"""
with open(path, "r") as stream:
try:
policy_config = yaml.safe_load(stream)
new_policies = PolicyCollection.from_data(
policy_config, self.options)
if new_policies:
for p in new_policies:
log.info("Loading Policy %s from %s" % (p.name, path))
p.validate()
policies.update({p.name: {'policy': p}})
# Update periodic and set event update flag
self.update_periodic(p)
if p.data.get('mode', {}).get('events'):
self.require_event_update = True
except Exception as exc:
log.error('Invalid policy file %s %s' % (path, exc))
def process(self, resources, event=None):
provider = clouds[self.manager.ctx.policy.provider_name]()
# if the embedded policy only specifies one region, or only
# being executed on a single region.
if self.embedded_policy.region or len(
self.manager.config.regions) <= 1:
if (self.embedded_policy.region and
self.embedded_policy.region != self.manager.config.region):
return []
self.embedded_policy.expand_variables(
self.embedded_policy.get_variables())
return not self.embedded_policy.poll() and resources or []
# For regional resources and multi-region execution, the policy matches if
# the resource is missing in any region.
found = {}
for p in provider.initialize_policies(
PolicyCollection([self.embedded_policy], self.manager.config),
self.manager.config):
p.expand_variables(p.get_variables())
p.validate()
found[p.options.region] = p.poll()
if not all(found.values()):
return resources
return []
def run(event, context):
# policies file should always be valid in functions so do loading naively
with open(context['config_file']) as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
log.error('Invalid policy config')
return False
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
# setup our auth file location on disk
options_overrides['authorization_file'] = context['auth_file']
# if output_dir specified use that, otherwise make a temp directory
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = get_tmp_output_dir()
# merge all our options in
options = Config.empty(**options_overrides)
load_resources()
options = Azure().initialize(options)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
try:
p.push(event, context)
except (CloudError, AzureHttpError) as error:
log.error("Unable to process policy: %s :: %s" % (p.name, error))
return True
def load_policy(self, path, policies):
"""
Loads a YAML file and prompts scheduling updates
:param path: Path to YAML file on disk
:param policies: Dictionary of policies to update
"""
with open(path, "r") as stream:
try:
policy_config = yaml.safe_load(stream)
new_policies = PolicyCollection.from_data(
policy_config, self.options)
if new_policies:
for p in new_policies:
log.info("Loading Policy %s from %s" % (p.name, path))
p.validate()
policies.update({p.name: {'policy': p}})
# Update periodic
policy_mode = p.data.get('mode', {}).get('type')
if policy_mode == CONTAINER_TIME_TRIGGER_MODE:
self.update_periodic(p)
elif policy_mode != CONTAINER_EVENT_TRIGGER_MODE:
log.warning(
"Unsupported policy mode for Azure Container Host: {}. "
"{} will not be run. "
"Supported policy modes include \"{}\" and \"{}\"."
.format(policy_mode, p.data['name'],
CONTAINER_EVENT_TRIGGER_MODE,
CONTAINER_TIME_TRIGGER_MODE))
except Exception as exc:
log.error('Invalid policy file %s %s' % (path, exc))
def run(event, context):
# policies file should always be valid in functions so do loading naively
with open(context['config_file']) as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# Initialize output directory
output_dir = os.environ.get(
'C7N_OUTPUT_DIR',
'/tmp/' + str(uuid.uuid4()))
if not os.path.exists(output_dir):
try:
os.mkdir(output_dir)
except OSError as error:
log.warning("Unable to make output directory: {}".format(error))
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
options_overrides['authorization_file'] = context['auth_file']
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = output_dir
options = Config.empty(**options_overrides)
load_resources()
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def dispatch_event(event, context):
error = event.get('detail', {}).get('errorCode')
if error and C7N_SKIP_EVTERR:
log.debug("Skipping failed operation: %s" % error)
return
# one time initialization for cold starts.
global policy_config, policy_data
if policy_config is None:
with open('config.json') as f:
policy_data = json.load(f)
policy_config = init_config(policy_data)
load_resources(StructureParser().get_resource_types(policy_data))
if C7N_DEBUG_EVENT:
event['debug'] = True
log.info("Processing event\n %s", format_event(event))
if not policy_data or not policy_data.get('policies'):
return False
policies = PolicyCollection.from_data(policy_data, policy_config)
for p in policies:
try:
# validation provides for an initialization point for
# some filters/actions.
p.validate()
p.push(event, context)
except Exception:
log.exception("error during policy execution")
if C7N_CATCH_ERR:
continue
raise
return True
def dispatch_event(event, context):
global account_id
error = event.get('detail', {}).get('errorCode')
if error:
log.debug("Skipping failed operation: %s" % error)
return
event['debug'] = True
if event['debug']:
log.info("Processing event\n %s", format_event(event))
# Policies file should always be valid in lambda so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# Initialize output directory, we've seen occassional perm issues with
# lambda on temp directory and changing unix execution users, so
# use a per execution temp space.
output_dir = os.environ.get('C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4()))
if not os.path.exists(output_dir):
try:
os.mkdir(output_dir)
except OSError as error:
log.warning("Unable to make output directory: {}".format(error))
# TODO. This enshrines an assumption of a single policy per lambda.
options_overrides = policy_config['policies'][0].get('mode', {}).get(
'execution-options', {})
# if using assume role in lambda ensure that the correct
# execution account is captured in options.
if 'assume_role' in options_overrides:
account_id = options_overrides['assume_role'].split(':')[4]
elif account_id is None:
session = boto3.Session()
account_id = get_account_id_from_sts(session)
# Historical compatibility with manually set execution options
# previously this was a boolean, its now a string value with the
# boolean flag triggering a string value of 'aws'
if 'metrics_enabled' in options_overrides and isinstance(
options_overrides['metrics_enabled'], bool):
options_overrides['metrics_enabled'] = 'aws'
options_overrides['account_id'] = account_id
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = output_dir
options = Config.empty(**options_overrides)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def dispatch_event(event, context):
error = event.get('detail', {}).get('errorCode')
if error and C7N_SKIP_EVTERR:
log.debug("Skipping failed operation: %s" % error)
return
if C7N_DEBUG_EVENT:
event['debug'] = True
log.info("Processing event\n %s", format_event(event))
# Policies file should always be valid in lambda so do loading naively
global policy_config
if policy_config is None:
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
options = init_config(policy_config)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
try:
# validation provides for an initialization point for
# some filters/actions.
p.validate()
p.push(event, context)
except Exception:
log.exception("error during policy execution")
if C7N_CATCH_ERR:
continue
raise
return True
def run_account(account, region, policies_config, output_path, cache_period,
dryrun, debug):
"""Execute a set of policies on an account.
"""
logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
CONN_CACHE.session = None
CONN_CACHE.time = None
output_path = os.path.join(output_path, account['name'], region)
if not os.path.exists(output_path):
os.makedirs(output_path)
cache_path = os.path.join(output_path, "c7n.cache")
bag = Bag.empty(region=region,
assume_role=account['role'],
cache_period=cache_period,
dryrun=dryrun,
output_dir=output_path,
account_id=account['account_id'],
metrics_enabled=False,
cache=cache_path,
log_group=None,
profile=None,
external_id=None)
policies = PolicyCollection.from_data(policies_config, bag)
policy_counts = {}
st = time.time()
with environ(**account_tags(account)):
for p in policies:
log.debug("Running policy:%s account:%s region:%s", p.name,
account['name'], region)
try:
resources = p.run()
policy_counts[p.name] = resources and len(resources) or 0
if not resources:
continue
log.info(
"Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
account['name'], region, p.name, len(resources),
time.time() - st)
except ClientError as e:
if e.response['Error']['Code'] == 'AccessDenied':
log.warning('Access denied account:%s region:%s',
account['name'], region)
return policy_counts
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
continue
except Exception as e:
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
if not debug:
continue
import traceback, pdb, sys
pdb.post_mortem(sys.exc_info()[-1])
raise
return policy_counts
def get_untagged_resources(self, rtype):
policy_data = {
'name': 'inventory-%s' % rtype,
'resource': rtype,
'filters': [{
'tag:{}'.format(self.creator_tag): 'absent'
}]
}
# Cloud Formation stacks can only be tagged in successful
# steady state.
if rtype == 'cfn':
policy_data['filters'].insert(
0, {
'type': 'value',
'key': 'StackStatus',
'op': 'in',
'value': ['UPDATE_COMPLETE', 'CREATE_COMPLETE']
})
policy = list(
PolicyCollection.from_data({'policies': [policy_data]},
self.config)).pop()
if 'tag' not in policy.resource_manager.action_registry:
return [], None
resources = policy.run()
return resources, policy.resource_manager
def dispatch_event(event, context):
error = event.get('detail', {}).get('errorCode')
if error:
log.debug("Skipping failed operation: %s" % error)
return
event['debug'] = True
if event['debug']:
log.info("Processing event\n %s", format_event(event))
# policies file should always be valid in lambda so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# TODO. This enshrines an assumption of a single policy per lambda.
options_overrides = policy_config[
'policies'][0].get('mode', {}).get('execution-options', {})
options = Config.empty(**options_overrides)
load_resources()
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def report_account(account, region, policies_config, output_path, debug):
cache_path = os.path.join(output_path, "c7n.cache")
output_path = os.path.join(output_path, account['name'], region)
config = Config.empty(
region=region,
output_dir=output_path,
account_id=account['account_id'], metrics_enabled=False,
cache=cache_path, log_group=None, profile=None, external_id=None)
if account.get('role'):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
records = []
for p in policies:
log.debug(
"Report policy:%s account:%s region:%s path:%s",
p.name, account['name'], region, output_path)
policy_records = fs_record_set(p.ctx.output_path, p.name)
for r in policy_records:
r['policy'] = p.name
r['region'] = p.options.region
r['account'] = account['name']
for t in account.get('tags', ()):
if ':' in t:
k, v = t.split(':', 1)
r[k] = v
records.extend(policy_records)
return records
def run(event, context):
# policies file should always be valid in functions so do loading naively
with open(context['config_file']) as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
log.error('Invalid policy config')
return False
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
# setup our auth file location on disk
options_overrides['authorization_file'] = context['auth_file']
# if output_dir specified use that, otherwise make a temp directory
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = get_tmp_output_dir()
# merge all our options in
options = Config.empty(**options_overrides)
load_resources()
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def initialize_policies(self, policy_collection, options):
"""Return a set of policies targetted to the given regions.
Supports symbolic regions like 'all'. This will automatically
filter out policies if their being targetted to a region that
does not support the service. Global services will target a
single region (us-east-1 if only all specified, else first
region in the list).
Note for region partitions (govcloud and china) an explicit
region from the partition must be passed in.
"""
from c7n.policy import Policy, PolicyCollection
policies = []
service_region_map, resource_service_map = get_service_region_map(
options.regions, policy_collection.resource_types)
for p in policy_collection:
available_regions = service_region_map.get(
resource_service_map.get(p.resource_type), ())
# its a global service/endpoint, use user provided region
# or us-east-1.
if not available_regions and options.regions:
candidates = [r for r in options.regions if r != 'all']
candidate = candidates and candidates[0] or 'us-east-1'
svc_regions = [candidate]
elif 'all' in options.regions:
svc_regions = available_regions
else:
svc_regions = options.regions
for region in svc_regions:
if available_regions and region not in available_regions:
level = ('all' in options.regions and logging.DEBUG
or logging.WARNING)
# TODO: fixme
policy_collection.log.log(
level,
"policy:%s resources:%s not available in region:%s",
p.name, p.resource_type, region)
continue
options_copy = copy.copy(options)
options_copy.region = str(region)
if len(options.regions
) > 1 or 'all' in options.regions and getattr(
options, 'output_dir', None):
options_copy.output_dir = (options.output_dir.rstrip('/') +
'/%s' % region)
policies.append(
Policy(
p.data,
options_copy,
session_factory=policy_collection.session_factory()))
return PolicyCollection(policies, options)
def cli(**kwargs):
policy_config = Config.empty()
policies = PolicyCollection([
p for p in load_policies(kwargs['config'], policy_config)
if p.provider_name == 'azure'
], policy_config)
session = policies.policies[0].session_factory()
web_client = session.client('azure.mgmt.web.WebSiteManagementClient')
deployments = {}
credentials_load_failed = 0
not_functions_policy = 0
for p in policies:
if not p.is_lambda:
not_functions_policy += 1
continue
try:
params = AzureFunctionMode(p).get_function_app_params()
creds = web_client.web_apps.list_publishing_credentials(
params.function_app_resource_group_name,
params.function_app_name).result()
deployments[p.name] = {'scm_uri': creds.scm_uri, 'status': None}
log.info('Retrieved deployment credentials for %s policy', p.name)
except Exception:
log.error(
'Unable to retrieve deployment credentials for %s policy',
p.name)
credentials_load_failed += 1
wait_for_remote_builds(deployments)
success = 0
fail = 0
not_found = 0
for name, params in deployments.items():
if params['status'] != DeploymentStatus.Succeeded:
log.info('%s: %s', name, params['status'])
if params['status'] == DeploymentStatus.Failed:
log.error('Build logs can be retrieved here: %s',
params['scm_uri'])
fail += 1
else:
not_found += 1
else:
success += 1
log.info(
'Policies total: %i, unable to load credentials: %i, not Functions mode: %i',
len(policies), credentials_load_failed, not_functions_policy)
log.info(
'Status not found can happen if Linux Consumption function was deployed'
'more than 2 hours ago.')
log.info(
'Deployments complete. Success: %i, Fail: %i, Status not found: %i',
success, fail, not_found)
def report_account(account, region, policies_config, output_path, cache_path,
debug):
output_path = os.path.join(output_path, account['name'], region)
cache_path = os.path.join(cache_path,
"%s-%s.cache" % (account['name'], region))
load_available()
config = Config.empty(region=region,
output_dir=output_path,
account_id=account['account_id'],
metrics_enabled=False,
cache=cache_path,
log_group=None,
profile=None,
external_id=None)
if account.get('role'):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
records = []
for p in policies:
# initializee policy execution context for output access
p.ctx.initialize()
log.debug("Report policy:%s account:%s region:%s path:%s", p.name,
account['name'], region, output_path)
if p.ctx.output.type == "s3":
delta = timedelta(days=1)
begin_date = datetime.now() - delta
policy_records = record_set(
p.session_factory, p.ctx.output.config['netloc'],
strip_output_path(p.ctx.output.config['path'], p.name),
begin_date)
else:
policy_records = fs_record_set(p.ctx.log_dir, p.name)
for r in policy_records:
r['policy'] = p.name
r['region'] = p.options.region
r['account'] = account['name']
for t in account.get('tags', ()):
if ':' in t:
k, v = t.split(':', 1)
r[k] = v
records.extend(policy_records)
return records
def dispatch_event(event, context):
global account_id
if account_id is None:
session = boto3.Session()
account_id = get_account_id_from_sts(session)
error = event.get('detail', {}).get('errorCode')
if error:
log.debug("Skipping failed operation: %s" % error)
return
event['debug'] = True
if event['debug']:
log.info("Processing event\n %s", format_event(event))
# policies file should always be valid in lambda so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# Initialize output directory, we've seen occassional perm issues with
# lambda on temp directory and changing unix execution users, so
# use a per execution temp space.
output_dir = os.environ.get(
'C7N_OUTPUT_DIR',
'/tmp/' + str(uuid.uuid4()))
if not os.path.exists(output_dir):
try:
os.mkdir(output_dir)
except OSError as error:
log.warning("Unable to make output directory: {}".format(error))
# TODO. This enshrines an assumption of a single policy per lambda.
options_overrides = policy_config[
'policies'][0].get('mode', {}).get('execution-options', {})
options_overrides['account_id'] = account_id
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = output_dir
options = Config.empty(**options_overrides)
load_resources()
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def dispatch_event(event, context):
global account_id
if account_id is None:
session = boto3.Session()
account_id = get_account_id_from_sts(session)
error = event.get('detail', {}).get('errorCode')
if error:
log.debug("Skipping failed operation: %s" % error)
return
event['debug'] = True
if event['debug']:
log.info("Processing event\n %s", format_event(event))
# policies file should always be valid in lambda so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# Initialize output directory, we've seen occassional perm issues with
# lambda on temp directory and changing unix execution users, so
# use a per execution temp space.
output_dir = os.environ.get('C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4()))
if not os.path.exists(output_dir):
try:
os.mkdir(output_dir)
except OSError as error:
log.warning("Unable to make output directory: {}".format(error))
# TODO. This enshrines an assumption of a single policy per lambda.
options_overrides = policy_config['policies'][0].get('mode', {}).get(
'execution-options', {})
options_overrides['account_id'] = account_id
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = output_dir
options = Config.empty(**options_overrides)
load_resources()
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def main():
parser = setup_parser()
options = parser.parse_args()
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(
level=log_level,
format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
logging.getLogger('botocore').setLevel(logging.ERROR)
logging.getLogger('urllib3').setLevel(logging.ERROR)
logging.getLogger('c7n.cache').setLevel(logging.WARNING)
if not options.policy_regex:
options.policy_regex = f"^{options.prefix}.*"
if not options.regions:
options.regions = [os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')]
files = []
files.extend(itertools.chain(*options.config_files))
files.extend(options.configs)
options.config_files = files
options.policy_tags = list(itertools.chain(*options.policy_tags))
if not files:
parser.print_help()
sys.exit(1)
policy_config = Config.empty(
regions=options.regions,
profile=options.profile,
assume_role=options.assume_role)
# use cloud provider to initialize policies to get region expansion
policies = AWS().initialize_policies(
PolicyCollection([
p for p in load_policies(
options, policy_config)
if p.provider_name == 'aws'],
policy_config),
policy_config)
resources_gc_prefix(options, policy_config, policies)
def run_account(account, region, policies_config, output_path, cache_period, dryrun, debug):
"""Execute a set of policies on an account.
"""
CONN_CACHE.session = None
CONN_CACHE.time = None
output_path = os.path.join(output_path, account['name'], region)
if not os.path.exists(output_path):
os.makedirs(output_path)
cache_path = os.path.join(output_path, "c7n.cache")
bag = Bag.empty(
region=region, assume_role=account['role'],
cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
account_id=account['account_id'], metrics_enabled=False,
cache=cache_path, log_group=None, profile=None, external_id=None)
policies = PolicyCollection.from_data(policies_config, bag)
policy_counts = {}
st = time.time()
with environ(**account_tags(account)):
for p in policies:
log.debug(
"Running policy:%s account:%s region:%s", p.name, account['name'], region)
try:
resources = p.run()
policy_counts[p.name] = resources and len(resources) or 0
if not resources:
continue
log.info("Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
account['name'], region, p.name, len(resources), time.time()-st)
except Exception as e:
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
if not debug:
continue
import traceback, pdb, sys
pdb.post_mortem(sys.exc_info()[-1])
raise
return policy_counts
def run(event, context, subscription_id=None):
# policies file should always be valid in functions so do loading naively
with open(context['config_file']) as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
log.error('Invalid policy config')
return False
options_overrides = \
policy_config['policies'][0].get('mode', {}).get('execution-options', {})
# setup our auth file location on disk
options_overrides['authorization_file'] = context['auth_file']
# if output_dir specified use that, otherwise make a temp directory
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = get_tmp_output_dir()
# merge all our options in
options = Config.empty(**options_overrides)
if subscription_id is not None:
options['account_id'] = subscription_id
load_resources(StructureParser().get_resource_types(policy_config))
options = Azure().initialize(options)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
try:
p.push(event, context)
except (CloudError, AzureHttpError) as error:
log.error("Unable to process policy: %s :: %s" %
(p.name, error))
reset_session_cache()
return True
def _load_policies(options):
validate = True
if 'skip_validation' in options:
validate = not options.skip_validation
if not validate:
log.debug('Policy validation disabled')
vars = _load_vars(options)
errors = 0
all_policies = PolicyCollection.from_data({}, options)
# for a default region for policy loading, we'll expand regions later.
options.region = ""
for fp in options.configs:
try:
collection = policy_load(options, fp, validate=validate, vars=vars)
except IOError:
log.error('policy file does not exist ({})'.format(fp))
errors += 1
continue
except yaml.YAMLError as e:
log.error(
"yaml syntax error loading policy file ({}) error:\n {}".format(
fp, e))
errors += 1
continue
except ValueError as e:
log.error('problem loading policy file ({}) error: {}'.format(
fp, str(e)))
errors += 1
continue
except PolicyValidationError as e:
log.error('invalid policy file: {} error: {}'.format(
fp, str(e)))
errors += 1
continue
if collection is None:
log.debug('Loaded file {}. Contained no policies.'.format(fp))
else:
log.debug(
'Loaded file {}. Contains {} policies'.format(
fp, len(collection)))
all_policies = all_policies + collection
if errors > 0:
log.error('Found {} errors. Exiting.'.format(errors))
sys.exit(1)
# filter by name and resource type
policies = all_policies.filter(
getattr(options, 'policy_filters', []),
getattr(options, 'resource_types', []))
# provider initialization
provider_policies = {}
for p in policies:
provider_policies.setdefault(p.provider_name, []).append(p)
policies = PolicyCollection.from_data({}, options)
for provider_name in provider_policies:
provider = clouds[provider_name]()
p_options = provider.initialize(options)
policies += provider.initialize_policies(
PolicyCollection(provider_policies[provider_name], p_options),
p_options)
if len(policies) == 0:
_print_no_policies_warning(options, all_policies)
# If we filtered out all the policies we want to exit with a
# non-zero status. But if the policy file is empty then continue
# on to the specific command to determine the exit status.
if len(all_policies) > 0:
sys.exit(1)
# Do not allow multiple policies in a region with the same name,
# even across files
policies_by_region = defaultdict(list)
for p in policies:
policies_by_region[p.options.region].append(p)
for region in policies_by_region.keys():
counts = Counter([p.name for p in policies_by_region[region]])
for policy, count in six.iteritems(counts):
if count > 1:
log.error("duplicate policy name '{}'".format(policy))
sys.exit(1)
# Variable expansion and non schema validation (not optional)
for p in policies:
p.expand_variables(p.get_variables())
p.validate()
return f(options, list(policies))
def run_account(account, region, policies_config, output_path,
cache_period, cache_path, metrics, dryrun, debug):
"""Execute a set of policies on an account.
"""
logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
CONN_CACHE.session = None
CONN_CACHE.time = None
# allow users to specify interpolated output paths
if '{' not in output_path:
output_path = os.path.join(output_path, account['name'], region)
cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['account_id'], region))
config = Config.empty(
region=region, cache=cache_path,
cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
account_id=account['account_id'], metrics_enabled=metrics,
log_group=None, profile=None, external_id=None)
env_vars = account_tags(account)
if account.get('role'):
if isinstance(account['role'], six.string_types):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
else:
env_vars.update(
_get_env_creds(get_session(account, 'custodian', region), region))
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
policy_counts = {}
success = True
st = time.time()
with environ(**env_vars):
for p in policies:
# Variable expansion and non schema validation (not optional)
p.expand_variables(p.get_variables(account.get('vars', {})))
p.validate()
log.debug(
"Running policy:%s account:%s region:%s",
p.name, account['name'], region)
try:
resources = p.run()
policy_counts[p.name] = resources and len(resources) or 0
if not resources:
continue
log.info(
"Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
account['name'], region, p.name, len(resources),
time.time() - st)
except ClientError as e:
success = False
if e.response['Error']['Code'] == 'AccessDenied':
log.warning('Access denied account:%s region:%s',
account['name'], region)
return policy_counts, success
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
continue
except Exception as e:
success = False
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
if not debug:
continue
import traceback, pdb, sys
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
raise
return policy_counts, success
def _load_policies(options):
load_resources()
vars = _load_vars(options)
errors = 0
all_policies = PolicyCollection.from_data({}, options)
# for a default region for policy loading, we'll expand regions later.
options.region = options.regions[0]
for fp in options.configs:
try:
collection = policy_load(options, fp, vars=vars)
except IOError:
log.error('policy file does not exist ({})'.format(fp))
errors += 1
continue
except ValueError as e:
log.error('problem loading policy file ({})'.format(e.message))
errors += 1
continue
if collection is None:
log.debug('Loaded file {}. Contained no policies.'.format(fp))
else:
log.debug(
'Loaded file {}. Contains {} policies'.format(
fp, len(collection)))
all_policies = all_policies + collection
if errors > 0:
log.error('Found {} errors. Exiting.'.format(errors))
sys.exit(1)
# filter by name and resource type
policies = all_policies.filter(
getattr(options, 'policy_filter', None),
getattr(options, 'resource_type', None))
# expand by region, this results in a separate policy instance per region of execution.
policies = policies.expand_regions(options.regions)
if len(policies) == 0:
_print_no_policies_warning(options, all_policies)
# If we filtered out all the policies we want to exit with a
# non-zero status. But if the policy file is empty then continue
# on to the specific command to determine the exit status.
if len(all_policies) > 0:
sys.exit(1)
# Do not allow multiple policies in a region with the same name,
# even across files
policies_by_region = defaultdict(list)
for p in policies:
policies_by_region[p.options.region].append(p)
for region in policies_by_region.keys():
counts = Counter([p.name for p in policies_by_region[region]])
for policy, count in counts.iteritems():
if count > 1:
log.error("duplicate policy name '{}'".format(policy))
sys.exit(1)
return f(options, list(policies))
def initialize_policies(self, policy_collection, options):
"""Return a set of policies targetted to the given regions.
Supports symbolic regions like 'all'. This will automatically
filter out policies if their being targetted to a region that
does not support the service. Global services will target a
single region (us-east-1 if only all specified, else first
region in the list).
Note for region partitions (govcloud and china) an explicit
region from the partition must be passed in.
"""
from c7n.policy import Policy, PolicyCollection
policies = []
service_region_map, resource_service_map = get_service_region_map(
options.regions, policy_collection.resource_types)
if 'all' in options.regions:
enabled_regions = set([
r['RegionName'] for r in get_profile_session(options).client(
'ec2').describe_regions(
Filters=[{
'Name': 'opt-in-status',
'Values': ['opt-in-not-required', 'opted-in']
}]).get('Regions')
])
for p in policy_collection:
if 'aws.' in p.resource_type:
_, resource_type = p.resource_type.split('.', 1)
else:
resource_type = p.resource_type
available_regions = service_region_map.get(
resource_service_map.get(resource_type), ())
# its a global service/endpoint, use user provided region
# or us-east-1.
if not available_regions and options.regions:
candidates = [r for r in options.regions if r != 'all']
candidate = candidates and candidates[0] or 'us-east-1'
svc_regions = [candidate]
elif 'all' in options.regions:
svc_regions = list(
set(available_regions).intersection(enabled_regions))
else:
svc_regions = options.regions
for region in svc_regions:
if available_regions and region not in available_regions:
level = ('all' in options.regions and logging.DEBUG
or logging.WARNING)
# TODO: fixme
policy_collection.log.log(
level,
"policy:%s resources:%s not available in region:%s",
p.name, p.resource_type, region)
continue
options_copy = copy.copy(options)
options_copy.region = str(region)
if len(options.regions
) > 1 or 'all' in options.regions and getattr(
options, 'output_dir', None):
options_copy.output_dir = join_output(
options.output_dir, region)
policies.append(
Policy(
p.data,
options_copy,
session_factory=policy_collection.session_factory()))
return PolicyCollection(
# order policies by region to minimize local session invalidation.
# note relative ordering of policies must be preserved, python sort
# is stable.
sorted(policies, key=operator.attrgetter('options.region')),
options)
def _load_policies(options):
load_resources()
vars = _load_vars(options)
errors = 0
all_policies = PolicyCollection.from_data({}, options)
# for a default region for policy loading, we'll expand regions later.
options.region = options.regions[0]
for fp in options.configs:
try:
collection = policy_load(options, fp, vars=vars)
except IOError:
log.error('policy file does not exist ({})'.format(fp))
errors += 1
continue
except ValueError as e:
log.error('problem loading policy file ({})'.format(e.message))
errors += 1
continue
if collection is None:
log.debug('Loaded file {}. Contained no policies.'.format(fp))
else:
log.debug('Loaded file {}. Contains {} policies'.format(
fp, len(collection)))
all_policies = all_policies + collection
if errors > 0:
log.error('Found {} errors. Exiting.'.format(errors))
sys.exit(1)
# filter by name and resource type
policies = all_policies.filter(getattr(options, 'policy_filter', None),
getattr(options, 'resource_type', None))
# expand by region, this results in a separate policy instance per region of execution.
policies = policies.expand_regions(options.regions)
if len(policies) == 0:
_print_no_policies_warning(options, all_policies)
# If we filtered out all the policies we want to exit with a
# non-zero status. But if the policy file is empty then continue
# on to the specific command to determine the exit status.
if len(all_policies) > 0:
sys.exit(1)
# Do not allow multiple policies in a region with the same name,
# even across files
policies_by_region = defaultdict(list)
for p in policies:
policies_by_region[p.options.region].append(p)
for region in policies_by_region.keys():
counts = Counter([p.name for p in policies_by_region[region]])
for policy, count in six.iteritems(counts):
if count > 1:
log.error("duplicate policy name '{}'".format(policy))
sys.exit(1)
return f(options, list(policies))
def load_policies(options, config):
policies = PolicyCollection([], config)
for f in options.config_files:
policies += policy_load(config, f).filter(options.policy_filter)
return policies
def run_account(account, region, policies_config, output_path,
cache_period, cache_path, metrics, dryrun, debug):
"""Execute a set of policies on an account.
"""
logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
CONN_CACHE.session = None
CONN_CACHE.time = None
load_available()
# allow users to specify interpolated output paths
if '{' not in output_path:
output_path = os.path.join(output_path, account['name'], region)
cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['account_id'], region))
config = Config.empty(
region=region, cache=cache_path,
cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
account_id=account['account_id'], metrics_enabled=metrics,
log_group=None, profile=None, external_id=None)
env_vars = account_tags(account)
if account.get('role'):
if isinstance(account['role'], six.string_types):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
else:
env_vars.update(
_get_env_creds(get_session(account, 'custodian', region), region))
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
policy_counts = {}
success = True
st = time.time()
with environ(**env_vars):
for p in policies:
# Variable expansion and non schema validation (not optional)
p.expand_variables(p.get_variables(account.get('vars', {})))
p.validate()
if p.region and p.region != region:
continue
log.debug(
"Running policy:%s account:%s region:%s",
p.name, account['name'], region)
try:
resources = p.run()
policy_counts[p.name] = resources and len(resources) or 0
if not resources:
continue
if not config.dryrun and p.execution_mode != 'pull':
log.info("Ran account:%s region:%s policy:%s provisioned time:%0.2f",
account['name'], region, p.name, time.time() - st)
continue
log.info(
"Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
account['name'], region, p.name, len(resources),
time.time() - st)
except ClientError as e:
success = False
if e.response['Error']['Code'] == 'AccessDenied':
log.warning('Access denied api:%s policy:%s account:%s region:%s',
e.operation_name, p.name, account['name'], region)
return policy_counts, success
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
continue
except Exception as e:
success = False
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
if not debug:
continue
import traceback, pdb, sys
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
raise
return policy_counts, success
def run_account(account, region, policies_config, output_path,
cache_period, metrics, dryrun, debug):
"""Execute a set of policies on an account.
"""
logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
CONN_CACHE.session = None
CONN_CACHE.time = None
output_path = os.path.join(output_path, account['name'], region)
if not os.path.exists(output_path):
os.makedirs(output_path)
cache_path = os.path.join(output_path, "c7n.cache")
config = Config.empty(
region=region,
cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
account_id=account['account_id'], metrics_enabled=metrics,
cache=cache_path, log_group=None, profile=None, external_id=None)
if account.get('role'):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
policy_counts = {}
st = time.time()
with environ(**account_tags(account)):
for p in policies:
log.debug(
"Running policy:%s account:%s region:%s",
p.name, account['name'], region)
try:
resources = p.run()
policy_counts[p.name] = resources and len(resources) or 0
if not resources:
continue
log.info(
"Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
account['name'], region, p.name, len(resources),
time.time() - st)
except ClientError as e:
if e.response['Error']['Code'] == 'AccessDenied':
log.warning('Access denied account:%s region:%s',
account['name'], region)
return policy_counts
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
continue
except Exception as e:
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
if not debug:
continue
import traceback, pdb, sys
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
raise
return policy_counts
def _load_policies(options):
validate = True
if 'skip_validation' in options:
validate = not options.skip_validation
if not validate:
log.debug('Policy validation disabled')
load_resources()
vars = _load_vars(options)
errors = 0
all_policies = PolicyCollection.from_data({}, options)
# for a default region for policy loading, we'll expand regions later.
options.region = ""
for fp in options.configs:
try:
collection = policy_load(options, fp, validate=validate, vars=vars)
except IOError:
log.error('policy file does not exist ({})'.format(fp))
errors += 1
continue
except ValueError as e:
log.error('problem loading policy file ({})'.format(e.message))
errors += 1
continue
if collection is None:
log.debug('Loaded file {}. Contained no policies.'.format(fp))
else:
log.debug(
'Loaded file {}. Contains {} policies'.format(
fp, len(collection)))
all_policies = all_policies + collection
if errors > 0:
log.error('Found {} errors. Exiting.'.format(errors))
sys.exit(1)
# filter by name and resource type
policies = all_policies.filter(
getattr(options, 'policy_filter', None),
getattr(options, 'resource_type', None))
# provider initialization
provider_policies = {}
for p in policies:
provider_policies.setdefault(p.provider_name, []).append(p)
policies = PolicyCollection.from_data({}, options)
for provider_name in provider_policies:
provider = clouds[provider_name]()
p_options = provider.initialize(options)
policies += provider.initialize_policies(
PolicyCollection(provider_policies[provider_name], p_options),
p_options)
if len(policies) == 0:
_print_no_policies_warning(options, all_policies)
# If we filtered out all the policies we want to exit with a
# non-zero status. But if the policy file is empty then continue
# on to the specific command to determine the exit status.
if len(all_policies) > 0:
sys.exit(1)
# Do not allow multiple policies in a region with the same name,
# even across files
policies_by_region = defaultdict(list)
for p in policies:
policies_by_region[p.options.region].append(p)
for region in policies_by_region.keys():
counts = Counter([p.name for p in policies_by_region[region]])
for policy, count in six.iteritems(counts):
if count > 1:
log.error("duplicate policy name '{}'".format(policy))
sys.exit(1)
# Variable expansion and non schema validation (not optional)
for p in policies:
p.expand_variables(p.get_variables())
p.validate()
return f(options, list(policies))
def dispatch_event(event, context):
global account_id
error = event.get('detail', {}).get('errorCode')
if error:
log.debug("Skipping failed operation: %s" % error)
return
event['debug'] = True
if event['debug']:
log.info("Processing event\n %s", format_event(event))
# Policies file should always be valid in lambda so do loading naively
with open('config.json') as f:
policy_config = json.load(f)
if not policy_config or not policy_config.get('policies'):
return False
# Initialize output directory, we've seen occassional perm issues with
# lambda on temp directory and changing unix execution users, so
# use a per execution temp space.
output_dir = os.environ.get(
'C7N_OUTPUT_DIR',
'/tmp/' + str(uuid.uuid4()))
if not os.path.exists(output_dir):
try:
os.mkdir(output_dir)
except OSError as error:
log.warning("Unable to make output directory: {}".format(error))
# TODO. This enshrines an assumption of a single policy per lambda.
options_overrides = policy_config[
'policies'][0].get('mode', {}).get('execution-options', {})
# if using assume role in lambda ensure that the correct
# execution account is captured in options.
if 'assume_role' in options_overrides:
account_id = options_overrides['assume_role'].split(':')[4]
elif account_id is None:
session = boto3.Session()
account_id = get_account_id_from_sts(session)
# Historical compatibility with manually set execution options
# previously this was a boolean, its now a string value with the
# boolean flag triggering a string value of 'aws'
if 'metrics_enabled' in options_overrides and isinstance(
options_overrides['metrics_enabled'], bool):
options_overrides['metrics_enabled'] = 'aws'
options_overrides['account_id'] = account_id
if 'output_dir' not in options_overrides:
options_overrides['output_dir'] = output_dir
options = Config.empty(**options_overrides)
policies = PolicyCollection.from_data(policy_config, options)
if policies:
for p in policies:
p.push(event, context)
return True
def load_policies(config_files, config):
policies = PolicyCollection([], config)
for f in config_files:
policies += policy_load(config, f)
return policies
