def report(config, output, use, output_dir, accounts, field, tags, region,
debug, verbose, policy, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(config,
use,
debug,
verbose,
accounts,
tags,
policy,
resource=resource)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
records = []
with executor(max_workers=16) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
account_regions = region or a['regions']
for r in account_regions:
futures[w.submit(report_account, a, r, custodian_config,
output_dir, debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning("Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug("Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Bag.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(factory.resource_type,
extra_fields=field,
include_default_fields=True,
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = csv.writer(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
def report_account(account, region, policies_config, output_path, debug):
cache_path = os.path.join(output_path, "c7n.cache")
output_path = os.path.join(output_path, account['name'], region)
bag = Bag.empty(region=region,
assume_role=account['role'],
output_dir=output_path,
account_id=account['account_id'],
metrics_enabled=False,
cache=cache_path,
log_group=None,
profile=None,
external_id=None)
policies = PolicyCollection.from_data(policies_config, bag)
records = []
for p in policies:
log.debug("Report policy:%s account:%s region:%s path:%s", p.name,
account['name'], region, output_path)
policy_records = fs_record_set(p.ctx.output_path, p.name)
for r in policy_records:
r['policy'] = p.name
r['region'] = p.options.region
r['account'] = account['name']
for t in account['tags']:
if ':' in t:
k, v = t.split(':', 1)
r[k] = v
records.extend(policy_records)
return records
def run_account(account, region, policies_config, output_path, cache_period,
dryrun, debug):
"""Execute a set of policies on an account.
"""
logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
CONN_CACHE.session = None
CONN_CACHE.time = None
output_path = os.path.join(output_path, account['name'], region)
if not os.path.exists(output_path):
os.makedirs(output_path)
cache_path = os.path.join(output_path, "c7n.cache")
bag = Bag.empty(region=region,
assume_role=account['role'],
cache_period=cache_period,
dryrun=dryrun,
output_dir=output_path,
account_id=account['account_id'],
metrics_enabled=False,
cache=cache_path,
log_group=None,
profile=None,
external_id=None)
policies = PolicyCollection.from_data(policies_config, bag)
policy_counts = {}
st = time.time()
with environ(**account_tags(account)):
for p in policies:
log.debug("Running policy:%s account:%s region:%s", p.name,
account['name'], region)
try:
resources = p.run()
policy_counts[p.name] = resources and len(resources) or 0
if not resources:
continue
log.info(
"Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
account['name'], region, p.name, len(resources),
time.time() - st)
except ClientError as e:
if e.response['Error']['Code'] == 'AccessDenied':
log.warning('Access denied account:%s region:%s',
account['name'], region)
return policy_counts
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
continue
except Exception as e:
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
if not debug:
continue
import traceback, pdb, sys
pdb.post_mortem(sys.exc_info()[-1])
raise
return policy_counts
def report(config, output, use, output_dir, accounts, field, tags, region, debug, verbose, policy, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(
config, use, debug, verbose, accounts, tags, policy, resource=resource)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
records = []
with executor(max_workers=16) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
account_regions = region or a['regions']
for r in account_regions:
futures[w.submit(
report_account,
a, r,
custodian_config,
output_dir,
debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning(
"Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug(
"Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Bag.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(
factory.resource_type,
extra_fields=field,
include_default_fields=True,
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = csv.writer(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
def report_account(account, region, policies_config, output_path, debug):
cache_path = os.path.join(output_path, "c7n.cache")
output_path = os.path.join(output_path, account['name'], region)
bag = Bag.empty(
region=region, assume_role=account['role'],
output_dir=output_path,
account_id=account['account_id'], metrics_enabled=False,
cache=cache_path, log_group=None, profile=None, external_id=None)
policies = PolicyCollection.from_data(policies_config, bag)
records = []
for p in policies:
log.debug(
"Report policy:%s account:%s region:%s path:%s",
p.name, account['name'], region, output_path)
policy_records = fs_record_set(p.ctx.output_path, p.name)
for r in policy_records:
r['policy'] = p.name
r['region'] = p.options.region
r['account'] = account['name']
for t in account['tags']:
if ':' in t:
k, v = t.split(':', 1)
r[k] = v
records.extend(policy_records)
return records
def get_resource_manager(self, session, region, resource_id, policy=None):
def session_factory(assume=False, region=None):
if region:
session.set_config_variable('region', region)
return session
ctx = ExecutionContext(
session_factory, Config({"name": "sphere11"}),
Config.empty(verbose=True))
cls = self.get_resource_class(resource_id)
return cls(ctx, policy and policy or {})
def run_account(account, region, policies_config, output_path, cache_period, dryrun, debug):
"""Execute a set of policies on an account.
"""
CONN_CACHE.session = None
CONN_CACHE.time = None
output_path = os.path.join(output_path, account['name'], region)
if not os.path.exists(output_path):
os.makedirs(output_path)
cache_path = os.path.join(output_path, "c7n.cache")
bag = Bag.empty(
region=region, assume_role=account['role'],
cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
account_id=account['account_id'], metrics_enabled=False,
cache=cache_path, log_group=None, profile=None, external_id=None)
policies = PolicyCollection.from_data(policies_config, bag)
policy_counts = {}
st = time.time()
with environ(**account_tags(account)):
for p in policies:
log.debug(
"Running policy:%s account:%s region:%s", p.name, account['name'], region)
try:
resources = p.run()
policy_counts[p.name] = resources and len(resources) or 0
if not resources:
continue
log.info("Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
account['name'], region, p.name, len(resources), time.time()-st)
except Exception as e:
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
if not debug:
continue
import traceback, pdb, sys
pdb.post_mortem(sys.exc_info()[-1])
raise
return policy_counts