def report(config, output, use, output_dir, accounts, field, tags, region,
debug, verbose, policy, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(config,
use,
debug,
verbose,
accounts,
tags,
policy,
resource=resource)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
records = []
with executor(max_workers=16) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
account_regions = region or a['regions']
for r in account_regions:
futures[w.submit(report_account, a, r, custodian_config,
output_dir, debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning("Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug("Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Bag.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(factory.resource_type,
extra_fields=field,
include_default_fields=True,
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = csv.writer(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
def report(config, output, use, output_dir, accounts, field, tags, region, debug, verbose, policy, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(
config, use, debug, verbose, accounts, tags, policy, resource=resource)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
records = []
with executor(max_workers=16) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
account_regions = region or a['regions']
for r in account_regions:
futures[w.submit(
report_account,
a, r,
custodian_config,
output_dir,
debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning(
"Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug(
"Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Bag.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(
factory.resource_type,
extra_fields=field,
include_default_fields=True,
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = csv.writer(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
def test_custom_fields(self):
# Test the ability to include custom fields.
extra_fields = [
"custom_field=CustomField",
"missing_field=MissingField",
"custom_tag=tag:CustomTag",
]
# First do a test with adding custom fields to the normal ones
formatter = Formatter(
EC2_POLICY.resource_manager,
extra_fields=extra_fields,
)
recs = [self.records['full']]
rows = [self.rows['full_custom']]
self.assertEqual(formatter.to_csv(recs), rows)
# Then do a test with only having custom fields
formatter = Formatter(
EC2_POLICY.resource_manager,
extra_fields=extra_fields,
no_default_fields=True,
)
recs = [self.records['full']]
rows = [self.rows['minimal_custom']]
self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self):
formatter = Formatter(ASG_POLICY.resource_manager)
tests = [(['full'], ['full']), (['minimal'], ['minimal']),
(['full', 'minimal'], ['full', 'minimal']),
(['full', 'duplicate', 'minimal'], ['full', 'minimal'])]
for rec_ids, row_ids in tests:
recs = map(lambda x: self.records[x], rec_ids)
rows = map(lambda x: self.rows[x], row_ids)
self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self):
# Test the extra headers for multi-policy
formatter = Formatter(EC2_POLICY.resource_manager, include_region=True, include_policy=True)
tests = [
(['minimal'], ['minimal_multipolicy']),
]
for rec_ids, row_ids in tests:
recs = list(map(lambda x: self.records[x], rec_ids))
rows = list(map(lambda x: self.rows[x], row_ids))
self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self):
# Test the extra headers for multi-policy
formatter = Formatter(EC2_POLICY.resource_manager, include_region=True, include_policy=True)
tests = [
(['minimal'], ['minimal_multipolicy']),
]
for rec_ids, row_ids in tests:
recs = list(map(lambda x: self.records[x], rec_ids))
rows = list(map(lambda x: self.rows[x], row_ids))
self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self):
formatter = Formatter(ELB_POLICY.resource_manager)
tests = [
(['full'], ['full']),
(['minimal'], ['minimal']),
(['full', 'minimal'], ['full', 'minimal']),
(['full', 'duplicate', 'minimal'], ['full', 'minimal'])]
for rec_ids, row_ids in tests:
recs = map(lambda x: self.records[x], rec_ids)
rows = map(lambda x: self.rows[x], row_ids)
self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self):
formatter = Formatter(ASG_POLICY.resource_manager.resource_type)
tests = [
(["full"], ["full"]),
(["minimal"], ["minimal"]),
(["full", "minimal"], ["full", "minimal"]),
(["full", "duplicate", "minimal"], ["full", "minimal"]),
]
for rec_ids, row_ids in tests:
recs = list(map(lambda x: self.records[x], rec_ids))
rows = list(map(lambda x: self.rows[x], row_ids))
self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self):
p = self.load_policy({"name": "report-test-elb", "resource": "elb"})
formatter = Formatter(p.resource_manager.resource_type)
tests = [
(["full"], ["full"]),
(["minimal"], ["minimal"]),
(["full", "minimal"], ["full", "minimal"]),
(["full", "duplicate", "minimal"], ["full", "minimal"]),
]
for rec_ids, row_ids in tests:
recs = list(map(lambda x: self.records[x], rec_ids))
rows = list(map(lambda x: self.rows[x], row_ids))
self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self):
# Test the extra headers for multi-policy
p = self.load_policy({"name": "report-test-ec2", "resource": "ec2"})
formatter = Formatter(
p.resource_manager.resource_type,
include_region=True,
include_policy=True,
)
tests = [(["minimal"], ["minimal_multipolicy"])]
for rec_ids, row_ids in tests:
recs = list(map(lambda x: self.records[x], rec_ids))
rows = list(map(lambda x: self.rows[x], row_ids))
self.assertEqual(formatter.to_csv(recs), rows)
def test_custom_fields(self):
# Test the ability to include custom fields.
extra_fields = [
"custom_field=CustomField",
"missing_field=MissingField",
"custom_tag=tag:CustomTag",
]
# First do a test with adding custom fields to the normal ones
formatter = Formatter(
EC2_POLICY.resource_manager,
extra_fields=extra_fields,
)
recs = [self.records['full']]
rows = [self.rows['full_custom']]
self.assertEqual(formatter.to_csv(recs), rows)
# Then do a test with only having custom fields
formatter = Formatter(
EC2_POLICY.resource_manager,
extra_fields=extra_fields,
include_default_fields=False,
)
recs = [self.records['full']]
rows = [self.rows['minimal_custom']]
self.assertEqual(formatter.to_csv(recs), rows)
def print_report(report_obj, options, account, region, report_name):
hash_str = '###################'
report_header = '\n\n%s Account: %s - Region: %s - Report %s %s\n' % (
hash_str, account, region, report_name, hash_str)
sys.stdout.write(report_header)
formatter = Formatter(
report_obj.resource_manager,
extra_fields=options.field,
no_default_fields=options.no_default_fields,
)
records = fs_record_set(report_obj.ctx.output_path, report_obj.name)
rows = formatter.to_csv(records)
if options.format == 'csv':
writer = csv.writer(options.raw, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
else:
# We special case CSV, and for other formats we pass to tabulate
print(tabulate(rows, formatter.headers(), tablefmt=options.format))
def test_default_csv(self):
self.patch(self.p.resource_manager.resource_type,
'default_report_fields', ())
formatter = Formatter(self.p.resource_manager.resource_type)
self.assertEqual(formatter.to_csv([self.records['full']]),
[['InstanceId-1', '', 'LaunchTime-1']])
