def report(policies, start_date, options, output_fh, raw_output_fh=None):
"""Format a policy's extant records into a report."""
regions = set([p.options.region for p in policies])
policy_names = set([p.name for p in policies])
formatter = Formatter(
policies[0].resource_manager.resource_type,
extra_fields=options.field,
include_default_fields=not options.no_default_fields,
include_region=len(regions) > 1,
include_policy=len(policy_names) > 1
)
records = []
for policy in policies:
# initialize policy execution context for output access
policy.ctx.initialize()
if policy.ctx.output.type == 's3':
policy_records = record_set(
policy.session_factory,
policy.ctx.output.config['netloc'],
policy.ctx.output.config['path'].strip('/'),
start_date)
else:
policy_records = fs_record_set(policy.ctx.log_dir, policy.name)
log.debug("Found %d records for region %s", len(policy_records), policy.options.region)
for record in policy_records:
record['policy'] = policy.name
record['region'] = policy.options.region
records += policy_records
rows = formatter.to_csv(records)
if options.format == 'csv':
writer = UnicodeWriter(output_fh, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
elif options.format == 'json':
print(dumps(records, indent=2))
else:
# We special case CSV, and for other formats we pass to tabulate
print(tabulate(rows, formatter.headers(), tablefmt=options.format))
if raw_output_fh is not None:
dumps(records, raw_output_fh, indent=2)
def report(policies, start_date, options, output_fh, raw_output_fh=None):
"""Format a policy's extant records into a report."""
regions = set([p.options.region for p in policies])
policy_names = set([p.name for p in policies])
formatter = Formatter(policies[0].resource_manager.resource_type,
extra_fields=options.field,
include_default_fields=not options.no_default_fields,
include_region=len(regions) > 1,
include_policy=len(policy_names) > 1)
records = []
for policy in policies:
# initialize policy execution context for output access
policy.ctx.initialize()
if policy.ctx.output.type == 's3':
policy_records = record_set(policy.session_factory,
policy.ctx.output.bucket,
policy.ctx.output.key_prefix,
start_date)
else:
policy_records = fs_record_set(policy.ctx.log_dir, policy.name)
log.debug("Found %d records for region %s", len(policy_records),
policy.options.region)
for record in policy_records:
record['policy'] = policy.name
record['region'] = policy.options.region
records += policy_records
rows = formatter.to_csv(records)
if options.format == 'csv':
writer = UnicodeWriter(output_fh, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
elif options.format == 'json':
print(dumps(records, indent=2))
else:
# We special case CSV, and for other formats we pass to tabulate
print(tabulate(rows, formatter.headers(), tablefmt=options.format))
if raw_output_fh is not None:
dumps(records, raw_output_fh, indent=2)
def report(policies, start_date, options, output_fh, raw_output_fh=None):
"""Format a policy's extant records into a report."""
regions = set([p.options.region for p in policies])
policy_names = set([p.name for p in policies])
formatter = Formatter(
policies[0].resource_manager.resource_type,
extra_fields=options.field,
include_default_fields=not options.no_default_fields,
include_region=len(regions) > 1,
include_policy=len(policy_names) > 1
)
records = []
for policy in policies:
if policy.ctx.output.use_s3():
policy_records = record_set(
policy.session_factory,
policy.ctx.output.bucket,
policy.ctx.output.key_prefix,
start_date)
else:
policy_records = fs_record_set(policy.ctx.output_path, policy.name)
log.debug("Found %d records for region %s", len(policy_records), policy.options.region)
for record in policy_records:
record['policy'] = policy.name
record['region'] = policy.options.region
records += policy_records
rows = formatter.to_csv(records)
if options.format == 'csv':
writer = UnicodeWriter(output_fh, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
else:
# We special case CSV, and for other formats we pass to tabulate
print(tabulate(rows, formatter.headers(), tablefmt=options.format))
if raw_output_fh is not None:
dumps(records, raw_output_fh, indent=2)
def report(config, output, use, output_dir, accounts,
field, no_default_fields, tags, region, debug, verbose,
policy, policy_tags, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(
config, use, debug, verbose, accounts, tags, policy,
resource=resource, policy_tags=policy_tags)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
elif not len(custodian_config['policies']) > 0:
raise ValueError("no matching policies found")
records = []
with executor(max_workers=WORKER_COUNT) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
for r in resolve_regions(region or a.get('regions', ())):
futures[w.submit(
report_account,
a, r,
custodian_config,
output_dir,
debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning(
"Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug(
"Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Config.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(
factory.resource_type,
extra_fields=field,
include_default_fields=not(no_default_fields),
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = UnicodeWriter(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
def report(config, output, use, output_dir, accounts,
field, no_default_fields, tags, region, debug, verbose,
policy, policy_tags, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(
config, use, debug, verbose, accounts, tags, policy,
resource=resource, policy_tags=policy_tags)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
elif not len(custodian_config['policies']) > 0:
raise ValueError("no matching policies found")
records = []
with executor(max_workers=WORKER_COUNT) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
for r in resolve_regions(region or a.get('regions', ())):
futures[w.submit(
report_account,
a, r,
custodian_config,
output_dir,
debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning(
"Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug(
"Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Config.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(
factory.resource_type,
extra_fields=field,
include_default_fields=not(no_default_fields),
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = UnicodeWriter(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)