def report(policies, start_date, options, output_fh, raw_output_fh=None): """Format a policy's extant records into a report.""" regions = set([p.options.region for p in policies]) policy_names = set([p.name for p in policies]) formatter = Formatter( policies[0].resource_manager.resource_type, extra_fields=options.field, include_default_fields=not options.no_default_fields, include_region=len(regions) > 1, include_policy=len(policy_names) > 1 ) records = [] for policy in policies: # initialize policy execution context for output access policy.ctx.initialize() if policy.ctx.output.type == 's3': policy_records = record_set( policy.session_factory, policy.ctx.output.config['netloc'], policy.ctx.output.config['path'].strip('/'), start_date) else: policy_records = fs_record_set(policy.ctx.log_dir, policy.name) log.debug("Found %d records for region %s", len(policy_records), policy.options.region) for record in policy_records: record['policy'] = policy.name record['region'] = policy.options.region records += policy_records rows = formatter.to_csv(records) if options.format == 'csv': writer = UnicodeWriter(output_fh, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows) elif options.format == 'json': print(dumps(records, indent=2)) else: # We special case CSV, and for other formats we pass to tabulate print(tabulate(rows, formatter.headers(), tablefmt=options.format)) if raw_output_fh is not None: dumps(records, raw_output_fh, indent=2)
def report(policies, start_date, options, output_fh, raw_output_fh=None): """Format a policy's extant records into a report.""" regions = set([p.options.region for p in policies]) policy_names = set([p.name for p in policies]) formatter = Formatter(policies[0].resource_manager.resource_type, extra_fields=options.field, include_default_fields=not options.no_default_fields, include_region=len(regions) > 1, include_policy=len(policy_names) > 1) records = [] for policy in policies: # initialize policy execution context for output access policy.ctx.initialize() if policy.ctx.output.type == 's3': policy_records = record_set(policy.session_factory, policy.ctx.output.bucket, policy.ctx.output.key_prefix, start_date) else: policy_records = fs_record_set(policy.ctx.log_dir, policy.name) log.debug("Found %d records for region %s", len(policy_records), policy.options.region) for record in policy_records: record['policy'] = policy.name record['region'] = policy.options.region records += policy_records rows = formatter.to_csv(records) if options.format == 'csv': writer = UnicodeWriter(output_fh, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows) elif options.format == 'json': print(dumps(records, indent=2)) else: # We special case CSV, and for other formats we pass to tabulate print(tabulate(rows, formatter.headers(), tablefmt=options.format)) if raw_output_fh is not None: dumps(records, raw_output_fh, indent=2)
def report(policies, start_date, options, output_fh, raw_output_fh=None): """Format a policy's extant records into a report.""" regions = set([p.options.region for p in policies]) policy_names = set([p.name for p in policies]) formatter = Formatter( policies[0].resource_manager.resource_type, extra_fields=options.field, include_default_fields=not options.no_default_fields, include_region=len(regions) > 1, include_policy=len(policy_names) > 1 ) records = [] for policy in policies: if policy.ctx.output.use_s3(): policy_records = record_set( policy.session_factory, policy.ctx.output.bucket, policy.ctx.output.key_prefix, start_date) else: policy_records = fs_record_set(policy.ctx.output_path, policy.name) log.debug("Found %d records for region %s", len(policy_records), policy.options.region) for record in policy_records: record['policy'] = policy.name record['region'] = policy.options.region records += policy_records rows = formatter.to_csv(records) if options.format == 'csv': writer = UnicodeWriter(output_fh, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows) else: # We special case CSV, and for other formats we pass to tabulate print(tabulate(rows, formatter.headers(), tablefmt=options.format)) if raw_output_fh is not None: dumps(records, raw_output_fh, indent=2)
def report(config, output, use, output_dir, accounts, field, no_default_fields, tags, region, debug, verbose, policy, policy_tags, format, resource): """report on a cross account policy execution.""" accounts_config, custodian_config, executor = init( config, use, debug, verbose, accounts, tags, policy, resource=resource, policy_tags=policy_tags) resource_types = set() for p in custodian_config.get('policies'): resource_types.add(p['resource']) if len(resource_types) > 1: raise ValueError("can only report on one resource type at a time") elif not len(custodian_config['policies']) > 0: raise ValueError("no matching policies found") records = [] with executor(max_workers=WORKER_COUNT) as w: futures = {} for a in accounts_config.get('accounts', ()): for r in resolve_regions(region or a.get('regions', ())): futures[w.submit( report_account, a, r, custodian_config, output_dir, debug)] = (a, r) for f in as_completed(futures): a, r = futures[f] if f.exception(): if debug: raise log.warning( "Error running policy in %s @ %s exception: %s", a['name'], r, f.exception()) records.extend(f.result()) log.debug( "Found %d records across %d accounts and %d policies", len(records), len(accounts_config['accounts']), len(custodian_config['policies'])) if format == 'json': dumps(records, output, indent=2) return prefix_fields = OrderedDict( (('Account', 'account'), ('Region', 'region'), ('Policy', 'policy'))) config = Config.empty() factory = resource_registry.get(list(resource_types)[0]) formatter = Formatter( factory.resource_type, extra_fields=field, include_default_fields=not(no_default_fields), include_region=False, include_policy=False, fields=prefix_fields) rows = formatter.to_csv(records, unique=False) writer = UnicodeWriter(output, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows)
def report(config, output, use, output_dir, accounts, field, no_default_fields, tags, region, debug, verbose, policy, policy_tags, format, resource): """report on a cross account policy execution.""" accounts_config, custodian_config, executor = init( config, use, debug, verbose, accounts, tags, policy, resource=resource, policy_tags=policy_tags) resource_types = set() for p in custodian_config.get('policies'): resource_types.add(p['resource']) if len(resource_types) > 1: raise ValueError("can only report on one resource type at a time") elif not len(custodian_config['policies']) > 0: raise ValueError("no matching policies found") records = [] with executor(max_workers=WORKER_COUNT) as w: futures = {} for a in accounts_config.get('accounts', ()): for r in resolve_regions(region or a.get('regions', ())): futures[w.submit( report_account, a, r, custodian_config, output_dir, debug)] = (a, r) for f in as_completed(futures): a, r = futures[f] if f.exception(): if debug: raise log.warning( "Error running policy in %s @ %s exception: %s", a['name'], r, f.exception()) records.extend(f.result()) log.debug( "Found %d records across %d accounts and %d policies", len(records), len(accounts_config['accounts']), len(custodian_config['policies'])) if format == 'json': dumps(records, output, indent=2) return prefix_fields = OrderedDict( (('Account', 'account'), ('Region', 'region'), ('Policy', 'policy'))) config = Config.empty() factory = resource_registry.get(list(resource_types)[0]) formatter = Formatter( factory.resource_type, extra_fields=field, include_default_fields=not(no_default_fields), include_region=False, include_policy=False, fields=prefix_fields) rows = formatter.to_csv(records, unique=False) writer = UnicodeWriter(output, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows)