def report(config, output, use, output_dir, accounts, field, tags, region, debug, verbose, policy, format, resource): """report on a cross account policy execution.""" accounts_config, custodian_config, executor = init(config, use, debug, verbose, accounts, tags, policy, resource=resource) resource_types = set() for p in custodian_config.get('policies'): resource_types.add(p['resource']) if len(resource_types) > 1: raise ValueError("can only report on one resource type at a time") records = [] with executor(max_workers=16) as w: futures = {} for a in accounts_config.get('accounts', ()): account_regions = region or a['regions'] for r in account_regions: futures[w.submit(report_account, a, r, custodian_config, output_dir, debug)] = (a, r) for f in as_completed(futures): a, r = futures[f] if f.exception(): if debug: raise log.warning("Error running policy in %s @ %s exception: %s", a['name'], r, f.exception()) records.extend(f.result()) log.debug("Found %d records across %d accounts and %d policies", len(records), len(accounts_config['accounts']), len(custodian_config['policies'])) if format == 'json': dumps(records, output, indent=2) return prefix_fields = OrderedDict( (('Account', 'account'), ('Region', 'region'), ('Policy', 'policy'))) config = Bag.empty() factory = resource_registry.get(list(resource_types)[0]) formatter = Formatter(factory.resource_type, extra_fields=field, include_default_fields=True, include_region=False, include_policy=False, fields=prefix_fields) rows = formatter.to_csv(records, unique=False) writer = csv.writer(output, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows)
def report(config, output, use, output_dir, accounts, field, tags, region, debug, verbose, policy, format, resource): """report on a cross account policy execution.""" accounts_config, custodian_config, executor = init( config, use, debug, verbose, accounts, tags, policy, resource=resource) resource_types = set() for p in custodian_config.get('policies'): resource_types.add(p['resource']) if len(resource_types) > 1: raise ValueError("can only report on one resource type at a time") records = [] with executor(max_workers=16) as w: futures = {} for a in accounts_config.get('accounts', ()): account_regions = region or a['regions'] for r in account_regions: futures[w.submit( report_account, a, r, custodian_config, output_dir, debug)] = (a, r) for f in as_completed(futures): a, r = futures[f] if f.exception(): if debug: raise log.warning( "Error running policy in %s @ %s exception: %s", a['name'], r, f.exception()) records.extend(f.result()) log.debug( "Found %d records across %d accounts and %d policies", len(records), len(accounts_config['accounts']), len(custodian_config['policies'])) if format == 'json': dumps(records, output, indent=2) return prefix_fields = OrderedDict( (('Account', 'account'), ('Region', 'region'), ('Policy', 'policy'))) config = Bag.empty() factory = resource_registry.get(list(resource_types)[0]) formatter = Formatter( factory.resource_type, extra_fields=field, include_default_fields=True, include_region=False, include_policy=False, fields=prefix_fields) rows = formatter.to_csv(records, unique=False) writer = csv.writer(output, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows)
def test_custom_fields(self): # Test the ability to include custom fields. extra_fields = [ "custom_field=CustomField", "missing_field=MissingField", "custom_tag=tag:CustomTag", ] # First do a test with adding custom fields to the normal ones formatter = Formatter( EC2_POLICY.resource_manager, extra_fields=extra_fields, ) recs = [self.records['full']] rows = [self.rows['full_custom']] self.assertEqual(formatter.to_csv(recs), rows) # Then do a test with only having custom fields formatter = Formatter( EC2_POLICY.resource_manager, extra_fields=extra_fields, no_default_fields=True, ) recs = [self.records['full']] rows = [self.rows['minimal_custom']] self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self): formatter = Formatter(ASG_POLICY.resource_manager) tests = [(['full'], ['full']), (['minimal'], ['minimal']), (['full', 'minimal'], ['full', 'minimal']), (['full', 'duplicate', 'minimal'], ['full', 'minimal'])] for rec_ids, row_ids in tests: recs = map(lambda x: self.records[x], rec_ids) rows = map(lambda x: self.rows[x], row_ids) self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self): # Test the extra headers for multi-policy formatter = Formatter(EC2_POLICY.resource_manager, include_region=True, include_policy=True) tests = [ (['minimal'], ['minimal_multipolicy']), ] for rec_ids, row_ids in tests: recs = list(map(lambda x: self.records[x], rec_ids)) rows = list(map(lambda x: self.rows[x], row_ids)) self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self): formatter = Formatter(ELB_POLICY.resource_manager) tests = [ (['full'], ['full']), (['minimal'], ['minimal']), (['full', 'minimal'], ['full', 'minimal']), (['full', 'duplicate', 'minimal'], ['full', 'minimal'])] for rec_ids, row_ids in tests: recs = map(lambda x: self.records[x], rec_ids) rows = map(lambda x: self.rows[x], row_ids) self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self): formatter = Formatter(ASG_POLICY.resource_manager.resource_type) tests = [ (["full"], ["full"]), (["minimal"], ["minimal"]), (["full", "minimal"], ["full", "minimal"]), (["full", "duplicate", "minimal"], ["full", "minimal"]), ] for rec_ids, row_ids in tests: recs = list(map(lambda x: self.records[x], rec_ids)) rows = list(map(lambda x: self.rows[x], row_ids)) self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self): p = self.load_policy({"name": "report-test-elb", "resource": "elb"}) formatter = Formatter(p.resource_manager.resource_type) tests = [ (["full"], ["full"]), (["minimal"], ["minimal"]), (["full", "minimal"], ["full", "minimal"]), (["full", "duplicate", "minimal"], ["full", "minimal"]), ] for rec_ids, row_ids in tests: recs = list(map(lambda x: self.records[x], rec_ids)) rows = list(map(lambda x: self.rows[x], row_ids)) self.assertEqual(formatter.to_csv(recs), rows)
def test_csv(self): # Test the extra headers for multi-policy p = self.load_policy({"name": "report-test-ec2", "resource": "ec2"}) formatter = Formatter( p.resource_manager.resource_type, include_region=True, include_policy=True, ) tests = [(["minimal"], ["minimal_multipolicy"])] for rec_ids, row_ids in tests: recs = list(map(lambda x: self.records[x], rec_ids)) rows = list(map(lambda x: self.rows[x], row_ids)) self.assertEqual(formatter.to_csv(recs), rows)
def test_custom_fields(self): # Test the ability to include custom fields. extra_fields = [ "custom_field=CustomField", "missing_field=MissingField", "custom_tag=tag:CustomTag", ] # First do a test with adding custom fields to the normal ones formatter = Formatter( EC2_POLICY.resource_manager, extra_fields=extra_fields, ) recs = [self.records['full']] rows = [self.rows['full_custom']] self.assertEqual(formatter.to_csv(recs), rows) # Then do a test with only having custom fields formatter = Formatter( EC2_POLICY.resource_manager, extra_fields=extra_fields, include_default_fields=False, ) recs = [self.records['full']] rows = [self.rows['minimal_custom']] self.assertEqual(formatter.to_csv(recs), rows)
def print_report(report_obj, options, account, region, report_name): hash_str = '###################' report_header = '\n\n%s Account: %s - Region: %s - Report %s %s\n' % ( hash_str, account, region, report_name, hash_str) sys.stdout.write(report_header) formatter = Formatter( report_obj.resource_manager, extra_fields=options.field, no_default_fields=options.no_default_fields, ) records = fs_record_set(report_obj.ctx.output_path, report_obj.name) rows = formatter.to_csv(records) if options.format == 'csv': writer = csv.writer(options.raw, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows) else: # We special case CSV, and for other formats we pass to tabulate print(tabulate(rows, formatter.headers(), tablefmt=options.format))
def test_default_csv(self): self.patch(self.p.resource_manager.resource_type, 'default_report_fields', ()) formatter = Formatter(self.p.resource_manager.resource_type) self.assertEqual(formatter.to_csv([self.records['full']]), [['InstanceId-1', '', 'LaunchTime-1']])