def get_service_region_map(regions, resource_types):
# we're not interacting with the apis just using the sdk meta information.
session = boto3.Session(
region_name='us-east-1',
aws_access_key_id='never',
aws_secret_access_key='found')
resource_service_map = {r: resources.get(r).resource_type.service
for r in resource_types if r != 'account'}
# support for govcloud and china, we only utilize these regions if they
# are explicitly passed in on the cli.
partition_regions = {}
for p in ('aws-cn', 'aws-us-gov'):
for r in session.get_available_regions('s3', partition_name=p):
partition_regions[r] = p
partitions = ['aws']
for r in regions:
if r in partition_regions:
partitions.append(partition_regions[r])
service_region_map = {}
for s in set(itertools.chain(resource_service_map.values())):
for partition in partitions:
service_region_map.setdefault(s, []).extend(
session.get_available_regions(s, partition_name=partition))
return service_region_map, resource_service_map
def get_resource_manager(self):
resource_type = self.data.get('resource')
factory = resources.get(resource_type)
if not factory:
raise ValueError(
"Invalid resource type: %s" % resource_type)
return factory(self.ctx, self.data)
def get_resource_manager(self):
resource_type = self.data.get('resource')
factory = resources.get(resource_type)
if not factory:
raise ValueError(
"Invalid resource type: %s" % resource_type)
return factory(self.ctx, self.data)
def report(config, output, use, output_dir, accounts, field, tags, region,
debug, verbose, policy, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(config,
use,
debug,
verbose,
accounts,
tags,
policy,
resource=resource)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
records = []
with executor(max_workers=16) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
account_regions = region or a['regions']
for r in account_regions:
futures[w.submit(report_account, a, r, custodian_config,
output_dir, debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning("Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug("Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Bag.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(factory.resource_type,
extra_fields=field,
include_default_fields=True,
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = csv.writer(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
def report(config, output, use, output_dir, accounts, field, tags, region, debug, verbose, policy, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(
config, use, debug, verbose, accounts, tags, policy, resource=resource)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
records = []
with executor(max_workers=16) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
account_regions = region or a['regions']
for r in account_regions:
futures[w.submit(
report_account,
a, r,
custodian_config,
output_dir,
debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning(
"Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug(
"Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Bag.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(
factory.resource_type,
extra_fields=field,
include_default_fields=True,
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = csv.writer(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
def get_service_region_map(regions, resource_types):
# we're not interacting with the apis just using the sdk meta information.
session = boto3.Session(
region_name='us-east-1',
aws_access_key_id='never',
aws_secret_access_key='found')
resource_service_map = {r: resources.get(r).resource_type.service
for r in resource_types if r != 'account'}
# support for govcloud and china, we only utilize these regions if they
# are explicitly passed in on the cli.
partition_regions = {}
for p in ('aws-cn', 'aws-us-gov'):
for r in session.get_available_regions('s3', partition_name=p):
partition_regions[r] = p
partitions = ['aws']
for r in regions:
if r in partition_regions:
partitions.append(partition_regions[r])
service_region_map = {}
for s in set(itertools.chain(resource_service_map.values())):
for partition in partitions:
service_region_map.setdefault(s, []).extend(
session.get_available_regions(s, partition_name=partition))
return service_region_map, resource_service_map
def expand_regions(self, regions):
"""Return a set of policies targetted to the given regions.
Supports symbolic regions like 'all'. This will automatically filter out policies
if their being targetted to a region that does not support the service. Global
services will target a single region (us-east-1 if only all specified, else
first region in the list).
"""
# we're not interacting with the apis just using the sdk meta information.
session = boto3.Session(region_name='us-east-1',
aws_access_key_id='never',
aws_secret_access_key='found')
resource_service_map = {
r: resources.get(r).resource_type.service
for r in self.resource_types if r != 'account'
}
service_region_map = {
s: session.get_available_regions(s)
for s in set(itertools.chain(resource_service_map.values()))
}
policies = []
for p in self.policies:
available_regions = service_region_map.get(
resource_service_map.get(p.resource_type), ())
# its a global service/endpoint, use user provided region or us-east-1.
if not available_regions and regions:
candidates = [r for r in regions if r != 'all']
candidate = candidates and candidates[0] or 'us-east-1'
svc_regions = [candidate]
elif 'all' in regions:
svc_regions = available_regions
else:
svc_regions = regions
for region in svc_regions:
if available_regions and region not in available_regions:
level = 'all' in self.options.regions and logging.DEBUG or logging.WARNING
self.log.log(
level,
"policy:%s resources:%s not available in region:%s",
p.name, p.resource_type, region)
continue
options_copy = copy.copy(self.options)
options_copy.region = str(region)
if len(regions) > 1 or 'all' in regions and getattr(
self.options, 'output_dir', None):
options_copy.output_dir = (
self.options.output_dir.rstrip('/') + '/%s' % region)
policies.append(
Policy(p.data,
options_copy,
session_factory=self.test_session_factory()))
return PolicyCollection(policies, self.options)
def _add_annotations(self, related_ids, resource):
resources = self.get_related([resource])
a_resources = resources.get(resource[self.RelatedIdsExpression], [])
akey = 'c7n:%s' % self.AnnotationKey
resource[akey] = a_resources