def get_sqs_queue_policies(
session: botocore.session.Session,
account_id: str,
region_allow_list: Optional[List[str]] = None,
region_deny_list: Optional[List[str]] = None) -> List[Policy]:
"""Using a botocore Session object, return a list of Policy objects representing the key policies of each
KMS key in this account.
The region allow/deny lists are mutually-exclusive (i.e. at least one of which has the value None) lists of
allowed/denied regions to pull data from.
"""
result = []
# Iterate through all regions of SQS where possible
for sqs_region in get_regions_to_search(session, 'sqs', region_allow_list,
region_deny_list):
try:
# Grab the queue names
queue_urls = []
sqsclient = session.create_client('sqs', region_name=sqs_region)
response = sqsclient.list_queues()
if 'QueueUrls' in response:
queue_urls.extend(response['QueueUrls'])
else:
continue
# Grab the queue policies
for queue_url in queue_urls:
queue_name = queue_url.split('/')[-1]
sqs_policy_response = sqsclient.get_queue_attributes(
QueueUrl=queue_url, AttributeNames=['Policy'])
if 'Policy' in sqs_policy_response:
sqs_policy_doc = json.loads(sqs_policy_response['Policy'])
result.append(
Policy(
'arn:aws:sqs:{}:{}:{}'.format(
sqs_region, account_id, queue_name),
queue_name, json.loads(sqs_policy_doc)))
logger.info('Caching policy for {}'.format(
'arn:aws:sqs:{}:{}:{}'.format(sqs_region, account_id,
queue_name)))
else:
result.append(
Policy(
'arn:aws:sqs:{}:{}:{}'.format(
sqs_region, account_id, queue_name),
queue_name, {
"Statement": [],
"Version": "2012-10-17"
}))
logger.info(
'Queue {} does not have a bucket policy, adding a "stub" policy instead.'
.format(queue_name))
except botocore.exceptions.ClientError as ex:
logger.info(
'Unable to search SQS in region {} for queues. The region may be disabled, or the current principal may not be authorized to access the service. Continuing.'
.format(sqs_region))
logger.debug('Exception was: {}'.format(ex))
return result
def return_edges(self,
nodes: List[Node],
region_allow_list: Optional[List[str]] = None,
region_deny_list: Optional[List[str]] = None,
scps: Optional[List[List[dict]]] = None,
client_args_map: Optional[dict] = None) -> List[Edge]:
"""Fulfills expected method return_edges."""
logger.info('Generating Edges based on EC2 Auto Scaling.')
if client_args_map is None:
asargs = {}
else:
asargs = client_args_map.get('autoscaling', {})
# Gather projects information for each region
autoscaling_clients = []
if self.session is not None:
as_regions = botocore_tools.get_regions_to_search(
self.session, 'autoscaling', region_allow_list,
region_deny_list)
for region in as_regions:
autoscaling_clients.append(
self.session.create_client('autoscaling',
region_name=region,
**asargs))
launch_configs = []
for as_client in autoscaling_clients:
logger.debug('Looking at region {}'.format(
as_client.meta.region_name))
try:
lc_paginator = as_client.get_paginator(
'describe_launch_configurations')
for page in lc_paginator.paginate():
if 'LaunchConfigurations' in page:
for launch_config in page['LaunchConfigurations']:
if 'IamInstanceProfile' in launch_config and launch_config[
'IamInstanceProfile']:
launch_configs.append({
'lc_arn':
launch_config['LaunchConfigurationARN'],
'lc_iip':
launch_config['IamInstanceProfile']
})
except ClientError as ex:
logger.warning(
'Unable to search region {} for launch configs. The region may be disabled, or the error may '
'be caused by an authorization issue. Continuing.'.format(
as_client.meta.region_name))
logger.debug('Exception details: {}'.format(ex))
result = generate_edges_locally(nodes, scps, launch_configs)
for edge in result:
logger.info("Found new edge: {}".format(edge.describe_edge()))
return result
def return_edges(self,
nodes: List[Node],
region_allow_list: Optional[List[str]] = None,
region_deny_list: Optional[List[str]] = None,
scps: Optional[List[List[dict]]] = None,
client_args_map: Optional[dict] = None) -> List[Edge]:
"""Fulfills expected method return_edges."""
logger.info('Pulling data on CloudFormation stacks.')
if client_args_map is None:
cfargs = {}
else:
cfargs = client_args_map.get('cloudformation', {})
# Grab existing stacks in each region
cloudformation_clients = []
if self.session is not None:
cf_regions = botocore_tools.get_regions_to_search(
self.session, 'cloudformation', region_allow_list,
region_deny_list)
for region in cf_regions:
cloudformation_clients.append(
self.session.create_client('cloudformation',
region_name=region,
**cfargs))
# grab existing cloudformation stacks
stack_list = []
for cf_client in cloudformation_clients:
logger.debug('Looking at region {}'.format(
cf_client.meta.region_name))
try:
paginator = cf_client.get_paginator('describe_stacks')
for page in paginator.paginate():
for stack in page['Stacks']:
if stack['StackStatus'] not in [
'CREATE_FAILED', 'DELETE_COMPLETE',
'DELETE_FAILED', 'DELETE_IN_PROGRESS'
]: # ignore unusable stacks
stack_list.append(stack)
except ClientError as ex:
logger.warning(
'Unable to search region {} for stacks. The region may be disabled, or the error may '
'be caused by an authorization issue. Continuing.'.format(
cf_client.meta.region_name))
logger.debug('Exception details: {}'.format(ex))
logger.info('Generating Edges based on data from CloudFormation.')
result = generate_edges_locally(nodes, stack_list, scps)
for edge in result:
logger.info("Found new edge: {}".format(edge.describe_edge()))
return result
def return_edges(self,
nodes: List[Node],
region_allow_list: Optional[List[str]] = None,
region_deny_list: Optional[List[str]] = None,
scps: Optional[List[List[dict]]] = None,
client_args_map: Optional[dict] = None) -> List[Edge]:
"""Fulfills expected method return_edges. If session object is None, runs checks in offline mode."""
logger.info('Pulling data on Lambda functions')
if client_args_map is None:
lambdaargs = {}
else:
lambdaargs = client_args_map.get('lambda', {})
lambda_clients = []
if self.session is not None:
lambda_regions = botocore_tools.get_regions_to_search(
self.session, 'lambda', region_allow_list, region_deny_list)
for region in lambda_regions:
lambda_clients.append(
self.session.create_client('lambda',
region_name=region,
**lambdaargs))
# grab existing lambda functions
function_list = []
for lambda_client in lambda_clients:
try:
paginator = lambda_client.get_paginator('list_functions')
for page in paginator.paginate(
PaginationConfig={'PageSize': 25}):
for func in page['Functions']:
function_list.append(func)
except ClientError as ex:
logger.warning(
'Unable to search region {} for stacks. The region may be disabled, or the error may '
'be caused by an authorization issue. Continuing.'.format(
lambda_client.meta.region_name))
logger.debug('Exception details: {}'.format(ex))
logger.info('Generating Edges based on Lambda data.')
logger.debug('Identified {} Lambda functions for processing'.format(
len(function_list)))
result = generate_edges_locally(nodes, function_list, scps)
for edge in result:
logger.info("Found new edge: {}".format(edge.describe_edge()))
return result
def get_sns_topic_policies(
session: botocore.session.Session,
region_allow_list: Optional[List[str]] = None,
region_deny_list: Optional[List[str]] = None,
client_args_map: Optional[dict] = None) -> List[Policy]:
"""Using a botocore Session object, return a list of Policy objects representing the topic policies of each
SNS topic in this account.
The region allow/deny lists are mutually-exclusive (i.e. at least one of which has the value None) lists of
allowed/denied regions to pull data from.
"""
result = []
snsargs = client_args_map.get('sns', {})
# Iterate through all regions of SNS where possible
for sns_region in get_regions_to_search(session, 'sns', region_allow_list,
region_deny_list):
try:
# Grab the topics
topics = []
snsclient = session.create_client('sns',
region_name=sns_region,
**snsargs)
sns_paginator = snsclient.get_paginator('list_topics')
for page in sns_paginator.paginate():
topics.extend([x['TopicArn'] for x in page['Topics']])
# Grab the topic policies
for topic in topics:
policy_str = snsclient.get_topic_attributes(
TopicArn=topic)['Attributes']['Policy']
result.append(
Policy(
topic,
topic.split(':')
[-1], # SNS Topic ARN Format: arn:<partition>:sns:<region>:<account>:<Topic Name>
json.loads(policy_str)))
logger.info('Caching policy for {}'.format(topic))
except botocore.exceptions.ClientError as ex:
logger.info(
'Unable to search SNS in region {} for topic policies. The region may be disabled, or the current principal may not be authorized to access the service. Continuing.'
.format(sns_region))
logger.debug('Exception was: {}'.format(ex))
continue
return result
def get_kms_key_policies(
session: botocore.session.Session,
region_allow_list: Optional[List[str]] = None,
region_deny_list: Optional[List[str]] = None,
client_args_map: Optional[dict] = None) -> List[Policy]:
"""Using a botocore Session object, return a list of Policy objects representing the key policies of each
KMS key in this account.
The region allow/deny lists are mutually-exclusive (i.e. at least one of which has the value None) lists of
allowed/denied regions to pull data from.
"""
result = []
kmsargs = client_args_map.get('kms', {})
# Iterate through all regions of KMS where possible
for kms_region in get_regions_to_search(session, 'kms', region_allow_list,
region_deny_list):
try:
# Grab the keys
cmks = []
kmsclient = session.create_client('kms',
region_name=kms_region,
**kmsargs)
kms_paginator = kmsclient.get_paginator('list_keys')
for page in kms_paginator.paginate():
cmks.extend([x['KeyArn'] for x in page['Keys']])
# Grab the key policies
for cmk in cmks:
policy_str = kmsclient.get_key_policy(
KeyId=cmk, PolicyName='default')['Policy']
result.append(
Policy(
cmk,
cmk.split('/')
[-1], # CMK ARN Format: arn:<partition>:kms:<region>:<account>:key/<Key ID>
json.loads(policy_str)))
logger.info('Caching policy for {}'.format(cmk))
except botocore.exceptions.ClientError as ex:
logger.info(
'Unable to search KMS in region {} for key policies. The region may be disabled, or the current principal may not be authorized to access the service. Continuing.'
.format(kms_region))
logger.debug('Exception was: {}'.format(ex))
continue
return result
def get_secrets_manager_policies(
session: botocore.session.Session,
region_allow_list: Optional[List[str]] = None,
region_deny_list: Optional[List[str]] = None) -> List[Policy]:
"""Using a botocore Session object, return a list of Policy objects representing the resource policies
of the secrets in AWS Secrets Manager.
The region allow/deny lists are mutually-exclusive (i.e. at least one of which has the value None) lists of
allowed/denied regions to pull data from.
"""
result = []
# Iterate through all regions of Secrets Manager where possible
for sm_region in get_regions_to_search(session, 'secretsmanager',
region_allow_list,
region_deny_list):
try:
# Grab the ARNs of the secrets in this region
secret_arns = []
smclient = session.create_client('secretsmanager',
region_name=sm_region)
list_secrets_paginator = smclient.get_paginator('list_secrets')
for page in list_secrets_paginator.paginate():
if 'SecretList' in page:
for entry in page['SecretList']:
if 'PrimaryRegion' in entry and entry[
'PrimaryRegion'] != sm_region:
continue # skip things we're supposed to find in other regions
secret_arns.append(entry['ARN'])
# Grab resource policies for each secret
for secret_arn in secret_arns:
sm_response = smclient.get_resource_policy(SecretId=secret_arn)
# verify that it is in the response and not None/empty
if 'ResourcePolicy' in sm_response and sm_response[
'ResourcePolicy']:
sm_policy_doc = json.loads(sm_response['ResourcePolicy'])
result.append(
Policy(secret_arn, sm_response['Name'], sm_policy_doc))
logger.info(
'Storing the resource policy for secret {}'.format(
secret_arn))
else:
result.append(
Policy(secret_arn, sm_response['Name'], {
"Statement": [],
"Version": "2012-10-17"
}))
logger.info(
'Secret {} does not have a resource policy, inserting a "stub" policy instead'
.format(secret_arn))
except botocore.exceptions.ClientError as ex:
logger.info(
'Unable to search Secrets Manager in region {} for secrets. The region may be disabled, or '
'the current principal may not be authorized to access the service. '
'Continuing.'.format(sm_region))
logger.debug('Exception was: {}'.format(ex))
return result
def return_edges(self,
nodes: List[Node],
region_allow_list: Optional[List[str]] = None,
region_deny_list: Optional[List[str]] = None,
scps: Optional[List[List[dict]]] = None) -> List[Edge]:
"""Fulfills expected method return_edges."""
logger.info('Generating Edges based on CodeBuild.')
# Gather projects information for each region
codebuild_clients = []
if self.session is not None:
cf_regions = botocore_tools.get_regions_to_search(
self.session, 'codebuild', region_allow_list, region_deny_list)
for region in cf_regions:
codebuild_clients.append(
self.session.create_client('codebuild',
region_name=region))
codebuild_projects = []
for cb_client in codebuild_clients:
logger.debug('Looking at region {}'.format(
cb_client.meta.region_name))
region_project_list_list = []
try:
# list the projects first, 50 at a time
paginator = cb_client.get_paginator('list_projects')
for page in paginator.paginate(
PaginationConfig={'MaxItems': 50}):
if 'projects' in page and len(page['projects']) > 0:
region_project_list_list.append(page['projects'])
for region_project_list in region_project_list_list:
batch_project_data = cb_client.batch_get_projects(
names=region_project_list) # no pagination
if 'projects' in batch_project_data:
for project_data in batch_project_data['projects']:
if 'serviceRole' in project_data:
codebuild_projects.append({
'project_arn':
project_data['arn'],
'project_role':
project_data['serviceRole'],
'project_tags':
project_data['tags']
})
except ClientError as ex:
logger.warning(
'Unable to search region {} for projects. The region may be disabled, or the error may '
'be caused by an authorization issue. Continuing.'.format(
cb_client.meta.region_name))
logger.debug('Exception details: {}'.format(ex))
result = generate_edges_locally(nodes, scps, codebuild_projects)
for edge in result:
logger.info("Found new edge: {}".format(edge.describe_edge()))
return result
