async def get_buckets(self):
try:
# If there are regions specified, try for each of them until one works.
# This is required in case there's an IAM policy that denies access to APIs on a regional basis,
# as per https://github.com/nccgroup/ScoutSuite/issues/727
region = None
if self.regions:
buckets = []
exception = ''
for region in self.regions:
try:
client = AWSFacadeUtils.get_client(
's3', self.session, region)
buckets = await run_concurrently(
lambda: client.list_buckets()['Buckets'])
except Exception as e:
exception = e
else:
break
if not buckets:
if exception:
print_exception(f'Failed to list buckets: {exception}')
return []
else:
client = AWSFacadeUtils.get_client('s3', self.session)
buckets = await run_concurrently(
lambda: client.list_buckets()['Buckets'])
except Exception as e:
print_exception(f'Failed to list buckets: {e}')
return []
else:
# We need first to retrieve bucket locations before retrieving bucket details
await get_and_set_concurrently(
[self._get_and_set_s3_bucket_location], buckets, region=region)
# Then we can retrieve bucket details concurrently
await get_and_set_concurrently([
self._get_and_set_s3_bucket_logging,
self._get_and_set_s3_bucket_versioning,
self._get_and_set_s3_bucket_webhosting,
self._get_and_set_s3_bucket_default_encryption,
self._get_and_set_s3_acls, self._get_and_set_s3_bucket_policy,
self._get_and_set_s3_bucket_tags,
self._get_and_set_s3_bucket_block_public_access
], buckets)
# Non-async post-processing
for bucket in buckets:
self._set_s3_bucket_secure_transport(bucket)
return buckets
async def _get_and_set_snapshot_attributes(self, snapshot: {},
region: str):
ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region)
snapshot['CreateVolumePermissions'] = await run_concurrently(
lambda: ec2_client.describe_snapshot_attribute(
Attribute='createVolumePermission',
SnapshotId=snapshot['SnapshotId'])['CreateVolumePermissions'])
async def _get_and_set_s3_acls(self, bucket: {}, key_name=None):
bucket_name = bucket['Name']
client = AWSFacadeUtils.get_client('s3', self.session, bucket['region'])
try:
grantees = {}
if key_name:
grants = await run_concurrently(lambda: client.get_object_acl(Bucket=bucket_name, Key=key_name))
else:
grants = await run_concurrently(lambda: client.get_bucket_acl(Bucket=bucket_name))
for grant in grants['Grants']:
if 'ID' in grant['Grantee']:
grantee = grant['Grantee']['ID']
display_name = grant['Grantee']['DisplayName'] if \
'DisplayName' in grant['Grantee'] else grant['Grantee']['ID']
elif 'URI' in grant['Grantee']:
grantee = grant['Grantee']['URI'].split('/')[-1]
display_name = self._s3_group_to_string(grant['Grantee']['URI'])
else:
grantee = display_name = 'Unknown'
permission = grant['Permission']
grantees.setdefault(grantee, {})
grantees[grantee]['DisplayName'] = display_name
if 'URI' in grant['Grantee']:
grantees[grantee]['URI'] = grant['Grantee']['URI']
grantees[grantee].setdefault('permissions', self._init_s3_permissions())
self._set_s3_permissions(grantees[grantee]['permissions'], permission)
bucket['grantees'] = grantees
except Exception as e:
print_exception('Failed to get ACL configuration for %s: %s' % (bucket_name, e))
bucket['grantees'] = {}
async def _get_and_set_load_balancer_attributes(self, load_balancer: dict,
region: str):
elbv2_client = AWSFacadeUtils.get_client('elbv2', self.session, region)
load_balancer['attributes'] = await run_concurrently(
lambda: elbv2_client.describe_load_balancer_attributes(
LoadBalancerArn=load_balancer['LoadBalancerArn'])['Attributes']
)
async def get_buckets(self):
client = AWSFacadeUtils.get_client('s3', self.session)
try:
buckets = await run_concurrently(
lambda: client.list_buckets()['Buckets'])
except Exception as e:
print_exception('Failed to list buckets: {}'.format(e))
return []
else:
# We need first to retrieve bucket locations before retrieving bucket details:
await get_and_set_concurrently(
[self._get_and_set_s3_bucket_location], buckets)
# Then we can retrieve bucket details concurrently:
await get_and_set_concurrently([
self._get_and_set_s3_bucket_logging,
self._get_and_set_s3_bucket_versioning,
self._get_and_set_s3_bucket_webhosting,
self._get_and_set_s3_bucket_default_encryption,
self._get_and_set_s3_acls, self._get_and_set_s3_bucket_policy
], buckets)
# Non-async post-processing:
for bucket in buckets:
self._set_s3_bucket_secure_transport(bucket)
return buckets
async def get_recorders(self, region: str):
client = AWSFacadeUtils.get_client('config', self.session, region)
try:
recorders = (await run_concurrently(
client.describe_configuration_recorders
))['ConfigurationRecorders']
except Exception as e:
print_exception('Failed to get Config recorders: {}'.format(e))
recorders = []
try:
recorder_statuses_list = \
(await run_concurrently(client.describe_configuration_recorder_status))['ConfigurationRecordersStatus']
except Exception as e:
print_exception(
'Failed to get Config recorder statuses: {}'.format(e))
else:
# To accelerate the mapping of the statuses, we preprocess the data by creating a
# <recorder_name: recorder_status> map. This prevents having to iterate over the list of statuses for each
# recorder.
recorder_statuses_map = {
recorder['name']: recorder
for recorder in recorder_statuses_list
}
for recorder in recorders:
recorder[
'ConfigurationRecordersStatus'] = recorder_statuses_map[
recorder['name']]
return recorders
async def _get_and_set_policy_details(self, policy):
client = AWSFacadeUtils.get_client('iam', self.session)
policy_version = await run_concurrently(
lambda: client.get_policy_version(
PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId']))
policy['PolicyDocument'] = policy_version['PolicyVersion']['Document']
policy['attached_to'] = {}
attached_entities = await AWSFacadeUtils.get_multiple_entities_from_all_pages(
'iam',
None,
self.session,
'list_entities_for_policy',
['PolicyGroups', 'PolicyRoles', 'PolicyUsers'],
PolicyArn=policy['Arn'])
for entity_type in attached_entities:
resource_type = entity_type.replace('Policy', '').lower()
if len(attached_entities[entity_type]):
policy['attached_to'][resource_type] = []
for entity in attached_entities[entity_type]:
name_field = entity_type.replace('Policy', '')[:-1] + 'Name'
resource_name = entity[name_field]
id_field = entity_type.replace('Policy', '')[:-1] + 'Id'
resource_id = entity[id_field]
policy['attached_to'][resource_type].append({
'name': resource_name,
'id': resource_id
})
async def get_vpcs(self, region: str):
ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region)
try:
return await run_concurrently(lambda: ec2_client.describe_vpcs()['Vpcs'])
except Exception as e:
print_exception('Failed to describe EC2 VPC: {}'.format(e))
return []
async def get_credential_reports(self):
client = AWSFacadeUtils.get_client('iam', self.session)
response = await run_concurrently(client.generate_credential_report)
if response['State'] != 'COMPLETE':
print_error('Failed to generate a credential report.')
return []
report = (await
run_concurrently(client.get_credential_report))['Content']
# The report is a CSV string. The first row contains the name of each column. The next rows
# each represent an individual account. This algorithm provides a simple initial parsing.
lines = report.splitlines()
keys = lines[0].decode('utf-8').split(',')
credential_reports = []
for line in lines[1:]:
credential_report = {}
values = line.decode('utf-8').split(',')
for key, value in zip(keys, values):
credential_report[key] = value
credential_reports.append(credential_report)
return credential_reports
async def _get_and_set_key_metadata(self, key: {}, region: str):
client = AWSFacadeUtils.get_client('kms', self.session, region)
try:
key['metadata'] = await run_concurrently(
lambda: client.describe_key(KeyId=key['KeyId']))
except Exception as e:
print_exception('Failed to describe KMS key: {}'.format(e))
async def _get_and_set_s3_bucket_default_encryption(self, bucket: {}):
bucket_name = bucket['Name']
client = AWSFacadeUtils.get_client('s3', self.session,
bucket['region'])
try:
config = await run_concurrently(
lambda: client.get_bucket_encryption(Bucket=bucket['Name']))
bucket['default_encryption_enabled'] = True
bucket['default_encryption_algorithm'] = config.get('ServerSideEncryptionConfiguration', {})\
.get('Rules', [{}])[0].get('ApplyServerSideEncryptionByDefault', {}).get('SSEAlgorithm')
bucket['default_encryption_key'] = config.get('ServerSideEncryptionConfiguration', {})\
.get('Rules', [{}])[0].get('ApplyServerSideEncryptionByDefault', {}).get('KMSMasterKeyID')
except ClientError as e:
if 'ServerSideEncryptionConfigurationNotFoundError' in e.response[
'Error']['Code']:
bucket['default_encryption_enabled'] = False
bucket['default_encryption_algorithm'] = None
bucket['default_encryption_key'] = None
else:
bucket['default_encryption_enabled'] = None
bucket['default_encryption_algorithm'] = None
bucket['default_encryption_key'] = None
print_exception(
f'Failed to get encryption configuration for {bucket_name}: {e}'
)
except Exception as e:
bucket['default_encryption'] = 'Unknown'
bucket['default_encryption_algorithm'] = None
bucket['default_encryption_key'] = None
print_exception(
f'Failed to get encryption configuration for {bucket_name}: {e}'
)
async def _get_and_set_tags(self, file_system: {}, region: str):
client = AWSFacadeUtils.get_client('efs', self.session, region)
try:
file_system['Tags'] = await run_concurrently(
lambda: client.describe_tags(FileSystemId=file_system['FileSystemId'])['Tags'])
except Exception as e:
print_exception('Failed to describe EFS tags: {}'.format(e))
async def _get_and_set_key_rotation_status(self, key: {}, region: str):
client = AWSFacadeUtils.get_client('kms', self.session, region)
try:
key['rotation_status'] = await run_concurrently(
lambda: client.get_key_rotation_status(KeyId=key['KeyId']))
except Exception as e:
print_exception('Failed to get KMS key rotation: {}'.format(e))
async def get_distributions(self):
client = AWSFacadeUtils.get_client('cloudfront',self.session)
# When no cloudfront distribution exists, we first need to initiate the creation
# of a new distributions generate_credential_report by calling
# client.list_distributions and then check for COMPLETE status before trying to download it:
aws_cloudfront_api_called, n_attempts = False, 3
try:
while not aws_cloudfront_api_called and n_attempts > 0:
response = await run_concurrently(client.list_distributions)
if 'ResponseMetadata' in response:
aws_cloudfront_api_called = True
else:
n_attempts -= 1
await asyncio.sleep(0.1) # Wait for 100ms before doing a new attempt.
except Exception as e:
print_exception('Failed to call aws cloudfront api: {}'.format(e))
return []
finally:
if not aws_cloudfront_api_called and n_attempts == 0:
print_exception('Failed to call aws cloudfront api in {} attempts'.format(n_attempts))
return []
try:
return response.get('DistributionList', {}).get('Items', [])
except Exception as e:
print_exception(f'Failed to get CloudFront distribution lists: {e}')
return []
async def _get_identity_dkim_attributes(self, identity_name: str,
region: str):
ses_client = AWSFacadeUtils.get_client('ses', self.session, region)
dkim_attributes = await run_concurrently(
lambda: ses_client.get_identity_dkim_attributes(
Identities=[identity_name])['DkimAttributes'][identity_name])
return identity_name, dkim_attributes
async def get_key_rotation_status(self, region: str, key_id: str):
client = AWSFacadeUtils.get_client('kms', self.session, region)
try:
return await run_concurrently(
lambda: client.get_key_rotation_status(KeyId=key_id))
except Exception as e:
print_exception('Failed to get KMS key rotation: {}'.format(e))
async def _get_and_set_inline_policies(self, resource, iam_resource_type):
client = AWSFacadeUtils.get_client('iam', self.session)
list_policy_method = getattr(client,
'list_' + iam_resource_type + '_policies')
resource_name = resource[iam_resource_type.title() + 'Name']
args = {iam_resource_type.title() + 'Name': resource_name}
resource['inline_policies'] = {}
policy_names = await run_concurrently(
lambda: list_policy_method(**args)['PolicyNames'])
if len(policy_names) == 0:
resource['inline_policies_count'] = 0
return
get_policy_method = getattr(client,
'get_' + iam_resource_type + '_policy')
tasks = {
asyncio.ensure_future(
run_concurrently(lambda: get_policy_method(**dict(
args, PolicyName=policy_name))))
for policy_name in policy_names
}
for task in asyncio.as_completed(tasks):
policy = await task
policy_name = policy['PolicyName']
policy_id = get_non_provider_id(policy_name)
policy_document = policy['PolicyDocument']
resource['inline_policies'][policy_id] = {}
resource['inline_policies'][policy_id][
'PolicyDocument'] = self._normalize_statements(policy_document)
resource['inline_policies'][policy_id]['name'] = policy_name
resource['inline_policies_count'] = len(resource['inline_policies'])
async def _get_and_set_key_aliases(self, key: {}, region: str):
client = AWSFacadeUtils.get_client('kms', self.session, region)
try:
response = await run_concurrently(
lambda: client.list_aliases(KeyId=key['KeyId']))
key['aliases'] = response.get('Aliases')
except Exception as e:
print_exception('Failed to get KMS aliases: {}'.format(e))
async def _get_and_set_key_policy(self, key: {}, region: str):
client = AWSFacadeUtils.get_client('kms', self.session, region)
try:
response = await run_concurrently(lambda: client.get_key_policy(
KeyId=key['KeyId'], PolicyName='default'))
key['policy'] = json.loads(response.get('Policy'))
except Exception as e:
print_exception('Failed to get KMS key policy: {}'.format(e))
async def _get_certificate(self, cert_arn: str, region: str):
client = AWSFacadeUtils.get_client('acm', self.session, region)
try:
return await run_concurrently(lambda: client.describe_certificate(
CertificateArn=cert_arn)['Certificate'])
except Exception as e:
print_exception(f'Failed to describe acm certificate: {e}')
raise
async def _get_and_set_topic_attributes(self, topic: {}, region: str):
sns_client = AWSFacadeUtils.get_client('sns', self.session, region)
try:
topic['attributes'] = await run_concurrently(
lambda: sns_client.get_topic_attributes(TopicArn=topic[
'TopicArn'])['Attributes'])
except Exception as e:
print_exception('Failed to get SNS topic attributes: {}'.format(e))
async def _get_and_set_selectors(self, trail: {}, region: str):
client = AWSFacadeUtils.get_client('cloudtrail', self.session, region)
try:
trail['EventSelectors'] = await run_concurrently(
lambda: client.get_event_selectors(TrailName=trail['TrailARN']
)['EventSelectors'])
except Exception as e:
print_exception(f'Failed to get CloudTrail event selectors: {e}')
async def get_images(self, region: str):
filters = [{'Name': 'owner-id', 'Values': [self.owner_id]}]
client = AWSFacadeUtils.get_client('ec2', self.session, region)
try:
return await run_concurrently(lambda: client.describe_images(Filters=filters)['Images'])
except Exception as e:
print_exception('Failed to get EC2 images: {}'.format(e))
return []
async def _get_and_set_mount_target_security_groups(self, mount_target: {}, region: str):
client = AWSFacadeUtils.get_client('efs', self.session, region)
try:
mount_target['SecurityGroups'] = \
await run_concurrently(lambda: client.describe_mount_target_security_groups(
MountTargetId=mount_target['MountTargetId'])['SecurityGroups'])
except Exception as e:
print_exception('Failed to describe EFS mount target security groups: {}'.format(e))
async def _get_and_set_user_access_keys(self, user: {}):
client = AWSFacadeUtils.get_client('iam', self.session)
try:
user['AccessKeys'] = await run_concurrently(
lambda: client.list_access_keys(UserName=user['UserName'])[
'AccessKeyMetadata'])
except Exception as e:
print_exception('Failed to list access keys: {}'.format(e))
async def _get_and_set_status(self, trail: {}, region: str):
client = AWSFacadeUtils.get_client('cloudtrail', self.session, region)
try:
trail_status = await run_concurrently(
lambda: client.get_trail_status(Name=trail['TrailARN']))
trail.update(trail_status)
except Exception as e:
print_exception(f'Failed to get CloudTrail trail status: {e}')
async def _get_and_set_template(self, stack: {}, region: str):
client = AWSFacadeUtils.get_client('cloudformation', self.session, region)
try:
stack['template'] = await run_concurrently(
lambda: client.get_template(StackName=stack['StackName'])['TemplateBody'])
except Exception as e:
print_exception('Failed to get CloudFormation template: {}'.format(e))
stack['template'] = None
async def _get_and_set_user_login_profile(self, user: {}):
client = AWSFacadeUtils.get_client('iam', self.session)
try:
user['LoginProfile'] = await run_concurrently(
lambda: client.get_login_profile(UserName=user['UserName'])[
'LoginProfile'])
except Exception:
pass
async def _get_and_set_user_mfa_devices(self, user: {}):
client = AWSFacadeUtils.get_client('iam', self.session)
try:
user['MFADevices'] = await run_concurrently(
lambda: client.list_mfa_devices(UserName=user['UserName'])[
'MFADevices'])
except Exception as e:
print_exception('Failed to list MFA devices: {}'.format(e))
async def _get_cluster(self, cluster_id: str, region: str):
client = AWSFacadeUtils.get_client('emr', self.session, region)
try:
return await run_concurrently(lambda: client.describe_cluster(
ClusterId=cluster_id)['Cluster'])
except Exception as e:
print_exception(f'Failed to describe EMR cluster: {e}')
raise
