async def get_buckets(self): try: # If there are regions specified, try for each of them until one works. # This is required in case there's an IAM policy that denies access to APIs on a regional basis, # as per https://github.com/nccgroup/ScoutSuite/issues/727 region = None if self.regions: buckets = [] exception = '' for region in self.regions: try: client = AWSFacadeUtils.get_client( 's3', self.session, region) buckets = await run_concurrently( lambda: client.list_buckets()['Buckets']) except Exception as e: exception = e else: break if not buckets: if exception: print_exception(f'Failed to list buckets: {exception}') return [] else: client = AWSFacadeUtils.get_client('s3', self.session) buckets = await run_concurrently( lambda: client.list_buckets()['Buckets']) except Exception as e: print_exception(f'Failed to list buckets: {e}') return [] else: # We need first to retrieve bucket locations before retrieving bucket details await get_and_set_concurrently( [self._get_and_set_s3_bucket_location], buckets, region=region) # Then we can retrieve bucket details concurrently await get_and_set_concurrently([ self._get_and_set_s3_bucket_logging, self._get_and_set_s3_bucket_versioning, self._get_and_set_s3_bucket_webhosting, self._get_and_set_s3_bucket_default_encryption, self._get_and_set_s3_acls, self._get_and_set_s3_bucket_policy, self._get_and_set_s3_bucket_tags, self._get_and_set_s3_bucket_block_public_access ], buckets) # Non-async post-processing for bucket in buckets: self._set_s3_bucket_secure_transport(bucket) return buckets
async def _get_and_set_snapshot_attributes(self, snapshot: {}, region: str): ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region) snapshot['CreateVolumePermissions'] = await run_concurrently( lambda: ec2_client.describe_snapshot_attribute( Attribute='createVolumePermission', SnapshotId=snapshot['SnapshotId'])['CreateVolumePermissions'])
async def _get_and_set_s3_acls(self, bucket: {}, key_name=None): bucket_name = bucket['Name'] client = AWSFacadeUtils.get_client('s3', self.session, bucket['region']) try: grantees = {} if key_name: grants = await run_concurrently(lambda: client.get_object_acl(Bucket=bucket_name, Key=key_name)) else: grants = await run_concurrently(lambda: client.get_bucket_acl(Bucket=bucket_name)) for grant in grants['Grants']: if 'ID' in grant['Grantee']: grantee = grant['Grantee']['ID'] display_name = grant['Grantee']['DisplayName'] if \ 'DisplayName' in grant['Grantee'] else grant['Grantee']['ID'] elif 'URI' in grant['Grantee']: grantee = grant['Grantee']['URI'].split('/')[-1] display_name = self._s3_group_to_string(grant['Grantee']['URI']) else: grantee = display_name = 'Unknown' permission = grant['Permission'] grantees.setdefault(grantee, {}) grantees[grantee]['DisplayName'] = display_name if 'URI' in grant['Grantee']: grantees[grantee]['URI'] = grant['Grantee']['URI'] grantees[grantee].setdefault('permissions', self._init_s3_permissions()) self._set_s3_permissions(grantees[grantee]['permissions'], permission) bucket['grantees'] = grantees except Exception as e: print_exception('Failed to get ACL configuration for %s: %s' % (bucket_name, e)) bucket['grantees'] = {}
async def _get_and_set_load_balancer_attributes(self, load_balancer: dict, region: str): elbv2_client = AWSFacadeUtils.get_client('elbv2', self.session, region) load_balancer['attributes'] = await run_concurrently( lambda: elbv2_client.describe_load_balancer_attributes( LoadBalancerArn=load_balancer['LoadBalancerArn'])['Attributes'] )
async def get_buckets(self): client = AWSFacadeUtils.get_client('s3', self.session) try: buckets = await run_concurrently( lambda: client.list_buckets()['Buckets']) except Exception as e: print_exception('Failed to list buckets: {}'.format(e)) return [] else: # We need first to retrieve bucket locations before retrieving bucket details: await get_and_set_concurrently( [self._get_and_set_s3_bucket_location], buckets) # Then we can retrieve bucket details concurrently: await get_and_set_concurrently([ self._get_and_set_s3_bucket_logging, self._get_and_set_s3_bucket_versioning, self._get_and_set_s3_bucket_webhosting, self._get_and_set_s3_bucket_default_encryption, self._get_and_set_s3_acls, self._get_and_set_s3_bucket_policy ], buckets) # Non-async post-processing: for bucket in buckets: self._set_s3_bucket_secure_transport(bucket) return buckets
async def get_recorders(self, region: str): client = AWSFacadeUtils.get_client('config', self.session, region) try: recorders = (await run_concurrently( client.describe_configuration_recorders ))['ConfigurationRecorders'] except Exception as e: print_exception('Failed to get Config recorders: {}'.format(e)) recorders = [] try: recorder_statuses_list = \ (await run_concurrently(client.describe_configuration_recorder_status))['ConfigurationRecordersStatus'] except Exception as e: print_exception( 'Failed to get Config recorder statuses: {}'.format(e)) else: # To accelerate the mapping of the statuses, we preprocess the data by creating a # <recorder_name: recorder_status> map. This prevents having to iterate over the list of statuses for each # recorder. recorder_statuses_map = { recorder['name']: recorder for recorder in recorder_statuses_list } for recorder in recorders: recorder[ 'ConfigurationRecordersStatus'] = recorder_statuses_map[ recorder['name']] return recorders
async def _get_and_set_policy_details(self, policy): client = AWSFacadeUtils.get_client('iam', self.session) policy_version = await run_concurrently( lambda: client.get_policy_version( PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId'])) policy['PolicyDocument'] = policy_version['PolicyVersion']['Document'] policy['attached_to'] = {} attached_entities = await AWSFacadeUtils.get_multiple_entities_from_all_pages( 'iam', None, self.session, 'list_entities_for_policy', ['PolicyGroups', 'PolicyRoles', 'PolicyUsers'], PolicyArn=policy['Arn']) for entity_type in attached_entities: resource_type = entity_type.replace('Policy', '').lower() if len(attached_entities[entity_type]): policy['attached_to'][resource_type] = [] for entity in attached_entities[entity_type]: name_field = entity_type.replace('Policy', '')[:-1] + 'Name' resource_name = entity[name_field] id_field = entity_type.replace('Policy', '')[:-1] + 'Id' resource_id = entity[id_field] policy['attached_to'][resource_type].append({ 'name': resource_name, 'id': resource_id })
async def get_vpcs(self, region: str): ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region) try: return await run_concurrently(lambda: ec2_client.describe_vpcs()['Vpcs']) except Exception as e: print_exception('Failed to describe EC2 VPC: {}'.format(e)) return []
async def get_credential_reports(self): client = AWSFacadeUtils.get_client('iam', self.session) response = await run_concurrently(client.generate_credential_report) if response['State'] != 'COMPLETE': print_error('Failed to generate a credential report.') return [] report = (await run_concurrently(client.get_credential_report))['Content'] # The report is a CSV string. The first row contains the name of each column. The next rows # each represent an individual account. This algorithm provides a simple initial parsing. lines = report.splitlines() keys = lines[0].decode('utf-8').split(',') credential_reports = [] for line in lines[1:]: credential_report = {} values = line.decode('utf-8').split(',') for key, value in zip(keys, values): credential_report[key] = value credential_reports.append(credential_report) return credential_reports
async def _get_and_set_key_metadata(self, key: {}, region: str): client = AWSFacadeUtils.get_client('kms', self.session, region) try: key['metadata'] = await run_concurrently( lambda: client.describe_key(KeyId=key['KeyId'])) except Exception as e: print_exception('Failed to describe KMS key: {}'.format(e))
async def _get_and_set_s3_bucket_default_encryption(self, bucket: {}): bucket_name = bucket['Name'] client = AWSFacadeUtils.get_client('s3', self.session, bucket['region']) try: config = await run_concurrently( lambda: client.get_bucket_encryption(Bucket=bucket['Name'])) bucket['default_encryption_enabled'] = True bucket['default_encryption_algorithm'] = config.get('ServerSideEncryptionConfiguration', {})\ .get('Rules', [{}])[0].get('ApplyServerSideEncryptionByDefault', {}).get('SSEAlgorithm') bucket['default_encryption_key'] = config.get('ServerSideEncryptionConfiguration', {})\ .get('Rules', [{}])[0].get('ApplyServerSideEncryptionByDefault', {}).get('KMSMasterKeyID') except ClientError as e: if 'ServerSideEncryptionConfigurationNotFoundError' in e.response[ 'Error']['Code']: bucket['default_encryption_enabled'] = False bucket['default_encryption_algorithm'] = None bucket['default_encryption_key'] = None else: bucket['default_encryption_enabled'] = None bucket['default_encryption_algorithm'] = None bucket['default_encryption_key'] = None print_exception( f'Failed to get encryption configuration for {bucket_name}: {e}' ) except Exception as e: bucket['default_encryption'] = 'Unknown' bucket['default_encryption_algorithm'] = None bucket['default_encryption_key'] = None print_exception( f'Failed to get encryption configuration for {bucket_name}: {e}' )
async def _get_and_set_tags(self, file_system: {}, region: str): client = AWSFacadeUtils.get_client('efs', self.session, region) try: file_system['Tags'] = await run_concurrently( lambda: client.describe_tags(FileSystemId=file_system['FileSystemId'])['Tags']) except Exception as e: print_exception('Failed to describe EFS tags: {}'.format(e))
async def _get_and_set_key_rotation_status(self, key: {}, region: str): client = AWSFacadeUtils.get_client('kms', self.session, region) try: key['rotation_status'] = await run_concurrently( lambda: client.get_key_rotation_status(KeyId=key['KeyId'])) except Exception as e: print_exception('Failed to get KMS key rotation: {}'.format(e))
async def get_distributions(self): client = AWSFacadeUtils.get_client('cloudfront',self.session) # When no cloudfront distribution exists, we first need to initiate the creation # of a new distributions generate_credential_report by calling # client.list_distributions and then check for COMPLETE status before trying to download it: aws_cloudfront_api_called, n_attempts = False, 3 try: while not aws_cloudfront_api_called and n_attempts > 0: response = await run_concurrently(client.list_distributions) if 'ResponseMetadata' in response: aws_cloudfront_api_called = True else: n_attempts -= 1 await asyncio.sleep(0.1) # Wait for 100ms before doing a new attempt. except Exception as e: print_exception('Failed to call aws cloudfront api: {}'.format(e)) return [] finally: if not aws_cloudfront_api_called and n_attempts == 0: print_exception('Failed to call aws cloudfront api in {} attempts'.format(n_attempts)) return [] try: return response.get('DistributionList', {}).get('Items', []) except Exception as e: print_exception(f'Failed to get CloudFront distribution lists: {e}') return []
async def _get_identity_dkim_attributes(self, identity_name: str, region: str): ses_client = AWSFacadeUtils.get_client('ses', self.session, region) dkim_attributes = await run_concurrently( lambda: ses_client.get_identity_dkim_attributes( Identities=[identity_name])['DkimAttributes'][identity_name]) return identity_name, dkim_attributes
async def get_key_rotation_status(self, region: str, key_id: str): client = AWSFacadeUtils.get_client('kms', self.session, region) try: return await run_concurrently( lambda: client.get_key_rotation_status(KeyId=key_id)) except Exception as e: print_exception('Failed to get KMS key rotation: {}'.format(e))
async def _get_and_set_inline_policies(self, resource, iam_resource_type): client = AWSFacadeUtils.get_client('iam', self.session) list_policy_method = getattr(client, 'list_' + iam_resource_type + '_policies') resource_name = resource[iam_resource_type.title() + 'Name'] args = {iam_resource_type.title() + 'Name': resource_name} resource['inline_policies'] = {} policy_names = await run_concurrently( lambda: list_policy_method(**args)['PolicyNames']) if len(policy_names) == 0: resource['inline_policies_count'] = 0 return get_policy_method = getattr(client, 'get_' + iam_resource_type + '_policy') tasks = { asyncio.ensure_future( run_concurrently(lambda: get_policy_method(**dict( args, PolicyName=policy_name)))) for policy_name in policy_names } for task in asyncio.as_completed(tasks): policy = await task policy_name = policy['PolicyName'] policy_id = get_non_provider_id(policy_name) policy_document = policy['PolicyDocument'] resource['inline_policies'][policy_id] = {} resource['inline_policies'][policy_id][ 'PolicyDocument'] = self._normalize_statements(policy_document) resource['inline_policies'][policy_id]['name'] = policy_name resource['inline_policies_count'] = len(resource['inline_policies'])
async def _get_and_set_key_aliases(self, key: {}, region: str): client = AWSFacadeUtils.get_client('kms', self.session, region) try: response = await run_concurrently( lambda: client.list_aliases(KeyId=key['KeyId'])) key['aliases'] = response.get('Aliases') except Exception as e: print_exception('Failed to get KMS aliases: {}'.format(e))
async def _get_and_set_key_policy(self, key: {}, region: str): client = AWSFacadeUtils.get_client('kms', self.session, region) try: response = await run_concurrently(lambda: client.get_key_policy( KeyId=key['KeyId'], PolicyName='default')) key['policy'] = json.loads(response.get('Policy')) except Exception as e: print_exception('Failed to get KMS key policy: {}'.format(e))
async def _get_certificate(self, cert_arn: str, region: str): client = AWSFacadeUtils.get_client('acm', self.session, region) try: return await run_concurrently(lambda: client.describe_certificate( CertificateArn=cert_arn)['Certificate']) except Exception as e: print_exception(f'Failed to describe acm certificate: {e}') raise
async def _get_and_set_topic_attributes(self, topic: {}, region: str): sns_client = AWSFacadeUtils.get_client('sns', self.session, region) try: topic['attributes'] = await run_concurrently( lambda: sns_client.get_topic_attributes(TopicArn=topic[ 'TopicArn'])['Attributes']) except Exception as e: print_exception('Failed to get SNS topic attributes: {}'.format(e))
async def _get_and_set_selectors(self, trail: {}, region: str): client = AWSFacadeUtils.get_client('cloudtrail', self.session, region) try: trail['EventSelectors'] = await run_concurrently( lambda: client.get_event_selectors(TrailName=trail['TrailARN'] )['EventSelectors']) except Exception as e: print_exception(f'Failed to get CloudTrail event selectors: {e}')
async def get_images(self, region: str): filters = [{'Name': 'owner-id', 'Values': [self.owner_id]}] client = AWSFacadeUtils.get_client('ec2', self.session, region) try: return await run_concurrently(lambda: client.describe_images(Filters=filters)['Images']) except Exception as e: print_exception('Failed to get EC2 images: {}'.format(e)) return []
async def _get_and_set_mount_target_security_groups(self, mount_target: {}, region: str): client = AWSFacadeUtils.get_client('efs', self.session, region) try: mount_target['SecurityGroups'] = \ await run_concurrently(lambda: client.describe_mount_target_security_groups( MountTargetId=mount_target['MountTargetId'])['SecurityGroups']) except Exception as e: print_exception('Failed to describe EFS mount target security groups: {}'.format(e))
async def _get_and_set_user_access_keys(self, user: {}): client = AWSFacadeUtils.get_client('iam', self.session) try: user['AccessKeys'] = await run_concurrently( lambda: client.list_access_keys(UserName=user['UserName'])[ 'AccessKeyMetadata']) except Exception as e: print_exception('Failed to list access keys: {}'.format(e))
async def _get_and_set_status(self, trail: {}, region: str): client = AWSFacadeUtils.get_client('cloudtrail', self.session, region) try: trail_status = await run_concurrently( lambda: client.get_trail_status(Name=trail['TrailARN'])) trail.update(trail_status) except Exception as e: print_exception(f'Failed to get CloudTrail trail status: {e}')
async def _get_and_set_template(self, stack: {}, region: str): client = AWSFacadeUtils.get_client('cloudformation', self.session, region) try: stack['template'] = await run_concurrently( lambda: client.get_template(StackName=stack['StackName'])['TemplateBody']) except Exception as e: print_exception('Failed to get CloudFormation template: {}'.format(e)) stack['template'] = None
async def _get_and_set_user_login_profile(self, user: {}): client = AWSFacadeUtils.get_client('iam', self.session) try: user['LoginProfile'] = await run_concurrently( lambda: client.get_login_profile(UserName=user['UserName'])[ 'LoginProfile']) except Exception: pass
async def _get_and_set_user_mfa_devices(self, user: {}): client = AWSFacadeUtils.get_client('iam', self.session) try: user['MFADevices'] = await run_concurrently( lambda: client.list_mfa_devices(UserName=user['UserName'])[ 'MFADevices']) except Exception as e: print_exception('Failed to list MFA devices: {}'.format(e))
async def _get_cluster(self, cluster_id: str, region: str): client = AWSFacadeUtils.get_client('emr', self.session, region) try: return await run_concurrently(lambda: client.describe_cluster( ClusterId=cluster_id)['Cluster']) except Exception as e: print_exception(f'Failed to describe EMR cluster: {e}') raise