def parse_file_system(self, global_params, region, file_system):
"""
:param global_params:
:param region: Name of the AWS region
:param file_system:
:return:
"""
fs_id = file_system.pop('FileSystemId')
if 'Name' in file_system:
file_system['name'] = file_system.pop('Name')
else:
file_system['name'] = None
# Get tags
file_system['tags'] = \
handle_truncated_response(api_clients[region].describe_tags, {'FileSystemId': fs_id}, ['Tags'])['Tags']
# Get mount targets
mount_targets = handle_truncated_response(
api_clients[region].describe_mount_targets,
{'FileSystemId': fs_id}, ['MountTargets'])['MountTargets']
file_system['mount_targets'] = {}
for mt in mount_targets:
mt_id = mt['MountTargetId']
file_system['mount_targets'][mt_id] = mt
# Get security groups
file_system['mount_targets'][mt_id]['security_groups'] = \
api_clients[region].describe_mount_target_security_groups(MountTargetId=mt_id)['SecurityGroups']
self.file_systems[fs_id] = file_system
def parse_users(self, user, params):
"""
Parse a single IAM user and fetch additional data
"""
if user['UserName'] in self.users:
return
api_client = params['api_client']
# Ensure consistent attribute names across resource types
user['id'] = user.pop('UserId')
user['name'] = user.pop('UserName')
user['arn'] = user.pop('Arn')
policies = self.__get_inline_policies(api_client, 'user', user['id'],
user['name'])
if len(policies):
user['inline_policies'] = policies
user['inline_policies_count'] = len(policies)
user['groups'] = []
groups = handle_truncated_response(api_client.list_groups_for_user,
{'UserName': user['name']},
['Groups'])['Groups']
for group in groups:
user['groups'].append(group['GroupName'])
try:
user['LoginProfile'] = api_client.get_login_profile(
UserName=user['name'])['LoginProfile']
except Exception:
pass
user['AccessKeys'] = api_client.list_access_keys(
UserName=user['name'])['AccessKeyMetadata']
user['MFADevices'] = api_client.list_mfa_devices(
UserName=user['name'])['MFADevices']
# TODO: Users signing certss
self.users[user['id']] = user
def parse_policies(self, fetched_policy, params):
"""
Parse a single IAM policy and fetch additional information
"""
api_client = params['api_client']
policy = {
'name': fetched_policy.pop('PolicyName'),
'id': fetched_policy.pop('PolicyId'),
'arn': fetched_policy.pop('Arn')
}
# Download version and document
policy_version = api_client.get_policy_version(
PolicyArn=policy['arn'],
VersionId=fetched_policy['DefaultVersionId'])
policy_version = policy_version['PolicyVersion']
policy['PolicyDocument'] = policy_version['Document']
# Get attached IAM entities
policy['attached_to'] = {}
attached_entities = handle_truncated_response(
api_client.list_entities_for_policy, {'PolicyArn': policy['arn']},
['PolicyGroups', 'PolicyRoles', 'PolicyUsers'])
for entity_type in attached_entities:
resource_type = entity_type.replace('Policy', '').lower()
if len(attached_entities[entity_type]):
policy['attached_to'][resource_type] = []
for entity in attached_entities[entity_type]:
name_field = entity_type.replace('Policy', '')[:-1] + 'Name'
resource_name = entity[name_field]
policy['attached_to'][resource_type].append(
{'name': resource_name})
# Save policy
self.policies[policy['id']] = policy
def parse_hosted_zones(self, hosted_zone, params):
"""
Parse a single Route53hosted_zoness hosted_zones
"""
# When resuming upon throttling error, skip if already fetched
hosted_zone_id = hosted_zone.pop('Id')
hosted_zone['name'] = hosted_zone.pop('Name')
api_client = params['api_client']
record_sets = handle_truncated_response(api_client.list_resource_record_sets, {'HostedZoneId': hosted_zone_id},
['ResourceRecordSets'])
hosted_zone.update(record_sets)
self.hosted_zones[hosted_zone_id] = hosted_zone
def _get_targets(self, response_attribute, api_client, method, list_params, ignore_list_error):
"""
Fetch the targets, required as each provider may have particularities
:return:
"""
if type(list_params) != list:
list_params = [list_params]
targets = []
for lp in list_params:
targets += handle_truncated_response(method, lp, [response_attribute])[response_attribute]
return targets
def parse_parameter_group(self, global_params, region, parameter_group):
parameter_group['arn'] = parameter_group.pop('DBParameterGroupArn')
parameter_group['name'] = parameter_group.pop('DBParameterGroupName')
api_client = api_clients[region]
try:
parameters = handle_truncated_response(api_client.describe_db_parameters,
{'DBParameterGroupName': parameter_group['name']}, ['Parameters'])[
'Parameters']
manage_dictionary(parameter_group, 'parameters', {})
for parameter in parameters:
if not parameter['IsModifiable']:
# Discard non-modifiable parameters
continue
parameter_name = parameter.pop('ParameterName')
parameter_group['parameters'][parameter_name] = parameter
except Exception as e:
print_exception(e)
print_error('Failed fetching DB parameters for %s' % parameter_group['name'])
# Save
parameter_group_id = self.get_non_provider_id(parameter_group['name'])
self.parameter_groups[parameter_group_id] = parameter_group
def get_s3_bucket_keys(api_client, bucket_name, bucket, check_encryption, check_acls):
"""
Get key-specific information (server-side encryption, acls, etc...)
:param api_client:
:param bucket_name:
:param bucket:
:param check_encryption:
:param check_acls:
:return:
"""
bucket['keys'] = []
keys = handle_truncated_response(api_client.list_objects, {'Bucket': bucket_name}, ['Contents'])
bucket['keys_count'] = len(keys['Contents'])
key_count = 0
# FIXME - commented for now as this method doesn't seem to be defined anywhere'
# update_status(key_count, bucket['keys_count'], 'keys')
for key in keys['Contents']:
key_count += 1
key['name'] = key.pop('Key')
key['LastModified'] = str(key['LastModified'])
if check_encryption:
try:
# The encryption configuration is only accessible via an HTTP header,
# only returned when requesting one object at a time...
k = api_client.get_object(Bucket=bucket_name, Key=key['name'])
key['ServerSideEncryption'] = k['ServerSideEncryption'] if 'ServerSideEncryption' in k else None
key['SSEKMSKeyId'] = k['SSEKMSKeyId'] if 'SSEKMSKeyId' in k else None
except Exception as e:
print_exception(e)
continue
if check_acls:
# noinspection PyBroadException
try:
key['grantees'] = get_s3_acls(api_client, bucket_name, bucket, key_name=key['name'])
except Exception:
continue
# Save it
bucket['keys'].append(key)
def parse_roles(self, fetched_role, params):
"""
Parse a single IAM role and fetch additional data
"""
role = {'instances_count': 'N/A'}
# When resuming upon throttling error, skip if already fetched
if fetched_role['RoleName'] in self.roles:
return
api_client = params['api_client']
# Ensure consistent attribute names across resource types
role['id'] = fetched_role.pop('RoleId')
role['name'] = fetched_role.pop('RoleName')
role['arn'] = fetched_role.pop('Arn')
# Get other attributes
get_keys(fetched_role, role, ['CreateDate', 'Path'])
# Get role policies
policies = self.__get_inline_policies(api_client, 'role', role['id'],
role['name'])
if len(policies):
role['inline_policies'] = policies
role['inline_policies_count'] = len(policies)
# Get instance profiles
profiles = handle_truncated_response(
api_client.list_instance_profiles_for_role,
{'RoleName': role['name']}, ['InstanceProfiles'])
manage_dictionary(role, 'instance_profiles', {})
for profile in profiles['InstanceProfiles']:
manage_dictionary(role['instance_profiles'],
profile['InstanceProfileId'], {})
role['instance_profiles'][
profile['InstanceProfileId']]['arn'] = profile['Arn']
role['instance_profiles'][profile['InstanceProfileId']][
'name'] = profile['InstanceProfileName']
# Get trust relationship
role['assume_role_policy'] = {}
role['assume_role_policy']['PolicyDocument'] = fetched_role.pop(
'AssumeRolePolicyDocument')
# Save role
self.roles[role['id']] = role
def parse_lb(self, global_params, region, lb):
"""
:param lb:
:param global_params:
:param region: Name of the AWS region
:return:
"""
lb['arn'] = lb.pop('LoadBalancerArn')
lb['name'] = lb.pop('LoadBalancerName')
vpc_id = lb.pop(
'VpcId') if 'VpcId' in lb and lb['VpcId'] else ec2_classic
manage_dictionary(self.vpcs, vpc_id,
VPCConfig(self.vpc_resource_types))
lb['security_groups'] = []
try:
for sg in lb['SecurityGroups']:
lb['security_groups'].append({'GroupId': sg})
lb.pop('SecurityGroups')
except Exception as e:
# Network load balancers do not have security groups
pass
lb['listeners'] = {}
# Get listeners
listeners = handle_truncated_response(
api_clients[region].describe_listeners,
{'LoadBalancerArn': lb['arn']}, ['Listeners'])['Listeners']
for listener in listeners:
listener.pop('ListenerArn')
listener.pop('LoadBalancerArn')
port = listener.pop('Port')
lb['listeners'][port] = listener
# Get attributes
lb['attributes'] = api_clients[
region].describe_load_balancer_attributes(
LoadBalancerArn=lb['arn'])['Attributes']
self.vpcs[vpc_id].lbs[self.get_non_provider_id(lb['name'])] = lb
def _fetch_targets(self, api_client, q, target):
"""
Make an API call defined in metadata.json.
Parse the returned object as implemented in the "parse_[object name]" method.
:param api_client:
:param q:
:param target:
:return:
"""
# Handle & format the target type
target_type, response_attribute, list_method_name, list_params, ignore_list_error = target
list_method = getattr(api_client, list_method_name)
try:
targets = handle_truncated_response(
list_method, list_params,
[response_attribute])[response_attribute]
except Exception as e:
if not ignore_list_error:
print_exception(e)
targets = []
setattr(self, '%s_count' % target_type, len(targets))
self.fetchstatuslogger.counts[target_type]['discovered'] += len(
targets)
region = api_client._client_config.region_name
# Queue resources
for target in targets:
# call callback methods
try:
callback = getattr(self, 'parse_%s' % target_type[0:-1])
except:
callback = self.store_target
target['scout2_target_type'] = target_type
if q:
# Add to the queue
q.put((callback, region, target))
def parse_parameter_group(self, global_params, region, parameter_group):
"""
Parse a single Redshift parameter group and fetch all of its parameters
:param global_params: Parameters shared for all regions
:param region: Name of the AWS region
:param parameter_group: Parameter group
"""
pg_name = parameter_group.pop('ParameterGroupName')
pg_id = self.get_non_provider_id(
pg_name) # Name could be used as only letters digits or hyphens
parameter_group['name'] = pg_name
parameter_group['parameters'] = {}
api_client = api_clients[region]
parameters = handle_truncated_response(
api_client.describe_cluster_parameters,
{'ParameterGroupName': pg_name}, ['Parameters'])['Parameters']
for parameter in parameters:
param = {
'value': parameter['ParameterValue'],
'source': parameter['Source']
}
parameter_group['parameters'][parameter['ParameterName']] = param
self.parameter_groups[pg_id] = parameter_group