def parse_file_system(self, global_params, region, file_system): """ :param global_params: :param region: Name of the AWS region :param file_system: :return: """ fs_id = file_system.pop('FileSystemId') if 'Name' in file_system: file_system['name'] = file_system.pop('Name') else: file_system['name'] = None # Get tags file_system['tags'] = \ handle_truncated_response(api_clients[region].describe_tags, {'FileSystemId': fs_id}, ['Tags'])['Tags'] # Get mount targets mount_targets = handle_truncated_response( api_clients[region].describe_mount_targets, {'FileSystemId': fs_id}, ['MountTargets'])['MountTargets'] file_system['mount_targets'] = {} for mt in mount_targets: mt_id = mt['MountTargetId'] file_system['mount_targets'][mt_id] = mt # Get security groups file_system['mount_targets'][mt_id]['security_groups'] = \ api_clients[region].describe_mount_target_security_groups(MountTargetId=mt_id)['SecurityGroups'] self.file_systems[fs_id] = file_system
def parse_users(self, user, params): """ Parse a single IAM user and fetch additional data """ if user['UserName'] in self.users: return api_client = params['api_client'] # Ensure consistent attribute names across resource types user['id'] = user.pop('UserId') user['name'] = user.pop('UserName') user['arn'] = user.pop('Arn') policies = self.__get_inline_policies(api_client, 'user', user['id'], user['name']) if len(policies): user['inline_policies'] = policies user['inline_policies_count'] = len(policies) user['groups'] = [] groups = handle_truncated_response(api_client.list_groups_for_user, {'UserName': user['name']}, ['Groups'])['Groups'] for group in groups: user['groups'].append(group['GroupName']) try: user['LoginProfile'] = api_client.get_login_profile( UserName=user['name'])['LoginProfile'] except Exception: pass user['AccessKeys'] = api_client.list_access_keys( UserName=user['name'])['AccessKeyMetadata'] user['MFADevices'] = api_client.list_mfa_devices( UserName=user['name'])['MFADevices'] # TODO: Users signing certss self.users[user['id']] = user
def parse_policies(self, fetched_policy, params): """ Parse a single IAM policy and fetch additional information """ api_client = params['api_client'] policy = { 'name': fetched_policy.pop('PolicyName'), 'id': fetched_policy.pop('PolicyId'), 'arn': fetched_policy.pop('Arn') } # Download version and document policy_version = api_client.get_policy_version( PolicyArn=policy['arn'], VersionId=fetched_policy['DefaultVersionId']) policy_version = policy_version['PolicyVersion'] policy['PolicyDocument'] = policy_version['Document'] # Get attached IAM entities policy['attached_to'] = {} attached_entities = handle_truncated_response( api_client.list_entities_for_policy, {'PolicyArn': policy['arn']}, ['PolicyGroups', 'PolicyRoles', 'PolicyUsers']) for entity_type in attached_entities: resource_type = entity_type.replace('Policy', '').lower() if len(attached_entities[entity_type]): policy['attached_to'][resource_type] = [] for entity in attached_entities[entity_type]: name_field = entity_type.replace('Policy', '')[:-1] + 'Name' resource_name = entity[name_field] policy['attached_to'][resource_type].append( {'name': resource_name}) # Save policy self.policies[policy['id']] = policy
def parse_hosted_zones(self, hosted_zone, params): """ Parse a single Route53hosted_zoness hosted_zones """ # When resuming upon throttling error, skip if already fetched hosted_zone_id = hosted_zone.pop('Id') hosted_zone['name'] = hosted_zone.pop('Name') api_client = params['api_client'] record_sets = handle_truncated_response(api_client.list_resource_record_sets, {'HostedZoneId': hosted_zone_id}, ['ResourceRecordSets']) hosted_zone.update(record_sets) self.hosted_zones[hosted_zone_id] = hosted_zone
def _get_targets(self, response_attribute, api_client, method, list_params, ignore_list_error): """ Fetch the targets, required as each provider may have particularities :return: """ if type(list_params) != list: list_params = [list_params] targets = [] for lp in list_params: targets += handle_truncated_response(method, lp, [response_attribute])[response_attribute] return targets
def parse_parameter_group(self, global_params, region, parameter_group): parameter_group['arn'] = parameter_group.pop('DBParameterGroupArn') parameter_group['name'] = parameter_group.pop('DBParameterGroupName') api_client = api_clients[region] try: parameters = handle_truncated_response(api_client.describe_db_parameters, {'DBParameterGroupName': parameter_group['name']}, ['Parameters'])[ 'Parameters'] manage_dictionary(parameter_group, 'parameters', {}) for parameter in parameters: if not parameter['IsModifiable']: # Discard non-modifiable parameters continue parameter_name = parameter.pop('ParameterName') parameter_group['parameters'][parameter_name] = parameter except Exception as e: print_exception(e) print_error('Failed fetching DB parameters for %s' % parameter_group['name']) # Save parameter_group_id = self.get_non_provider_id(parameter_group['name']) self.parameter_groups[parameter_group_id] = parameter_group
def get_s3_bucket_keys(api_client, bucket_name, bucket, check_encryption, check_acls): """ Get key-specific information (server-side encryption, acls, etc...) :param api_client: :param bucket_name: :param bucket: :param check_encryption: :param check_acls: :return: """ bucket['keys'] = [] keys = handle_truncated_response(api_client.list_objects, {'Bucket': bucket_name}, ['Contents']) bucket['keys_count'] = len(keys['Contents']) key_count = 0 # FIXME - commented for now as this method doesn't seem to be defined anywhere' # update_status(key_count, bucket['keys_count'], 'keys') for key in keys['Contents']: key_count += 1 key['name'] = key.pop('Key') key['LastModified'] = str(key['LastModified']) if check_encryption: try: # The encryption configuration is only accessible via an HTTP header, # only returned when requesting one object at a time... k = api_client.get_object(Bucket=bucket_name, Key=key['name']) key['ServerSideEncryption'] = k['ServerSideEncryption'] if 'ServerSideEncryption' in k else None key['SSEKMSKeyId'] = k['SSEKMSKeyId'] if 'SSEKMSKeyId' in k else None except Exception as e: print_exception(e) continue if check_acls: # noinspection PyBroadException try: key['grantees'] = get_s3_acls(api_client, bucket_name, bucket, key_name=key['name']) except Exception: continue # Save it bucket['keys'].append(key)
def parse_roles(self, fetched_role, params): """ Parse a single IAM role and fetch additional data """ role = {'instances_count': 'N/A'} # When resuming upon throttling error, skip if already fetched if fetched_role['RoleName'] in self.roles: return api_client = params['api_client'] # Ensure consistent attribute names across resource types role['id'] = fetched_role.pop('RoleId') role['name'] = fetched_role.pop('RoleName') role['arn'] = fetched_role.pop('Arn') # Get other attributes get_keys(fetched_role, role, ['CreateDate', 'Path']) # Get role policies policies = self.__get_inline_policies(api_client, 'role', role['id'], role['name']) if len(policies): role['inline_policies'] = policies role['inline_policies_count'] = len(policies) # Get instance profiles profiles = handle_truncated_response( api_client.list_instance_profiles_for_role, {'RoleName': role['name']}, ['InstanceProfiles']) manage_dictionary(role, 'instance_profiles', {}) for profile in profiles['InstanceProfiles']: manage_dictionary(role['instance_profiles'], profile['InstanceProfileId'], {}) role['instance_profiles'][ profile['InstanceProfileId']]['arn'] = profile['Arn'] role['instance_profiles'][profile['InstanceProfileId']][ 'name'] = profile['InstanceProfileName'] # Get trust relationship role['assume_role_policy'] = {} role['assume_role_policy']['PolicyDocument'] = fetched_role.pop( 'AssumeRolePolicyDocument') # Save role self.roles[role['id']] = role
def parse_lb(self, global_params, region, lb): """ :param lb: :param global_params: :param region: Name of the AWS region :return: """ lb['arn'] = lb.pop('LoadBalancerArn') lb['name'] = lb.pop('LoadBalancerName') vpc_id = lb.pop( 'VpcId') if 'VpcId' in lb and lb['VpcId'] else ec2_classic manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) lb['security_groups'] = [] try: for sg in lb['SecurityGroups']: lb['security_groups'].append({'GroupId': sg}) lb.pop('SecurityGroups') except Exception as e: # Network load balancers do not have security groups pass lb['listeners'] = {} # Get listeners listeners = handle_truncated_response( api_clients[region].describe_listeners, {'LoadBalancerArn': lb['arn']}, ['Listeners'])['Listeners'] for listener in listeners: listener.pop('ListenerArn') listener.pop('LoadBalancerArn') port = listener.pop('Port') lb['listeners'][port] = listener # Get attributes lb['attributes'] = api_clients[ region].describe_load_balancer_attributes( LoadBalancerArn=lb['arn'])['Attributes'] self.vpcs[vpc_id].lbs[self.get_non_provider_id(lb['name'])] = lb
def _fetch_targets(self, api_client, q, target): """ Make an API call defined in metadata.json. Parse the returned object as implemented in the "parse_[object name]" method. :param api_client: :param q: :param target: :return: """ # Handle & format the target type target_type, response_attribute, list_method_name, list_params, ignore_list_error = target list_method = getattr(api_client, list_method_name) try: targets = handle_truncated_response( list_method, list_params, [response_attribute])[response_attribute] except Exception as e: if not ignore_list_error: print_exception(e) targets = [] setattr(self, '%s_count' % target_type, len(targets)) self.fetchstatuslogger.counts[target_type]['discovered'] += len( targets) region = api_client._client_config.region_name # Queue resources for target in targets: # call callback methods try: callback = getattr(self, 'parse_%s' % target_type[0:-1]) except: callback = self.store_target target['scout2_target_type'] = target_type if q: # Add to the queue q.put((callback, region, target))
def parse_parameter_group(self, global_params, region, parameter_group): """ Parse a single Redshift parameter group and fetch all of its parameters :param global_params: Parameters shared for all regions :param region: Name of the AWS region :param parameter_group: Parameter group """ pg_name = parameter_group.pop('ParameterGroupName') pg_id = self.get_non_provider_id( pg_name) # Name could be used as only letters digits or hyphens parameter_group['name'] = pg_name parameter_group['parameters'] = {} api_client = api_clients[region] parameters = handle_truncated_response( api_client.describe_cluster_parameters, {'ParameterGroupName': pg_name}, ['Parameters'])['Parameters'] for parameter in parameters: param = { 'value': parameter['ParameterValue'], 'source': parameter['Source'] } parameter_group['parameters'][parameter['ParameterName']] = param self.parameter_groups[pg_id] = parameter_group