def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_log_id, callback_args): attached_resource = current_config['resource_id'] if attached_resource.startswith('vpc-'): vpc_path = combine_paths( current_path[0:4], ['vpcs', attached_resource]) try: attached_vpc = get_object_at(self, vpc_path) except Exception: print_debug( 'It appears that the flow log %s is attached to a resource that was previously deleted (%s).' % ( flow_log_id, attached_resource)) return manage_dictionary(attached_vpc, 'flow_logs', []) if flow_log_id not in attached_vpc['flow_logs']: attached_vpc['flow_logs'].append(flow_log_id) for subnet_id in attached_vpc['subnets']: manage_dictionary( attached_vpc['subnets'][subnet_id], 'flow_logs', []) if flow_log_id not in attached_vpc['subnets'][subnet_id]['flow_logs']: attached_vpc['subnets'][subnet_id]['flow_logs'].append( flow_log_id) elif attached_resource.startswith('subnet-'): subnet_path = combine_paths(current_path[0:4], ['vpcs', self.subnet_map[attached_resource]['vpc_id'], 'subnets', attached_resource]) subnet = get_object_at(self, subnet_path) manage_dictionary(subnet, 'flow_logs', []) if flow_log_id not in subnet['flow_logs']: subnet['flow_logs'].append(flow_log_id) else: print_exception('Resource %s attached to flow logs is not handled' % attached_resource)
def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_log_id, callback_args): attached_resource = current_config['ResourceId'] if attached_resource.startswith('vpc-'): vpc_path = combine_paths(current_path[0:4], ['vpcs', attached_resource]) try: attached_vpc = get_object_at(self, vpc_path) except Exception as e: printDebug( 'It appears that the flow log %s is attached to a resource that was previously deleted (%s).' % ( flow_log_id, attached_resource)) return manage_dictionary(attached_vpc, 'flow_logs', []) if flow_log_id not in attached_vpc['flow_logs']: attached_vpc['flow_logs'].append(flow_log_id) for subnet_id in attached_vpc['subnets']: manage_dictionary(attached_vpc['subnets'][subnet_id], 'flow_logs', []) if flow_log_id not in attached_vpc['subnets'][subnet_id]['flow_logs']: attached_vpc['subnets'][subnet_id]['flow_logs'].append(flow_log_id) elif attached_resource.startswith('subnet-'): subnet_path = combine_paths(current_path[0:4], ['vpcs', self.subnet_map[attached_resource]['vpc_id'], 'subnets', attached_resource]) subnet = get_object_at(self, subnet_path) manage_dictionary(subnet, 'flow_logs', []) if flow_log_id not in subnet['flow_logs']: subnet['flow_logs'].append(flow_log_id) # TODO this is pre-merge (from Loic) code # all_vpcs = get_object_at(self, combine_paths(current_path[0:2], ['vpcs'])) # for vpc in self.services['vpc']: # if attached_resource in all_vpcs[vpc]['subnets']: # manage_dictionary(all_vpcs[vpc]['subnets'][attached_resource], 'flow_logs', []) # if flow_log_id not in all_vpcs[vpc]['subnets'][attached_resource]['flow_logs']: # all_vpcs[vpc]['subnets'][attached_resource]['flow_logs'].append(flow_log_id) # break else: printError('Resource %s attached to flow logs is not handled' % attached_resource)
def match_network_acls_and_subnets_callback(self, current_config, path, current_path, acl_id, callback_args): for association in current_config['Associations']: subnet_path = current_path[:-1] + \ ['subnets', association['SubnetId']] subnet = get_object_at(self, subnet_path) subnet['network_acl'] = acl_id
def _security_group_to_attack_surface(self, attack_surface_config, public_ip, current_path, security_groups, listeners=None): listeners = [] if listeners is None else listeners manage_dictionary(attack_surface_config, public_ip, {'protocols': {}}) instance_path = current_path[:-3] if 'ec2' in self.service_list: # validate that the service was included in run for sg_id in security_groups: sg_path = copy.deepcopy(current_path[0:6]) sg_path[1] = 'ec2' sg_path.append('security_groups') sg_path.append(sg_id) sg_path.append('rules') sg_path.append('ingress') ingress_rules = get_object_at(self, sg_path) for p in ingress_rules['protocols']: for port in ingress_rules['protocols'][p]['ports']: if len(listeners) == 0 and 'cidrs' in ingress_rules[ 'protocols'][p]['ports'][port]: manage_dictionary( attack_surface_config[public_ip]['protocols'], p, {'ports': {}}) manage_dictionary( attack_surface_config[public_ip]['protocols'] [p]['ports'], port, {'cidrs': []}) attack_surface_config[public_ip]['protocols'][p]['ports'][port]['cidrs'] += \ ingress_rules['protocols'][p]['ports'][port]['cidrs'] else: ports = port.split('-') if len(ports) > 1: port_min = int(ports[0]) port_max = int(ports[1]) elif port == 'N/A': port_min = port_max = None elif port == 'ALL': port_min = 0 port_max = 65535 elif p == 'ICMP': port_min = port_max = None else: port_min = port_max = int(port) for listener in listeners: if (port_min and port_max) and port_min < int(listener) < port_max and \ 'cidrs' in ingress_rules['protocols'][p]['ports'][port]: manage_dictionary( attack_surface_config[public_ip] ['protocols'], p, {'ports': {}}) manage_dictionary( attack_surface_config[public_ip] ['protocols'][p]['ports'], str(listener), {'cidrs': []}) attack_surface_config[public_ip]['protocols'][p]['ports'][str(listener)]['cidrs'] += \ ingress_rules['protocols'][p]['ports'][port]['cidrs']
def set_emr_vpc_ids_callback(self, current_config, path, current_path, vpc_id, callback_args): if vpc_id != 'EMR-UNKNOWN-VPC': return region = current_path[3] vpc_id = sg_id = subnet_id = None pop_list = [] for cluster_id in current_config['clusters']: cluster = current_config['clusters'][cluster_id] if 'EmrManagedMasterSecurityGroup' in cluster[ 'Ec2InstanceAttributes']: sg_id = cluster['Ec2InstanceAttributes'][ 'EmrManagedMasterSecurityGroup'] elif 'RequestedEc2SubnetIds' in cluster['Ec2InstanceAttributes']: subnet_id = cluster['Ec2InstanceAttributes'][ 'RequestedEc2SubnetIds'] else: print_exception( 'Unable to determine VPC id for EMR cluster %s' % str(cluster_id)) continue if sg_id in self.sg_map: vpc_id = self.sg_map[sg_id]['vpc_id'] pop_list.append(cluster_id) else: sid_found = False if subnet_id: for sid in subnet_id: if sid in self.subnet_map: vpc_id = self.subnet_map[sid]['vpc_id'] pop_list.append(cluster_id) sid_found = True if not sid_found: print_exception( 'Unable to determine VPC id for %s' % (str(subnet_id) if subnet_id else str(sg_id))) continue if vpc_id: region_vpcs_config = get_object_at(self, current_path) manage_dictionary(region_vpcs_config, vpc_id, {'clusters': {}}) region_vpcs_config[vpc_id]['clusters'][cluster_id] = cluster for cluster_id in pop_list: current_config['clusters'].pop(cluster_id) if len(current_config['clusters']) == 0: callback_args['clear_list'].append(region)
def parse_elb_policies_callback(self, current_config, path, current_path, region_id, callback_args): region_config = get_object_at([ 'services', 'elb', ] + current_path + [region_id]) region_config['elb_policies'] = current_config['elb_policies'] for policy_id in region_config['elb_policies']: if region_config['elb_policies'][policy_id][ 'PolicyTypeName'] != 'SSLNegotiationPolicyType': continue # protocols, options, ciphers policy = region_config['elb_policies'][policy_id] protocols = {} options = {} ciphers = {} for attribute in policy['PolicyAttributeDescriptions']: if attribute['AttributeName'] in [ 'Protocol-SSLv3', 'Protocol-TLSv1', 'Protocol-TLSv1.1', 'Protocol-TLSv1.2' ]: protocols[attribute['AttributeName']] = attribute[ 'AttributeValue'] elif attribute['AttributeName'] in [ 'Server-Defined-Cipher-Order' ]: options[attribute['AttributeName']] = attribute[ 'AttributeValue'] elif attribute['AttributeName'] == 'Reference-Security-Policy': policy['reference_security_policy'] = attribute[ 'AttributeValue'] else: ciphers[attribute['AttributeName']] = attribute[ 'AttributeValue'] policy['protocols'] = protocols policy['options'] = options policy['ciphers'] = ciphers
def match_security_groups_and_resources_callback(self, current_config, path, current_path, resource_id, callback_args): service = current_path[1] original_resource_path = combine_paths(copy.deepcopy(current_path), [resource_id]) resource = get_object_at(self, original_resource_path) if not 'resource_id_path' in callback_args: resource_type = current_path[-1] resource_path = copy.deepcopy(current_path) resource_path.append(resource_id) else: resource_path = combine_paths(copy.deepcopy(current_path), callback_args['resource_id_path']) resource_id = resource_path[-1] resource_type = resource_path[-2] if 'status_path' in callback_args: status_path = combine_paths(copy.deepcopy(original_resource_path), callback_args['status_path']) resource_status = get_object_at(self, status_path).replace('.', '_') else: resource_status = None unknown_vpc_id = True if current_path[4] != 'vpcs' else False # Issue 89 & 91 : can instances have no security group? try: try: sg_attribute = get_object_at( resource, callback_args['sg_list_attribute_name']) except: return if type(sg_attribute) != list: sg_attribute = [sg_attribute] for resource_sg in sg_attribute: if type(resource_sg) == dict: sg_id = resource_sg[callback_args['sg_id_attribute_name']] else: sg_id = resource_sg if unknown_vpc_id: vpc_id = self.sg_map[sg_id]['vpc_id'] sg_base_path = copy.deepcopy(current_path[0:4]) sg_base_path[1] = 'ec2' sg_base_path = sg_base_path + [ 'vpcs', vpc_id, 'security_groups' ] else: sg_base_path = copy.deepcopy(current_path[0:6]) sg_base_path[1] = 'ec2' sg_base_path.append('security_groups') sg_path = copy.deepcopy(sg_base_path) sg_path.append(sg_id) sg = get_object_at(self, sg_path) # Add usage information manage_dictionary(sg, 'used_by', {}) manage_dictionary(sg['used_by'], service, {}) manage_dictionary(sg['used_by'][service], 'resource_type', {}) manage_dictionary(sg['used_by'][service]['resource_type'], resource_type, {} if resource_status else []) if resource_status: manage_dictionary( sg['used_by'][service]['resource_type'][resource_type], resource_status, []) if not resource_id in sg['used_by'][service][ 'resource_type'][resource_type][resource_status]: sg['used_by'][service]['resource_type'][resource_type][ resource_status].append(resource_id) else: sg['used_by'][service]['resource_type'][ resource_type].append(resource_id) except Exception as e: region = current_path[3] vpc_id = current_path[5] if vpc_id == ec2_classic and resource_type == 'elbs': pass else: printError('Failed to parse %s in %s in %s' % (resource_type, vpc_id, region)) printException(e)
def _process_metadata_callbacks(self): """ Iterates through each type of resource and, when callbacks have been configured in the config metadata, recurse through each resource and calls each callback. :param self.config: The entire AWS configuration object :return: None """ # Service-level summaries for service_group in self.metadata: for service in self.metadata[service_group]: if service == 'summaries': continue # Reset external attack surface if 'summaries' in self.metadata[service_group][service]: for summary in self.metadata[service_group][service]['summaries']: if summary == 'external attack surface' and \ service in self.services and \ 'external_attack_surface' in self.services[service]: self.services[service].pop('external_attack_surface') # Reset all global summaries if hasattr(self, 'service_groups'): del self.service_groups # Resources for resource_type in self.metadata[service_group][service]['resources']: if 'callbacks' in self.metadata[service_group][service]['resources'][resource_type]: current_path = ['services', service] target_path = self.metadata[service_group][service]['resources'][resource_type][ 'path'].replace('.id', '').split('.')[2:] callbacks = self.metadata[service_group][service]['resources'][resource_type][ 'callbacks'] self._new_go_to_and_do(self.services[service], target_path, current_path, callbacks) # Summaries if 'summaries' in self.metadata[service_group][service]: for summary in self.metadata[service_group][service]['summaries']: if 'callbacks' in self.metadata[service_group][service]['summaries'][summary]: current_path = ['services', service] for callback in self.metadata[service_group][service]['summaries'][summary][ 'callbacks']: callback_name = callback[0] callback_args = copy.deepcopy(callback[1]) target_path = callback_args.pop('path').replace('.id', '').split('.')[2:] callbacks = [[callback_name, callback_args]] self._new_go_to_and_do(self.services[service], target_path, current_path, callbacks) # Group-level summaries for service_group in self.metadata: if 'summaries' in self.metadata[service_group]: for summary in self.metadata[service_group]['summaries']: current_path = ['services', service] for callback in self.metadata[service_group]['summaries'][summary]['callbacks']: callback_name = callback[0] callback_args = copy.deepcopy(callback[1]) target_path = self.metadata[service_group]['summaries'][summary]['path'].split('.') # quick fix as legacy Scout2 expects "self" to be a dict target_object = self for p in target_path: self.manage_object(target_object, p, {}) if type(target_object) == dict: target_object = target_object[p] else: target_object = getattr(target_object, p) if callback_name == 'merge': for service in self.metadata[service_group]: if service == 'summaries': continue if 'summaries' in self.metadata[service_group][service] and \ summary in self.metadata[service_group][service]['summaries']: try: source = get_object_at(self, self.metadata[service_group][service]['summaries'][summary]['path'].split('.')) except Exception as e: source = {} target_object.update(source) return None
def match_security_groups_and_resources_callback(self, current_config, path, current_path, resource_id, callback_args): if 'ec2' in self.service_list: # validate that the service was included in run service = current_path[1] original_resource_path = combine_paths( copy.deepcopy(current_path), [resource_id]) resource = get_object_at(self, original_resource_path) if 'resource_id_path' not in callback_args: resource_type = current_path[-1] resource_path = copy.deepcopy(current_path) resource_path.append(resource_id) else: resource_path = combine_paths(copy.deepcopy( current_path), callback_args['resource_id_path']) resource_id = resource_path[-1] resource_type = resource_path[-2] if 'status_path' in callback_args: status_path = combine_paths(copy.deepcopy( original_resource_path), callback_args['status_path']) resource_status = get_object_at(self, status_path).replace('.', '_') else: resource_status = None unknown_vpc_id = True if current_path[4] != 'vpcs' else False # Issue 89 & 91 : can instances have no security group? try: try: sg_attribute = get_object_at( resource, callback_args['sg_list_attribute_name']) except Exception as e: return if type(sg_attribute) != list: sg_attribute = [sg_attribute] for resource_sg in sg_attribute: if type(resource_sg) == dict: sg_id = resource_sg[callback_args['sg_id_attribute_name']] else: sg_id = resource_sg if unknown_vpc_id: vpc_id = self.sg_map[sg_id]['vpc_id'] sg_base_path = copy.deepcopy(current_path[0:4]) sg_base_path[1] = 'ec2' sg_base_path = sg_base_path + \ ['vpcs', vpc_id, 'security_groups'] else: sg_base_path = copy.deepcopy(current_path[0:6]) sg_base_path[1] = 'ec2' sg_base_path.append('security_groups') sg_path = copy.deepcopy(sg_base_path) sg_path.append(sg_id) sg = get_object_at(self, sg_path) # Add usage information manage_dictionary(sg, 'used_by', {}) manage_dictionary(sg['used_by'], service, {}) manage_dictionary(sg['used_by'][service], 'resource_type', {}) manage_dictionary(sg['used_by'][service]['resource_type'], resource_type, { } if resource_status else []) if resource_status: manage_dictionary( sg['used_by'][service]['resource_type'][resource_type], resource_status, []) if resource_id not in sg['used_by'][service]['resource_type'][resource_type][resource_status]: sg['used_by'][service]['resource_type'][resource_type][resource_status].append( {'id': resource_id, 'name': resource['name']}) else: sg['used_by'][service]['resource_type'][resource_type].append( {'id': resource_id, 'name': resource['name']}) except Exception as e: if resource_type == 'elbs' and current_path[5] == ec2_classic: pass elif not self.services['ec2']: # service not included in run pass elif not str(e): print_exception(f'Failed to parse {resource_type}') else: print_exception(f'Failed to parse {resource_type}: {e}')