def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_log_id, callback_args):
attached_resource = current_config['resource_id']
if attached_resource.startswith('vpc-'):
vpc_path = combine_paths(
current_path[0:4], ['vpcs', attached_resource])
try:
attached_vpc = get_object_at(self, vpc_path)
except Exception:
print_debug(
'It appears that the flow log %s is attached to a resource that was previously deleted (%s).' % (
flow_log_id, attached_resource))
return
manage_dictionary(attached_vpc, 'flow_logs', [])
if flow_log_id not in attached_vpc['flow_logs']:
attached_vpc['flow_logs'].append(flow_log_id)
for subnet_id in attached_vpc['subnets']:
manage_dictionary(
attached_vpc['subnets'][subnet_id], 'flow_logs', [])
if flow_log_id not in attached_vpc['subnets'][subnet_id]['flow_logs']:
attached_vpc['subnets'][subnet_id]['flow_logs'].append(
flow_log_id)
elif attached_resource.startswith('subnet-'):
subnet_path = combine_paths(current_path[0:4],
['vpcs', self.subnet_map[attached_resource]['vpc_id'], 'subnets',
attached_resource])
subnet = get_object_at(self, subnet_path)
manage_dictionary(subnet, 'flow_logs', [])
if flow_log_id not in subnet['flow_logs']:
subnet['flow_logs'].append(flow_log_id)
else:
print_exception('Resource %s attached to flow logs is not handled' % attached_resource)
def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_log_id, callback_args):
attached_resource = current_config['ResourceId']
if attached_resource.startswith('vpc-'):
vpc_path = combine_paths(current_path[0:4], ['vpcs', attached_resource])
try:
attached_vpc = get_object_at(self, vpc_path)
except Exception as e:
printDebug(
'It appears that the flow log %s is attached to a resource that was previously deleted (%s).' % (
flow_log_id, attached_resource))
return
manage_dictionary(attached_vpc, 'flow_logs', [])
if flow_log_id not in attached_vpc['flow_logs']:
attached_vpc['flow_logs'].append(flow_log_id)
for subnet_id in attached_vpc['subnets']:
manage_dictionary(attached_vpc['subnets'][subnet_id], 'flow_logs', [])
if flow_log_id not in attached_vpc['subnets'][subnet_id]['flow_logs']:
attached_vpc['subnets'][subnet_id]['flow_logs'].append(flow_log_id)
elif attached_resource.startswith('subnet-'):
subnet_path = combine_paths(current_path[0:4],
['vpcs', self.subnet_map[attached_resource]['vpc_id'], 'subnets', attached_resource])
subnet = get_object_at(self, subnet_path)
manage_dictionary(subnet, 'flow_logs', [])
if flow_log_id not in subnet['flow_logs']:
subnet['flow_logs'].append(flow_log_id)
# TODO this is pre-merge (from Loic) code
# all_vpcs = get_object_at(self, combine_paths(current_path[0:2], ['vpcs']))
# for vpc in self.services['vpc']:
# if attached_resource in all_vpcs[vpc]['subnets']:
# manage_dictionary(all_vpcs[vpc]['subnets'][attached_resource], 'flow_logs', [])
# if flow_log_id not in all_vpcs[vpc]['subnets'][attached_resource]['flow_logs']:
# all_vpcs[vpc]['subnets'][attached_resource]['flow_logs'].append(flow_log_id)
# break
else:
printError('Resource %s attached to flow logs is not handled' % attached_resource)
def match_network_acls_and_subnets_callback(self, current_config, path,
current_path, acl_id,
callback_args):
for association in current_config['Associations']:
subnet_path = current_path[:-1] + \
['subnets', association['SubnetId']]
subnet = get_object_at(self, subnet_path)
subnet['network_acl'] = acl_id
def _security_group_to_attack_surface(self,
attack_surface_config,
public_ip,
current_path,
security_groups,
listeners=None):
listeners = [] if listeners is None else listeners
manage_dictionary(attack_surface_config, public_ip, {'protocols': {}})
instance_path = current_path[:-3]
if 'ec2' in self.service_list: # validate that the service was included in run
for sg_id in security_groups:
sg_path = copy.deepcopy(current_path[0:6])
sg_path[1] = 'ec2'
sg_path.append('security_groups')
sg_path.append(sg_id)
sg_path.append('rules')
sg_path.append('ingress')
ingress_rules = get_object_at(self, sg_path)
for p in ingress_rules['protocols']:
for port in ingress_rules['protocols'][p]['ports']:
if len(listeners) == 0 and 'cidrs' in ingress_rules[
'protocols'][p]['ports'][port]:
manage_dictionary(
attack_surface_config[public_ip]['protocols'],
p, {'ports': {}})
manage_dictionary(
attack_surface_config[public_ip]['protocols']
[p]['ports'], port, {'cidrs': []})
attack_surface_config[public_ip]['protocols'][p]['ports'][port]['cidrs'] += \
ingress_rules['protocols'][p]['ports'][port]['cidrs']
else:
ports = port.split('-')
if len(ports) > 1:
port_min = int(ports[0])
port_max = int(ports[1])
elif port == 'N/A':
port_min = port_max = None
elif port == 'ALL':
port_min = 0
port_max = 65535
elif p == 'ICMP':
port_min = port_max = None
else:
port_min = port_max = int(port)
for listener in listeners:
if (port_min and port_max) and port_min < int(listener) < port_max and \
'cidrs' in ingress_rules['protocols'][p]['ports'][port]:
manage_dictionary(
attack_surface_config[public_ip]
['protocols'], p, {'ports': {}})
manage_dictionary(
attack_surface_config[public_ip]
['protocols'][p]['ports'],
str(listener), {'cidrs': []})
attack_surface_config[public_ip]['protocols'][p]['ports'][str(listener)]['cidrs'] += \
ingress_rules['protocols'][p]['ports'][port]['cidrs']
def set_emr_vpc_ids_callback(self, current_config, path, current_path,
vpc_id, callback_args):
if vpc_id != 'EMR-UNKNOWN-VPC':
return
region = current_path[3]
vpc_id = sg_id = subnet_id = None
pop_list = []
for cluster_id in current_config['clusters']:
cluster = current_config['clusters'][cluster_id]
if 'EmrManagedMasterSecurityGroup' in cluster[
'Ec2InstanceAttributes']:
sg_id = cluster['Ec2InstanceAttributes'][
'EmrManagedMasterSecurityGroup']
elif 'RequestedEc2SubnetIds' in cluster['Ec2InstanceAttributes']:
subnet_id = cluster['Ec2InstanceAttributes'][
'RequestedEc2SubnetIds']
else:
print_exception(
'Unable to determine VPC id for EMR cluster %s' %
str(cluster_id))
continue
if sg_id in self.sg_map:
vpc_id = self.sg_map[sg_id]['vpc_id']
pop_list.append(cluster_id)
else:
sid_found = False
if subnet_id:
for sid in subnet_id:
if sid in self.subnet_map:
vpc_id = self.subnet_map[sid]['vpc_id']
pop_list.append(cluster_id)
sid_found = True
if not sid_found:
print_exception(
'Unable to determine VPC id for %s' %
(str(subnet_id) if subnet_id else str(sg_id)))
continue
if vpc_id:
region_vpcs_config = get_object_at(self, current_path)
manage_dictionary(region_vpcs_config, vpc_id, {'clusters': {}})
region_vpcs_config[vpc_id]['clusters'][cluster_id] = cluster
for cluster_id in pop_list:
current_config['clusters'].pop(cluster_id)
if len(current_config['clusters']) == 0:
callback_args['clear_list'].append(region)
def parse_elb_policies_callback(self, current_config, path, current_path,
region_id, callback_args):
region_config = get_object_at([
'services',
'elb',
] + current_path + [region_id])
region_config['elb_policies'] = current_config['elb_policies']
for policy_id in region_config['elb_policies']:
if region_config['elb_policies'][policy_id][
'PolicyTypeName'] != 'SSLNegotiationPolicyType':
continue
# protocols, options, ciphers
policy = region_config['elb_policies'][policy_id]
protocols = {}
options = {}
ciphers = {}
for attribute in policy['PolicyAttributeDescriptions']:
if attribute['AttributeName'] in [
'Protocol-SSLv3', 'Protocol-TLSv1', 'Protocol-TLSv1.1',
'Protocol-TLSv1.2'
]:
protocols[attribute['AttributeName']] = attribute[
'AttributeValue']
elif attribute['AttributeName'] in [
'Server-Defined-Cipher-Order'
]:
options[attribute['AttributeName']] = attribute[
'AttributeValue']
elif attribute['AttributeName'] == 'Reference-Security-Policy':
policy['reference_security_policy'] = attribute[
'AttributeValue']
else:
ciphers[attribute['AttributeName']] = attribute[
'AttributeValue']
policy['protocols'] = protocols
policy['options'] = options
policy['ciphers'] = ciphers
def match_security_groups_and_resources_callback(self, current_config,
path, current_path,
resource_id,
callback_args):
service = current_path[1]
original_resource_path = combine_paths(copy.deepcopy(current_path),
[resource_id])
resource = get_object_at(self, original_resource_path)
if not 'resource_id_path' in callback_args:
resource_type = current_path[-1]
resource_path = copy.deepcopy(current_path)
resource_path.append(resource_id)
else:
resource_path = combine_paths(copy.deepcopy(current_path),
callback_args['resource_id_path'])
resource_id = resource_path[-1]
resource_type = resource_path[-2]
if 'status_path' in callback_args:
status_path = combine_paths(copy.deepcopy(original_resource_path),
callback_args['status_path'])
resource_status = get_object_at(self,
status_path).replace('.', '_')
else:
resource_status = None
unknown_vpc_id = True if current_path[4] != 'vpcs' else False
# Issue 89 & 91 : can instances have no security group?
try:
try:
sg_attribute = get_object_at(
resource, callback_args['sg_list_attribute_name'])
except:
return
if type(sg_attribute) != list:
sg_attribute = [sg_attribute]
for resource_sg in sg_attribute:
if type(resource_sg) == dict:
sg_id = resource_sg[callback_args['sg_id_attribute_name']]
else:
sg_id = resource_sg
if unknown_vpc_id:
vpc_id = self.sg_map[sg_id]['vpc_id']
sg_base_path = copy.deepcopy(current_path[0:4])
sg_base_path[1] = 'ec2'
sg_base_path = sg_base_path + [
'vpcs', vpc_id, 'security_groups'
]
else:
sg_base_path = copy.deepcopy(current_path[0:6])
sg_base_path[1] = 'ec2'
sg_base_path.append('security_groups')
sg_path = copy.deepcopy(sg_base_path)
sg_path.append(sg_id)
sg = get_object_at(self, sg_path)
# Add usage information
manage_dictionary(sg, 'used_by', {})
manage_dictionary(sg['used_by'], service, {})
manage_dictionary(sg['used_by'][service], 'resource_type', {})
manage_dictionary(sg['used_by'][service]['resource_type'],
resource_type, {} if resource_status else [])
if resource_status:
manage_dictionary(
sg['used_by'][service]['resource_type'][resource_type],
resource_status, [])
if not resource_id in sg['used_by'][service][
'resource_type'][resource_type][resource_status]:
sg['used_by'][service]['resource_type'][resource_type][
resource_status].append(resource_id)
else:
sg['used_by'][service]['resource_type'][
resource_type].append(resource_id)
except Exception as e:
region = current_path[3]
vpc_id = current_path[5]
if vpc_id == ec2_classic and resource_type == 'elbs':
pass
else:
printError('Failed to parse %s in %s in %s' %
(resource_type, vpc_id, region))
printException(e)
def _process_metadata_callbacks(self):
"""
Iterates through each type of resource and, when callbacks have been
configured in the config metadata, recurse through each resource and calls
each callback.
:param self.config: The entire AWS configuration object
:return: None
"""
# Service-level summaries
for service_group in self.metadata:
for service in self.metadata[service_group]:
if service == 'summaries':
continue
# Reset external attack surface
if 'summaries' in self.metadata[service_group][service]:
for summary in self.metadata[service_group][service]['summaries']:
if summary == 'external attack surface' and \
service in self.services and \
'external_attack_surface' in self.services[service]:
self.services[service].pop('external_attack_surface')
# Reset all global summaries
if hasattr(self, 'service_groups'):
del self.service_groups
# Resources
for resource_type in self.metadata[service_group][service]['resources']:
if 'callbacks' in self.metadata[service_group][service]['resources'][resource_type]:
current_path = ['services', service]
target_path = self.metadata[service_group][service]['resources'][resource_type][
'path'].replace('.id', '').split('.')[2:]
callbacks = self.metadata[service_group][service]['resources'][resource_type][
'callbacks']
self._new_go_to_and_do(self.services[service],
target_path,
current_path,
callbacks)
# Summaries
if 'summaries' in self.metadata[service_group][service]:
for summary in self.metadata[service_group][service]['summaries']:
if 'callbacks' in self.metadata[service_group][service]['summaries'][summary]:
current_path = ['services', service]
for callback in self.metadata[service_group][service]['summaries'][summary][
'callbacks']:
callback_name = callback[0]
callback_args = copy.deepcopy(callback[1])
target_path = callback_args.pop('path').replace('.id', '').split('.')[2:]
callbacks = [[callback_name, callback_args]]
self._new_go_to_and_do(self.services[service],
target_path,
current_path,
callbacks)
# Group-level summaries
for service_group in self.metadata:
if 'summaries' in self.metadata[service_group]:
for summary in self.metadata[service_group]['summaries']:
current_path = ['services', service]
for callback in self.metadata[service_group]['summaries'][summary]['callbacks']:
callback_name = callback[0]
callback_args = copy.deepcopy(callback[1])
target_path = self.metadata[service_group]['summaries'][summary]['path'].split('.')
# quick fix as legacy Scout2 expects "self" to be a dict
target_object = self
for p in target_path:
self.manage_object(target_object, p, {})
if type(target_object) == dict:
target_object = target_object[p]
else:
target_object = getattr(target_object, p)
if callback_name == 'merge':
for service in self.metadata[service_group]:
if service == 'summaries':
continue
if 'summaries' in self.metadata[service_group][service] and \
summary in self.metadata[service_group][service]['summaries']:
try:
source = get_object_at(self,
self.metadata[service_group][service]['summaries'][summary]['path'].split('.'))
except Exception as e:
source = {}
target_object.update(source)
return None
def match_security_groups_and_resources_callback(self, current_config, path, current_path, resource_id,
callback_args):
if 'ec2' in self.service_list: # validate that the service was included in run
service = current_path[1]
original_resource_path = combine_paths(
copy.deepcopy(current_path), [resource_id])
resource = get_object_at(self, original_resource_path)
if 'resource_id_path' not in callback_args:
resource_type = current_path[-1]
resource_path = copy.deepcopy(current_path)
resource_path.append(resource_id)
else:
resource_path = combine_paths(copy.deepcopy(
current_path), callback_args['resource_id_path'])
resource_id = resource_path[-1]
resource_type = resource_path[-2]
if 'status_path' in callback_args:
status_path = combine_paths(copy.deepcopy(
original_resource_path), callback_args['status_path'])
resource_status = get_object_at(self, status_path).replace('.', '_')
else:
resource_status = None
unknown_vpc_id = True if current_path[4] != 'vpcs' else False
# Issue 89 & 91 : can instances have no security group?
try:
try:
sg_attribute = get_object_at(
resource, callback_args['sg_list_attribute_name'])
except Exception as e:
return
if type(sg_attribute) != list:
sg_attribute = [sg_attribute]
for resource_sg in sg_attribute:
if type(resource_sg) == dict:
sg_id = resource_sg[callback_args['sg_id_attribute_name']]
else:
sg_id = resource_sg
if unknown_vpc_id:
vpc_id = self.sg_map[sg_id]['vpc_id']
sg_base_path = copy.deepcopy(current_path[0:4])
sg_base_path[1] = 'ec2'
sg_base_path = sg_base_path + \
['vpcs', vpc_id, 'security_groups']
else:
sg_base_path = copy.deepcopy(current_path[0:6])
sg_base_path[1] = 'ec2'
sg_base_path.append('security_groups')
sg_path = copy.deepcopy(sg_base_path)
sg_path.append(sg_id)
sg = get_object_at(self, sg_path)
# Add usage information
manage_dictionary(sg, 'used_by', {})
manage_dictionary(sg['used_by'], service, {})
manage_dictionary(sg['used_by'][service], 'resource_type', {})
manage_dictionary(sg['used_by'][service]['resource_type'], resource_type, {
} if resource_status else [])
if resource_status:
manage_dictionary(
sg['used_by'][service]['resource_type'][resource_type], resource_status, [])
if resource_id not in sg['used_by'][service]['resource_type'][resource_type][resource_status]:
sg['used_by'][service]['resource_type'][resource_type][resource_status].append(
{'id': resource_id, 'name': resource['name']})
else:
sg['used_by'][service]['resource_type'][resource_type].append(
{'id': resource_id, 'name': resource['name']})
except Exception as e:
if resource_type == 'elbs' and current_path[5] == ec2_classic:
pass
elif not self.services['ec2']: # service not included in run
pass
elif not str(e):
print_exception(f'Failed to parse {resource_type}')
else:
print_exception(f'Failed to parse {resource_type}: {e}')
