def _load_associations(self): if len(self.associates) == 0: return self._print( f"[*] Adding {self.__class__.__name__} associative relationships") edges = Elements() for resource in self.get("Resource"): references = {} label = [l for l in resource.labels() if l != "Resource"][0] # Find references to other resources in the form of a dictionary (refs) self._references(resource.properties(), references) # Create an edge, for all known associations (as defined by self.rels). for rel in [ r for r in self.associates if r[0] == label or r[1] == label ]: i = 1 if label == rel[0] else 0 # Get a list of foreign keys that we must be capable of referencing # in order to create an association fk = [ a for a in re.compile("{([A-Za-z]+)}").findall( RESOURCES.definition(rel[i])) if a not in ["Account", "Region"] ] if not all([k in references.keys() for k in fk]): continue # TODO: Handle Types that make use of more than one # variable identifier if len(fk) != 1: raise NotImplementedError fk = fk[0] for v in list(references[fk]): # Find the first resource matching the reference r = next((r for r in self if re.compile( RESOURCES.definition(rel[i]). format(Account=self.account_id, Region=self.session.region_name, **{ **{ x: list(y)[0] for x, y in references.items( ) if len(y) == 1 }, **{ fk: v } })).match(r.id()) is not None), None) if r is None: # print("Failed to match (%s: %s) against any resources" % (k, v)) # print("Its likely that the resource was missed during ingestion") continue # Delete the properties that are responsible for the edge's existence. properties = self._extract_property_value( resource.properties(), fk) # Even though direction is irrelavent when dealing with Associative # edges, neo4j is directed. We need to ensure the direction is kept # in order to eliminate duplicate edges. (source, target) = (resource, r) if i == 1 else (r, resource) edge = Associative(properties={"Name": "Attached"}, source=source, target=target) opposite_edge = Associative( properties={"Name": "Attached"}, source=target, target=source) if (edge not in self and opposite_edge not in self) and edge not in edges: edges.append(edge) self.extend(edges)
def load_associatives(self): if len(self.associations) == 0: return def set_references(references, item, key=None): if isinstance(item, list): [set_references(references, i) for i in item] elif isinstance(item, dict): [set_references(references, v, k) for k, v in item.items()] elif (key is not None and any([isinstance(item, t) for t in [str, int, bool]]) and len(str(item)) > 0): if key not in references: references[key] = set() references[key].update([item]) for resource in self.console.tasklist( f"Adding Associative relationships", self.get("Resource"), done="Added Associative relationships"): # Extract reference key-value pairs from this resource's # properties (if we need to) prop_refs = {} # Extract reference key-value pairs from this resource's ARN: regex = re.compile(RESOURCES[resource.label()]) matches = regex.match(resource.id()) arn_refs = { k: set([matches.group(k)]) for k in regex.groupindex.keys() } if matches is not None else {} # For each of the resource types associated with this resource type for rt in [[rt for rt in association if rt != resource.label()][0] for association in self.associations if resource.label() in association]: refs = {} required = list(re.compile(RESOURCES[rt]).groupindex.keys()) # We have all the information we need using just the ARN if all([k in arn_refs for k in required]): refs = arn_refs else: # Check the resource's properties (once) if len(prop_refs) == 0: set_references(prop_refs, resource.properties()) # Use property and ARN refs (ARN values take precedence) refs = { **{ k: v for k, v in prop_refs.items() if k in required }, **{k: v for k, v in arn_refs.items() if k in required}, } # There isn't enough information to create a reference ARN if not all([k in refs for k in required]): continue # Hopefully, this never happens if not all([len(v) == 1 for v in refs.values()]): continue # Construct a reference ARN and get the associated resource arn = RESOURCES.types[rt].format( **{k: list(v)[0] for k, v in refs.items()}) associate = next((r for r in self if r.id() == arn), None) if associate is None: # self.console.debug(f"Couldn't create association: resource ({arn}), " # f"referenced by {resource}, doesn't exist ") continue (source, target) = sorted((resource, associate), key=lambda r: r.id()) self.add( Associative(properties={"Name": "Attached"}, source=source, target=target))
def _load_resource_collection(self, collection, boto_base_resource=None, awspx_base_resource=None): resources = [] label = "AWS::{service}::{type}".format( service=self.__class__.__name__.capitalize(), type=getattr(boto_base_resource, collection)._model.resource.type) # Known cases of misaligned naming conventions (boto and lib.aws.RESOURCES) if label.endswith("Version"): label = label[:-7] elif label == "AWS::Ec2::KeyPairInfo": label = "AWS::Ec2::KeyPair" # Skip excluded types or types unknown to us if label not in RESOURCES.types \ or (not self.run_all and label not in self.run): return try: # Note: Depending on the stage at which the exception is thrown, we # may miss certain resources. resources = [ r for r in getattr(boto_base_resource, collection).all() ] except Exception as e: print(f"[!] Couldn't load {collection} of {boto_base_resource} " "-- probably due to a resource based policy or something.") for resource in resources: # Get properties properties = resource.meta.data if properties is None: continue # Get Arn and Name arn = self._get_resource_arn(resource, boto_base_resource) name = self._get_resource_name(resource) # Include or exclude this ARN if self.only_arns and arn not in self.only_arns: continue elif self.except_arns and arn in self.except_arns: continue properties["Arn"] = arn properties["Name"] = name if not (properties["Arn"] and properties["Name"]): print(label) print(json.dumps(properties, indent=2, default=str)) raise ValueError # Create resource r = Resource(labels=[label], properties=properties) if r not in self: self._print(f"[*] Adding {r}") self.append(r) # Add associative relationship with parent if awspx_base_resource: assocs = [set(a) for a in self.associates] if {awspx_base_resource.labels()[0], r.labels()[0]} in assocs \ or not self.associates: e = Associative(properties={"Name": "Attached"}, source=r, target=awspx_base_resource) oe = Associative(properties={"Name": "Attached"}, source=awspx_base_resource, target=r) if not (e in self or oe in self): self.append(e) # Load resources from this one's collections if self._get_collections(resource): self._load_resources(resource, r) # Return when we've seen all explicit resources if self.only_arns and all( [r in map(lambda x: x.id(), self) for r in self.only_arns]): return
def _load_resource_collection(self, collection, boto_base_resource=None, awspx_base_resource=None): resources = [] try: # Note: Depending on the stage at which the exception is thrown, we # may miss certain resources. resources = [ r for r in getattr(boto_base_resource, collection).all() ] except Exception as e: print(f"Couldn't load {collection} of {boto_base_resource} " "- probably due to a resource based policy or something.") for resource in resources: # Get properties properties = resource.meta.data if properties is None: continue # Get label resource_type = resource.__class__.__name__.split(".")[-1] label = self._get_resource_type_label(resource_type) # Get Arn and Name arn = self._get_resource_arn(resource, boto_base_resource) name = self._get_resource_name(resource) # Include or exclude this ARN if self.only_arns and arn not in self.only_arns: continue elif self.except_arns and arn in self.except_arns: continue properties["Arn"] = arn properties["Name"] = name if not (properties["Arn"] and properties["Name"]): print(label) print(json.dumps(properties, indent=2, default=str)) raise ValueError # Create resource r = Resource(labels=[label], properties=properties) if r not in self: print(f" \-> Adding {r}") self.append(r) # Add associative relationship with parent if awspx_base_resource: assocs = [set(a) for a in self.associates] if {awspx_base_resource.labels()[0], r.labels()[0]} in assocs \ or not self.associates: e = Associative(properties={"Name": "Attached"}, source=r, target=awspx_base_resource) oe = Associative(properties={"Name": "Attached"}, source=awspx_base_resource, target=r) if not (e in self or oe in self): self.append(e) # Load resources from this one's collections if self._get_collections(resource): self._load_resources(resource, r) # Return when we've seen all explicit resources if self.only_arns and all( [r in map(lambda x: x.id(), self) for r in self.only_arns]): return