Exemple #1
0
    def _load_associations(self):

        if len(self.associates) == 0:
            return

        self._print(
            f"[*] Adding {self.__class__.__name__} associative relationships")

        edges = Elements()

        for resource in self.get("Resource"):

            references = {}
            label = [l for l in resource.labels() if l != "Resource"][0]

            # Find references to other resources in the form of a dictionary (refs)

            self._references(resource.properties(), references)

            # Create an edge, for all known associations (as defined by self.rels).

            for rel in [
                    r for r in self.associates
                    if r[0] == label or r[1] == label
            ]:

                i = 1 if label == rel[0] else 0

                # Get a list of foreign keys that we must be capable of referencing
                # in order to create an association

                fk = [
                    a for a in re.compile("{([A-Za-z]+)}").findall(
                        RESOURCES.definition(rel[i]))
                    if a not in ["Account", "Region"]
                ]

                if not all([k in references.keys() for k in fk]):
                    continue

                # TODO: Handle Types that make use of more than one
                # variable identifier

                if len(fk) != 1:
                    raise NotImplementedError

                fk = fk[0]

                for v in list(references[fk]):

                    # Find the first resource matching the reference

                    r = next((r
                              for r in self if re.compile(
                                  RESOURCES.definition(rel[i]).
                                  format(Account=self.account_id,
                                         Region=self.session.region_name,
                                         **{
                                             **{
                                                 x: list(y)[0]
                                                 for x, y in references.items(
                                                 ) if len(y) == 1
                                             },
                                             **{
                                                 fk: v
                                             }
                                         })).match(r.id()) is not None), None)

                    if r is None:
                        # print("Failed to match (%s: %s) against any resources" % (k, v))
                        # print("Its likely that the resource was missed during ingestion")
                        continue

                    # Delete the properties that are responsible for the edge's existence.

                    properties = self._extract_property_value(
                        resource.properties(), fk)

                    # Even though direction is irrelavent when dealing with Associative
                    # edges, neo4j is directed. We need to ensure the direction is kept
                    # in order to eliminate duplicate edges.

                    (source, target) = (resource, r) if i == 1 else (r,
                                                                     resource)

                    edge = Associative(properties={"Name": "Attached"},
                                       source=source,
                                       target=target)
                    opposite_edge = Associative(
                        properties={"Name": "Attached"},
                        source=target,
                        target=source)

                    if (edge not in self and opposite_edge
                            not in self) and edge not in edges:
                        edges.append(edge)

        self.extend(edges)
Exemple #2
0
    def load_associatives(self):

        if len(self.associations) == 0:
            return

        def set_references(references, item, key=None):

            if isinstance(item, list):
                [set_references(references, i) for i in item]

            elif isinstance(item, dict):
                [set_references(references, v, k) for k, v in item.items()]

            elif (key is not None
                  and any([isinstance(item, t) for t in [str, int, bool]])
                  and len(str(item)) > 0):

                if key not in references:
                    references[key] = set()

                references[key].update([item])

        for resource in self.console.tasklist(
                f"Adding Associative relationships",
                self.get("Resource"),
                done="Added Associative relationships"):

            # Extract reference key-value pairs from this resource's
            # properties (if we need to)
            prop_refs = {}

            # Extract reference key-value pairs from this resource's ARN:
            regex = re.compile(RESOURCES[resource.label()])
            matches = regex.match(resource.id())
            arn_refs = {
                k: set([matches.group(k)])
                for k in regex.groupindex.keys()
            } if matches is not None else {}

            # For each of the resource types associated with this resource type
            for rt in [[rt for rt in association if rt != resource.label()][0]
                       for association in self.associations
                       if resource.label() in association]:

                refs = {}
                required = list(re.compile(RESOURCES[rt]).groupindex.keys())

                # We have all the information we need using just the ARN
                if all([k in arn_refs for k in required]):
                    refs = arn_refs
                else:
                    # Check the resource's properties (once)
                    if len(prop_refs) == 0:
                        set_references(prop_refs, resource.properties())

                    # Use property and ARN refs (ARN values take precedence)
                    refs = {
                        **{
                            k: v
                            for k, v in prop_refs.items() if k in required
                        },
                        **{k: v
                           for k, v in arn_refs.items() if k in required},
                    }

                    # There isn't enough information to create a reference ARN
                    if not all([k in refs for k in required]):
                        continue

                # Hopefully, this never happens
                if not all([len(v) == 1 for v in refs.values()]):
                    continue

                # Construct a reference ARN and get the associated resource
                arn = RESOURCES.types[rt].format(
                    **{k: list(v)[0]
                       for k, v in refs.items()})

                associate = next((r for r in self if r.id() == arn), None)

                if associate is None:
                    # self.console.debug(f"Couldn't create association: resource ({arn}), "
                    #                    f"referenced by {resource}, doesn't exist ")
                    continue

                (source, target) = sorted((resource, associate),
                                          key=lambda r: r.id())

                self.add(
                    Associative(properties={"Name": "Attached"},
                                source=source,
                                target=target))
Exemple #3
0
    def _load_resource_collection(self,
                                  collection,
                                  boto_base_resource=None,
                                  awspx_base_resource=None):

        resources = []

        label = "AWS::{service}::{type}".format(
            service=self.__class__.__name__.capitalize(),
            type=getattr(boto_base_resource, collection)._model.resource.type)

        # Known cases of misaligned naming conventions (boto and lib.aws.RESOURCES)
        if label.endswith("Version"):
            label = label[:-7]
        elif label == "AWS::Ec2::KeyPairInfo":
            label = "AWS::Ec2::KeyPair"

        # Skip excluded types or types unknown to us
        if label not in RESOURCES.types \
                or (not self.run_all and label not in self.run):
            return

        try:
            # Note: Depending on the stage at which the exception is thrown, we
            # may miss certain resources.

            resources = [
                r for r in getattr(boto_base_resource, collection).all()
            ]

        except Exception as e:
            print(f"[!] Couldn't load {collection} of {boto_base_resource} "
                  "-- probably due to a resource based policy or something.")

        for resource in resources:

            # Get properties
            properties = resource.meta.data

            if properties is None:
                continue

            # Get Arn and Name
            arn = self._get_resource_arn(resource, boto_base_resource)
            name = self._get_resource_name(resource)

            # Include or exclude this ARN
            if self.only_arns and arn not in self.only_arns:
                continue
            elif self.except_arns and arn in self.except_arns:
                continue

            properties["Arn"] = arn
            properties["Name"] = name

            if not (properties["Arn"] and properties["Name"]):
                print(label)
                print(json.dumps(properties, indent=2, default=str))
                raise ValueError

            # Create resource
            r = Resource(labels=[label], properties=properties)
            if r not in self:
                self._print(f"[*] Adding {r}")
                self.append(r)

            # Add associative relationship with parent
            if awspx_base_resource:
                assocs = [set(a) for a in self.associates]
                if {awspx_base_resource.labels()[0], r.labels()[0]} in assocs \
                        or not self.associates:
                    e = Associative(properties={"Name": "Attached"},
                                    source=r,
                                    target=awspx_base_resource)
                    oe = Associative(properties={"Name": "Attached"},
                                     source=awspx_base_resource,
                                     target=r)
                    if not (e in self or oe in self):
                        self.append(e)

            # Load resources from this one's collections
            if self._get_collections(resource):
                self._load_resources(resource, r)

            # Return when we've seen all explicit resources
            if self.only_arns and all(
                [r in map(lambda x: x.id(), self) for r in self.only_arns]):
                return
Exemple #4
0
    def _load_resource_collection(self,
                                  collection,
                                  boto_base_resource=None,
                                  awspx_base_resource=None):

        resources = []

        try:

            # Note: Depending on the stage at which the exception is thrown, we
            # may miss certain resources.

            resources = [
                r for r in getattr(boto_base_resource, collection).all()
            ]

        except Exception as e:
            print(f"Couldn't load {collection} of {boto_base_resource} "
                  "- probably due to a resource based policy or something.")

        for resource in resources:

            # Get properties
            properties = resource.meta.data
            if properties is None:
                continue

            # Get label
            resource_type = resource.__class__.__name__.split(".")[-1]
            label = self._get_resource_type_label(resource_type)

            # Get Arn and Name
            arn = self._get_resource_arn(resource, boto_base_resource)
            name = self._get_resource_name(resource)

            # Include or exclude this ARN
            if self.only_arns and arn not in self.only_arns:
                continue
            elif self.except_arns and arn in self.except_arns:
                continue

            properties["Arn"] = arn
            properties["Name"] = name

            if not (properties["Arn"] and properties["Name"]):
                print(label)
                print(json.dumps(properties, indent=2, default=str))
                raise ValueError

            # Create resource
            r = Resource(labels=[label], properties=properties)
            if r not in self:
                print(f" \-> Adding {r}")
                self.append(r)

            # Add associative relationship with parent
            if awspx_base_resource:
                assocs = [set(a) for a in self.associates]
                if {awspx_base_resource.labels()[0], r.labels()[0]} in assocs \
                        or not self.associates:
                    e = Associative(properties={"Name": "Attached"},
                                    source=r,
                                    target=awspx_base_resource)
                    oe = Associative(properties={"Name": "Attached"},
                                     source=awspx_base_resource,
                                     target=r)
                    if not (e in self or oe in self):
                        self.append(e)

            # Load resources from this one's collections
            if self._get_collections(resource):
                self._load_resources(resource, r)

            # Return when we've seen all explicit resources
            if self.only_arns and all(
                [r in map(lambda x: x.id(), self) for r in self.only_arns]):
                return