def reference_handler(self,iobject, fact, attr_info, add_fact_kargs):
        """
        Handler for facts that contain a reference to a fact.
        See below in the comment regarding the fact_handler_list
        for a description of the signature of handler functions.

        As shown below in the handler list, this handler is called
        when a attribute with key '@idref' on the fact's node
        is detected -- this attribute signifies that this fact does not contain
        a value but points to another object. Thus, we try to retrieve that
        object from the database. If it exists, fine -- if not, then the
        call to 'create_iobject' returns a PLACEHOLDER object.

        We further create/refer to the fitting fact data type:
        we want the fact data type to express that the fact is
        a reference to an object.
        """

        (namespace_uri,uid) = (self.identifier_ns_uri,attr_info['idref'])


        # We are always able to extract the timestamp from the referencing node, because for OpenIOC,
        # all references are created by DINGO's generic import, and the import writes the timestamp
        # information into the created reference.

        timestamp = attr_info['@timestamp']

        # The following either retrieves an already existing object of given ID and timestamp
        # or creates a placeholder object.

        (target_mantis_obj, existed) = MantisImporter.create_iobject(
            uid=uid,
            identifier_ns_uri=namespace_uri,
            timestamp=timestamp)

        logger.debug("Creation of Placeholder for %s %s returned %s" % (namespace_uri,uid,existed))

        # What remains to be done is to write the reference to the created placeholder object

        add_fact_kargs['value_iobject_id'] = Identifier.objects.get(uid=uid,namespace__uri=namespace_uri)

        # Handlers have to return 'True', otherwise the fact will not be created.

        return True
Exemplo n.º 2
0
    def xml_import(self,
                   filepath=None,
                   xml_content=None,
                   markings=None,
                   identifier_ns_uri=None,
                   **kwargs):
        """
        Import a iodef XML  from file <filepath>.
        You can provide:

        - a list of markings with which all generated Information Objects
           will be associated (e.g., in order to provide provenance function)

        - The uri of a namespace of the identifiers for the generated information objects.
          This namespace identifiers the 'owner' of the object. For iodef import, this
          should not be necessary, because the XML schema makes sure that each
          Inicdent is associated with ownership information via the 'name' attribute.

        The kwargs are not read -- they are present to allow the use of the
        DingoImportCommand class for easy definition of commandline import commands
        (the class passes all command line arguments to the xml_import function, so
        without the **kwargs parameter, an error would occur.
        """

        # Clear state in case xml_import is used several times

        self.__init__()

        # Initialize  default arguments

        # '[]' would be mutable, so we initialize here
        if not markings:
            markings = []

        # Initalizing here allows us to also get the default namespace when
        # explicitly passing 'None' as parameter.

        if identifier_ns_uri:
            self.identifier_ns_uri = identifier_ns_uri

        # Use the generic XML import customized for  OpenIOC import
        # to turn XML into DingoObjDicts

        import_result = MantisImporter.xml_import(
            xml_fname=filepath,
            xml_content=xml_content,
            ns_mapping=self.namespace_dict,
            embedded_predicate=self.embedding_pred,
            id_and_revision_extractor=self.id_and_revision_extractor,
            transformer=self.transformer,
            keep_attrs_in_created_reference=False,
        )

        # The result is of the following form::
        #
        #
        #   {'id_and_rev_info': Id and revision info of top-level element; for iodef, we always have
        #                       {'id':None, 'timestamp':None}, because the  <IODEF-Document> element
        #                       carries no identifier or timestamp
        #    'elt_name': Element name of top-level element, for iodef always 'IODEF-Document'
        #    'dict_repr': Dictionary representation of IODEF XML, minus the embedded Incident objects
        #    'embedded_objects': List of embedded objects, as dictionary
        #                           {"id_and_revision_info": id and revision info of extracted object,
        #                            "elt_name": Element name (for IODEF always 'Incident'),
        #                            "dict_repr" :  dictionary representation of XML of embedded object
        #                           }
        #    'unprocessed' : List of unprocessed embedded objects (not used for iodef import
        #    'file_content': Content of imported file (or, if content was passed instead of a file name,
        #                    the original content)}

        id_and_rev_info = import_result['id_and_rev_info']
        elt_name = import_result['elt_name']
        elt_dict = import_result['dict_repr']

        embedded_objects = import_result['embedded_objects']

        default_ns = self.namespace_dict.get(elt_dict.get('@@ns', None))

        # Here, we could try to extract the family name and version from
        # the namespace information, but we do not do that for now.

        ns_info = search_by_re_list(self.RE_LIST_NS_TYPE_FROM_NS_URL,
                                    default_ns)

        if ns_info:
            if 'family' in ns_info:
                self.iobject_family_name = ns_info['family']
            if 'revision' in ns_info:
                self.iobject_family_revision_name = ns_info['revision']

        # Initialize stack with import_results.

        # First, the result from the top-level import
        pending_stack = [(id_and_rev_info, elt_name, elt_dict)]

        # Then the embedded objects
        for embedded_object in embedded_objects:
            id_and_rev_info = embedded_object['id_and_rev_info']
            elt_name = embedded_object['elt_name']
            elt_dict = embedded_object['dict_repr']
            pending_stack.append((id_and_rev_info, elt_name, elt_dict))

        for (id_and_rev_info, elt_name, elt_dict) in pending_stack:
            # call the importer that turns DingoObjDicts into Information Objects in the database

            if id_and_rev_info['timestamp']:
                ts = id_and_rev_info['timestamp']
            else:
                ts = self.create_timestamp

            iobject_type_name = elt_name

            ns_info = search_by_re_list(self.RE_LIST_NS_TYPE_FROM_NS_URL,
                                        default_ns)

            iobject_type_namespace_uri = None
            iobject_type_revision_name = None

            if ns_info:
                if 'family_ns' in ns_info:
                    iobject_type_namespace_uri = ns_info['family_ns']
                if 'revision' in ns_info:
                    iobject_type_revision_name = ns_info['revision']

            if not iobject_type_namespace_uri:
                iobject_type_namespace_uri = self.namespace_dict.get(
                    elt_dict.get('@@ns', None), DINGOS_GENERIC_FAMILY_NAME)

            if not id_and_rev_info['id']:
                logger.error(
                    "Attempt to import object (element name %s) without id -- object is ignored"
                    % elt_name)
                continue

            MantisImporter.create_iobject(
                iobject_family_name=self.iobject_family_name,
                iobject_family_revision_name=self.iobject_family_revision_name,
                iobject_type_name=iobject_type_name,
                iobject_type_namespace_uri=iobject_type_namespace_uri,
                iobject_type_revision_name=iobject_type_revision_name,
                iobject_data=elt_dict,
                uid=id_and_rev_info['id'].split(":")[0],
                identifier_ns_uri=id_and_rev_info['id'].split(":")[1],
                timestamp=ts,
                create_timestamp=self.create_timestamp,
                markings=markings,
                config_hooks={
                    'special_ft_handler': self.fact_handler_list(),
                    'datatype_extractor': self.datatype_extractor,
                    'attr_ignore_predicate': self.attr_ignore_predicate
                },
                namespace_dict=self.namespace_dict,
            )
Exemplo n.º 3
0
    def xml_import(self,
                   filepath=None,
                   xml_content=None,
                   markings=None,
                   identifier_ns_uri=None,
                   uid = None,
                   **kwargs):
        """
        Import a TEMPLATE XML  from file <filepath>.
        You can provide:

        - a list of markings with which all generated Information Objects
           will be associated (e.g., in order to provide provenance function)

        - The uri of a namespace of the identifiers for the generated information objects.
          This namespace identifiers the 'owner' of the object. For example, if importing
          IOCs published by Mandiant (e.g., as part of the APT1 report), chose an namespace
          such  as 'mandiant.com' or similar (and be consistent about it, when importing
          other stuff published by Mandiant).

        The kwargs are not read -- they are present to allow the use of the
        DingoImportCommand class for easy definition of commandline import commands
        (the class passes all command line arguments to the xml_import function, so
        without the **kwargs parameter, an error would occur.
        """

        # Clear state in case xml_import is used several times

        self.__init__()

        # Initialize  default arguments

        # '[]' would be mutable, so we initialize here
        if not markings:
            markings = []

        # Initalizing here allows us to also get the default namespace when
        # explicitly passing 'None' as parameter.

        if identifier_ns_uri:
            self.identifier_ns_uri = identifier_ns_uri


        # Use the generic XML import customized for  OpenIOC import
        # to turn XML into DingoObjDicts

        import_result =  MantisImporter.xml_import(xml_fname=filepath,
                                                   xml_content=xml_content,
                                                   ns_mapping=self.namespace_dict,
                                                   embedded_predicate=self.embedding_pred,
                                                   id_and_revision_extractor=self.id_and_revision_extractor,
                                                   transformer=self.transformer,
                                                   keep_attrs_in_created_reference=False,
                                                  )



        id_and_rev_info = import_result['id_and_rev_info']
        elt_name = import_result['elt_name']
        elt_dict = import_result['dict_repr']
        file_content = import_result['file_content']

        if uid:
            id_and_rev_info['id'] = uid
        else:
            id_and_rev_info['id'] = hashlib.sha256(file_content).hexdigest()


        embedded_objects = import_result['embedded_objects']

        default_ns = self.namespace_dict.get(elt_dict.get('@@ns',None))

        # Export family information.
        family_info_dict = {}
        if family_info_dict:
            self.iobject_family_name=family_info_dict['family']
            self.iobject_family_revision_name=family_info_dict['revision']


        # Initialize stack with import_results.

        # First, the result from the top-level import
        pending_stack = [(id_and_rev_info, elt_name,elt_dict)]

        # Then the embedded objects
        for embedded_object in  embedded_objects:
            id_and_rev_info = embedded_object['id_and_rev_info']
            elt_name = embedded_object['elt_name']
            elt_dict = embedded_object['dict_repr']
            pending_stack.append((id_and_rev_info,elt_name,elt_dict))

        if id_and_rev_info['timestamp']:
            ts = id_and_rev_info['timestamp']
        else:
            ts = self.create_timestamp

        for (id_and_rev_info, elt_name, elt_dict) in pending_stack:
            # call the importer that turns DingoObjDicts into Information Objects in the database
            iobject_type_name = elt_name
            iobject_type_namespace_uri = self.namespace_dict.get(elt_dict.get('@@ns',None),DINGOS_GENERIC_FAMILY_NAME)

            MantisImporter.create_iobject(iobject_family_name = self.iobject_family_name,
                                          iobject_family_revision_name= self.iobject_family_revision_name,
                                          iobject_type_name=iobject_type_name,
                                          iobject_type_namespace_uri=iobject_type_namespace_uri,
                                          iobject_type_revision_name= '',
                                          iobject_data=elt_dict,
                                          uid=id_and_rev_info['id'],
                                          identifier_ns_uri= identifier_ns_uri,
                                          timestamp = ts,
                                          create_timestamp = self.create_timestamp,
                                          markings=markings,
                                          config_hooks = {'special_ft_handler' : self.fact_handler_list(),
                                                         'datatype_extractor' : self.datatype_extractor,
                                                         'attr_ignore_predicate' : self.attr_ignore_predicate},
                                          namespace_dict=self.namespace_dict,
                                          )
    def xml_import(self,
                   filepath=None,
                   xml_content=None,
                   markings=None,
                   identifier_ns_uri=None,
                   **kwargs):
        """
        Import a iodef XML  from file <filepath>.
        You can provide:

        - a list of markings with which all generated Information Objects
           will be associated (e.g., in order to provide provenance function)

        - The uri of a namespace of the identifiers for the generated information objects.
          This namespace identifiers the 'owner' of the object. For iodef import, this
          should not be necessary, because the XML schema makes sure that each
          Inicdent is associated with ownership information via the 'name' attribute.

        The kwargs are not read -- they are present to allow the use of the
        DingoImportCommand class for easy definition of commandline import commands
        (the class passes all command line arguments to the xml_import function, so
        without the **kwargs parameter, an error would occur.
        """

        # Clear state in case xml_import is used several times

        self.__init__()

        # Initialize  default arguments

        # '[]' would be mutable, so we initialize here
        if not markings:
            markings = []

        # Initalizing here allows us to also get the default namespace when
        # explicitly passing 'None' as parameter.

        if identifier_ns_uri:
            self.identifier_ns_uri = identifier_ns_uri

        # Use the generic XML import customized for  OpenIOC import
        # to turn XML into DingoObjDicts

        import_result = MantisImporter.xml_import(xml_fname=filepath,
                                                  xml_content=xml_content,
                                                  ns_mapping=self.namespace_dict,
                                                  embedded_predicate=self.embedding_pred,
                                                  id_and_revision_extractor=self.id_and_revision_extractor,
                                                  transformer=self.transformer,
                                                  keep_attrs_in_created_reference=False,
        )

        # The result is of the following form::
        #
        #
        #   {'id_and_rev_info': Id and revision info of top-level element; for iodef, we always have
        #                       {'id':None, 'timestamp':None}, because the  <IODEF-Document> element
        #                       carries no identifier or timestamp
        #    'elt_name': Element name of top-level element, for iodef always 'IODEF-Document'
        #    'dict_repr': Dictionary representation of IODEF XML, minus the embedded Incident objects
        #    'embedded_objects': List of embedded objects, as dictionary
        #                           {"id_and_revision_info": id and revision info of extracted object,
        #                            "elt_name": Element name (for IODEF always 'Incident'),
        #                            "dict_repr" :  dictionary representation of XML of embedded object
        #                           }
        #    'unprocessed' : List of unprocessed embedded objects (not used for iodef import
        #    'file_content': Content of imported file (or, if content was passed instead of a file name,
        #                    the original content)}

        id_and_rev_info = import_result['id_and_rev_info']
        elt_name = import_result['elt_name']
        elt_dict = import_result['dict_repr']

        embedded_objects = import_result['embedded_objects']

        default_ns = self.namespace_dict.get(elt_dict.get('@@ns', None))

        # Here, we could try to extract the family name and version from
        # the namespace information, but we do not do that for now.

        ns_info = search_by_re_list(self.RE_LIST_NS_TYPE_FROM_NS_URL,default_ns)

        if ns_info:
            if 'family' in ns_info:
                self.iobject_family_name = ns_info['family']
            if 'revision' in ns_info:
                self.iobject_family_revision_name = ns_info['revision']

        # Initialize stack with import_results.

        # First, the result from the top-level import
        pending_stack = [(id_and_rev_info, elt_name, elt_dict)]

        # Then the embedded objects
        for embedded_object in embedded_objects:
            id_and_rev_info = embedded_object['id_and_rev_info']
            elt_name = embedded_object['elt_name']
            elt_dict = embedded_object['dict_repr']
            pending_stack.append((id_and_rev_info, elt_name, elt_dict))


        for (id_and_rev_info, elt_name, elt_dict) in pending_stack:
            # call the importer that turns DingoObjDicts into Information Objects in the database

            if id_and_rev_info['timestamp']:
                ts = id_and_rev_info['timestamp']
            else:
                ts = self.create_timestamp

            iobject_type_name = elt_name

            ns_info = search_by_re_list(self.RE_LIST_NS_TYPE_FROM_NS_URL,default_ns)

            iobject_type_namespace_uri = None
            iobject_type_revision_name = None

            if ns_info:
                if 'family_ns' in ns_info:
                    iobject_type_namespace_uri = ns_info['family_ns']
                if 'revision' in ns_info:
                    iobject_type_revision_name = ns_info['revision']

            if not iobject_type_namespace_uri:
                iobject_type_namespace_uri = self.namespace_dict.get(elt_dict.get('@@ns', None), DINGOS_GENERIC_FAMILY_NAME)

            if not id_and_rev_info['id']:
                logger.error("Attempt to import object (element name %s) without id -- object is ignored" % elt_name)
                continue

            MantisImporter.create_iobject(iobject_family_name=self.iobject_family_name,
                                          iobject_family_revision_name=self.iobject_family_revision_name,
                                          iobject_type_name=iobject_type_name,
                                          iobject_type_namespace_uri=iobject_type_namespace_uri,
                                          iobject_type_revision_name=iobject_type_revision_name,
                                          iobject_data=elt_dict,
                                          uid=id_and_rev_info['id'].split(":")[0],
                                          identifier_ns_uri=id_and_rev_info['id'].split(":")[1],
                                          timestamp=ts,
                                          create_timestamp=self.create_timestamp,
                                          markings=markings,
                                          config_hooks={'special_ft_handler': self.fact_handler_list(),
                                                        'datatype_extractor': self.datatype_extractor,
                                                        'attr_ignore_predicate': self.attr_ignore_predicate},
                                          namespace_dict=self.namespace_dict,
            )
    def xml_import(self,
                   filepath=None,
                   xml_content=None,
                   markings=None,
                   identifier_ns_uri=None,
                   initialize_importer=True,
                   **kwargs):
        """
        Import an OpenIOC indicator xml (root element 'ioc') from file <filepath> or
        from a string <xml_content>

        You can provide:

        - a list of markings with which all generated Information Objects
           will be associated (e.g., in order to provide provenance function)

        - The uri of a namespace of the identifiers for the generated information objects.
          This namespace identifiers the 'owner' of the object. For example, if importing
          IOCs published by Mandiant (e.g., as part of the APT1 report), chose an namespace
          such  as 'mandiant.com' or similar (and be consistent about it, when importing
          other stuff published by Mandiant).

        The kwargs are not read -- they are present to allow the use of the
        DingoImportCommand class for easy definition of commandline import commands
        (the class passes all command line arguments to the xml_import function, so
        without the **kwargs parameter, an error would occur.
        """


        if initialize_importer:
            # Clear state in case xml_import is used several times, but keep namespace info
            self.__init__()

        # Initialize  default arguments

        # '[]' would be mutable, so we initialize here
        if not markings:
            markings = []

        # Initializing here allows us to also get the default namespace when
        # explicitly passing 'None' as parameter.

        if identifier_ns_uri:
            self.identifier_ns_uri = identifier_ns_uri

        # Use the generic XML import customized for  OpenIOC import
        # to turn XML into DingoObjDicts

        import_result =  MantisImporter.xml_import(xml_fname=filepath,
                                                   xml_content=xml_content,
                                                   ns_mapping=self.namespace_dict,
                                                   embedded_predicate=self.openioc_embedding_pred,
                                                   id_and_revision_extractor=self.id_and_revision_extractor,
                                                   transformer=self.transformer,
                                                   keep_attrs_in_created_reference=False,
                                                   )


        # The MANTIS/DINGOS xml importer returns then the following structure::
        #
        #
        #    {'id_and_rev_info': Id and revision info of top-level element of the form
        #        {'id': ... , 'timestamp': ...}
        #    'elt_name': Element name of top-level element
        #    'dict_repr': Dictionary representation of XML, minus the embedded objects -- for
        #                 those, an 'idref' reference has been generated
        #    'embedded_objects': List of embedded objects, as dictionary
        #                       {"id_and_revision_info": id and revision info of extracted object,
        #                        "elt_name": Element name,
        #                        "dict_repr" :  dictionary representation of XML of embedded object
        #                       }
        #    'unprocessed' : List of unprocessed embedded objects (as libxml2 Node object)
        #                    (e.g. for handover to other importer
        #    'file_content': Content of imported file (or, if content was passed instead of a file name,
        #                                                                                         the original content)}


        id_and_rev_info = import_result['id_and_rev_info']
        elt_name = import_result['elt_name']
        elt_dict = import_result['dict_repr']

        embedded_objects = import_result['embedded_objects']

        default_ns = self.namespace_dict.get(elt_dict.get('@@ns',None),'http://schemas.mandiant.com/unknown/ioc')

        # Export family information.
        family_info_dict = search_by_re_list(self.RE_LIST_NS_TYPE_FROM_NS_URL,default_ns)
        if family_info_dict:
            self.iobject_family_name="%s.mandiant.com" % family_info_dict['family']
            self.iobject_family_revision_name=family_info_dict['revision']


        # Initialize stack with import_results.

        # First, the result from the top-level import
        pending_stack = deque()

        pending_stack.append((id_and_rev_info, elt_name,elt_dict))

        # Then the embedded objects
        while embedded_objects:
            embedded_object = embedded_objects.pop()
            id_and_rev_info = embedded_object['id_and_rev_info']
            elt_name = embedded_object['elt_name']
            elt_dict = embedded_object['dict_repr']
            pending_stack.append((id_and_rev_info,elt_name,elt_dict))

        if id_and_rev_info['timestamp']:
            ts = id_and_rev_info['timestamp']
        else:
            ts = self.create_timestamp

        while pending_stack:
            (id_and_rev_info, elt_name, elt_dict) = pending_stack.pop()

            # Call the importer that turns DingoObjDicts into Information Objects in the database
            iobject_type_name = elt_name
            iobject_type_namespace_uri = self.namespace_dict.get(elt_dict.get('@@ns',None),DINGOS_GENERIC_FAMILY_NAME)

            MantisImporter.create_iobject(iobject_family_name = self.iobject_family_name,
                                          iobject_family_revision_name= self.iobject_family_revision_name,
                                          iobject_type_name=iobject_type_name,
                                          iobject_type_namespace_uri=iobject_type_namespace_uri,
                                          iobject_type_revision_name= '',
                                          iobject_data=elt_dict,
                                          uid=id_and_rev_info['id'],
                                          identifier_ns_uri= self.identifier_ns_uri,
                                          timestamp = ts,
                                          create_timestamp = self.create_timestamp,
                                          markings=markings,
                                          config_hooks = {'special_ft_handler' : self.fact_handler_list(),
                                                          'datatype_extractor' : self.datatype_extractor,
                                                          'attr_ignore_predicate' : self.attr_ignore_predicate},
                                          namespace_dict=self.namespace_dict,
                                          )