class ReportImporter: def __init__(self) -> None: # Instantiate the connector helper from config base_path = os.path.dirname(os.path.abspath(__file__)) config_file_path = base_path + "/../config.yml" config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.helper = OpenCTIConnectorHelper(config) self.create_indicator = get_config_variable( "IMPORT_DOCUMENT_CREATE_INDICATOR", ["import_document", "create_indicator"], config, ) # Load Entity and Observable configs observable_config_file = base_path + "/config/observable_config.ini" entity_config_file = base_path + "/config/entity_config.ini" if os.path.isfile(observable_config_file) and os.path.isfile( entity_config_file): self.observable_config = self._parse_config( observable_config_file, Observable) else: raise FileNotFoundError(f"{observable_config_file} was not found") if os.path.isfile(entity_config_file): self.entity_config = self._parse_config(entity_config_file, EntityConfig) else: raise FileNotFoundError(f"{entity_config_file} was not found") def _process_message(self, data: Dict) -> str: self.helper.log_info("Processing new message") file_name = self._download_import_file(data) entity_id = data.get("entity_id", None) bypass_validation = data.get("bypass_validation", False) entity = (self.helper.api.stix_domain_object.read( id=entity_id) if entity_id is not None else None) if self.helper.get_only_contextual() and entity is None: return "Connector is only contextual and entity is not defined. Nothing was imported" # Retrieve entity set from OpenCTI entity_indicators = self._collect_stix_objects(self.entity_config) # Parse report parser = ReportParser(self.helper, entity_indicators, self.observable_config) parsed = parser.run_parser(file_name, data["file_mime"]) os.remove(file_name) if not parsed: return "No information extracted from report" # Process parsing results self.helper.log_debug("Results: {}".format(parsed)) observables, entities = self._process_parsing_results(parsed, entity) # Send results to OpenCTI observable_cnt = self._process_parsed_objects(entity, observables, entities, bypass_validation, file_name) entity_cnt = len(entities) if self.helper.get_validate_before_import() and not bypass_validation: return "Generated bundle sent for validation" else: return ( f"Sent {observable_cnt} observables, 1 report update and {entity_cnt} entity connections as stix " f"bundle for worker import ") def start(self) -> None: self.helper.listen(self._process_message) def _download_import_file(self, data: Dict) -> str: file_fetch = data["file_fetch"] file_uri = self.helper.opencti_url + file_fetch # Downloading and saving file to connector self.helper.log_info("Importing the file " + file_uri) file_name = os.path.basename(file_fetch) file_content = self.helper.api.fetch_opencti_file(file_uri, True) with open(file_name, "wb") as f: f.write(file_content) return file_name def _collect_stix_objects( self, entity_config_list: List[EntityConfig]) -> List[Entity]: base_func = self.helper.api entity_list = [] for entity_config in entity_config_list: func_format = entity_config.stix_class try: custom_function = getattr(base_func, func_format) entries = custom_function.list( getAll=True, filters=entity_config.filter, customAttributes=entity_config.custom_attributes, ) entity_list += entity_config.convert_to_entity( entries, self.helper) except AttributeError: e = "Selected parser format is not supported: {}".format( func_format) raise NotImplementedError(e) return entity_list @staticmethod def _parse_config(config_file: str, file_class: Callable) -> List[BaseModel]: config = MyConfigParser() config.read(config_file) config_list = [] for section, content in config.as_dict().items(): content["name"] = section config_object = file_class(**content) config_list.append(config_object) return config_list def _process_parsing_results( self, parsed: List[Dict], context_entity: Dict) -> (List[SimpleObservable], List[str]): observables = [] entities = [] if context_entity is not None: object_markings = [ x["standard_id"] for x in context_entity.get("objectMarking", []) ] # external_references = [x['standard_id'] for x in report.get('externalReferences', [])] # labels = [x['standard_id'] for x in report.get('objectLabel', [])] author = context_entity.get("createdBy") else: object_markings = [] author = None if author is not None: author = author.get("standard_id", None) for match in parsed: if match[RESULT_FORMAT_TYPE] == OBSERVABLE_CLASS: if match[RESULT_FORMAT_CATEGORY] == "Vulnerability.name": entity = self.helper.api.vulnerability.read( filters={ "key": "name", "values": [match[RESULT_FORMAT_MATCH]] }) if entity is None: self.helper.log_info( f"Vulnerability with name '{match[RESULT_FORMAT_MATCH]}' could not be " f"found. Is the CVE Connector activated?") continue entities.append(entity["standard_id"]) elif match[ RESULT_FORMAT_CATEGORY] == "Attack-Pattern.x_mitre_id": entity = self.helper.api.attack_pattern.read( filters={ "key": "x_mitre_id", "values": [match[RESULT_FORMAT_MATCH]], }) if entity is None: self.helper.log_info( f"AttackPattern with MITRE ID '{match[RESULT_FORMAT_MATCH]}' could not be " f"found. Is the MITRE Connector activated?") continue entities.append(entity["standard_id"]) else: observable = SimpleObservable( id=OpenCTIStix2Utils.generate_random_stix_id( "x-opencti-simple-observable"), key=match[RESULT_FORMAT_CATEGORY], value=match[RESULT_FORMAT_MATCH], x_opencti_create_indicator=self.create_indicator, object_marking_refs=object_markings, created_by_ref=author, # labels=labels, # external_references=external_references ) observables.append(observable) elif match[RESULT_FORMAT_TYPE] == ENTITY_CLASS: entities.append(match[RESULT_FORMAT_MATCH]) else: self.helper.log_info("Odd data received: {}".format(match)) return observables, entities def _process_parsed_objects( self, entity: Dict, observables: List, entities: List, bypass_validation: bool, file_name: str, ) -> int: if len(observables) == 0 and len(entities) == 0: return 0 if entity is not None and entity["entity_type"] == "Report": report = Report( id=entity["standard_id"], name=entity["name"], description=entity["description"], published=self.helper.api.stix2.format_date(entity["created"]), report_types=entity["report_types"], object_refs=observables + entities, allow_custom=True, ) observables.append(report) elif entity is not None: # TODO, relate all object to the entity entity_stix_bundle = self.helper.api.stix2.export_entity( entity["entity_type"], entity["id"]) observables = observables + entity_stix_bundle["objects"] else: timestamp = int(time.time()) now = datetime.utcfromtimestamp(timestamp) report = Report( name=file_name, description="Automatic import", published=now, report_types=["threat-report"], object_refs=observables + entities, allow_custom=True, ) observables.append(report) bundles_sent = [] if len(observables) > 0: bundle = Bundle(objects=observables, allow_custom=True).serialize() bundles_sent = self.helper.send_stix2_bundle( bundle=bundle, update=True, bypass_validation=bypass_validation, file_name=file_name + ".json", entity_id=entity["id"] if entity is not None else None, ) # len() - 1 because the report update increases the count by one return len(bundles_sent) - 1
class ImportFilePdfObservables: def __init__(self): # Instantiate the connector helper from config config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml" config = ( yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {} ) self.helper = OpenCTIConnectorHelper(config) self.create_indicator = get_config_variable( "PDF_OBSERVABLES_CREATE_INDICATOR", ["pdf_observables", "create_indicator"], config, ) def _process_message(self, data): file_fetch = data["file_fetch"] file_uri = self.helper.opencti_url + file_fetch file_name = os.path.basename(file_fetch) entity_id = data["entity_id"] # Get context is_context = entity_id is not None and len(entity_id) > 0 if self.helper.get_only_contextual() and not is_context: raise ValueError( "No context defined, connector is get_only_contextual true" ) self.helper.log_info("Importing the file " + file_uri) # Get the file file_content = self.helper.api.fetch_opencti_file(file_uri, True) # Write the file f = open(file_name, "wb") f.write(file_content) f.close() # Parse bundle_objects = [] i = 0 parser = iocp.IOC_Parser(None, "pdf", True, "pdfminer", "json") parsed = parser.parse(file_name) os.remove(file_name) if parsed != []: for file in parsed: if file != None: for page in file: if page != []: for match in page: resolved_match = self.resolve_match(match) if resolved_match: observable = SimpleObservable( id=OpenCTIStix2Utils.generate_random_stix_id( "x-opencti-simple-observable" ), key=resolved_match["type"], value=resolved_match["value"], x_opencti_create_indicator=self.create_indicator, ) bundle_objects.append(observable) i += 1 else: self.helper.log_error("Could not parse the report!") if is_context: entity = self.helper.api.stix_domain_object.read(id=entity_id) if entity is not None: if entity["entity_type"] == "Report" and len(bundle_objects) > 0: report = Report( id=entity["standard_id"], name=entity["name"], description=entity["description"], published=self.helper.api.stix2.format_date(entity["created"]), report_types=entity["report_types"], object_refs=bundle_objects, ) bundle_objects.append(report) bundle = Bundle(objects=bundle_objects).serialize() bundles_sent = self.helper.send_stix2_bundle(bundle) return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import" # Start the main loop def start(self): self.helper.listen(self._process_message) def resolve_match(self, match): types = { "MD5": "File.hashes.MD5", "SHA1": "File.hashes.SHA-1", "SHA256": "File.hashes.SHA-256", "Filename": "File.name", "IP": "IPv4-Addr.value", "Host": "X-OpenCTI-Hostname.value", "Filepath": "File.path", "URL": "Url.value", "Email": "Email-Addr.value", } type = match["type"] value = match["match"] if type in types: resolved_type = types[type] if resolved_type == "IPv4-Addr.value": # Demilitarized IP if "[.]" in value: value = value.replace("[.]", ".") type_0 = self.detect_ip_version(value) elif resolved_type == "Url.value": # Demilitarized URL if "hxxp://" in value: value = value.replace("hxxp://", "http://") if "hxxps://" in value: value = value.replace("hxxps://", "https://") if "hxxxs://" in value: value = value.replace("hxxxs://", "https://") if "[.]" in value: value = value.replace("[.]", ".") type_0 = resolved_type elif resolved_type == "X-OpenCTI-Hostname.value": # Demilitarized Host if "[.]" in value: value = value.replace("[.]", ".") type_0 = resolved_type else: type_0 = resolved_type return {"type": type_0, "value": value} else: return False def detect_ip_version(self, value): if len(value) > 16: return "IPv6-Addr.value" else: return "IPv4-Addr.value"
class ReportImporter: def __init__(self) -> None: # Instantiate the connector helper from config base_path = os.path.dirname(os.path.abspath(__file__)) config_file_path = base_path + "/../config.yml" config = ( yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {} ) self.helper = OpenCTIConnectorHelper(config) self.create_indicator = get_config_variable( "IMPORT_REPORT_CREATE_INDICATOR", ["import_report", "create_indicator"], config, ) # Load Entity and Observable configs observable_config_file = base_path + "/config/observable_config.ini" entity_config_file = base_path + "/config/entity_config.ini" if os.path.isfile(observable_config_file) and os.path.isfile( entity_config_file ): self.observable_config = self._parse_config( observable_config_file, Observable ) else: raise FileNotFoundError(f"{observable_config_file} was not found") if os.path.isfile(entity_config_file): self.entity_config = self._parse_config(entity_config_file, EntityConfig) else: raise FileNotFoundError(f"{entity_config_file} was not found") def _process_message(self, data: Dict) -> str: file_name = self._download_import_file(data) entity_id = data.get("entity_id", None) if self._check_context(entity_id): raise ValueError( "No context defined, connector is get_only_contextual true" ) # Retrieve entity set from OpenCTI entity_indicators = self._collect_stix_objects(self.entity_config) # Parse peport parser = ReportParser(self.helper, entity_indicators, self.observable_config) parsed = parser.run_parser(file_name, data["file_mime"]) os.remove(file_name) if not parsed: return "No information extracted from report" # Process parsing results self.helper.log_debug("Results: {}".format(parsed)) observables, entities = self._process_parsing_results(parsed) report = self.helper.api.report.read(id=entity_id) # Send results to OpenCTI observable_cnt = self._process_observables(report, observables) entity_cnt = self._process_entities(report, entities) return f"Sent {observable_cnt} stix bundle(s) and {entity_cnt} entity connections for worker import" def start(self) -> None: self.helper.listen(self._process_message) def _download_import_file(self, data: Dict) -> str: file_fetch = data["file_fetch"] file_uri = self.helper.opencti_url + file_fetch # Downloading and saving file to connector self.helper.log_info("Importing the file " + file_uri) file_name = os.path.basename(file_fetch) file_content = self.helper.api.fetch_opencti_file(file_uri, True) with open(file_name, "wb") as f: f.write(file_content) return file_name def _check_context(self, entity_id: str) -> bool: is_context = entity_id and len(entity_id) > 0 return self.helper.get_only_contextual() and not is_context def _collect_stix_objects( self, entity_config_list: List[EntityConfig] ) -> List[Entity]: base_func = self.helper.api entity_list = [] for entity_config in entity_config_list: func_format = entity_config.stix_class try: custom_function = getattr(base_func, func_format) entries = custom_function.list( getAll=True, filters=entity_config.filter ) entity_list += entity_config.convert_to_entity(entries) except AttributeError: e = "Selected parser format is not supported: {}".format(func_format) raise NotImplementedError(e) return entity_list @staticmethod def _parse_config(config_file: str, file_class: Callable) -> List[BaseModel]: config = MyConfigParser() config.read(config_file) config_list = [] for section, content in config.as_dict().items(): content["name"] = section config_object = file_class(**content) config_list.append(config_object) return config_list def _process_parsing_results( self, parsed: List[Dict] ) -> (List[SimpleObservable], List[str]): observables = [] entities = [] for match in parsed: if match[RESULT_FORMAT_TYPE] == OBSERVABLE_CLASS: # Hardcoded exceptions since SimpleObservable doesn't support those types yet if match[RESULT_FORMAT_CATEGORY] == "Vulnerability.name": observable = self.helper.api.vulnerability.read( filters={"key": "name", "values": [match[RESULT_FORMAT_MATCH]]} ) if observable is None: self.helper.log_info( f"Vulnerability with name '{match[RESULT_FORMAT_MATCH]}' could not be " f"found. Is the CVE Connector activated?" ) continue elif match[RESULT_FORMAT_CATEGORY] == "Attack-Pattern.x_mitre_id": observable = self.helper.api.attack_pattern.read( filters={ "key": "x_mitre_id", "values": [match[RESULT_FORMAT_MATCH]], } ) if observable is None: self.helper.log_info( f"AttackPattern with MITRE ID '{match[RESULT_FORMAT_MATCH]}' could not be " f"found. Is the MITRE Connector activated?" ) continue else: observable = self.helper.api.stix_cyber_observable.create( simple_observable_id=OpenCTIStix2Utils.generate_random_stix_id( "x-opencti-simple-observable" ), simple_observable_key=match[RESULT_FORMAT_CATEGORY], simple_observable_value=match[RESULT_FORMAT_MATCH], createIndicator=self.create_indicator, ) observables.append(observable["id"]) elif match[RESULT_FORMAT_TYPE] == ENTITY_CLASS: entities.append(match[RESULT_FORMAT_MATCH]) else: self.helper.log_info("Odd data received: {}".format(match)) return observables, entities def _process_observables(self, report: Dict, observables: List) -> int: if report is None: self.helper.log_error( "No report found! This is a purely contextual connector and this should not happen" ) if len(observables) == 0: return 0 report = self.helper.api.report.create( id=report["standard_id"], name=report["name"], description=report["description"], published=self.helper.api.stix2.format_date(report["created"]), report_types=report["report_types"], update=True, ) for observable in observables: self.helper.api.report.add_stix_object_or_stix_relationship( id=report["id"], stixObjectOrStixRelationshipId=observable ) return len(observables) def _process_entities(self, report: Dict, entities: List) -> int: if report: for stix_object in entities: self.helper.api.report.add_stix_object_or_stix_relationship( id=report["id"], stixObjectOrStixRelationshipId=stix_object ) return len(entities)
class ImportFilePdfObservables: def __init__(self): # Instantiate the connector helper from config config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml" config = ( yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {} ) self.helper = OpenCTIConnectorHelper(config) self.create_indicator = get_config_variable( "PDF_OBSERVABLES_CREATE_INDICATOR", ["pdf_observables", "create_indicator"], config, ) self.types = { "MD5": "File.hashes.MD5", "SHA1": "File.hashes.SHA-1", "SHA256": "File.hashes.SHA-256", "Filename": "File.name", "IP": "IPv4-Addr.value", "DomainName": "Domain-Name.value", # Directory is not yet fully functional # "Directory": "Directory.path", "URL": "Url.value", "Email": "Email-Addr.value", "CVE": "Vulnerability.name", "Registry": "Windows-Registry-Key.key", } def _process_message(self, data): file_fetch = data["file_fetch"] file_uri = self.helper.opencti_url + file_fetch file_name = os.path.basename(file_fetch) entity_id = data.get("entity_id", None) self.helper.log_info(entity_id) # Get context is_context = entity_id is not None and len(entity_id) > 0 self.helper.log_info("Context: {}".format(is_context)) self.helper.log_info( "get_only_contextual: {}".format(self.helper.get_only_contextual()) ) if self.helper.get_only_contextual() and not is_context: raise ValueError( "No context defined, connector is get_only_contextual true" ) self.helper.log_info("Importing the file " + file_uri) # Get the file file_content = self.helper.api.fetch_opencti_file(file_uri, True) # Write the file f = open(file_name, "wb") f.write(file_content) f.close() # Parse bundle_objects = [] stix_objects = set() i = 0 custom_indicators = self._get_entities() mime_type = self._get_mime_type(file_name) print(mime_type) if mime_type is None: raise ValueError("Invalid file format of {}".format(file_name)) parser = iocp.IOC_Parser( None, mime_type, True, "pdfminer", "json", custom_indicators=custom_indicators, ) parsed = parser.parse(file_name) os.remove(file_name) if parsed != []: for file in parsed: if file != None: for page in file: if page != []: for match in page: resolved_match = self._resolve_match(match) if resolved_match: # For the creation of relationships if resolved_match[ "type" ] not in self.types.values() and self._is_uuid( resolved_match["value"] ): stix_objects.add(resolved_match["value"]) # For CVEs since SimpleObservable doesn't support Vulnerabilities yet elif resolved_match["type"] == "Vulnerability.name": vulnerability = Vulnerability( name=resolved_match["value"] ) bundle_objects.append(vulnerability) # Other observables elif resolved_match["type"] in self.types.values(): observable = SimpleObservable( id=OpenCTIStix2Utils.generate_random_stix_id( "x-opencti-simple-observable" ), key=resolved_match["type"], value=resolved_match["value"], x_opencti_create_indicator=self.create_indicator, ) bundle_objects.append(observable) else: self.helper.log_info( "Odd data received: {}".format( resolved_match ) ) i += 1 else: self.helper.log_error("Could not parse the report!") if is_context: entity = self.helper.api.stix_domain_object.read(id=entity_id) if entity is not None: if entity["entity_type"] == "Report" and len(bundle_objects) > 0: report = Report( id=entity["standard_id"], name=entity["name"], description=entity["description"], published=self.helper.api.stix2.format_date(entity["created"]), report_types=entity["report_types"], object_refs=bundle_objects, ) bundle_objects.append(report) bundles_sent = [] if len(bundle_objects) > 0: bundle = Bundle(objects=bundle_objects).serialize() bundles_sent = self.helper.send_stix2_bundle(bundle) if len(stix_objects) > 0 and entity_id is not None: report = self.helper.api.report.read(id=entity_id) if report: for stix_object in stix_objects: self.helper.log_info(stix_object) self.helper.api.report.add_stix_object_or_stix_relationship( id=report["id"], stixObjectOrStixRelationshipId=stix_object ) return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import" def start(self): self.helper.listen(self._process_message) def _resolve_match(self, match): type = match["type"] value = match["match"] if type in self.types: resolved_type = self.types[type] if resolved_type == "IPv4-Addr.value": # Demilitarized IP if "[.]" in value: value = value.replace("[.]", ".") type_0 = self._detect_ip_version(value) elif resolved_type == "Url.value": # Demilitarized URL if "hxxp://" in value: value = value.replace("hxxp://", "http://") if "hxxps://" in value: value = value.replace("hxxps://", "https://") if "hxxxs://" in value: value = value.replace("hxxxs://", "https://") if "[.]" in value: value = value.replace("[.]", ".") type_0 = resolved_type elif resolved_type == "DomainName.value": # Demilitarized Host if "[.]" in value: value = value.replace("[.]", ".") type_0 = resolved_type else: type_0 = resolved_type return {"type": type_0, "value": value} elif self._is_uuid(value): return {"type": type, "value": value} else: self.helper.log_info("Some odd info received: {}".format(match)) return False def _detect_ip_version(self, value): if len(value) > 16: return "IPv6-Addr.value" else: return "IPv4-Addr.value" def _is_uuid(self, value): try: uuid.UUID(value) except ValueError: return False return True def _get_entities(self): setup = { "attack_pattern": { "entity_filter": None, "entity_fields": ["x_mitre_id"], "type": "entity", }, "identity": { "entity_filter": None, "entity_fields": ["aliases", "name"], "type": "entity", }, "location": { "entity_filter": [{"key": "entity_type", "values": ["Country"]}], "entity_fields": ["aliases", "name"], "type": "entity", }, "intrusion_set": { "entity_filter": None, "entity_fields": ["aliases", "name"], "type": "entity", }, "malware": { "entity_filter": None, "entity_fields": ["aliases", "name"], "type": "entity", }, "tool": { "entity_filter": None, "entity_fields": ["aliases", "name"], "type": "entity", }, } return self._collect_stix_objects(setup) def _collect_stix_objects(self, setup_dict: Dict): base_func = self.helper.api information_list = {} for entity, args in setup_dict.items(): func_format = entity try: custom_function = getattr(base_func, func_format) entries = custom_function.list( getAll=True, filters=args["entity_filter"] ) information_list[entity] = self._make_1d_list( entries, args["entity_fields"] ) except AttributeError: e = "Selected parser format is not supported: %s" % func_format raise NotImplementedError(e) return information_list def _make_1d_list(self, values, keys): items = {} for item in values: _id = item.get("id") sub_items = set() if ( item.get("externalReferences", None) is None or len(item["externalReferences"]) == 0 ): continue for key in keys: elem = item.get(key, []) if elem: if type(elem) == list: sub_items.update(elem) elif type(elem) == str: sub_items.add(elem) items[_id] = sub_items return items def _get_mime_type(self, file_name): mime = magic.Magic(mime=True) translation = { "application/pdf": "pdf", # 'text/html': 'html', # 'text/plain': 'txt' } mimetype = mime.from_file(file_name) return translation.get(mimetype, None)