def _process_parsed_objects( self, entity: Dict, observables: List, entities: List, bypass_validation: bool, file_name: str, ) -> int: if len(observables) == 0 and len(entities) == 0: return 0 if entity is not None and entity["entity_type"] == "Report": report = Report( id=entity["standard_id"], name=entity["name"], description=entity["description"], published=self.helper.api.stix2.format_date(entity["created"]), report_types=entity["report_types"], object_refs=observables + entities, allow_custom=True, ) observables.append(report) elif entity is not None: # TODO, relate all object to the entity entity_stix_bundle = self.helper.api.stix2.export_entity( entity["entity_type"], entity["id"]) observables = observables + entity_stix_bundle["objects"] else: timestamp = int(time.time()) now = datetime.utcfromtimestamp(timestamp) report = Report( name=file_name, description="Automatic import", published=now, report_types=["threat-report"], object_refs=observables + entities, allow_custom=True, ) observables.append(report) bundles_sent = [] if len(observables) > 0: bundle = Bundle(objects=observables, allow_custom=True).serialize() bundles_sent = self.helper.send_stix2_bundle( bundle=bundle, update=True, bypass_validation=bypass_validation, file_name=file_name + ".json", entity_id=entity["id"] if entity is not None else None, ) # len() - 1 because the report update increases the count by one return len(bundles_sent) - 1
def _process_message(self, data: Dict) -> str: file_fetch = data["file_fetch"] file_uri = self.helper.opencti_url + file_fetch # Downloading and saving file to connector self.helper.log_info("Importing the file " + file_uri) observable = SimpleObservable( id=OpenCTIStix2Utils.generate_random_stix_id( "x-opencti-simple-observable"), key=self.data["simple_observable_key"], value=self.data["simple_observable_value"], ) bundle_objects = [observable] entity_id = data.get("entity_id", None) report = self.helper.api.report.read(id=entity_id) report = Report( id=report["standard_id"], name=report["name"], description=report["description"], published=self.helper.api.stix2.format_date(report["published"]), report_types=report["report_types"], object_refs=bundle_objects, ) bundle_objects.append(report) # create stix bundle bundle = Bundle(objects=bundle_objects).serialize() # send data self.helper.send_stix2_bundle(bundle=bundle) return "foo"
def create_report( name: str, published: datetime, objects: List[Union[_DomainObject, _RelationshipObject]], created_by: Optional[Identity] = None, created: Optional[datetime] = None, modified: Optional[datetime] = None, description: Optional[str] = None, report_types: Optional[List[str]] = None, labels: Optional[List[str]] = None, confidence: Optional[int] = None, external_references: Optional[List[ExternalReference]] = None, object_markings: Optional[List[MarkingDefinition]] = None, x_opencti_report_status: Optional[int] = None, ) -> Report: """Create a report.""" return Report( id=_create_random_identifier("report"), created_by_ref=created_by, created=created, modified=modified, name=name, description=description, report_types=report_types, published=published, object_refs=objects, labels=labels, confidence=confidence, external_references=external_references, object_marking_refs=object_markings, custom_properties={X_OPENCTI_REPORT_STATUS: x_opencti_report_status}, allow_custom=True, )
def _process_parsed_objects(self, report: Dict, observables: List, entities: List) -> int: if len(observables) == 0 and len(entities) == 0: return 0 report = Report( id=report["standard_id"], name=report["name"], description=report["description"], published=self.helper.api.stix2.format_date(report["created"]), report_types=report["report_types"], object_refs=observables + entities, ) observables.append(report) bundles_sent = [] if len(observables) > 0: bundle = Bundle(objects=observables).serialize() bundles_sent = self.helper.send_stix2_bundle( bundle=bundle, update=True, ) # len() - 1 because the report update increases the count by one return len(bundles_sent) - 1
def _create_report(self, object_refs: List[STIXDomainObject]) -> Report: external_references = [] for reference in self.pulse.references: if not reference: continue external_reference = create_external_reference( self.source_name, reference) external_references.append(external_reference) tags = [] for pulse_tag in self.pulse.tags: tag = create_tag(self.source_name, pulse_tag, "#" + "%06x" % random.randint(0, 0xFFFFFF)) tags.append(tag) return Report( created_by_ref=self.author, name=self.pulse.name, description=self.pulse.description, published=self.pulse.created, object_refs=object_refs, labels=["threat-report"], external_references=external_references, object_marking_refs=self.object_marking_refs, custom_properties={ CustomProperties.REPORT_CLASS: self.report_type, CustomProperties.OBJECT_STATUS: self.report_status, CustomProperties.SRC_CONF_LEVEL: self.confidence_level, CustomProperties.TAG_TYPE: tags, }, )
def test_crate_bundle(): stix_splitter = OpenCTIStix2Splitter() report = Report( report_types=["campaign"], name="Bad Cybercrime", published="2016-04-06T20:03:00.000Z", object_refs=["indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7"], ).serialize() observables = [report] bundle = stix_splitter.stix2_create_bundle(observables, use_json=False, event_version=None) for key in ["type", "id", "spec_version", "objects"]: assert key in bundle assert len(bundle.keys()) == 4 bundle = stix_splitter.stix2_create_bundle(observables, use_json=False, event_version=1) for key in [ "type", "id", "spec_version", "objects", "x_opencti_event_version" ]: assert key in bundle assert len(bundle.keys()) == 5
def _process_message(self, data): file_fetch = data["file_fetch"] file_uri = self.helper.opencti_url + file_fetch file_name = os.path.basename(file_fetch) entity_id = data["entity_id"] # Get context is_context = entity_id is not None and len(entity_id) > 0 if self.helper.get_only_contextual() and not is_context: raise ValueError( "No context defined, connector is get_only_contextual true" ) self.helper.log_info("Importing the file " + file_uri) # Get the file file_content = self.helper.api.fetch_opencti_file(file_uri, True) # Write the file f = open(file_name, "wb") f.write(file_content) f.close() # Parse bundle_objects = [] i = 0 parser = iocp.IOC_Parser(None, "pdf", True, "pdfminer", "json") parsed = parser.parse(file_name) os.remove(file_name) if parsed != []: for file in parsed: if file != None: for page in file: if page != []: for match in page: resolved_match = self.resolve_match(match) if resolved_match: observable = SimpleObservable( id=OpenCTIStix2Utils.generate_random_stix_id( "x-opencti-simple-observable" ), key=resolved_match["type"], value=resolved_match["value"], x_opencti_create_indicator=self.create_indicator, ) bundle_objects.append(observable) i += 1 else: self.helper.log_error("Could not parse the report!") if is_context: entity = self.helper.api.stix_domain_object.read(id=entity_id) if entity is not None: if entity["entity_type"] == "Report" and len(bundle_objects) > 0: report = Report( id=entity["standard_id"], name=entity["name"], description=entity["description"], published=self.helper.api.stix2.format_date(entity["created"]), report_types=entity["report_types"], object_refs=bundle_objects, ) bundle_objects.append(report) bundle = Bundle(objects=bundle_objects).serialize() bundles_sent = self.helper.send_stix2_bundle(bundle) return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import"
def run(self, work_id: str, state: Mapping[str, Any]) -> Mapping[str, Any]: """Run the importation of the article.""" self.work_id = work_id created = parser.parse(self.article["createdDate"]) # RisIQ API does not always provide the `publishedDate`. # If it does not exist, take the value of the `createdDate` instead. published = ( parser.parse(self.article["publishedDate"]) if self.article["publishedDate"] is not None else created ) indicators = itertools.chain( *[ self._process_indicator(indicator) for indicator in self.article["indicators"] ] ) indicators = utils.deduplicate(list(indicators)) # Return the initial state if we don't have any indicators. if not indicators: self.helper.log_info("No indicator in article, report will not be created.") return state self.helper.log_debug(f"Number of indicators: {len(indicators)}") # Check if all indicators' TLP marking are `TLP_WHITE`. report_tlp = TLP_WHITE if TLP_AMBER in [i["object_marking_refs"][0] for i in indicators]: report_tlp = TLP_AMBER report = Report( type="report", name=self.article.get("title", "RiskIQ Threat Report"), description=self.article["summary"], report_types=["threat-report"], created_by_ref=self.author, created=created, published=published, lang="en", labels=self.article["tags"], object_refs=indicators, object_marking_refs=report_tlp, external_references=[ { "source_name": "riskiq", "url": self.article["link"], "external_id": self.article["guid"], } ], allow_custom=True, ) self.helper.log_debug(f"[RiskIQ] Report = {report}") bundle = Bundle(objects=indicators + [report, self.author], allow_custom=True) self.helper.log_info("[RiskIQ] Sending report STIX2 bundle") self._send_bundle(bundle) return self._create_state(created)
def _update_report(self, bundle: List, entity_id: int) -> List: report = self.helper.api.report.read(id=entity_id) # The entity_id can be any SDO if report: report = Report( id=report["standard_id"], name=report["name"], description=report["description"], published=self.helper.api.stix2.format_date(report["created"]), report_types=report["report_types"], object_refs=bundle, ) bundle.append(report) return bundle
def _create_report(self, object_refs: List[_DomainObject]) -> Report: external_references = self._create_report_external_references() tags = self._create_report_tags() return Report( created_by_ref=self.author, name=self.pulse.name, description=self.pulse.description, published=self.pulse.created, object_refs=object_refs, labels=["threat-report"], external_references=external_references, object_marking_refs=self.object_marking_refs, custom_properties={ CustomProperties.REPORT_CLASS: self.report_type, CustomProperties.OBJECT_STATUS: self.report_status, CustomProperties.SRC_CONF_LEVEL: self.confidence_level, CustomProperties.TAG_TYPE: tags, }, )
def process_events(self, events): for event in events: self.helper.log_info("Processing event " + event["Event"]["uuid"]) ### Default variables added_markings = [] added_entities = [] added_object_refs = [] ### Pre-process # Author author = Identity( name=event["Event"]["Orgc"]["name"], identity_class="organization" ) # Elements event_elements = self.prepare_elements(event["Event"]["Galaxy"], author) # Markings if "Tag" in event["Event"]: event_markings = self.resolve_markings(event["Event"]["Tag"]) else: event_markings = [TLP_WHITE] # Tags event_tags = [] if "Tag" in event["Event"]: event_tags = self.resolve_tags(event["Event"]["Tag"]) # ExternalReference event_external_reference = ExternalReference( source_name=self.helper.connect_name, external_id=event["Event"]["uuid"], url=self.misp_url + "/events/view/" + event["Event"]["uuid"], ) ### Get indicators event_external_references = [event_external_reference] indicators = [] # Get attributes for attribute in event["Event"]["Attribute"]: indicator = self.process_attribute( author, event_elements, event_markings, [], attribute ) if attribute["type"] == "link": event_external_references.append( ExternalReference( source_name=attribute["category"], external_id=attribute["uuid"], url=attribute["value"], ) ) if indicator is not None: indicators.append(indicator) # Get attributes of objects objects_relationships = [] for object in event["Event"]["Object"]: attribute_external_references = [] for attribute in object["Attribute"]: if attribute["type"] == "link": attribute_external_references.append( ExternalReference( source_name=attribute["category"], external_id=attribute["uuid"], url=attribute["value"], ) ) object_attributes = [] for attribute in object["Attribute"]: indicator = self.process_attribute( author, event_elements, event_markings, attribute_external_references, attribute, ) if indicator is not None: indicators.append(indicator) if ( object["meta-category"] == "file" and indicator["indicator"].x_opencti_observable_type in FILETYPES ): object_attributes.append(indicator) objects_relationships.extend( self.process_observable_relations(object_attributes, []) ) ### Prepare the bundle bundle_objects = [author] object_refs = [] # Add event markings for event_marking in event_markings: if event_marking["id"] not in added_markings: bundle_objects.append(event_marking) added_markings.append(event_marking["id"]) # Add event elements all_event_elements = ( event_elements["intrusion_sets"] + event_elements["malwares"] + event_elements["tools"] + event_elements["attack_patterns"] ) for event_element in all_event_elements: if event_element["name"] not in added_object_refs: object_refs.append(event_element) added_object_refs.append(event_element["name"]) if event_element["name"] not in added_entities: bundle_objects.append(event_element) added_entities.append(event_element["name"]) # Add indicators for indicator in indicators: if indicator["indicator"]["id"] not in added_object_refs: object_refs.append(indicator["indicator"]) added_object_refs.append(indicator["indicator"]["id"]) if indicator["indicator"]["id"] not in added_entities: bundle_objects.append(indicator["indicator"]) added_entities.append(indicator["indicator"]["id"]) # Add attribute markings for attribute_marking in indicator["markings"]: if attribute_marking["id"] not in added_markings: bundle_objects.append(attribute_marking) added_markings.append(attribute_marking["id"]) # Add attribute elements all_attribute_elements = ( indicator["attribute_elements"]["intrusion_sets"] + indicator["attribute_elements"]["malwares"] + indicator["attribute_elements"]["tools"] + indicator["attribute_elements"]["attack_patterns"] ) for attribute_element in all_attribute_elements: if attribute_element["name"] not in added_object_refs: object_refs.append(attribute_element) added_object_refs.append(attribute_element["name"]) if attribute_element["name"] not in added_entities: bundle_objects.append(attribute_element) added_entities.append(attribute_element["name"]) # Add attribute relationships for relationship in indicator["relationships"]: object_refs.append(relationship) bundle_objects.append(relationship) # Add object_relationships for object_relationship in objects_relationships: bundle_objects.append(object_relationship) ### Create the report if needed if self.misp_create_report and len(object_refs) > 0: report = Report( name=event["Event"]["info"], description=event["Event"]["info"], published=parse(event["Event"]["date"]), created_by_ref=author, object_marking_refs=event_markings, labels=["threat-report"], object_refs=object_refs, external_references=event_external_references, custom_properties={ "x_opencti_report_class": self.misp_report_class, "x_opencti_object_status": 2, "x_opencti_tags": event_tags, }, ) bundle_objects.append(report) bundle = Bundle(objects=bundle_objects).serialize() self.helper.log_info("Sending event STIX2 bundle") self.helper.send_stix2_bundle( bundle, None, self.update_existing_data, False )
def process_events(self, events): for event in events: ### Default variables added_markings = [] added_entities = [] added_object_refs = [] ### Pre-process # Author author = Identity(name=event['Event']['Orgc']['name'], identity_class='organization') # Elements event_elements = self.prepare_elements(event['Event']['Galaxy']) # Markings if 'Tag' in event['Event']: event_markings = self.resolve_markings(event['Event']['Tag']) else: event_markings = [TLP_WHITE] # ExternalReference event_external_reference = ExternalReference( source_name=self.helper.connect_name, external_id=event['Event']['uuid'], url=self.misp_url + '/events/view/' + event['Event']['uuid']) ### Get indicators indicators = [] # Get attributes for attribute in event['Event']['Attribute']: indicator = self.process_attribute(author, event_elements, event_markings, attribute) if indicator is not None: indicators.append(indicator) # Get attributes of objects objects_relationships = [] for object in event['Event']['Object']: object_attributes = [] for attribute in object['Attribute']: indicator = self.process_attribute(author, event_elements, event_markings, attribute) if indicator is not None: indicators.append(indicator) if object['meta-category'] == 'file' and indicator[ 'indicator'].x_opencti_observable_type in FILETYPES: object_attributes.append(indicator) objects_relationships.extend(self.process_observable_relations(object_attributes, [])) ### Prepare the bundle bundle_objects = [author] object_refs = [] # Add event markings for event_marking in event_markings: if event_marking['id'] not in added_markings: bundle_objects.append(event_marking) added_markings.append(event_marking['id']) # Add event elements all_event_elements = \ event_elements['intrusion_sets'] + \ event_elements['malwares'] + \ event_elements['tools'] + \ event_elements['attack_patterns'] for event_element in all_event_elements: if event_element['name'] not in added_object_refs: object_refs.append(event_element) added_object_refs.append(event_element['name']) if event_element['name'] not in added_entities: bundle_objects.append(event_element) added_entities.append(event_element['name']) # Add indicators for indicator in indicators: if indicator['indicator']['id'] not in added_object_refs: object_refs.append(indicator['indicator']) added_object_refs.append(indicator['indicator']['id']) if indicator['indicator']['id'] not in added_entities: bundle_objects.append(indicator['indicator']) added_entities.append(indicator['indicator']['id']) # Add attribute markings for attribute_marking in indicator['markings']: if attribute_marking['id'] not in added_markings: bundle_objects.append(attribute_marking) added_markings.append(attribute_marking['id']) # Add attribute elements all_attribute_elements = \ indicator['attribute_elements']['intrusion_sets'] + \ indicator['attribute_elements']['malwares'] + \ indicator['attribute_elements']['tools'] + \ indicator['attribute_elements']['attack_patterns'] for attribute_element in all_attribute_elements: if attribute_element['name'] not in added_object_refs: object_refs.append(attribute_element) added_object_refs.append(attribute_element['name']) if attribute_element['name'] not in added_entities: bundle_objects.append(attribute_element) added_entities.append(attribute_element['name']) # Add attribute relationships for relationship in indicator['relationships']: object_refs.append(relationship) bundle_objects.append(relationship) # Add object_relationships for object_relationship in objects_relationships: bundle_objects.append(object_relationship) ### Create the report if needed if self.misp_create_report and len(object_refs) > 0: report = Report( name=event['Event']['info'], description=event['Event']['info'], published=parse(event['Event']['date']), created_by_ref=author, object_marking_refs=event_markings, labels=['threat-report'], object_refs=object_refs, external_references=[event_external_reference], custom_properties={ 'x_opencti_report_class': self.misp_report_class, 'x_opencti_object_status': 2 } ) bundle_objects.append(report) bundle = Bundle(objects=bundle_objects).serialize() self.helper.send_stix2_bundle(bundle, None, self.update_existing_data, False)
def process_events(self, events): for event in events: generic_actor = ThreatActor( name='Unknown threats', labels=['threat-actor'], description= 'All unknown threats are represented by this pseudo threat actor. This entity helps to organize knowledge and indicators that could not be attributed to any other threats.' ) added_threats = [] added_markings = [] # Default values author = Identity(name=event['Event']['Orgc']['name'], identity_class='organization') report_threats = self.prepare_threats(event['Event']['Galaxy']) if 'Tag' in event['Event']: report_markings = self.resolve_markings(event['Event']['Tag']) else: report_markings = [] reference_misp = ExternalReference( source_name=self.helper.connect_name, url=self.misp_url + '/events/view/' + event['Event']['uuid']) # Get all attributes indicators = [] for attribute in event['Event']['Attribute']: indicator = self.process_attribute(author, report_threats, attribute, generic_actor) if indicator is not None: indicators.append(indicator) # get all attributes of object for object in event['Event']['Object']: for attribute in object['Attribute']: indicator = self.process_attribute(author, report_threats, attribute, generic_actor) if indicator is not None: indicators.append(indicator) bundle_objects = [author] report_refs = [] for report_marking in report_markings: if report_marking['id'] not in added_markings: bundle_objects.append(report_marking) added_markings.append(report_marking['id']) for report_threat in report_threats: report_refs.append(report_threat) bundle_objects.append(report_threat) added_threats.append(report_threat['name']) for indicator in indicators: report_refs.append(indicator['indicator']) bundle_objects.append(indicator['indicator']) for attribute_threat in indicator['attribute_threats']: if attribute_threat['name'] not in added_threats: report_refs.append(attribute_threat) bundle_objects.append(attribute_threat) added_threats.append(attribute_threat['name']) for marking in indicator['markings']: if marking['id'] not in added_markings: bundle_objects.append(marking) added_markings.append(marking['id']) for relationship in indicator['relationships']: report_refs.append(relationship) bundle_objects.append(relationship) if len(report_refs) > 0: report = Report(name=event['Event']['info'], description=event['Event']['info'], published=parse(event['Event']['date']), created_by_ref=author, object_marking_refs=report_markings, labels=['threat-report'], object_refs=report_refs, external_references=[reference_misp], custom_properties={ 'x_opencti_report_class': 'Threat Report' }) bundle_objects.append(report) bundle = Bundle(objects=bundle_objects).serialize() self.helper.send_stix2_bundle(bundle) if self.misp_untag_event: self.misp.untag(event['Event']['uuid'], self.misp_tag) self.misp.tag(event['Event']['uuid'], self.misp_imported_tag)
def _process_message(self, data): file_fetch = data["file_fetch"] file_uri = self.helper.opencti_url + file_fetch file_name = os.path.basename(file_fetch) entity_id = data.get("entity_id", None) self.helper.log_info(entity_id) # Get context is_context = entity_id is not None and len(entity_id) > 0 self.helper.log_info("Context: {}".format(is_context)) self.helper.log_info( "get_only_contextual: {}".format(self.helper.get_only_contextual()) ) if self.helper.get_only_contextual() and not is_context: raise ValueError( "No context defined, connector is get_only_contextual true" ) self.helper.log_info("Importing the file " + file_uri) # Get the file file_content = self.helper.api.fetch_opencti_file(file_uri, True) # Write the file f = open(file_name, "wb") f.write(file_content) f.close() # Parse bundle_objects = [] stix_objects = set() i = 0 custom_indicators = self._get_entities() mime_type = self._get_mime_type(file_name) print(mime_type) if mime_type is None: raise ValueError("Invalid file format of {}".format(file_name)) parser = iocp.IOC_Parser( None, mime_type, True, "pdfminer", "json", custom_indicators=custom_indicators, ) parsed = parser.parse(file_name) os.remove(file_name) if parsed != []: for file in parsed: if file != None: for page in file: if page != []: for match in page: resolved_match = self._resolve_match(match) if resolved_match: # For the creation of relationships if resolved_match[ "type" ] not in self.types.values() and self._is_uuid( resolved_match["value"] ): stix_objects.add(resolved_match["value"]) # For CVEs since SimpleObservable doesn't support Vulnerabilities yet elif resolved_match["type"] == "Vulnerability.name": vulnerability = Vulnerability( name=resolved_match["value"] ) bundle_objects.append(vulnerability) # Other observables elif resolved_match["type"] in self.types.values(): observable = SimpleObservable( id=OpenCTIStix2Utils.generate_random_stix_id( "x-opencti-simple-observable" ), key=resolved_match["type"], value=resolved_match["value"], x_opencti_create_indicator=self.create_indicator, ) bundle_objects.append(observable) else: self.helper.log_info( "Odd data received: {}".format( resolved_match ) ) i += 1 else: self.helper.log_error("Could not parse the report!") if is_context: entity = self.helper.api.stix_domain_object.read(id=entity_id) if entity is not None: if entity["entity_type"] == "Report" and len(bundle_objects) > 0: report = Report( id=entity["standard_id"], name=entity["name"], description=entity["description"], published=self.helper.api.stix2.format_date(entity["created"]), report_types=entity["report_types"], object_refs=bundle_objects, ) bundle_objects.append(report) bundles_sent = [] if len(bundle_objects) > 0: bundle = Bundle(objects=bundle_objects).serialize() bundles_sent = self.helper.send_stix2_bundle(bundle) if len(stix_objects) > 0 and entity_id is not None: report = self.helper.api.report.read(id=entity_id) if report: for stix_object in stix_objects: self.helper.log_info(stix_object) self.helper.api.report.add_stix_object_or_stix_relationship( id=report["id"], stixObjectOrStixRelationshipId=stix_object ) return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import"