class RFEnricher: """Class for enriching an attribute with data from Recorded Future. The enrichment data is returned as a custom MISP object. """ def __init__(self, attribute_props: dict): self.event = MISPEvent() self.enrichment_object = MISPObject("Recorded Future Enrichment") description = ("An object containing the enriched attribute and " "related entities from Recorded Future.") self.enrichment_object.from_dict(**{ "meta-category": "misc", "description": description, "distribution": 0 }) # Create a copy of enriched attribute to add tags to temp_attr = MISPAttribute() temp_attr.from_dict(**attribute_props) self.enriched_attribute = MISPAttribute() self.enriched_attribute.from_dict(**{ "value": temp_attr.value, "type": temp_attr.type, "distribution": 0 }) self.related_attributes: List[Tuple[str, MISPAttribute]] = [] self.color_picker = RFColors() self.galaxy_finder = GalaxyFinder() # Mapping from MISP-type to RF-type self.type_to_rf_category = { "ip": "ip", "ip-src": "ip", "ip-dst": "ip", "ip-src|port": "ip", "ip-dst|port": "ip", "domain": "domain", "hostname": "domain", "md5": "hash", "sha1": "hash", "sha256": "hash", "uri": "url", "url": "url", "vulnerability": "vulnerability", "weakness": "vulnerability", } # Related entities have 'Related' as part of the word and Links entities from RF # portrayed as related attributes in MISP self.related_attribute_types = [ "RelatedIpAddress", "RelatedInternetDomainName", "RelatedHash", "RelatedEmailAddress", "RelatedCyberVulnerability", "IpAddress", "InternetDomainName", "Hash", "EmailAddress", "CyberVulnerability", ] # Related entities have 'Related' as part of the word and and Links entities from RF portrayed as tags in MISP self.galaxy_tag_types = [ "RelatedMalware", "RelatedThreatActor", "Threat Actor", "MitreAttackIdentifier", "Malware", ] def enrich(self) -> None: """Run the enrichment.""" category = self.type_to_rf_category.get(self.enriched_attribute.type, "") enriched_attribute_value = self.enriched_attribute.value # If enriched attribute has a port we need to remove that port # since RF do not support enriching ip addresses with port if self.enriched_attribute.type in ["ip-src|port", "ip-dst|port"]: enriched_attribute_value = enriched_attribute_value.split("|")[0] json_response = GLOBAL_REQUEST_HANDLER.rf_lookup( category, enriched_attribute_value) response = json.loads(json_response.content) try: # Add risk score and risk rules as tags to the enriched attribute risk_score = response["data"]["risk"]["score"] hex_color = self.color_picker.riskscore_color(risk_score) tag_name = f'recorded-future:risk-score="{risk_score}"' self.add_tag(tag_name, hex_color) risk_criticality = response["data"]["risk"]["criticalityLabel"] hex_color = self.color_picker.criticality_color(risk_criticality) tag_name = f'recorded-future:criticality="{risk_criticality}"' self.add_tag(tag_name, hex_color) for evidence in response["data"]["risk"]["evidenceDetails"]: risk_rule = evidence["rule"] criticality = evidence["criticality"] hex_color = self.color_picker.riskrule_color(criticality) tag_name = f'recorded-future:risk-rule="{risk_rule}"' self.add_tag(tag_name, hex_color) links_data = response["data"].get("links", {}).get("hits") # Check if we have error in links response. If yes, then user do not have right module enabled in token links_access_error = response["data"].get("links", {}).get("error") galaxy_tags = [] if not links_access_error: for hit in links_data: for section in hit["sections"]: for sec_list in section["lists"]: entity_type = sec_list["type"]["name"] for entity in sec_list["entities"]: if entity_type in self.galaxy_tag_types: galaxy = self.galaxy_finder.find_galaxy_match( entity["name"], entity_type) if galaxy and galaxy not in galaxy_tags: galaxy_tags.append(galaxy) else: self.add_attribute(entity["name"], entity_type) else: # Retrieve related entities for related_entity in response["data"]["relatedEntities"]: related_type = related_entity["type"] if related_type in self.related_attribute_types: # Related entities returned as additional attributes for related in related_entity["entities"]: # filter those entities that have count bigger than 4, to reduce noise # because there can be a huge list of related entities if int(related["count"]) > 4: indicator = related["entity"]["name"] self.add_attribute(indicator, related_type) elif related_type in self.galaxy_tag_types: # Related entities added as galaxy-tags to the enriched attribute galaxy_tags = [] for related in related_entity["entities"]: # filter those entities that have count bigger than 4, to reduce noise # because there can be a huge list of related entities if int(related["count"]) > 4: indicator = related["entity"]["name"] galaxy = self.galaxy_finder.find_galaxy_match( indicator, related_type) # Handle deduplication of galaxy tags if galaxy and galaxy not in galaxy_tags: galaxy_tags.append(galaxy) for galaxy in galaxy_tags: self.add_tag(galaxy) except KeyError: misperrors[ "error"] = "Unexpected format in Recorded Future api response." raise def add_attribute(self, indicator: str, indicator_type: str) -> None: """Helper method for adding an indicator to the attribute list.""" out_type = self.get_output_type(indicator_type, indicator) attribute = MISPAttribute() attribute.from_dict(**{ "value": indicator, "type": out_type, "distribution": 0 }) self.related_attributes.append((indicator_type, attribute)) def add_tag(self, tag_name: str, hex_color: str = None) -> None: """Helper method for adding a tag to the enriched attribute.""" tag = MISPTag() tag_properties = {"name": tag_name} if hex_color: tag_properties["colour"] = hex_color tag.from_dict(**tag_properties) self.enriched_attribute.add_tag(tag) def get_output_type(self, related_type: str, indicator: str) -> str: """Helper method for translating a Recorded Future related type to a MISP output type.""" output_type = "text" if related_type in ["RelatedIpAddress", "IpAddress"]: output_type = "ip-dst" elif related_type in [ "RelatedInternetDomainName", "InternetDomainName" ]: output_type = "domain" elif related_type in ["RelatedHash", "Hash"]: hash_len = len(indicator) if hash_len == 64: output_type = "sha256" elif hash_len == 40: output_type = "sha1" elif hash_len == 32: output_type = "md5" elif related_type in ["RelatedEmailAddress", "EmailAddress"]: output_type = "email-src" elif related_type in [ "RelatedCyberVulnerability", "CyberVulnerability" ]: signature = indicator.split("-")[0] if signature == "CVE": output_type = "vulnerability" elif signature == "CWE": output_type = "weakness" elif related_type == "MalwareSignature": output_type = "malware-sample" elif related_type == "Organization": output_type = "target-org" elif related_type == "Username": output_type = "target-user" return output_type def get_results(self) -> dict: """Build and return the enrichment results.""" self.enrichment_object.add_attribute("Enriched attribute", **self.enriched_attribute) for related_type, attribute in self.related_attributes: self.enrichment_object.add_attribute(related_type, **attribute) self.event.add_object(**self.enrichment_object) event = json.loads(self.event.to_json()) result = {key: event[key] for key in ["Object"] if key in event} return {"results": result}
class RFEnricher: """Class for enriching an attribute with data from Recorded Future. The enrichment data is returned as a custom MISP object. """ def __init__(self, api_token: str, attribute_props: dict): self.api_token = api_token self.event = MISPEvent() self.enrichment_object = MISPObject('Recorded Future Enrichment') self.enrichment_object.from_dict( **{ 'meta-category': 'misc', 'description': 'An object containing the enriched attribute and related ' 'entities from Recorded Future.', 'distribution': 0 }) # Create a copy of enriched attribute to add tags to temp_attr = MISPAttribute() temp_attr.from_dict(**attribute_props) self.enriched_attribute = MISPAttribute() self.enriched_attribute.from_dict(**{ 'value': temp_attr.value, 'type': temp_attr.type, 'distribution': 0 }) self.related_attributes = [] self.color_picker = RFColors() self.galaxy_finder = GalaxyFinder() # Mapping from MISP-type to RF-type self.type_to_rf_category = { 'ip': 'ip', 'ip-src': 'ip', 'ip-dst': 'ip', 'domain': 'domain', 'hostname': 'domain', 'md5': 'hash', 'sha1': 'hash', 'sha256': 'hash', 'uri': 'url', 'url': 'url', 'vulnerability': 'vulnerability', 'weakness': 'vulnerability' } # Related entities from RF portrayed as related attributes in MISP self.related_attribute_types = [ 'RelatedIpAddress', 'RelatedInternetDomainName', 'RelatedHash', 'RelatedEmailAddress', 'RelatedCyberVulnerability' ] # Related entities from RF portrayed as tags in MISP self.galaxy_tag_types = ['RelatedMalware', 'RelatedThreatActor'] def enrich(self): """Run the enrichment.""" category = self.type_to_rf_category.get(self.enriched_attribute.type) try: response = rf_lookup(self.api_token, category, self.enriched_attribute.value) json_response = json.loads(response.content) except requests.HTTPError as error: misperrors['error'] = f'Error when requesting data from Recorded Future. ' \ f'{error.response} : {error.response.reason}' raise error try: # Add risk score and risk rules as tags to the enriched attribute risk_score = json_response['data']['risk']['score'] hex_color = self.color_picker.riskscore_color(risk_score) tag_name = f'recorded-future:risk-score="{risk_score}"' self.add_tag(tag_name, hex_color) for evidence in json_response['data']['risk']['evidenceDetails']: risk_rule = evidence['rule'] criticality = evidence['criticality'] hex_color = self.color_picker.riskrule_color(criticality) tag_name = f'recorded-future:risk-rule="{risk_rule}"' self.add_tag(tag_name, hex_color) # Retrieve related entities for related_entity in json_response['data']['relatedEntities']: related_type = related_entity['type'] if related_type in self.related_attribute_types: # Related entities returned as additional attributes for related in related_entity['entities']: if int(related["count"]) > 4: indicator = related['entity']['name'] self.add_related_attribute(indicator, related_type) elif related_type in self.galaxy_tag_types: # Related entities added as galaxy-tags to the enriched attribute galaxy_tags = [] for related in related_entity['entities']: if int(related["count"]) > 4: indicator = related['entity']['name'] galaxy = self.galaxy_finder.find_galaxy_match( indicator, related_type) # Handle deduplication of galaxy tags if galaxy and galaxy not in galaxy_tags: galaxy_tags.append(galaxy) for galaxy in galaxy_tags: self.add_tag(galaxy) except KeyError as error: misperrors[ 'error'] = 'Unexpected format in Recorded Future api response.' raise error def add_related_attribute(self, indicator: str, related_type: str) -> None: """Helper method for adding an indicator to the related attribute list.""" out_type = self.get_output_type(related_type, indicator) attribute = MISPAttribute() attribute.from_dict(**{ 'value': indicator, 'type': out_type, 'distribution': 0 }) self.related_attributes.append((related_type, attribute)) def add_tag(self, tag_name: str, hex_color: str = None) -> None: """Helper method for adding a tag to the enriched attribute.""" tag = MISPTag() tag_properties = {'name': tag_name} if hex_color: tag_properties['colour'] = hex_color tag.from_dict(**tag_properties) self.enriched_attribute.add_tag(tag) def get_output_type(self, related_type: str, indicator: str) -> str: """Helper method for translating a Recorded Future related type to a MISP output type.""" output_type = 'text' if related_type == 'RelatedIpAddress': output_type = 'ip-dst' elif related_type == 'RelatedInternetDomainName': output_type = 'domain' elif related_type == 'RelatedHash': hash_len = len(indicator) if hash_len == 64: output_type = 'sha256' elif hash_len == 40: output_type = 'sha1' elif hash_len == 32: output_type = 'md5' elif related_type == 'RelatedEmailAddress': output_type = 'email-src' elif related_type == 'RelatedCyberVulnerability': signature = indicator.split('-')[0] if signature == 'CVE': output_type = 'vulnerability' elif signature == 'CWE': output_type = 'weakness' return output_type def get_results(self) -> dict: """Build and return the enrichment results.""" self.enrichment_object.add_attribute('Enriched attribute', **self.enriched_attribute) for related_type, attribute in self.related_attributes: self.enrichment_object.add_attribute(related_type, **attribute) self.event.add_object(**self.enrichment_object) event = json.loads(self.event.to_json()) result = {key: event[key] for key in ['Object'] if key in event} return {'results': result}