def _create_attribute(self, result, attribute_type):
attribute = MISPAttribute()
attribute.from_dict(type=attribute_type,
value=result['value'],
**self.attribute)
if result['classification'] != 'UNCLASSIFIED':
attribute.add_tag(result['classification'].lower())
self.misp_event.add_attribute(**attribute)
return {
'referenced_uuid': attribute.uuid,
'relationship_type': '-'.join(result['context'].lower().split(' '))
}
def create_attribute(self,
attribute_type,
attribute_value,
attribute_tags=list()):
attribute = MISPAttribute()
attribute.from_dict(**{
'type': attribute_type,
'value': attribute_value,
'to_ids': False
})
for tag in attribute_tags:
attribute.add_tag(tag)
self.misp_event.add_attribute(**attribute)
return attribute.uuid
def map_attribute(self, row):
malware_info = self.get_malware_info(row)
misp_attribute = MISPAttribute()
value = row[2].strip().strip('"')
fs = datetime.strptime(row[1].strip().strip('"'), '%Y-%m-%d %H:%M:%S')
misp_attribute.first_seen = fs
misp_attribute.last_seen = fs
misp_attribute.type = "url"
if row[4].strip().strip('"') == "malware_download":
misp_attribute.add_tag('kill-chain:Delivery')
if malware_info['ft'] is not None:
misp_attribute.add_tag(malware_info['ft'])
misp_attribute.value = value
misp_attribute.comment = row[6].strip().strip('"')
return misp_attribute
class RFEnricher:
"""Class for enriching an attribute with data from Recorded Future.
The enrichment data is returned as a custom MISP object.
"""
def __init__(self, attribute_props: dict):
self.event = MISPEvent()
self.enrichment_object = MISPObject("Recorded Future Enrichment")
description = ("An object containing the enriched attribute and "
"related entities from Recorded Future.")
self.enrichment_object.from_dict(**{
"meta-category": "misc",
"description": description,
"distribution": 0
})
# Create a copy of enriched attribute to add tags to
temp_attr = MISPAttribute()
temp_attr.from_dict(**attribute_props)
self.enriched_attribute = MISPAttribute()
self.enriched_attribute.from_dict(**{
"value": temp_attr.value,
"type": temp_attr.type,
"distribution": 0
})
self.related_attributes: List[Tuple[str, MISPAttribute]] = []
self.color_picker = RFColors()
self.galaxy_finder = GalaxyFinder()
# Mapping from MISP-type to RF-type
self.type_to_rf_category = {
"ip": "ip",
"ip-src": "ip",
"ip-dst": "ip",
"ip-src|port": "ip",
"ip-dst|port": "ip",
"domain": "domain",
"hostname": "domain",
"md5": "hash",
"sha1": "hash",
"sha256": "hash",
"uri": "url",
"url": "url",
"vulnerability": "vulnerability",
"weakness": "vulnerability",
}
# Related entities have 'Related' as part of the word and Links entities from RF
# portrayed as related attributes in MISP
self.related_attribute_types = [
"RelatedIpAddress",
"RelatedInternetDomainName",
"RelatedHash",
"RelatedEmailAddress",
"RelatedCyberVulnerability",
"IpAddress",
"InternetDomainName",
"Hash",
"EmailAddress",
"CyberVulnerability",
]
# Related entities have 'Related' as part of the word and and Links entities from RF portrayed as tags in MISP
self.galaxy_tag_types = [
"RelatedMalware",
"RelatedThreatActor",
"Threat Actor",
"MitreAttackIdentifier",
"Malware",
]
def enrich(self) -> None:
"""Run the enrichment."""
category = self.type_to_rf_category.get(self.enriched_attribute.type,
"")
enriched_attribute_value = self.enriched_attribute.value
# If enriched attribute has a port we need to remove that port
# since RF do not support enriching ip addresses with port
if self.enriched_attribute.type in ["ip-src|port", "ip-dst|port"]:
enriched_attribute_value = enriched_attribute_value.split("|")[0]
json_response = GLOBAL_REQUEST_HANDLER.rf_lookup(
category, enriched_attribute_value)
response = json.loads(json_response.content)
try:
# Add risk score and risk rules as tags to the enriched attribute
risk_score = response["data"]["risk"]["score"]
hex_color = self.color_picker.riskscore_color(risk_score)
tag_name = f'recorded-future:risk-score="{risk_score}"'
self.add_tag(tag_name, hex_color)
risk_criticality = response["data"]["risk"]["criticalityLabel"]
hex_color = self.color_picker.criticality_color(risk_criticality)
tag_name = f'recorded-future:criticality="{risk_criticality}"'
self.add_tag(tag_name, hex_color)
for evidence in response["data"]["risk"]["evidenceDetails"]:
risk_rule = evidence["rule"]
criticality = evidence["criticality"]
hex_color = self.color_picker.riskrule_color(criticality)
tag_name = f'recorded-future:risk-rule="{risk_rule}"'
self.add_tag(tag_name, hex_color)
links_data = response["data"].get("links", {}).get("hits")
# Check if we have error in links response. If yes, then user do not have right module enabled in token
links_access_error = response["data"].get("links", {}).get("error")
galaxy_tags = []
if not links_access_error:
for hit in links_data:
for section in hit["sections"]:
for sec_list in section["lists"]:
entity_type = sec_list["type"]["name"]
for entity in sec_list["entities"]:
if entity_type in self.galaxy_tag_types:
galaxy = self.galaxy_finder.find_galaxy_match(
entity["name"], entity_type)
if galaxy and galaxy not in galaxy_tags:
galaxy_tags.append(galaxy)
else:
self.add_attribute(entity["name"],
entity_type)
else:
# Retrieve related entities
for related_entity in response["data"]["relatedEntities"]:
related_type = related_entity["type"]
if related_type in self.related_attribute_types:
# Related entities returned as additional attributes
for related in related_entity["entities"]:
# filter those entities that have count bigger than 4, to reduce noise
# because there can be a huge list of related entities
if int(related["count"]) > 4:
indicator = related["entity"]["name"]
self.add_attribute(indicator, related_type)
elif related_type in self.galaxy_tag_types:
# Related entities added as galaxy-tags to the enriched attribute
galaxy_tags = []
for related in related_entity["entities"]:
# filter those entities that have count bigger than 4, to reduce noise
# because there can be a huge list of related entities
if int(related["count"]) > 4:
indicator = related["entity"]["name"]
galaxy = self.galaxy_finder.find_galaxy_match(
indicator, related_type)
# Handle deduplication of galaxy tags
if galaxy and galaxy not in galaxy_tags:
galaxy_tags.append(galaxy)
for galaxy in galaxy_tags:
self.add_tag(galaxy)
except KeyError:
misperrors[
"error"] = "Unexpected format in Recorded Future api response."
raise
def add_attribute(self, indicator: str, indicator_type: str) -> None:
"""Helper method for adding an indicator to the attribute list."""
out_type = self.get_output_type(indicator_type, indicator)
attribute = MISPAttribute()
attribute.from_dict(**{
"value": indicator,
"type": out_type,
"distribution": 0
})
self.related_attributes.append((indicator_type, attribute))
def add_tag(self, tag_name: str, hex_color: str = None) -> None:
"""Helper method for adding a tag to the enriched attribute."""
tag = MISPTag()
tag_properties = {"name": tag_name}
if hex_color:
tag_properties["colour"] = hex_color
tag.from_dict(**tag_properties)
self.enriched_attribute.add_tag(tag)
def get_output_type(self, related_type: str, indicator: str) -> str:
"""Helper method for translating a Recorded Future related type to a MISP output type."""
output_type = "text"
if related_type in ["RelatedIpAddress", "IpAddress"]:
output_type = "ip-dst"
elif related_type in [
"RelatedInternetDomainName", "InternetDomainName"
]:
output_type = "domain"
elif related_type in ["RelatedHash", "Hash"]:
hash_len = len(indicator)
if hash_len == 64:
output_type = "sha256"
elif hash_len == 40:
output_type = "sha1"
elif hash_len == 32:
output_type = "md5"
elif related_type in ["RelatedEmailAddress", "EmailAddress"]:
output_type = "email-src"
elif related_type in [
"RelatedCyberVulnerability", "CyberVulnerability"
]:
signature = indicator.split("-")[0]
if signature == "CVE":
output_type = "vulnerability"
elif signature == "CWE":
output_type = "weakness"
elif related_type == "MalwareSignature":
output_type = "malware-sample"
elif related_type == "Organization":
output_type = "target-org"
elif related_type == "Username":
output_type = "target-user"
return output_type
def get_results(self) -> dict:
"""Build and return the enrichment results."""
self.enrichment_object.add_attribute("Enriched attribute",
**self.enriched_attribute)
for related_type, attribute in self.related_attributes:
self.enrichment_object.add_attribute(related_type, **attribute)
self.event.add_object(**self.enrichment_object)
event = json.loads(self.event.to_json())
result = {key: event[key] for key in ["Object"] if key in event}
return {"results": result}
class RFEnricher:
"""Class for enriching an attribute with data from Recorded Future.
The enrichment data is returned as a custom MISP object.
"""
def __init__(self, api_token: str, attribute_props: dict):
self.api_token = api_token
self.event = MISPEvent()
self.enrichment_object = MISPObject('Recorded Future Enrichment')
self.enrichment_object.from_dict(
**{
'meta-category':
'misc',
'description':
'An object containing the enriched attribute and related '
'entities from Recorded Future.',
'distribution':
0
})
# Create a copy of enriched attribute to add tags to
temp_attr = MISPAttribute()
temp_attr.from_dict(**attribute_props)
self.enriched_attribute = MISPAttribute()
self.enriched_attribute.from_dict(**{
'value': temp_attr.value,
'type': temp_attr.type,
'distribution': 0
})
self.related_attributes = []
self.color_picker = RFColors()
self.galaxy_finder = GalaxyFinder()
# Mapping from MISP-type to RF-type
self.type_to_rf_category = {
'ip': 'ip',
'ip-src': 'ip',
'ip-dst': 'ip',
'domain': 'domain',
'hostname': 'domain',
'md5': 'hash',
'sha1': 'hash',
'sha256': 'hash',
'uri': 'url',
'url': 'url',
'vulnerability': 'vulnerability',
'weakness': 'vulnerability'
}
# Related entities from RF portrayed as related attributes in MISP
self.related_attribute_types = [
'RelatedIpAddress', 'RelatedInternetDomainName', 'RelatedHash',
'RelatedEmailAddress', 'RelatedCyberVulnerability'
]
# Related entities from RF portrayed as tags in MISP
self.galaxy_tag_types = ['RelatedMalware', 'RelatedThreatActor']
def enrich(self):
"""Run the enrichment."""
category = self.type_to_rf_category.get(self.enriched_attribute.type)
try:
response = rf_lookup(self.api_token, category,
self.enriched_attribute.value)
json_response = json.loads(response.content)
except requests.HTTPError as error:
misperrors['error'] = f'Error when requesting data from Recorded Future. ' \
f'{error.response} : {error.response.reason}'
raise error
try:
# Add risk score and risk rules as tags to the enriched attribute
risk_score = json_response['data']['risk']['score']
hex_color = self.color_picker.riskscore_color(risk_score)
tag_name = f'recorded-future:risk-score="{risk_score}"'
self.add_tag(tag_name, hex_color)
for evidence in json_response['data']['risk']['evidenceDetails']:
risk_rule = evidence['rule']
criticality = evidence['criticality']
hex_color = self.color_picker.riskrule_color(criticality)
tag_name = f'recorded-future:risk-rule="{risk_rule}"'
self.add_tag(tag_name, hex_color)
# Retrieve related entities
for related_entity in json_response['data']['relatedEntities']:
related_type = related_entity['type']
if related_type in self.related_attribute_types:
# Related entities returned as additional attributes
for related in related_entity['entities']:
if int(related["count"]) > 4:
indicator = related['entity']['name']
self.add_related_attribute(indicator, related_type)
elif related_type in self.galaxy_tag_types:
# Related entities added as galaxy-tags to the enriched attribute
galaxy_tags = []
for related in related_entity['entities']:
if int(related["count"]) > 4:
indicator = related['entity']['name']
galaxy = self.galaxy_finder.find_galaxy_match(
indicator, related_type)
# Handle deduplication of galaxy tags
if galaxy and galaxy not in galaxy_tags:
galaxy_tags.append(galaxy)
for galaxy in galaxy_tags:
self.add_tag(galaxy)
except KeyError as error:
misperrors[
'error'] = 'Unexpected format in Recorded Future api response.'
raise error
def add_related_attribute(self, indicator: str, related_type: str) -> None:
"""Helper method for adding an indicator to the related attribute list."""
out_type = self.get_output_type(related_type, indicator)
attribute = MISPAttribute()
attribute.from_dict(**{
'value': indicator,
'type': out_type,
'distribution': 0
})
self.related_attributes.append((related_type, attribute))
def add_tag(self, tag_name: str, hex_color: str = None) -> None:
"""Helper method for adding a tag to the enriched attribute."""
tag = MISPTag()
tag_properties = {'name': tag_name}
if hex_color:
tag_properties['colour'] = hex_color
tag.from_dict(**tag_properties)
self.enriched_attribute.add_tag(tag)
def get_output_type(self, related_type: str, indicator: str) -> str:
"""Helper method for translating a Recorded Future related type to a MISP output type."""
output_type = 'text'
if related_type == 'RelatedIpAddress':
output_type = 'ip-dst'
elif related_type == 'RelatedInternetDomainName':
output_type = 'domain'
elif related_type == 'RelatedHash':
hash_len = len(indicator)
if hash_len == 64:
output_type = 'sha256'
elif hash_len == 40:
output_type = 'sha1'
elif hash_len == 32:
output_type = 'md5'
elif related_type == 'RelatedEmailAddress':
output_type = 'email-src'
elif related_type == 'RelatedCyberVulnerability':
signature = indicator.split('-')[0]
if signature == 'CVE':
output_type = 'vulnerability'
elif signature == 'CWE':
output_type = 'weakness'
return output_type
def get_results(self) -> dict:
"""Build and return the enrichment results."""
self.enrichment_object.add_attribute('Enriched attribute',
**self.enriched_attribute)
for related_type, attribute in self.related_attributes:
self.enrichment_object.add_attribute(related_type, **attribute)
self.event.add_object(**self.enrichment_object)
event = json.loads(self.event.to_json())
result = {key: event[key] for key in ['Object'] if key in event}
return {'results': result}
def tag_artifact_attribute(self, attribute: MISPAttribute) -> None:
if self.is_ioc:
attribute.add_tag('vmray:artifact="IOC"')
if self.verdict:
attribute.add_tag(f'vmray:verdict="{self.verdict}"')
def add_tag_to_attribute(self, attr: MISPAttribute, tag: str) -> MISPAttribute:
if self.make_sure_tag_exists(tag):
attr.add_tag(tag)
return attr
def ip_attribute(category, type, value):
attribute = MISPAttribute()
attribute.category = category
attribute.org = "RST Cloud"
attribute.type = type
if value['ip']:
if value['ip']['v4']:
attribute.value = value['ip']['v4']
attribute.add_tag("rstcloud:asn:firstip=" +
str(value['asn']['firstip']['netv4']))
attribute.add_tag("rstcloud:asn:lastip=" +
str(value['asn']['lastip']['netv4']))
else:
if value['ip']['v6']:
attribute.value = value['ip']['v6']
attribute.add_tag("rstcloud:asn:firstip=" +
str(value['asn']['firstip']['netv6']))
attribute.add_tag("rstcloud:asn:lastip=" +
str(value['asn']['lastip']['netv6']))
attribute.add_tag("rstcloud:asn:number=" + str(value['asn']['num']))
attribute.comment = listToString(value['src']['str'])
attribute.first_seen = value['fseen']
attribute.last_seen = value['lseen']
attribute.timestamp = value['collect']
attribute.distribution = distribution_level
attribute.add_tag("rstcloud:score:total=" + str(value['score']['total']))
for rsttag in value['tags']['str']:
attribute.add_tag("rstcloud:tag=" + str(rsttag))
if value['asn']['cloud']:
attribute.add_tag("rstcloud:cloudprovider=" +
str(value['asn']['cloud']))
if value['asn']['domains']:
attribute.add_tag("rstcloud:number_of_hosted_domains=" +
str(value['asn']['domains']))
attribute.add_tag("rstcloud:org=" + str(value['asn']['org']))
attribute.add_tag("rstcloud:isp=" + str(value['asn']['isp']))
attribute.add_tag("rstcloud:geo.city=" + str(value['geo']['city']))
attribute.add_tag("rstcloud:geo.region=" + str(value['geo']['region']))
attribute.add_tag("rstcloud:geo.country=" + str(value['geo']['country']))
attribute.add_tag("rstcloud:score:total=" + str(value['score']['total']))
attribute.add_tag("rstcloud:false-positive:alarm=" +
str(value['fp']['alarm']))
if value['fp']['descr']:
attribute.add_tag("rstcloud:false-positive:description=" +
str(value['fp']['descr']))
return attribute
def domain_attribute(category, type, value):
attribute = MISPAttribute()
attribute.category = category
attribute.type = type
attribute.value = value['domain']
attribute.comment = listToString(value['src']['str'])
attribute.first_seen = value['fseen']
attribute.last_seen = value['lseen']
attribute.timestamp = value['collect']
attribute.distribution = distribution_level
attribute.add_tag("rstcloud:score:total=" + str(value['score']['total']))
for rsttag in value['tags']['str']:
attribute.add_tag("rstcloud:tag=" + str(rsttag))
if value['resolved'] and value['resolved']['whois']:
if value['resolved']['whois']['age'] > 0:
attribute.add_tag("rstcloud:whois:created=" +
str(value['resolved']['whois']['created']))
attribute.add_tag("rstcloud:whois:updated=" +
str(value['resolved']['whois']['updated']))
attribute.add_tag("rstcloud:whois:expires=" +
str(value['resolved']['whois']['expires']))
attribute.add_tag("rstcloud:whois:age=" +
str(value['resolved']['whois']['age']))
if value['resolved']['whois']['registrar'] and value['resolved'][
'whois']['registrar'] != 'unknown':
attribute.add_tag("rstcloud:whois:registrar=" +
str(value['resolved']['whois']['registrar']))
if value['resolved']['whois']['registrar'] and value['resolved'][
'whois']['registrant'] != 'unknown':
attribute.add_tag("rstcloud:whois:registrant=" +
str(value['resolved']['whois']['registrant']))
attribute.add_tag("rstcloud:score:total=" + str(value['score']['total']))
attribute.add_tag("rstcloud:false-positive:alarm=" +
str(value['fp']['alarm']))
if value['fp']['descr']:
attribute.add_tag("rstcloud:false-positive:description=" +
str(value['fp']['descr']))
return attribute
