def find_hashes(htype):
r = mymisp.search(controller='attributes', type_attribute=htype)
echeck(r)
if not r.get('response'):
return
for a in r['response']['Attribute']:
attribute = MISPAttribute(mymisp.describe_types)
attribute.from_dict(**a)
if '|' in attribute.type and '|' in attribute.value:
c, value = attribute.value.split('|')
comment = '{} - {}'.format(attribute.comment, c)
else:
comment = attribute.comment
value = attribute.value
mhash = value.replace(':', ';')
mfile = 'MISP event {} {}'.format(a['event_id'], comment.replace(':', ';').replace('\r', '').replace('\n', ''))
print('{}:*:{}:73'.format(mhash, mfile))
def handler(q=False):
if q is False:
return False
request = json.loads(q)
if request.get('config'):
if (request['config'].get('api_id') is
None) or (request['config'].get('api_secret') is None):
misperrors['error'] = "Censys API credentials are missing"
return misperrors
else:
misperrors['error'] = "Please provide config options"
return misperrors
api_id = request['config']['api_id']
api_secret = request['config']['api_secret']
if not request.get('attribute') or not check_input_attribute(
request['attribute']):
return {
'error':
f'{standard_error_message}, which should contain at least a type, a value and an uuid.'
}
attribute = request['attribute']
if not any(input_type == attribute['type']
for input_type in mispattributes['input']):
return {'error': 'Unsupported attribute type.'}
attribute = MISPAttribute()
attribute.from_dict(**request['attribute'])
# Lists to accomodate multi-types attribute
conn = list()
types = list()
values = list()
results = list()
if "|" in attribute.type:
t_1, t_2 = attribute.type.split('|')
v_1, v_2 = attribute.value.split('|')
# We cannot use the port information
if t_2 == "port":
types.append(t_1)
values.append(v_1)
else:
types = [t_1, t_2]
values = [v_1, v_2]
else:
types.append(attribute.type)
values.append(attribute.value)
for t in types:
# ip, ip-src or ip-dst
if t[:2] == "ip":
conn.append(
censys.ipv4.CensysIPv4(api_id=api_id, api_secret=api_secret))
elif t == 'domain' or t == "hostname":
conn.append(
censys.websites.CensysWebsites(api_id=api_id,
api_secret=api_secret))
elif 'x509-fingerprint' in t:
conn.append(
censys.certificates.CensysCertificates(api_id=api_id,
api_secret=api_secret))
found = True
for c in conn:
val = values.pop(0)
try:
r = c.view(val)
results.append(parse_response(r, attribute))
found = True
except censys.base.CensysNotFoundException:
found = False
except Exception:
misperrors['error'] = "Connection issue"
return misperrors
if not found:
misperrors['error'] = "Nothing could be found on Censys"
return misperrors
return {'results': remove_duplicates(results)}
class PassiveSSLParser():
def __init__(self, attribute, authentication):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.pssl = pypssl.PyPSSL(basic_auth=authentication)
self.cert_hash = 'x509-fingerprint-sha1'
self.cert_type = 'pem'
self.mapping = {
'issuer': ('text', 'issuer'),
'keylength': ('text', 'pubkey-info-size'),
'not_after': ('datetime', 'validity-not-after'),
'not_before': ('datetime', 'validity-not-before'),
'subject': ('text', 'subject')
}
def get_results(self):
if hasattr(self, 'result'):
return self.result
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object')}
return {'results': results}
def parse(self):
value = self.attribute.value.split(
'|')[0] if '|' in self.attribute.type else self.attribute.value
try:
results = self.pssl.query(value)
except Exception:
self.result = {
'error':
'There is an authentication error, please make sure you supply correct credentials.'
}
return
if not results:
self.result = {'error': 'Not found'}
return
if 'error' in results:
self.result = {'error': results['error']}
return
for ip_address, certificates in results.items():
ip_uuid = self._handle_ip_attribute(ip_address)
for certificate in certificates['certificates']:
self._handle_certificate(certificate, ip_uuid)
def _handle_certificate(self, certificate, ip_uuid):
x509 = MISPObject('x509')
x509.add_attribute(self.cert_hash,
type=self.cert_hash,
value=certificate)
cert_details = self.pssl.fetch_cert(certificate)
info = cert_details['info']
for feature, mapping in self.mapping.items():
attribute_type, object_relation = mapping
x509.add_attribute(object_relation,
type=attribute_type,
value=info[feature])
x509.add_attribute(self.cert_type, type='text', value=self.cert_type)
x509.add_reference(ip_uuid, 'seen-by')
self.misp_event.add_object(**x509)
def _handle_ip_attribute(self, ip_address):
if ip_address == self.attribute.value:
return self.attribute.uuid
ip_attribute = MISPAttribute()
ip_attribute.from_dict(**{
'type': self.attribute.type,
'value': ip_address
})
self.misp_event.add_attribute(**ip_attribute)
return ip_attribute.uuid
def parse_network_interactions(self):
domaininfo = self.data['domaininfo']
if domaininfo:
for domain in domaininfo['domain']:
if domain['@ip'] != 'unknown':
domain_object = MISPObject('domain-ip')
for key, mapping in domain_object_mapping.items():
attribute_type, object_relation = mapping
domain_object.add_attribute(
object_relation, **{
'type': attribute_type,
'value': domain[key]
})
self.misp_event.add_object(**domain_object)
reference = dict(referenced_uuid=domain_object.uuid,
relationship_type='contacts')
self.add_process_reference(domain['@targetid'],
domain['@currentpath'],
reference)
else:
attribute = MISPAttribute()
attribute.from_dict(**{
'type': 'domain',
'value': domain['@name']
})
self.misp_event.add_attribute(**attribute)
reference = dict(referenced_uuid=attribute.uuid,
relationship_type='contacts')
self.add_process_reference(domain['@targetid'],
domain['@currentpath'],
reference)
ipinfo = self.data['ipinfo']
if ipinfo:
for ip in ipinfo['ip']:
attribute = MISPAttribute()
attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip']})
self.misp_event.add_attribute(**attribute)
reference = dict(referenced_uuid=attribute.uuid,
relationship_type='contacts')
self.add_process_reference(ip['@targetid'], ip['@currentpath'],
reference)
urlinfo = self.data['urlinfo']
if urlinfo:
for url in urlinfo['url']:
target_id = int(url['@targetid'])
current_path = url['@currentpath']
attribute = MISPAttribute()
attribute_dict = {'type': 'url', 'value': url['@name']}
if target_id != -1 and current_path != 'unknown':
self.references[self.process_references[(
target_id, current_path)]].append({
'referenced_uuid':
attribute.uuid,
'relationship_type':
'contacts'
})
else:
attribute_dict[
'comment'] = 'From Memory - Enriched via the joe_import module'
attribute.from_dict(**attribute_dict)
self.misp_event.add_attribute(**attribute)
class VirusTotalParser():
def __init__(self):
super(VirusTotalParser, self).__init__()
self.misp_event = MISPEvent()
def declare_variables(self, apikey, attribute):
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.apikey = apikey
def get_result(self):
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object') if (key in event and event[key])
}
return {'results': results}
def parse_urls(self, query_result):
for feature in ('detected_urls', 'undetected_urls'):
if feature in query_result:
for url in query_result[feature]:
value = url['url'] if isinstance(url, dict) else url[0]
self.misp_event.add_attribute('url', value)
def parse_resolutions(self, resolutions, subdomains=None, uuids=None):
domain_ip_object = MISPObject('domain-ip')
if self.attribute.type == 'domain':
domain_ip_object.add_attribute('domain',
type='domain',
value=self.attribute.value)
attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address')
else:
domain_ip_object.add_attribute('ip',
type='ip-dst',
value=self.attribute.value)
attribute_type, relation, key = ('domain', 'domain', 'hostname')
for resolution in resolutions:
domain_ip_object.add_attribute(relation,
type=attribute_type,
value=resolution[key])
if subdomains:
for subdomain in subdomains:
attribute = MISPAttribute()
attribute.from_dict(**dict(type='domain', value=subdomain))
self.misp_event.add_attribute(**attribute)
domain_ip_object.add_reference(attribute.uuid, 'subdomain')
if uuids:
for uuid in uuids:
domain_ip_object.add_reference(uuid, 'sibling-of')
self.misp_event.add_object(**domain_ip_object)
def parse_vt_object(self, query_result):
if query_result['response_code'] == 1:
vt_object = MISPObject('virustotal-report')
vt_object.add_attribute('permalink',
type='link',
value=query_result['permalink'])
detection_ratio = '{}/{}'.format(query_result['positives'],
query_result['total'])
vt_object.add_attribute('detection-ratio',
type='text',
value=detection_ratio)
self.misp_event.add_object(**vt_object)
def get_query_result(self, query_type):
params = {query_type: self.attribute.value, 'apikey': self.apikey}
return requests.get(self.base_url, params=params)
class RFEnricher:
"""Class for enriching an attribute with data from Recorded Future.
The enrichment data is returned as a custom MISP object.
"""
def __init__(self, api_token: str, attribute_props: dict):
self.api_token = api_token
self.event = MISPEvent()
self.enrichment_object = MISPObject('Recorded Future Enrichment')
self.enrichment_object.from_dict(
**{
'meta-category':
'misc',
'description':
'An object containing the enriched attribute and related '
'entities from Recorded Future.',
'distribution':
0
})
# Create a copy of enriched attribute to add tags to
temp_attr = MISPAttribute()
temp_attr.from_dict(**attribute_props)
self.enriched_attribute = MISPAttribute()
self.enriched_attribute.from_dict(**{
'value': temp_attr.value,
'type': temp_attr.type,
'distribution': 0
})
self.related_attributes = []
self.color_picker = RFColors()
self.galaxy_finder = GalaxyFinder()
# Mapping from MISP-type to RF-type
self.type_to_rf_category = {
'ip': 'ip',
'ip-src': 'ip',
'ip-dst': 'ip',
'domain': 'domain',
'hostname': 'domain',
'md5': 'hash',
'sha1': 'hash',
'sha256': 'hash',
'uri': 'url',
'url': 'url',
'vulnerability': 'vulnerability',
'weakness': 'vulnerability'
}
# Related entities from RF portrayed as related attributes in MISP
self.related_attribute_types = [
'RelatedIpAddress', 'RelatedInternetDomainName', 'RelatedHash',
'RelatedEmailAddress', 'RelatedCyberVulnerability'
]
# Related entities from RF portrayed as tags in MISP
self.galaxy_tag_types = ['RelatedMalware', 'RelatedThreatActor']
def enrich(self):
"""Run the enrichment."""
category = self.type_to_rf_category.get(self.enriched_attribute.type)
try:
response = rf_lookup(self.api_token, category,
self.enriched_attribute.value)
json_response = json.loads(response.content)
except requests.HTTPError as error:
misperrors['error'] = f'Error when requesting data from Recorded Future. ' \
f'{error.response} : {error.response.reason}'
raise error
try:
# Add risk score and risk rules as tags to the enriched attribute
risk_score = json_response['data']['risk']['score']
hex_color = self.color_picker.riskscore_color(risk_score)
tag_name = f'recorded-future:risk-score="{risk_score}"'
self.add_tag(tag_name, hex_color)
for evidence in json_response['data']['risk']['evidenceDetails']:
risk_rule = evidence['rule']
criticality = evidence['criticality']
hex_color = self.color_picker.riskrule_color(criticality)
tag_name = f'recorded-future:risk-rule="{risk_rule}"'
self.add_tag(tag_name, hex_color)
# Retrieve related entities
for related_entity in json_response['data']['relatedEntities']:
related_type = related_entity['type']
if related_type in self.related_attribute_types:
# Related entities returned as additional attributes
for related in related_entity['entities']:
if int(related["count"]) > 4:
indicator = related['entity']['name']
self.add_related_attribute(indicator, related_type)
elif related_type in self.galaxy_tag_types:
# Related entities added as galaxy-tags to the enriched attribute
galaxy_tags = []
for related in related_entity['entities']:
if int(related["count"]) > 4:
indicator = related['entity']['name']
galaxy = self.galaxy_finder.find_galaxy_match(
indicator, related_type)
# Handle deduplication of galaxy tags
if galaxy and galaxy not in galaxy_tags:
galaxy_tags.append(galaxy)
for galaxy in galaxy_tags:
self.add_tag(galaxy)
except KeyError as error:
misperrors[
'error'] = 'Unexpected format in Recorded Future api response.'
raise error
def add_related_attribute(self, indicator: str, related_type: str) -> None:
"""Helper method for adding an indicator to the related attribute list."""
out_type = self.get_output_type(related_type, indicator)
attribute = MISPAttribute()
attribute.from_dict(**{
'value': indicator,
'type': out_type,
'distribution': 0
})
self.related_attributes.append((related_type, attribute))
def add_tag(self, tag_name: str, hex_color: str = None) -> None:
"""Helper method for adding a tag to the enriched attribute."""
tag = MISPTag()
tag_properties = {'name': tag_name}
if hex_color:
tag_properties['colour'] = hex_color
tag.from_dict(**tag_properties)
self.enriched_attribute.add_tag(tag)
def get_output_type(self, related_type: str, indicator: str) -> str:
"""Helper method for translating a Recorded Future related type to a MISP output type."""
output_type = 'text'
if related_type == 'RelatedIpAddress':
output_type = 'ip-dst'
elif related_type == 'RelatedInternetDomainName':
output_type = 'domain'
elif related_type == 'RelatedHash':
hash_len = len(indicator)
if hash_len == 64:
output_type = 'sha256'
elif hash_len == 40:
output_type = 'sha1'
elif hash_len == 32:
output_type = 'md5'
elif related_type == 'RelatedEmailAddress':
output_type = 'email-src'
elif related_type == 'RelatedCyberVulnerability':
signature = indicator.split('-')[0]
if signature == 'CVE':
output_type = 'vulnerability'
elif signature == 'CWE':
output_type = 'weakness'
return output_type
def get_results(self) -> dict:
"""Build and return the enrichment results."""
self.enrichment_object.add_attribute('Enriched attribute',
**self.enriched_attribute)
for related_type, attribute in self.related_attributes:
self.enrichment_object.add_attribute(related_type, **attribute)
self.event.add_object(**self.enrichment_object)
event = json.loads(self.event.to_json())
result = {key: event[key] for key in ['Object'] if key in event}
return {'results': result}
class MVAPI():
def __init__(self, attribute, api_key, client_id, client_secret):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.base_url = 'https://api.mvision.mcafee.com'
self.session = requests.Session()
self.api_key = api_key
auth = (client_id, client_secret)
self.logging()
self.auth(auth)
def logging(self):
self.logger = logging.getLogger('logs')
self.logger.setLevel('INFO')
handler = logging.StreamHandler()
formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(message)s")
handler.setFormatter(formatter)
self.logger.addHandler(handler)
def auth(self, auth):
iam_url = "https://iam.mcafee-cloud.com/iam/v1.1/token"
headers = {
'x-api-key': self.api_key,
'Content-Type': 'application/vnd.api+json'
}
payload = {
"grant_type": "client_credentials",
"scope": "ins.user ins.suser ins.ms.r"
}
res = self.session.post(iam_url,
headers=headers,
auth=auth,
data=payload)
if res.status_code != 200:
self.logger.error(
'Could not authenticate to get the IAM token: {0} - {1}'.
format(res.status_code, res.text))
sys.exit()
else:
self.logger.info('Successful authenticated.')
access_token = res.json()['access_token']
headers['Authorization'] = 'Bearer ' + access_token
self.session.headers = headers
def search_ioc(self):
filters = {
'filter[type][eq]':
self.attribute.type,
'filter[value]':
self.attribute.value,
'fields':
'id, type, value, coverage, uid, is_coat, is_sdb_dirty, category, comment, campaigns, threat, prevalence'
}
res = self.session.get(self.base_url + '/insights/v2/iocs',
params=filters)
if res.ok:
if len(res.json()['data']) == 0:
self.logger.info('No Hash details in MVISION Insights found.')
else:
self.logger.info(
'Successfully retrieved MVISION Insights details.')
self.logger.debug(res.text)
return res.json()
else:
self.logger.error('Error in search_ioc. HTTP {0} - {1}'.format(
str(res.status_code), res.text))
sys.exit()
def prep_result(self, ioc):
res = ioc['data'][0]
results = []
# Parse out Attribute Category
category_attr = {
'type':
'text',
'object_relation':
'text',
'value':
'Attribute Category: {0}'.format(res['attributes']['category'])
}
results.append(category_attr)
# Parse out Attribute Comment
comment_attr = {
'type': 'text',
'object_relation': 'text',
'value':
'Attribute Comment: {0}'.format(res['attributes']['comment'])
}
results.append(comment_attr)
# Parse out Attribute Dat Coverage
cover_attr = {
'type':
'text',
'object_relation':
'text',
'value':
'Dat Version Coverage: {0}'.format(
res['attributes']['coverage']['dat_version']['min'])
}
results.append(cover_attr)
# Parse out if Dirty
cover_attr = {
'type': 'text',
'object_relation': 'text',
'value': 'Is Dirty: {0}'.format(res['attributes']['is-sdb-dirty'])
}
results.append(cover_attr)
# Parse our targeted countries
countries_dict = []
countries = res['attributes']['prevalence']['countries']
for country in countries:
countries_dict.append(country['iso_code'])
country_attr = {
'type': 'text',
'object_relation': 'text',
'value': 'Targeted Countries: {0}'.format(countries_dict)
}
results.append(country_attr)
# Parse out targeted sectors
sectors_dict = []
sectors = res['attributes']['prevalence']['sectors']
for sector in sectors:
sectors_dict.append(sector['sector'])
sector_attr = {
'type': 'text',
'object_relation': 'text',
'value': 'Targeted Sectors: {0}'.format(sectors_dict)
}
results.append(sector_attr)
# Parse out Threat Classification
threat_class_attr = {
'type':
'text',
'object_relation':
'text',
'value':
'Threat Classification: {0}'.format(
res['attributes']['threat']['classification'])
}
results.append(threat_class_attr)
# Parse out Threat Name
threat_name_attr = {
'type': 'text',
'object_relation': 'text',
'value':
'Threat Name: {0}'.format(res['attributes']['threat']['name'])
}
results.append(threat_name_attr)
# Parse out Threat Severity
threat_sev_attr = {
'type':
'text',
'object_relation':
'text',
'value':
'Threat Severity: {0}'.format(
res['attributes']['threat']['severity'])
}
results.append(threat_sev_attr)
# Parse out Attribute ID
attr_id = {
'type': 'text',
'object_relation': 'text',
'value': 'Attribute ID: {0}'.format(res['id'])
}
results.append(attr_id)
# Parse out Campaign Relationships
campaigns = ioc['included']
for campaign in campaigns:
campaign_attr = {
'type': 'campaign-name',
'object_relation': 'campaign-name',
'value': campaign['attributes']['name']
}
results.append(campaign_attr)
mv_insights_obj = MISPObject(name='MVISION Insights Details')
for mvi_res in results:
mv_insights_obj.add_attribute(**mvi_res)
mv_insights_obj.add_reference(self.attribute.uuid,
'mvision-insights-details')
self.misp_event.add_object(mv_insights_obj)
event = json.loads(self.misp_event.to_json())
results_mvi = {
key: event[key]
for key in ('Attribute', 'Object') if (key in event and event[key])
}
return {'results': results_mvi}
class ShodanParser():
def __init__(self, attribute):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.ip_address_mapping = {
'asn': {
'type': 'AS',
'object_relation': 'asn'
},
'city': {
'type': 'text',
'object_relation': 'city'
},
'country_code': {
'type': 'text',
'object_relation': 'country-code'
},
'country_name': {
'type': 'text',
'object_relation': 'country'
},
'isp': {
'type': 'text',
'object_relation': 'ISP'
},
'latitude': {
'type': 'float',
'object_relation': 'latitude'
},
'longitude': {
'type': 'float',
'object_relation': 'longitude'
},
'org': {
'type': 'text',
'object_relation': 'organization'
},
'postal_code': {
'type': 'text',
'object_relation': 'zipcode'
},
'region_code': {
'type': 'text',
'object_relation': 'region-code'
}
}
self.ip_port_mapping = {
'domains': {
'type': 'domain',
'object_relation': 'domain'
},
'hostnames': {
'type': 'hostname',
'object_relation': 'hostname'
}
}
self.vulnerability_mapping = {
'cvss': {
'type': 'float',
'object_relation': 'cvss-score'
},
'summary': {
'type': 'text',
'object_relation': 'summary'
}
}
self.x509_mapping = {
'bits': {
'type': 'text',
'object_relation': 'pubkey-info-size'
},
'expires': {
'type': 'datetime',
'object_relation': 'validity-not-after'
},
'issued': {
'type': 'datetime',
'object_relation': 'validity-not-before'
},
'issuer': {
'type': 'text',
'object_relation': 'issuer'
},
'serial': {
'type': 'text',
'object_relation': 'serial-number'
},
'sig_alg': {
'type': 'text',
'object_relation': 'signature_algorithm'
},
'subject': {
'type': 'text',
'object_relation': 'subject'
},
'type': {
'type': 'text',
'object_relation': 'pubkey-info-algorithm'
},
'version': {
'type': 'text',
'object_relation': 'version'
}
}
def query_shodan(self, apikey):
# Query Shodan and get the results in a json blob
api = shodan.Shodan(apikey)
query_results = api.host(self.attribute.value)
# Parse the information about the IP address used as input
ip_address_attributes = []
for feature, mapping in self.ip_address_mapping.items():
if query_results.get(feature):
attribute = {'value': query_results[feature]}
attribute.update(mapping)
ip_address_attributes.append(attribute)
if ip_address_attributes:
ip_address_object = MISPObject('ip-api-address')
for attribute in ip_address_attributes:
ip_address_object.add_attribute(**attribute)
ip_address_object.add_attribute(**self._get_source_attribute())
ip_address_object.add_reference(self.attribute.uuid, 'describes')
self.misp_event.add_object(ip_address_object)
# Parse the hostnames / domains and ports associated with the IP address
if query_results.get('ports'):
ip_port_object = MISPObject('ip-port')
ip_port_object.add_attribute(**self._get_source_attribute())
feature = self.attribute.type.split('-')[1]
for port in query_results['ports']:
attribute = {
'type': 'port',
'object_relation': f'{feature}-port',
'value': port
}
ip_port_object.add_attribute(**attribute)
for feature, mapping in self.ip_port_mapping.items():
for value in query_results.get(feature, []):
attribute = {'value': value}
attribute.update(mapping)
ip_port_object.add_attribute(**attribute)
ip_port_object.add_reference(self.attribute.uuid, 'extends')
self.misp_event.add_object(ip_port_object)
else:
if any(
query_results.get(feature)
for feature in ('domains', 'hostnames')):
domain_ip_object = MISPObject('domain-ip')
domain_ip_object.add_attribute(**self._get_source_attribute())
for feature in ('domains', 'hostnames'):
for value in query_results[feature]:
attribute = {
'type': 'domain',
'object_relation': 'domain',
'value': value
}
domain_ip_object.add_attribute(**attribute)
domain_ip_object.add_reference(self.attribute.uuid, 'extends')
self.misp_event.add_object(domain_ip_object)
# Parse data within the "data" field
if query_results.get('vulns'):
vulnerabilities = {}
for data in query_results['data']:
# Parse vulnerabilities
if data.get('vulns'):
for cve, vulnerability in data['vulns'].items():
if cve not in vulnerabilities:
vulnerabilities[cve] = vulnerability
# Also parse the certificates
if data.get('ssl'):
self._parse_cert(data['ssl'])
for cve, vulnerability in vulnerabilities.items():
vulnerability_object = MISPObject('vulnerability')
vulnerability_object.add_attribute(**{
'type': 'vulnerability',
'object_relation': 'id',
'value': cve
})
for feature, mapping in self.vulnerability_mapping.items():
if vulnerability.get(feature):
attribute = {'value': vulnerability[feature]}
attribute.update(mapping)
vulnerability_object.add_attribute(**attribute)
if vulnerability.get('references'):
for reference in vulnerability['references']:
vulnerability_object.add_attribute(
**{
'type': 'link',
'object_relation': 'references',
'value': reference
})
vulnerability_object.add_reference(self.attribute.uuid,
'vulnerability-of')
self.misp_event.add_object(vulnerability_object)
for cve_id in query_results['vulns']:
if cve_id not in vulnerabilities:
attribute = {'type': 'vulnerability', 'value': cve_id}
self.misp_event.add_attribute(**attribute)
else:
# We have no vulnerability data, we only check if we have
# certificates within the "data" field
for data in query_results['data']:
if data.get('ssl'):
self._parse_cert(data['ssl']['cert'])
def get_result(self):
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object') if (key in event and event[key])
}
return {'results': results}
# When we want to add the IP address information in objects such as the
# domain-ip or ip-port objects referencing the input IP address attribute
def _get_source_attribute(self):
return {
'type': self.attribute.type,
'object_relation': self.attribute.type,
'value': self.attribute.value
}
def _parse_cert(self, certificate):
x509_object = MISPObject('x509')
for feature in ('serial', 'sig_alg', 'version'):
if certificate.get(feature):
attribute = {'value': certificate[feature]}
attribute.update(self.x509_mapping[feature])
x509_object.add_attribute(**attribute)
# Parse issuer and subject value
for feature in ('issuer', 'subject'):
if certificate.get(feature):
attribute_value = (
f'{identifier}={value}'
for identifier, value in certificate[feature].items())
attribute = {'value': f'/{"/".join(attribute_value)}'}
attribute.update(self.x509_mapping[feature])
x509_object.add_attribute(**attribute)
# Parse datetime attributes
for feature in ('expires', 'issued'):
if certificate.get(feature):
attribute = {
'value':
datetime.strptime(certificate[feature], '%Y%m%d%H%M%SZ')
}
attribute.update(self.x509_mapping[feature])
x509_object.add_attribute(**attribute)
# Parse fingerprints
if certificate.get('fingerprint'):
for hash_type, hash_value in certificate['fingerprint'].items():
x509_object.add_attribute(
**{
'type': f'x509-fingerprint-{hash_type}',
'object_relation': f'x509-fingerprint-{hash_type}',
'value': hash_value
})
# Parse public key related info
if certificate.get('pubkey'):
for feature, value in certificate['pubkey'].items():
attribute = {'value': value}
attribute.update(self.x509_mapping[feature])
x509_object.add_attribute(**attribute)
x509_object.add_reference(self.attribute.uuid, 'identifies')
self.misp_event.add_object(x509_object)
class VirusTotalParser(object):
def __init__(self, apikey):
self.apikey = apikey
self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report"
self.misp_event = MISPEvent()
self.parsed_objects = {}
self.input_types_mapping = {'ip-src': self.parse_ip, 'ip-dst': self.parse_ip,
'domain': self.parse_domain, 'hostname': self.parse_domain,
'md5': self.parse_hash, 'sha1': self.parse_hash,
'sha256': self.parse_hash, 'sha512': self.parse_hash,
'url': self.parse_url}
def query_api(self, attribute):
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
return self.input_types_mapping[self.attribute.type](self.attribute.value, recurse=True)
def get_result(self):
event = json.loads(self.misp_event.to_json())['Event']
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
return {'results': results}
################################################################################
#### Main parsing functions #### # noqa
################################################################################
def parse_domain(self, domain, recurse=False):
req = requests.get(self.base_url.format('domain'), params={'apikey': self.apikey, 'domain': domain})
if req.status_code != 200:
return req.status_code
req = req.json()
hash_type = 'sha256'
whois = 'whois'
feature_types = {'communicating': 'communicates-with',
'downloaded': 'downloaded-from',
'referrer': 'referring'}
siblings = (self.parse_siblings(domain) for domain in req['domain_siblings'])
uuid = self.parse_resolutions(req['resolutions'], req['subdomains'], siblings)
for feature_type, relationship in feature_types.items():
for feature in ('undetected_{}_samples', 'detected_{}_samples'):
for sample in req.get(feature.format(feature_type), []):
status_code = self.parse_hash(sample[hash_type], False, uuid, relationship)
if status_code != 200:
return status_code
if req.get(whois):
whois_object = MISPObject(whois)
whois_object.add_attribute('text', type='text', value=req[whois])
self.misp_event.add_object(**whois_object)
return self.parse_related_urls(req, recurse, uuid)
def parse_hash(self, sample, recurse=False, uuid=None, relationship=None):
req = requests.get(self.base_url.format('file'), params={'apikey': self.apikey, 'resource': sample})
status_code = req.status_code
if req.status_code == 200:
req = req.json()
vt_uuid = self.parse_vt_object(req)
file_attributes = []
for hash_type in ('md5', 'sha1', 'sha256'):
if req.get(hash_type):
file_attributes.append({'type': hash_type, 'object_relation': hash_type,
'value': req[hash_type]})
if file_attributes:
file_object = MISPObject('file')
for attribute in file_attributes:
file_object.add_attribute(**attribute)
file_object.add_reference(vt_uuid, 'analyzed-with')
if uuid and relationship:
file_object.add_reference(uuid, relationship)
self.misp_event.add_object(**file_object)
return status_code
def parse_ip(self, ip, recurse=False):
req = requests.get(self.base_url.format('ip-address'), params={'apikey': self.apikey, 'ip': ip})
if req.status_code != 200:
return req.status_code
req = req.json()
if req.get('asn'):
asn_mapping = {'network': ('ip-src', 'subnet-announced'),
'country': ('text', 'country')}
asn_object = MISPObject('asn')
asn_object.add_attribute('asn', type='AS', value=req['asn'])
for key, value in asn_mapping.items():
if req.get(key):
attribute_type, relation = value
asn_object.add_attribute(relation, type=attribute_type, value=req[key])
self.misp_event.add_object(**asn_object)
uuid = self.parse_resolutions(req['resolutions']) if req.get('resolutions') else None
return self.parse_related_urls(req, recurse, uuid)
def parse_url(self, url, recurse=False, uuid=None):
req = requests.get(self.base_url.format('url'), params={'apikey': self.apikey, 'resource': url})
status_code = req.status_code
if req.status_code == 200:
req = req.json()
vt_uuid = self.parse_vt_object(req)
if not recurse:
feature = 'url'
url_object = MISPObject(feature)
url_object.add_attribute(feature, type=feature, value=url)
url_object.add_reference(vt_uuid, 'analyzed-with')
if uuid:
url_object.add_reference(uuid, 'hosted-in')
self.misp_event.add_object(**url_object)
return status_code
################################################################################
#### Additional parsing functions #### # noqa
################################################################################
def parse_related_urls(self, query_result, recurse, uuid=None):
if recurse:
for feature in ('detected_urls', 'undetected_urls'):
if feature in query_result:
for url in query_result[feature]:
value = url['url'] if isinstance(url, dict) else url[0]
status_code = self.parse_url(value, False, uuid)
if status_code != 200:
return status_code
else:
for feature in ('detected_urls', 'undetected_urls'):
if feature in query_result:
for url in query_result[feature]:
value = url['url'] if isinstance(url, dict) else url[0]
self.misp_event.add_attribute('url', value)
return 200
def parse_resolutions(self, resolutions, subdomains=None, uuids=None):
domain_ip_object = MISPObject('domain-ip')
if self.attribute.type == 'domain':
domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value)
attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address')
else:
domain_ip_object.add_attribute('ip', type='ip-dst', value=self.attribute.value)
attribute_type, relation, key = ('domain', 'domain', 'hostname')
for resolution in resolutions:
domain_ip_object.add_attribute(relation, type=attribute_type, value=resolution[key])
if subdomains:
for subdomain in subdomains:
attribute = MISPAttribute()
attribute.from_dict(**dict(type='domain', value=subdomain))
self.misp_event.add_attribute(**attribute)
domain_ip_object.add_reference(attribute.uuid, 'subdomain')
if uuids:
for uuid in uuids:
domain_ip_object.add_reference(uuid, 'sibling-of')
self.misp_event.add_object(**domain_ip_object)
return domain_ip_object.uuid
def parse_siblings(self, domain):
attribute = MISPAttribute()
attribute.from_dict(**dict(type='domain', value=domain))
self.misp_event.add_attribute(**attribute)
return attribute.uuid
def parse_vt_object(self, query_result):
vt_object = MISPObject('virustotal-report')
vt_object.add_attribute('permalink', type='link', value=query_result['permalink'])
detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total'])
vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio)
self.misp_event.add_object(**vt_object)
return vt_object.uuid
data = json.load(json_file)
if 'Attribute' in data.get("response")[0].get("Event"):
attributes = data.get("response")[0].get("Event").get("Attribute")
for attribute in attributes:
misp_tag = []
if 'Tag' in attribute:
for tag in attribute.get('Tag'):
misp_tag.append(tag.get('name'))
mispattribute = MISPAttribute()
mispattribute.from_dict(
**{
'value': attribute.get("value"),
'category': attribute.get("category"),
'type': attribute.get("type"),
'to_ids': attribute.get("to_ids"),
'comment': attribute.get("comment"),
'Tag': misp_tag
})
res = api.add_attribute(event, mispattribute)
time.sleep(insert_sleep)
count_attributes = count_attributes + 1
if 'Object' in data.get("response")[0].get("Event"):
objects = data.get("response")[0].get("Event").get("Object")
for obj in objects:
misp_object = MISPObject(obj.get('name'))
if 'Attribute' in obj:
for attribute in obj.get('Attribute'):
misp_object.add_attribute(
class CytomicParser():
def __init__(self, attribute, config_object):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.config_object = config_object
if self.config_object:
self.token = self.get_token()
else:
sys.exit('Missing configuration')
def get_token(self):
try:
scope = self.config_object['scope']
grant_type = self.config_object['grant_type']
username = self.config_object['username']
password = self.config_object['password']
token_url = self.config_object['token_url']
clientid = self.config_object['clientid']
clientsecret = self.config_object['clientsecret']
if scope and grant_type and username and password:
data = {
'scope': scope,
'grant_type': grant_type,
'username': username,
'password': password
}
if token_url and clientid and clientsecret:
access_token_response = requests.post(
token_url,
data=data,
verify=False,
allow_redirects=False,
auth=(clientid, clientsecret))
tokens = json.loads(access_token_response.text)
if 'access_token' in tokens:
return tokens['access_token']
else:
self.result = {'error': 'No token received.'}
return
else:
self.result = {
'error':
'No token_url, clientid or clientsecret supplied.'
}
return
else:
self.result = {
'error':
'No scope, grant_type, username or password supplied.'
}
return
except Exception:
self.result = {'error': 'Unable to connect to token_url.'}
return
def get_results(self):
if hasattr(self, 'result'):
return self.result
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object')}
return {'results': results}
def parse(self, searchkey):
if self.token:
endpoint_fileinformation = self.config_object[
'endpoint_fileinformation']
endpoint_machines = self.config_object['endpoint_machines']
endpoint_machines_client = self.config_object[
'endpoint_machines_client']
query_machines = self.config_object['query_machines']
query_machine_info = self.config_object['query_machine_info']
# Update endpoint URLs
query_endpoint_fileinformation = endpoint_fileinformation.format(
md5=searchkey)
query_endpoint_machines = endpoint_machines.format(md5=searchkey)
# API calls
api_call_headers = {'Authorization': 'Bearer ' + self.token}
result_query_endpoint_fileinformation = requests.get(
query_endpoint_fileinformation,
headers=api_call_headers,
verify=False)
json_result_query_endpoint_fileinformation = json.loads(
result_query_endpoint_fileinformation.text)
if json_result_query_endpoint_fileinformation:
cytomic_object = MISPObject('cytomic-orion-file')
cytomic_object.add_attribute(
'fileName',
type='text',
value=json_result_query_endpoint_fileinformation[
'fileName'])
cytomic_object.add_attribute(
'fileSize',
type='text',
value=json_result_query_endpoint_fileinformation[
'fileSize'])
cytomic_object.add_attribute(
'last-seen',
type='datetime',
value=json_result_query_endpoint_fileinformation[
'lastSeen'])
cytomic_object.add_attribute(
'first-seen',
type='datetime',
value=json_result_query_endpoint_fileinformation[
'firstSeen'])
cytomic_object.add_attribute(
'classification',
type='text',
value=json_result_query_endpoint_fileinformation[
'classification'])
cytomic_object.add_attribute(
'classificationName',
type='text',
value=json_result_query_endpoint_fileinformation[
'classificationName'])
self.misp_event.add_object(**cytomic_object)
result_query_endpoint_machines = requests.get(
query_endpoint_machines,
headers=api_call_headers,
verify=False)
json_result_query_endpoint_machines = json.loads(
result_query_endpoint_machines.text)
if query_machines and json_result_query_endpoint_machines and len(
json_result_query_endpoint_machines) > 0:
for machine in json_result_query_endpoint_machines:
if query_machine_info and machine['muid']:
query_endpoint_machines_client = endpoint_machines_client.format(
muid=machine['muid'])
result_endpoint_machines_client = requests.get(
query_endpoint_machines_client,
headers=api_call_headers,
verify=False)
json_result_endpoint_machines_client = json.loads(
result_endpoint_machines_client.text)
if json_result_endpoint_machines_client:
cytomic_machine_object = MISPObject(
'cytomic-orion-machine')
clienttag = [{
'name':
json_result_endpoint_machines_client[
'clientName']
}]
cytomic_machine_object.add_attribute(
'machineName',
type='target-machine',
value=json_result_endpoint_machines_client[
'machineName'],
Tag=clienttag)
cytomic_machine_object.add_attribute(
'machineMuid',
type='text',
value=machine['muid'])
cytomic_machine_object.add_attribute(
'clientName',
type='target-org',
value=json_result_endpoint_machines_client[
'clientName'],
Tag=clienttag)
cytomic_machine_object.add_attribute(
'clientId',
type='text',
value=machine['clientId'])
cytomic_machine_object.add_attribute(
'machinePath',
type='text',
value=machine['lastPath'])
cytomic_machine_object.add_attribute(
'first-seen',
type='datetime',
value=machine['firstSeen'])
cytomic_machine_object.add_attribute(
'last-seen',
type='datetime',
value=machine['lastSeen'])
cytomic_machine_object.add_attribute(
'creationDate',
type='datetime',
value=json_result_endpoint_machines_client[
'creationDate'])
cytomic_machine_object.add_attribute(
'clientCreationDateUTC',
type='datetime',
value=json_result_endpoint_machines_client[
'clientCreationDateUTC'])
cytomic_machine_object.add_attribute(
'lastSeenUtc',
type='datetime',
value=json_result_endpoint_machines_client[
'lastSeenUtc'])
self.misp_event.add_object(
**cytomic_machine_object)
else:
self.result = {'error': 'No (valid) token.'}
return
class XforceExchange():
def __init__(self, attribute, apikey, apipassword):
self.base_url = "https://api.xforce.ibmcloud.com"
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self._apikey = apikey
self._apipassword = apipassword
self.result = {}
self.objects = defaultdict(dict)
self.status_mapping = {
403:
"Access denied, please check if your authentication is valid and if you did not reach the limit of queries.",
404: "No result found for your query."
}
def parse(self):
mapping = {
'url': '_parse_url',
'vulnerability': '_parse_vulnerability'
}
mapping.update(dict.fromkeys(('md5', 'sha1', 'sha256'), '_parse_hash'))
mapping.update(dict.fromkeys(('domain', 'hostname'), '_parse_dns'))
mapping.update(dict.fromkeys(('ip-src', 'ip-dst'), '_parse_ip'))
to_call = mapping[self.attribute.type]
getattr(self, to_call)(self.attribute.value)
def get_result(self):
if not self.misp_event.objects:
if 'error' not in self.result:
self.result[
'error'] = "No additional data found on Xforce Exchange."
return self.result
self.misp_event.add_attribute(**self.attribute)
event = json.loads(self.misp_event.to_json())
result = {
key: event[key]
for key in ('Attribute', 'Object') if (key in event and event[key])
}
return {'results': result}
def _api_call(self, url):
try:
result = requests.get(url,
auth=HTTPBasicAuth(self._apikey,
self._apipassword))
except Exception as e:
self.result['error'] = e
return
status_code = result.status_code
if status_code != 200:
try:
self.result['error'] = self.status_mapping[status_code]
except KeyError:
self.result['error'] = 'An error with the API has occurred.'
return
return result.json()
def _create_file(self, malware, relationship):
file_object = MISPObject('file')
for key, relation in zip(('filepath', 'md5'), ('filename', 'md5')):
file_object.add_attribute(relation, malware[key])
file_object.add_reference(self.attribute.uuid, relationship)
return file_object
def _create_url(self, malware):
url_object = MISPObject('url')
for key, relation in zip(('uri', 'domain'), ('url', 'domain')):
url_object.add_attribute(relation, malware[key])
attributes = tuple(f'{attribute.object_relation}_{attribute.value}'
for attribute in url_object.attributes)
if attributes in self.objects['url']:
del url_object
return self.objects['url'][attributes]
url_uuid = url_object.uuid
self.misp_event.add_object(**url_object)
self.objects['url'][attributes] = url_uuid
return url_uuid
def _fetch_types(self, value):
if self.attribute.type in ('ip-src', 'ip-dst'):
return 'ip', 'domain', self.attribute.value
return 'domain', 'ip', value
def _handle_file(self, malware, relationship):
file_object = self._create_file(malware, relationship)
attributes = tuple(f'{attribute.object_relation}_{attribute.value}'
for attribute in file_object.attributes)
if attributes in self.objects['file']:
self.objects['file'][attributes].add_reference(
self._create_url(malware), 'dropped-by')
del file_object
return
file_object.add_reference(self._create_url(malware), 'dropped-by')
self.objects['file'][attributes] = file_object
self.misp_event.add_object(**file_object)
def _parse_dns(self, value):
dns_result = self._api_call(f'{self.base_url}/resolve/{value}')
if dns_result.get('Passive') and dns_result['Passive'].get('records'):
itype, ftype, value = self._fetch_types(
dns_result['Passive']['query'])
misp_object = MISPObject('domain-ip')
misp_object.add_attribute(itype, value)
for record in dns_result['Passive']['records']:
misp_object.add_attribute(ftype, record['value'])
misp_object.add_reference(self.attribute.uuid, 'related-to')
self.misp_event.add_object(**misp_object)
def _parse_hash(self, value):
malware_result = self._api_call(f'{self.base_url}/malware/{value}')
if malware_result and malware_result.get('malware'):
malware_report = malware_result['malware']
for malware in malware_report.get('origins',
{}).get('CnCServers',
{}).get('rows', []):
self._handle_file(malware, 'related-to')
def _parse_ip(self, value):
self._parse_dns(value)
self._parse_malware(value, 'ipr')
def _parse_malware(self, value, feature):
malware_result = self._api_call(
f'{self.base_url}/{feature}/malware/{value}')
if malware_result and malware_result.get('malware'):
for malware in malware_result['malware']:
self._handle_file(malware, 'associated-with')
def _parse_url(self, value):
self._parse_dns(value)
self._parse_malware(value, 'url')
def _parse_vulnerability(self, value):
vulnerability_result = self._api_call(
f'{self.base_url}/vulnerabilities/search/{value}')
if vulnerability_result:
for vulnerability in vulnerability_result:
misp_object = MISPObject('vulnerability')
for code in vulnerability['stdcode']:
misp_object.add_attribute('id', code)
for feature, relation in zip(
('title', 'description', 'temporal_score'),
('summary', 'description', 'cvss-score')):
misp_object.add_attribute(relation, vulnerability[feature])
for reference in vulnerability['references']:
misp_object.add_attribute('references',
reference['link_target'])
misp_object.add_reference(self.attribute.uuid, 'related-to')
self.misp_event.add_object(**misp_object)
class VirusTotalParser():
def __init__(self):
super(VirusTotalParser, self).__init__()
self.misp_event = MISPEvent()
self.proxies = None
def declare_variables(self, apikey, attribute):
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.apikey = apikey
def get_result(self):
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object') if (key in event and event[key])
}
return {'results': results}
def parse_urls(self, query_result):
for feature in ('detected_urls', 'undetected_urls'):
if feature in query_result:
for url in query_result[feature]:
value = url['url'] if isinstance(url, dict) else url[0]
self.misp_event.add_attribute('url', value)
def parse_resolutions(self, resolutions, subdomains=None, uuids=None):
domain_ip_object = MISPObject('domain-ip')
if self.attribute.type in ('domain', 'hostname'):
domain_ip_object.add_attribute('domain',
type='domain',
value=self.attribute.value)
attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address')
else:
domain_ip_object.add_attribute('ip',
type='ip-dst',
value=self.attribute.value)
attribute_type, relation, key = ('domain', 'domain', 'hostname')
for resolution in resolutions:
domain_ip_object.add_attribute(relation,
type=attribute_type,
value=resolution[key])
if subdomains:
for subdomain in subdomains:
attribute = MISPAttribute()
attribute.from_dict(**dict(type='domain', value=subdomain))
self.misp_event.add_attribute(**attribute)
domain_ip_object.add_reference(attribute.uuid, 'subdomain')
if uuids:
for uuid in uuids:
domain_ip_object.add_reference(uuid, 'sibling-of')
self.misp_event.add_object(**domain_ip_object)
def parse_vt_object(self, query_result):
if query_result['response_code'] == 1:
vt_object = MISPObject('virustotal-report')
vt_object.add_attribute('permalink',
type='link',
value=query_result['permalink'])
detection_ratio = '{}/{}'.format(query_result['positives'],
query_result['total'])
vt_object.add_attribute('detection-ratio',
type='text',
value=detection_ratio)
self.misp_event.add_object(**vt_object)
def get_query_result(self, query_type):
params = {query_type: self.attribute.value, 'apikey': self.apikey}
return requests.get(self.base_url, params=params, proxies=self.proxies)
def set_proxy_settings(self, config: dict) -> dict:
"""Returns proxy settings in the requests format.
If no proxy settings are set, return None."""
proxies = None
host = config.get('proxy_host')
port = config.get('proxy_port')
username = config.get('proxy_username')
password = config.get('proxy_password')
if host:
if not port:
misperrors['error'] = 'The virustotal_public_proxy_host config is set, ' \
'please also set the virustotal_public_proxy_port.'
raise KeyError
parsed = urlparse(host)
if 'http' in parsed.scheme:
scheme = 'http'
else:
scheme = parsed.scheme
netloc = parsed.netloc
host = f'{netloc}:{port}'
if username:
if not password:
misperrors['error'] = 'The virustotal_public_proxy_username config is set, ' \
'please also set the virustotal_public_proxy_password.'
raise KeyError
auth = f'{username}:{password}'
host = auth + '@' + host
proxies = {
'http': f'{scheme}://{host}',
'https': f'{scheme}://{host}'
}
self.proxies = proxies
return True
class TruSTARParser:
ENTITY_TYPE_MAPPINGS = {
'BITCOIN_ADDRESS': "btc",
'CIDR_BLOCK': "ip-src",
'CVE': "vulnerability",
'URL': "url",
'EMAIL_ADDRESS': "email-src",
'SOFTWARE': "filename",
'IP': "ip-src",
'MALWARE': "malware-type",
'MD5': "md5",
'REGISTRY_KEY': "regkey",
'SHA1': "sha1",
'SHA256': "sha256"
}
REPORT_BASE_URL = "https://station.trustar.co/constellation/reports/{}"
CLIENT_METATAG = "MISP-{}".format(pymisp.__version__)
def __init__(self, attribute, config):
config['enclave_ids'] = config.get('enclave_ids',
"").strip().split(',')
config['client_metatag'] = self.CLIENT_METATAG
self.ts_client = TruStar(config=config)
self.misp_event = MISPEvent()
self.misp_attribute = MISPAttribute()
self.misp_attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.misp_attribute)
def get_results(self):
"""
Returns the MISP Event enriched with TruSTAR indicator summary data.
"""
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object') if (key in event and event[key])
}
return {'results': results}
def generate_trustar_links(self, entity_value):
"""
Generates links to TruSTAR reports if they exist.
:param entity_value: <str> Value of entity.
"""
report_links = list()
trustar_reports = self.ts_client.search_reports(entity_value)
for report in trustar_reports:
report_links.append(self.REPORT_BASE_URL.format(report.id))
return report_links
def parse_indicator_summary(self, summaries):
"""
Converts a response from the TruSTAR /1.3/indicators/summaries endpoint
a MISP trustar_report object and adds the summary data and links as attributes.
:param summaries: <generator> A TruSTAR Python SDK Page.generator object for generating
indicator summaries pages.
"""
for summary in summaries:
trustar_obj = MISPObject('trustar_report')
indicator_type = summary.indicator_type
indicator_value = summary.value
if indicator_type in self.ENTITY_TYPE_MAPPINGS:
trustar_obj.add_attribute(
indicator_type,
attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type],
value=indicator_value)
trustar_obj.add_attribute("INDICATOR_SUMMARY",
attribute_type="text",
value=json.dumps(summary.to_dict(),
sort_keys=True,
indent=4))
report_links = self.generate_trustar_links(indicator_value)
for link in report_links:
trustar_obj.add_attribute("REPORT_LINK",
attribute_type="link",
value=link)
self.misp_event.add_object(**trustar_obj)
def parse_siblings(self, domain):
attribute = MISPAttribute()
attribute.from_dict(**dict(type='domain', value=domain))
self.misp_event.add_attribute(**attribute)
return attribute.uuid
class TruSTARParser:
ENTITY_TYPE_MAPPINGS = {
'BITCOIN_ADDRESS': "btc",
'CIDR_BLOCK': "ip-src",
'CVE': "vulnerability",
'URL': "url",
'EMAIL_ADDRESS': "email-src",
'SOFTWARE': "filename",
'IP': "ip-src",
'MALWARE': "malware-type",
'MD5': "md5",
'REGISTRY_KEY': "regkey",
'SHA1': "sha1",
'SHA256': "sha256"
}
# Relevant fields from each TruSTAR endpoint
SUMMARY_FIELDS = ["severityLevel", "source", "score", "attributes"]
METADATA_FIELDS = ["sightings", "firstSeen", "lastSeen", "tags"]
REPORT_BASE_URL = "https://station.trustar.co/constellation/reports/{}"
CLIENT_METATAG = f"MISP-{pymisp.__version__}"
def __init__(self, attribute, config):
config['enclave_ids'] = config.get('enclave_ids',
"").strip().split(',')
config['client_metatag'] = self.CLIENT_METATAG
self.ts_client = TruStar(config=config)
self.misp_event = MISPEvent()
self.misp_attribute = MISPAttribute()
self.misp_attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.misp_attribute)
def get_results(self):
"""
Returns the MISP Event enriched with TruSTAR indicator summary data.
"""
try:
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object')
if (key in event and event[key])
}
return {'results': results}
except Exception as e:
misperrors[
'error'] += f" -- Encountered issue serializing enrichment data -- {e}"
return misperrors
def generate_trustar_link(self, entity_type, entity_value):
"""
Generates link to TruSTAR report of entity.
:param entity_type: <str> Type of entity.
:param entity_value: <str> Value of entity.
:return: <str> Link to indicator report in TruSTAR platform.
"""
report_id = b64encode(
quote(f"{entity_type}|{entity_value}").encode()).decode()
return self.REPORT_BASE_URL.format(report_id)
@staticmethod
def extract_tags(enrichment_report):
"""
Extracts tags from the enrichment report in order to add them
to the TruSTAR MISP Object. Removes tags from report to avoid
redundancy.
:param: <OrderedDict> Enrichment data.
:return: <list> List of tags.
"""
if enrichment_report and enrichment_report.get('tags'):
return [tag.get('name') for tag in enrichment_report.pop('tags')]
return None
def generate_enrichment_report(self, summary, metadata):
"""
Extracts desired fields from summary and metadata reports and
generates an enrichment report.
:param summary: <trustar.IndicatorSummary> Indicator summary report.
:param metadata: <trustar.Indicator> Indicator metadata report.
:return: <str> Enrichment report.
"""
# Preserve order of fields as they exist in SUMMARY_FIELDS and METADATA_FIELDS
enrichment_report = OrderedDict()
if summary:
summary_dict = summary.to_dict()
enrichment_report.update({
field: summary_dict[field]
for field in self.SUMMARY_FIELDS if summary_dict.get(field)
})
if metadata:
metadata_dict = metadata.to_dict()
enrichment_report.update({
field: metadata_dict[field]
for field in self.METADATA_FIELDS if metadata_dict.get(field)
})
return enrichment_report
def parse_indicator_summary(self, indicator, summary, metadata):
"""
Pulls enrichment data from the TruSTAR /indicators/summaries and /indicators/metadata endpoints
and creates a MISP trustar_report.
:param indicator: <str> Value of the attribute
:summary: <trustar.IndicatorSummary> Indicator summary response object.
:metadata: <trustar.Indicator> Indicator response object.
"""
# Verify that the indicator type is supported by TruSTAR
if summary and summary.indicator_type in self.ENTITY_TYPE_MAPPINGS:
indicator_type = summary.indicator_type
elif metadata and metadata.type in self.ENTITY_TYPE_MAPPINGS:
indicator_type = metadata.type
else:
misperrors['error'] += " -- Attribute not found or not supported"
raise Exception
try:
# Extract most relevant fields from indicator summary and metadata responses
enrichment_report = self.generate_enrichment_report(
summary, metadata)
tags = self.extract_tags(enrichment_report)
if enrichment_report:
# Create MISP trustar_report object and populate it with enrichment data
trustar_obj = MISPObject('trustar_report')
trustar_obj.add_attribute(
indicator_type,
attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type],
value=indicator)
trustar_obj.add_attribute("INDICATOR_SUMMARY",
attribute_type="text",
value=json.dumps(enrichment_report,
indent=4))
report_link = self.generate_trustar_link(
indicator_type, indicator)
trustar_obj.add_attribute("REPORT_LINK",
attribute_type="link",
value=report_link)
self.misp_event.add_object(**trustar_obj)
elif not tags:
# If enrichment report is empty and there are no tags, nothing to add to attribute
raise Exception("No relevant data found")
if tags:
for tag in tags:
self.misp_event.add_attribute_tag(tag, indicator)
except Exception as e:
misperrors[
'error'] += f" -- Error enriching attribute {indicator} -- {e}"
raise e
class APIVoidParser():
def __init__(self, attribute):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.url = 'https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&'
def get_results(self):
if hasattr(self, 'result'):
return self.result
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object')}
return {'results': results}
def parse_domain(self, apikey):
feature = 'dnslookup'
if requests.get(f'{self.url.format(feature, apikey)}stats').json(
)['credits_remained'] < 0.13:
self.result = {
'error':
'You do not have enough APIVoid credits to proceed your request.'
}
return
mapping = {
'A': 'resolution-of',
'MX': 'mail-server-of',
'NS': 'server-name-of'
}
dnslookup = requests.get(
f'{self.url.format(feature, apikey)}action=dns-any&host={self.attribute.value}'
).json()
for item in dnslookup['data']['records']['items']:
record_type = item['type']
try:
relationship = mapping[record_type]
except KeyError:
continue
self._handle_dns_record(item, record_type, relationship)
ssl = requests.get(
f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}'
).json()
self._parse_ssl_certificate(ssl['data']['certificate'])
def _handle_dns_record(self, item, record_type, relationship):
dns_record = MISPObject('dns-record')
dns_record.add_attribute('queried-domain',
type='domain',
value=item['host'])
attribute_type, feature = ('ip-dst',
'ip') if record_type == 'A' else ('domain',
'target')
dns_record.add_attribute(f'{record_type.lower()}-record',
type=attribute_type,
value=item[feature])
dns_record.add_reference(self.attribute.uuid, relationship)
self.misp_event.add_object(**dns_record)
def _parse_ssl_certificate(self, certificate):
x509 = MISPObject('x509')
fingerprint = 'x509-fingerprint-sha1'
x509.add_attribute(fingerprint,
type=fingerprint,
value=certificate['fingerprint'])
x509_mapping = {
'subject': {
'name': ('text', 'subject')
},
'issuer': {
'common_name': ('text', 'issuer')
},
'signature': {
'serial': ('text', 'serial-number')
},
'validity': {
'valid_from': ('datetime', 'validity-not-before'),
'valid_to': ('datetime', 'validity-not-after')
}
}
certificate = certificate['details']
for feature, subfeatures in x509_mapping.items():
for subfeature, mapping in subfeatures.items():
attribute_type, relation = mapping
x509.add_attribute(relation,
type=attribute_type,
value=certificate[feature][subfeature])
x509.add_reference(self.attribute.uuid, 'seen-by')
self.misp_event.add_object(**x509)
