def import_all(self, stations_short_names, interval, data_type):
object_creator = getattr(self, f'{interval}_flask_{data_type}')
if data_type == 'co2':
base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/'
elif data_type in ['c13', 'o18']:
base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/'
for station in stations_short_names:
url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv'
infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations'
filepath = self.fetch(url)
if not filepath:
continue
update = True
event = self.get_existing_event_to_update(infofield)
if event:
location = event.get_objects_by_name('geolocation')[0]
if not event:
event = MISPEvent()
event.info = infofield
event.add_tag(getattr(self, f'tag_{station}')())
location = getattr(self, f'geolocation_{station}')()
event.add_object(location)
event.add_attribute('link', f'http://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}')
update = False
object_creator(event, location, filepath, update)
if update:
self.misp.update_event(event)
else:
self.misp.add_event(event)
class PassiveDNSParser():
def __init__(self, attribute, authentication):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.pdns = pypdns.PyPDNS(basic_auth=authentication)
def get_results(self):
if hasattr(self, 'result'):
return self.result
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object')}
return {'results': results}
def parse(self, value):
try:
results = self.pdns.query(self.attribute.value)
except Exception:
self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'}
return
mapping = {'count': 'counter', 'origin': 'text',
'time_first': 'datetime', 'rrtype': 'text',
'rrname': 'text', 'rdata': 'text',
'time_last': 'datetime'}
for result in results:
pdns_object = MISPObject('passive-dns')
for relation, attribute_type in mapping.items():
pdns_object.add_attribute(relation, type=attribute_type, value=result[relation])
pdns_object.add_reference(self.attribute.uuid, 'associated-to')
self.misp_event.add_object(**pdns_object)
def create_list_of_objs_to_export(l_obj, r_type='json'):
all_obj_to_export = set()
set_relationship = set()
for obj in l_obj:
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, obj,
obj.get('lvl', 1))
# create MISP objects
dict_misp_obj = create_all_misp_obj(all_obj_to_export, set_relationship)
# create object relationships
for obj_global_id_1, obj_global_id_2 in set_relationship:
dict_relationship = get_relationship_between_global_obj(
obj_global_id_1, obj_global_id_2)
if dict_relationship:
obj_src = dict_misp_obj[dict_relationship['src']]
obj_dest = dict_misp_obj[dict_relationship['dest']]
obj_src.add_reference(obj_dest.uuid, dict_relationship['relation'],
'add a comment')
event = MISPEvent()
event.info = 'AIL framework export'
event.uuid = str(uuid.uuid4())
for obj_global_id in dict_misp_obj:
misp_obj = dict_misp_obj[obj_global_id]
AILObjects.create_map_obj_event_uuid(event.uuid, obj_global_id)
AILObjects.create_map_obj_uuid_golbal_id(event.uuid, obj_global_id)
if misp_obj:
# add object to event
event.add_object(dict_misp_obj[obj_global_id])
return event
def create_complex_event(self):
event = MISPEvent()
event.info = 'Complex Event'
event.distribution = Distribution.all_communities
event.add_tag('tlp:white')
event.add_attribute('ip-src', '8.8.8.8')
event.add_attribute('ip-dst', '8.8.8.9')
event.add_attribute('domain', 'google.com')
event.add_attribute('md5', '3c656da41f4645f77e3ec3281b63dd43')
event.attributes[0].distribution = Distribution.your_organisation_only
event.attributes[1].distribution = Distribution.this_community_only
event.attributes[2].distribution = Distribution.connected_communities
event.attributes[0].add_tag('tlp:red')
event.attributes[1].add_tag('tlp:amber')
event.attributes[2].add_tag('tlp:green')
obj = MISPObject('file')
obj.distribution = Distribution.connected_communities
obj.add_attribute('filename', 'testfile')
obj.add_attribute('md5', '3c656da41f4645f77e3ec3281b63dd44')
obj.attributes[0].distribution = Distribution.your_organisation_only
event.add_object(obj)
return event
def main():
parser = argparse.ArgumentParser(
description="Test of adding event to MISP")
parser.add_argument("mwdb_user", help="Mwdb username")
parser.add_argument("mwdb_pass", help="Mwdb password")
parser.add_argument("config", help="Config")
parser.add_argument("misp_url", help="Misp url")
parser.add_argument("misp_key", help="Misp key")
args = parser.parse_args()
mwdb = Malwarecage()
mwdb.login(args.mwdb_user, args.mwdb_pass)
try:
cfg = mwdb.query_config(args.config)
iocs = parse(cfg.family, cfg.cfg)
except FamilyNotSupportedYetError:
logging.info("Family %s not supported yet...", cfg.family)
return
if not iocs:
# Nothing actionable found - skip the config
return
event = MISPEvent()
event.add_tag(f"mwdb:family:{cfg.family}")
event.info = f"Malware configuration ({cfg.family})"
for o in iocs.to_misp():
event.add_object(o)
misp = PyMISP(args.misp_url, args.misp_key, False)
misp.add_event(event)
def process(self, task: Task) -> None: # type: ignore
config = task.get_payload("config")
family = task.headers["family"]
dhash = config_dhash(config)
# Parse the config using iocextract library
iocs = parse(family, config)
if not iocs:
# Nothing actionable found - skip the config
return
# Upload structured data to MISP
event = MISPEvent()
event.uuid = str(uuid5(self.CONFIG_NAMESPACE, dhash))
event.add_tag(f"mwdb:family:{family}")
event.info = f"Malware configuration ({family})"
if self.mwdb_url is not None:
event.add_attribute("link", f"{self.mwdb_url}/config/{dhash}")
for o in iocs.to_misp():
event.add_object(o)
misp = ExpandedPyMISP(self.misp_url, self.misp_key, self.misp_verifycert)
misp.add_event(event)
class PassiveSSLParser():
def __init__(self, attribute, authentication):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.pssl = pypssl.PyPSSL(basic_auth=authentication)
self.cert_hash = 'x509-fingerprint-sha1'
self.cert_type = 'pem'
self.mapping = {'issuer': ('text', 'issuer'),
'keylength': ('text', 'pubkey-info-size'),
'not_after': ('datetime', 'validity-not-after'),
'not_before': ('datetime', 'validity-not-before'),
'subject': ('text', 'subject')}
def get_results(self):
if hasattr(self, 'result'):
return self.result
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object')}
return {'results': results}
def parse(self):
value = self.attribute.value.split('|')[0] if '|' in self.attribute.type else self.attribute.value
try:
results = self.pssl.query(value)
except Exception:
self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'}
return
if not results:
self.result = {'error': 'Not found'}
return
for ip_address, certificates in results.items():
ip_uuid = self._handle_ip_attribute(ip_address)
for certificate in certificates['certificates']:
self._handle_certificate(certificate, ip_uuid)
def _handle_certificate(self, certificate, ip_uuid):
x509 = MISPObject('x509')
x509.add_attribute(self.cert_hash, type=self.cert_hash, value=certificate)
cert_details = self.pssl.fetch_cert(certificate)
info = cert_details['info']
for feature, mapping in self.mapping.items():
attribute_type, object_relation = mapping
x509.add_attribute(object_relation, type=attribute_type, value=info[feature])
x509.add_attribute(self.cert_type, type='text', value=self.cert_type)
x509.add_reference(ip_uuid, 'seen-by')
self.misp_event.add_object(**x509)
def _handle_ip_attribute(self, ip_address):
if ip_address == self.attribute.value:
return self.attribute.uuid
ip_attribute = MISPAttribute()
ip_attribute.from_dict(**{'type': self.attribute.type, 'value': ip_address})
self.misp_event.add_attribute(**ip_attribute)
return ip_attribute.uuid
def test_to_dict_json_format(self):
misp_event = MISPEvent()
av_signature_object = MISPObject("av-signature")
av_signature_object.add_attribute("signature", "EICAR")
av_signature_object.add_attribute("software", "ClamAv")
misp_event.add_object(av_signature_object)
self.assertEqual(json.loads(misp_event.to_json()),
misp_event.to_dict(json_format=True))
def addNewObjectsExistingJson(mispVehicle, mispGeolocation):
existing_event = MISPEvent()
existing_event.load_file('data.json')
existing_event.add_object(mispVehicle)
existing_event.add_object(mispGeolocation)
open("data.json", 'a').close()
with open("data.json", "w") as fichier:
fichier.write(
existing_event.to_json(indent=2)) # écriture de l'évènement
print("Nouveau véhicule ajouté au json existant")
class VulnerabilityParser():
def __init__(self, vulnerability):
self.vulnerability = vulnerability
self.misp_event = MISPEvent()
self.vulnerability_mapping = {
'id': ('text', 'id'),
'summary': ('text', 'summary'),
'vulnerable_configuration_cpe_2_2':
('text', 'vulnerable_configuration'),
'Modified': ('datetime', 'modified'),
'Published': ('datetime', 'published'),
'references': ('link', 'references'),
'cvss': ('float', 'cvss-score')
}
def get_result(self):
event = json.loads(self.misp_event.to_json())['Event']
results = {
key: event[key]
for key in ('Attribute', 'Object') if (key in event and event[key])
}
return {'results': results}
def parse_vulnerability_information(self):
vulnerability_object = MISPObject('vulnerability')
for feature in ('id', 'summary', 'Modified', 'cvss'):
value = self.vulnerability.get(feature)
if value:
attribute_type, relation = self.vulnerability_mapping[feature]
vulnerability_object.add_attribute(
relation, **{
'type': attribute_type,
'value': value
})
if 'Published' in self.vulnerability:
vulnerability_object.add_attribute(
'published', **{
'type': 'datetime',
'value': self.vulnerability['Published']
})
vulnerability_object.add_attribute(
'state', **{
'type': 'text',
'value': 'Published'
})
for feature in ('references', 'vulnerable_configuration_cpe_2_2'):
if feature in self.vulnerability:
attribute_type, relation = self.vulnerability_mapping[feature]
for value in self.vulnerability[feature]:
vulnerability_object.add_attribute(
relation, **{
'type': attribute_type,
'value': value
})
self.misp_event.add_object(**vulnerability_object)
class VirusTotalParser():
def __init__(self):
super(VirusTotalParser, self).__init__()
self.misp_event = MISPEvent()
def declare_variables(self, apikey, attribute):
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.apikey = apikey
def get_result(self):
event = json.loads(self.misp_event.to_json())['Event']
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
return {'results': results}
def parse_urls(self, query_result):
for feature in ('detected_urls', 'undetected_urls'):
if feature in query_result:
for url in query_result[feature]:
value = url['url'] if isinstance(url, dict) else url[0]
self.misp_event.add_attribute('url', value)
def parse_resolutions(self, resolutions, subdomains=None, uuids=None):
domain_ip_object = MISPObject('domain-ip')
if self.attribute.type == 'domain':
domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value)
attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address')
else:
domain_ip_object.add_attribute('ip', type='ip-dst', value=self.attribute.value)
attribute_type, relation, key = ('domain', 'domain', 'hostname')
for resolution in resolutions:
domain_ip_object.add_attribute(relation, type=attribute_type, value=resolution[key])
if subdomains:
for subdomain in subdomains:
attribute = MISPAttribute()
attribute.from_dict(**dict(type='domain', value=subdomain))
self.misp_event.add_attribute(**attribute)
domain_ip_object.add_reference(attribute.uuid, 'subdomain')
if uuids:
for uuid in uuids:
domain_ip_object.add_reference(uuid, 'sibling-of')
self.misp_event.add_object(**domain_ip_object)
def parse_vt_object(self, query_result):
vt_object = MISPObject('virustotal-report')
vt_object.add_attribute('permalink', type='link', value=query_result['permalink'])
detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total'])
vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio)
self.misp_event.add_object(**vt_object)
def get_query_result(self, query_type):
params = {query_type: self.attribute.value, 'apikey': self.apikey}
return requests.get(self.base_url, params=params)
def create_response(original_attribute: dict, software: str, signature: Optional[str] = None) -> dict:
misp_event = MISPEvent()
if signature:
misp_event.add_attribute(**original_attribute)
av_signature_object = MISPObject("av-signature")
av_signature_object.add_attribute("signature", signature)
av_signature_object.add_attribute("software", software)
av_signature_object.add_reference(original_attribute["uuid"], "belongs-to")
misp_event.add_object(av_signature_object)
event = json.loads(misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
return {"results": results}
def export(self, cache: 'CaptureCache', is_public_instance: bool=False) -> MISPEvent:
'''Export a capture in MISP format. You can POST the return of this method
directly to a MISP instance and it will create an event.'''
public_domain = get_config('generic', 'public_domain')
event = MISPEvent()
event.info = f'Lookyloo Capture ({cache.url})'
lookyloo_link: MISPAttribute = event.add_attribute('link', f'https://{public_domain}/tree/{cache.uuid}') # type: ignore
if not is_public_instance:
lookyloo_link.distribution = 0
initial_url = URLObject(cache.url)
initial_url.comment = 'Submitted URL'
self.__misp_add_ips_to_URLObject(initial_url, cache.tree.root_hartree.hostname_tree)
redirects: List[URLObject] = []
for nb, url in enumerate(cache.redirects):
if url == cache.url:
continue
obj = URLObject(url)
obj.comment = f'Redirect {nb}'
self.__misp_add_ips_to_URLObject(obj, cache.tree.root_hartree.hostname_tree)
redirects.append(obj)
if redirects:
redirects[-1].comment = f'Last redirect ({nb})'
if redirects:
prec_object = initial_url
for u_object in redirects:
prec_object.add_reference(u_object, 'redirects-to')
prec_object = u_object
initial_obj = event.add_object(initial_url)
initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo')
for u_object in redirects:
event.add_object(u_object)
final_redirect = event.objects[-1]
try:
fo = FileObject(pseudofile=cache.tree.root_hartree.rendered_node.body, filename=cache.tree.root_hartree.rendered_node.filename)
fo.comment = 'Content received for the final redirect (before rendering)'
fo.add_reference(final_redirect, 'loaded-by', 'URL loading that content')
event.add_object(fo)
except Har2TreeError:
pass
except AttributeError:
# No `body` in rendered node
pass
return event
def parse_response(response):
mapping = {
'file_name': {
'type': 'filename',
'object_relation': 'filename'
},
'file_size': {
'type': 'size-in-bytes',
'object_relation': 'size-in-bytes'
},
'file_type_mime': {
'type': 'mime-type',
'object_relation': 'mimetype'
},
'md5_hash': {
'type': 'md5',
'object_relation': 'md5'
},
'sha1_hash': {
'type': 'sha1',
'object_relation': 'sha1'
},
'sha256_hash': {
'type': 'sha256',
'object_relation': 'sha256'
},
'ssdeep': {
'type': 'ssdeep',
'object_relation': 'ssdeep'
}
}
misp_event = MISPEvent()
for data in response:
misp_object = MISPObject('file')
for feature, attribute in mapping.items():
if feature in data:
misp_attribute = {'value': data[feature]}
misp_attribute.update(attribute)
misp_object.add_attribute(**misp_attribute)
misp_event.add_object(**misp_object)
return {
'results': {
'Object': [
json.loads(misp_object.to_json())
for misp_object in misp_event.objects
]
}
}
def parse_result(attribute, values):
event = MISPEvent()
initial_attribute = MISPAttribute()
initial_attribute.from_dict(**attribute)
event.add_attribute(**initial_attribute)
mapping = {'asn': ('AS', 'asn'), 'prefix': ('ip-src', 'subnet-announced')}
print(values)
for last_seen, response in values['response'].items():
asn = MISPObject('asn')
asn.add_attribute('last-seen', **{'type': 'datetime', 'value': last_seen})
for feature, attribute_fields in mapping.items():
attribute_type, object_relation = attribute_fields
asn.add_attribute(object_relation, **{'type': attribute_type, 'value': response[feature]})
asn.add_reference(initial_attribute.uuid, 'related-to')
event.add_object(**asn)
event = json.loads(event.to_json())
return {key: event[key] for key in ('Attribute', 'Object')}
class HashlookupParser():
def __init__(self, attribute, hashlookupresult, api_url):
self.attribute = attribute
self.hashlookupresult = hashlookupresult
self.api_url = api_url
self.misp_event = MISPEvent()
self.misp_event.add_attribute(**attribute)
self.references = defaultdict(list)
def get_result(self):
if self.references:
self.__build_references()
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
return {'results': results}
def parse_hashlookup_information(self):
hashlookup_object = MISPObject('hashlookup')
if 'source' in self.hashlookupresult:
hashlookup_object.add_attribute('source', **{'type': 'text', 'value': self.hashlookupresult['source']})
if 'KnownMalicious' in self.hashlookupresult:
hashlookup_object.add_attribute('KnownMalicious', **{'type': 'text', 'value': self.hashlookupresult['KnownMalicious']})
hashlookup_object.add_attribute('MD5', **{'type': 'md5', 'value': self.hashlookupresult['MD5']})
hashlookup_object.add_attribute('SHA-1', **{'type': 'sha1', 'value': self.hashlookupresult['SHA-1']})
if 'SSDEEP' in self.hashlookupresult:
hashlookup_object.add_attribute('SSDEEP', **{'type': 'ssdeep', 'value': self.hashlookupresult['SSDEEP']})
if 'TLSH' in self.hashlookupresult:
hashlookup_object.add_attribute('TLSH', **{'type': 'tlsh', 'value': self.hashlookupresult['TLSH']})
if 'FileName' in self.hashlookupresult:
hashlookup_object.add_attribute('FileName', **{'type': 'filename', 'value': self.hashlookupresult['FileName']})
if 'FileSize' in self.hashlookupresult:
hashlookup_object.add_attribute('FileSize', **{'type': 'size-in-bytes', 'value': self.hashlookupresult['FileSize']})
hashlookup_object.add_reference(self.attribute['uuid'], 'related-to')
self.misp_event.add_object(hashlookup_object)
def __build_references(self):
for object_uuid, references in self.references.items():
for misp_object in self.misp_event.objects:
if misp_object.uuid == object_uuid:
for reference in references:
misp_object.add_reference(**reference)
break
class FarsightDnsdbParser():
def __init__(self, attribute):
self.attribute = attribute
self.misp_event = MISPEvent()
self.misp_event.add_attribute(**attribute)
self.passivedns_mapping = {
'bailiwick': {'type': 'text', 'object_relation': 'bailiwick'},
'count': {'type': 'counter', 'object_relation': 'count'},
'raw_rdata': {'type': 'text', 'object_relation': 'raw_rdata'},
'rdata': {'type': 'text', 'object_relation': 'rdata'},
'rrname': {'type': 'text', 'object_relation': 'rrname'},
'rrtype': {'type': 'text', 'object_relation': 'rrtype'},
'time_first': {'type': 'datetime', 'object_relation': 'time_first'},
'time_last': {'type': 'datetime', 'object_relation': 'time_last'},
'zone_time_first': {'type': 'datetime', 'object_relation': 'zone_time_first'},
'zone_time_last': {'type': 'datetime', 'object_relation': 'zone_time_last'}
}
self.comment = 'Result from an %s lookup on DNSDB about the %s: %s'
def parse_passivedns_results(self, query_response):
for query_type, results in query_response.items():
comment = self.comment % (query_type, TYPE_TO_FEATURE[self.attribute['type']], self.attribute['value'])
for result in results:
passivedns_object = MISPObject('passive-dns')
if result.get('rdata') and isinstance(result['rdata'], list):
for rdata in result.pop('rdata'):
passivedns_object.add_attribute(**self._parse_attribute(comment, 'rdata', rdata))
for feature, value in result.items():
passivedns_object.add_attribute(**self._parse_attribute(comment, feature, value))
passivedns_object.add_reference(self.attribute['uuid'], 'related-to')
self.misp_event.add_object(passivedns_object)
def get_results(self):
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object')}
return {'results': results}
def _parse_attribute(self, comment, feature, value):
attribute = {'value': value, 'comment': comment}
attribute.update(self.passivedns_mapping[feature])
return attribute
def import_all(self, stations_short_names, interval, data_type):
object_creator = getattr(self, f'{interval}_flask_{data_type}')
if data_type == 'co2':
base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/'
elif data_type in ['c13', 'o18']:
base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/'
for station in stations_short_names:
url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv'
infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations'
filepath = self.fetch(url)
if not filepath:
continue
if infofield in self.scrippts_meta:
event = MISPEvent()
event.load_file(str(self.output_dir / self.scrippts_meta[infofield]))
location = event.get_objects_by_name('geolocation')[0]
update = True
else:
event = MISPEvent()
event.uuid = str(uuid4())
event.info = infofield
event.Orgc = self.misp_org
event.add_tag(getattr(self, f'tag_{station}')())
location = getattr(self, f'geolocation_{station}')()
event.add_object(location)
event.add_attribute('link', f'https://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}')
update = False
with self.scrippts_meta_file.open('a') as f:
writer = csv.writer(f)
writer.writerow([infofield, f'{event.uuid}.json'])
object_creator(event, location, filepath, update)
if update:
# Bump the publish timestamp
event.publish_timestamp = datetime.datetime.timestamp(datetime.datetime.now())
feed_output = event.to_feed(with_meta=False)
with (self.output_dir / f'{event.uuid}.json').open('w') as f:
# json.dump(feed_output, f, indent=2, sort_keys=True) # For testing
json.dump(feed_output, f)
def test_feed(self):
me = MISPEvent()
me.info = 'Test feed'
org = MISPOrganisation()
org.name = 'TestOrg'
org.uuid = '123478'
me.Orgc = org
me.add_attribute('ip-dst', '8.8.8.8')
obj = me.add_object(name='file')
obj.add_attributes('filename', *['foo.exe', 'bar.exe'])
h = hashlib.new('md5')
h.update(b'8.8.8.8')
hash_attr_val = h.hexdigest()
feed = me.to_feed(with_meta=True)
self.assertEqual(feed['Event']['_hashes'][0], hash_attr_val)
self.assertEqual(feed['Event']['_manifest'][me.uuid]['info'], 'Test feed')
self.assertEqual(len(feed['Event']['Object'][0]['Attribute']), 2)
class GoAmlParser():
def __init__(self):
self.misp_event = MISPEvent()
def read_xml(self, data):
self.tree = ET.fromstring(data)
def parse_xml(self):
self.first_itteration()
for t in self.tree.findall('transaction'):
self.itterate(t, 'transaction')
def first_itteration(self):
submission_date = self.tree.find('submission_date').text.split('+')[0]
self.misp_event.timestamp = int(
time.mktime(time.strptime(submission_date, "%Y-%m-%dT%H:%M:%S")))
for node in goAMLobjects['report']['nodes']:
element = self.tree.find(node)
if element is not None:
self.itterate(element, element.tag)
def itterate(self,
tree,
aml_type,
referencing_uuid=None,
relationship_type=None):
objects = goAMLobjects[aml_type]
referenced_uuid = referencing_uuid
rel = relationship_type
if aml_type not in nodes_to_ignore:
try:
mapping = goAMLmapping[aml_type]
misp_object = MISPObject(name=mapping['misp_name'])
for leaf in objects['leaves']:
element = tree.find(leaf)
if element is not None:
object_relation = mapping[element.tag]
attribute = {
'object_relation': object_relation,
'value': element.text
}
misp_object.add_attribute(**attribute)
if aml_type == 'transaction':
for node in objects['nodes']:
element = tree.find(node)
if element is not None:
self.fill_transaction(element, element.tag,
misp_object)
self.misp_event.add_object(misp_object)
last_object = self.misp_event.objects[-1]
referenced_uuid = last_object.uuid
if referencing_uuid and relationship_type:
referencing_object = self.misp_event.get_object_by_uuid(
referencing_uuid)
referencing_object.add_reference(referenced_uuid, rel,
None, **last_object)
except KeyError:
pass
for node in objects['nodes']:
element = tree.find(node)
if element is not None:
tag = element.tag
if tag in relationship_to_keep:
rel = tag[2:] if tag.startswith('t_') else tag
self.itterate(element,
element.tag,
referencing_uuid=referenced_uuid,
relationship_type=rel)
@staticmethod
def fill_transaction(element, tag, misp_object):
if 't_from' in tag:
from_funds = element.find('from_funds_code').text
from_funds_attribute = {
'object_relation': 'from-funds-code',
'value': from_funds
}
misp_object.add_attribute(**from_funds_attribute)
from_country = element.find('from_country').text
from_country_attribute = {
'object_relation': 'from-country',
'value': from_country
}
misp_object.add_attribute(**from_country_attribute)
if 't_to' in tag:
to_funds = element.find('to_funds_code').text
to_funds_attribute = {
'object_relation': 'to-funds-code',
'value': to_funds
}
misp_object.add_attribute(**to_funds_attribute)
to_country = element.find('to_country').text
to_country_attribute = {
'object_relation': 'to-country',
'value': to_country
}
misp_object.add_attribute(**to_country_attribute)
fieldnames=args.fieldnames, has_fieldnames=has_fieldnames)
objects = csv_loader.load()
if args.dump:
for o in objects:
print(o.to_json())
else:
if offline:
print('You are in offline mode, quitting.')
else:
misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert)
if args.new_event:
event = MISPEvent()
event.info = args.new_event
for o in objects:
event.add_object(**o)
new_event = misp.add_event(event)
if isinstance(new_event, str):
print(new_event)
elif 'id' in new_event:
print(f'Created new event {new_event.id}')
else:
print('Something went wrong:')
print(new_event)
else:
for o in objects:
new_object = misp.add_object(args.update_event, o)
if isinstance(new_object, str):
print(new_object)
elif new_object.attributes:
print(f'New {new_object.name} object added to {args.update_event}')
class StixParser():
def __init__(self):
self.misp_event = MISPEvent()
self.event = []
self.misp_event['Galaxy'] = []
def loadEvent(self, args):
try:
filename = os.path.join(os.path.dirname(args[0]), args[1])
tempFile = open(filename, 'r', encoding='utf-8')
self.filename = filename
event = json.loads(tempFile.read())
self.stix_version = 'stix {}'.format(event.get('spec_version'))
for o in event.get('objects'):
try:
try:
self.event.append(stix2.parse(o, allow_custom=True))
except:
self.parse_custom(o)
except:
pass
if not self.event:
print(
json.dumps({
'success':
0,
'message':
'There is no valid STIX object to import'
}))
sys.exit(1)
try:
event_distribution = args[2]
if not isinstance(event_distribution, int):
event_distribution = int(
event_distribution) if event_distribution.isdigit(
) else 5
except:
event_distribution = 5
try:
attribute_distribution = args[3]
if attribute_distribution != 'event' and not isinstance(
attribute_distribution, int):
attribute_distribution = int(
attribute_distribution
) if attribute_distribution.isdigit() else 5
except:
attribute_distribution = 5
self.misp_event.distribution = event_distribution
self.__attribute_distribution = event_distribution if attribute_distribution == 'event' else attribute_distribution
self.load_mapping()
except:
print(
json.dumps({
'success': 0,
'message': 'The STIX file could not be read'
}))
sys.exit(1)
def parse_custom(self, obj):
custom_object_type = obj.pop('type')
labels = obj['labels']
try:
@stix2.CustomObject(
custom_object_type,
[('id', stix2.properties.StringProperty(required=True)),
('x_misp_timestamp',
stix2.properties.StringProperty(required=True)),
('labels', stix2.properties.ListProperty(labels,
required=True)),
('x_misp_value',
stix2.properties.StringProperty(required=True)),
('created_by_ref',
stix2.properties.StringProperty(required=True)),
('x_misp_comment', stix2.properties.StringProperty()),
('x_misp_category', stix2.properties.StringProperty())])
class Custom(object):
def __init__(self, **kwargs):
return
custom = Custom(**obj)
except:
@stix2.CustomObject(
custom_object_type,
[('id', stix2.properties.StringProperty(required=True)),
('x_misp_timestamp',
stix2.properties.StringProperty(required=True)),
('labels', stix2.properties.ListProperty(labels,
required=True)),
('x_misp_values',
stix2.properties.DictionaryProperty(required=True)),
('created_by_ref',
stix2.properties.StringProperty(required=True)),
('x_misp_comment', stix2.properties.StringProperty()),
('x_misp_category', stix2.properties.StringProperty())])
class Custom(object):
def __init__(self, **kwargs):
return
custom = Custom(**obj)
self.event.append(stix2.parse(custom))
def load_mapping(self):
self.objects_mapping = {
'asn': {
'observable': observable_asn,
'pattern': pattern_asn
},
'domain-ip': {
'observable': observable_domain_ip,
'pattern': pattern_domain_ip
},
'email': {
'observable': self.observable_email,
'pattern': self.pattern_email
},
'file': {
'observable': observable_file,
'pattern': self.pattern_file
},
'ip-port': {
'observable': observable_ip_port,
'pattern': pattern_ip_port
},
'network-socket': {
'observable': observable_socket,
'pattern': pattern_socket
},
'process': {
'observable': observable_process,
'pattern': pattern_process
},
'registry-key': {
'observable': observable_regkey,
'pattern': pattern_regkey
},
'url': {
'observable': observable_url,
'pattern': pattern_url
},
'WindowsPEBinaryFile': {
'observable': self.observable_pe,
'pattern': self.pattern_pe
},
'x509': {
'observable': observable_x509,
'pattern': pattern_x509
}
}
def handler(self):
self.outputname = '{}.stix2'.format(self.filename)
if self.from_misp():
self.buildMispDict()
else:
self.version_attribute = {
'type': 'text',
'object_relation': 'version',
'value': self.stix_version
}
self.buildExternalDict()
self.set_distribution()
def from_misp(self):
for o in self.event:
if o._type == 'report' and 'misp:tool="misp2stix2"' in o.get(
'labels'):
index = self.event.index(o)
self.report = self.event.pop(index)
return True
return False
def buildMispDict(self):
self.parse_identity()
self.parse_report()
for o in self.event:
try:
object_type = o._type
except:
object_type = o['type']
labels = o.get('labels')
if object_type in galaxy_types:
self.parse_galaxy(o, labels)
elif object_type == 'course-of-action':
self.parse_course_of_action(o)
elif 'x-misp-object' in object_type:
if 'from_object' in labels:
self.parse_custom_object(o)
else:
self.parse_custom_attribute(o, labels)
else:
if 'from_object' in labels:
self.parse_object(o, labels)
else:
self.parse_attribute(o, labels)
def parse_identity(self):
identity = self.event.pop(0)
org = {'name': identity.get('name')}
self.misp_event['Org'] = org
def parse_report(self):
report = self.report
self.misp_event.info = report.get('name')
if report.get('published'):
self.misp_event.publish_timestamp = self.getTimestampfromDate(
report.get('published'))
if hasattr(report, 'labels'):
labels = report['labels']
for l in labels:
self.misp_event.add_tag(l)
if hasattr(report, 'external_references'):
ext_refs = report['external_references']
for e in ext_refs:
link = {"type": "link"}
comment = e.get('source_name')
try:
comment = comment.split('url - ')[1]
except:
pass
if comment:
link['comment'] = comment
link['value'] = e.get('url')
self.misp_event.add_attribute(**link)
def parse_galaxy(self, o, labels):
galaxy_type = self.get_misp_type(labels)
tag = labels[1]
value = tag.split(':')[1].split('=')[1]
galaxy_description, cluster_description = o.get('description').split(
'|')
galaxy = {
'type':
galaxy_type,
'name':
o.get('name'),
'description':
galaxy_description,
'GalaxyCluster': [{
'type': galaxy_type,
'value': value,
'tag_name': tag,
'description': cluster_description
}]
}
self.misp_event['Galaxy'].append(galaxy)
def parse_course_of_action(self, o):
misp_object = MISPObject('course-of-action')
if 'name' in o:
attribute = {
'type': 'text',
'object_relation': 'name',
'value': o.get('name')
}
misp_object.add_attribute(**attribute)
else:
return
if 'description' in o:
attribute = {
'type': 'text',
'object_relation': 'description',
'value': o.get('description')
}
misp_object.add_attribute(**attribute)
self.misp_event.add_object(**misp_object)
def parse_custom_object(self, o):
name = o.get('type').split('x-misp-object-')[1]
timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp'))
category = o.get('category')
attributes = []
values = o.get('x_misp_values')
for v in values:
attribute_type, object_relation = v.split('_')
attribute = {
'type': attribute_type,
'value': values.get(v),
'object_relation': object_relation
}
attributes.append(attribute)
misp_object = {
'name': name,
'timestamp': timestamp,
'meta-category': category,
'Attribute': attributes
}
self.misp_event.add_object(**misp_object)
def parse_custom_attribute(self, o, labels):
attribute_type = o.get('type').split('x-misp-object-')[1]
if attribute_type not in misp_types:
attribute_type = attribute_type.replace('-', '|')
timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp'))
to_ids = bool(labels[1].split('=')[1])
value = o.get('x_misp_value')
category = self.get_misp_category(labels)
attribute = {
'type': attribute_type,
'timestamp': timestamp,
'to_ids': to_ids,
'value': value,
'category': category
}
self.misp_event.add_attribute(**attribute)
def parse_object(self, o, labels):
object_type = self.get_misp_type(labels)
name = 'file' if object_type == 'WindowsPEBinaryFile' else object_type
object_category = self.get_misp_category(labels)
stix_type = o._type
misp_object = MISPObject(name)
misp_object['meta-category'] = object_category
if stix_type == 'indicator':
pattern = o.get('pattern').replace('\\\\', '\\').split(' AND ')
pattern[0] = pattern[0][1:]
pattern[-1] = pattern[-1][:-1]
attributes = self.objects_mapping[object_type]['pattern'](pattern)
if stix_type == 'observed-data':
observable = o.get('objects')
attributes = self.objects_mapping[object_type]['observable'](
observable)
if isinstance(attributes, tuple):
attributes, pe_uuid = attributes
misp_object.add_reference(pe_uuid, 'included-in')
for attribute in attributes:
misp_object.add_attribute(**attribute)
misp_object.to_ids = bool(labels[1].split('=')[1])
self.misp_event.add_object(**misp_object)
def parse_attribute(self, o, labels):
attribute_type = self.get_misp_type(labels)
attribute_category = self.get_misp_category(labels)
attribute = {'type': attribute_type, 'category': attribute_category}
stix_type = o._type
if stix_type == 'vulnerability':
value = o.get('name')
else:
if stix_type == 'indicator':
o_date = o.get('valid_from')
pattern = o.get('pattern').replace('\\\\', '\\')
value = self.parse_pattern_with_data(
pattern) if attribute_type in (
'malware-sample',
'attachment') else self.parse_pattern(pattern)
attribute['to_ids'] = True
else:
o_date = o.get('first_observed')
observable = o.get('objects')
try:
value = self.parse_observable(observable, attribute_type)
except:
print('{}: {}'.format(attribute_type, observable))
attribute['to_ids'] = False
attribute['timestamp'] = self.getTimestampfromDate(o_date)
if 'description' in o:
attribute['comment'] = o.get('description')
if isinstance(value, tuple):
value, data = value
attribute['data'] = io.BytesIO(data.encode())
attribute['value'] = value
self.misp_event.add_attribute(**attribute)
@staticmethod
def observable_email(observable):
attributes = []
addresses = {}
files = {}
for o_key, o_dict in observable.items():
part_type = o_dict._type
if part_type == 'email-addr':
addresses[o_key] = o_dict.get('value')
elif part_type == 'file':
files[o_key] = o_dict.get('name')
else:
message = dict(o_dict)
attributes.append({
'type': 'email-src',
'object_relation': 'from',
'value': addresses[message.pop('from_ref')],
'to_ids': False
})
for ref in ('to_refs', 'cc_refs'):
if ref in message:
for item in message.pop(ref):
mapping = email_mapping[ref]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': addresses[item],
'to_ids': False
})
if 'body_multipart' in message:
for f in message.pop('body_multipart'):
attributes.append({
'type': 'email-attachment',
'object_relation': 'attachment',
'value': files[f.get('body_raw_ref')],
'to_ids': False
})
for m_key, m_value in message.items():
if m_key == 'additional_header_fields':
for field_key, field_value in m_value.items():
mapping = email_mapping[field_key]
if field_key == 'Reply-To':
for rt in field_value:
attributes.append({
'type':
mapping['type'],
'object_relation':
mapping['relation'],
'value':
rt,
'to_ids':
False
})
else:
attributes.append({
'type':
mapping['type'],
'object_relation':
mapping['relation'],
'value':
field_value,
'to_ids':
False
})
else:
try:
mapping = email_mapping[m_key]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': m_value,
'to_ids': False
})
except:
if m_key.startswith("x_misp_attachment_"):
attribute_type, relation = m_key.split(
"x_misp_")[1].split("_")
attributes.append({
'type':
attribute_type,
'object_relation':
relation,
'to_ids':
False,
'value':
m_value['value'],
'data':
io.BytesIO(m_value['data'].encode())
})
elif "x_misp_" in m_key:
attribute_type, relation = m_key.split(
"x_misp_")[1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': m_value,
'to_ids': False
})
return attributes
@staticmethod
def pattern_email(pattern):
attributes = []
attachments = defaultdict(dict)
for p in pattern:
p_type, p_value = p.split(' = ')
try:
mapping = email_mapping[p_type]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': p_value[1:-1],
'to_ids': True
})
except KeyError:
if p_type.startswith("email-message:'x_misp_attachment_"):
relation, field = p_type.split('.')
relation = relation.split(':')[1][1:-1]
attachments[relation][field] = p_value[1:-1]
elif "x_misp_" in p_type:
attribute_type, relation = p_type.split(
"x_misp_")[1][:-1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': p_value[1:-1],
'to_ids': True
})
for a_key, a_dict in attachments.items():
_, _, attribute_type, relation = a_key.split('_')
attributes.append({
'type': attribute_type,
'object_relation': relation,
'to_ids': True,
'value': a_dict['value'],
'data': io.BytesIO(a_dict['data'].encode())
})
return attributes
@staticmethod
def pattern_file(pattern):
attributes = []
malware_sample = {}
for p in pattern:
p_type, p_value = p.split(' = ')
if p_type == 'artifact:payload_bin':
malware_sample['data'] = p_value
elif p_type in ("file:name", "file:hashes.'md5'"):
try:
mapping = file_mapping[p_type]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': p_value[1:-1],
'to_ids': True
})
malware_sample['filename'] = p_value[1:-1]
except KeyError:
attributes.append({
'type': 'md5',
'object_relation': 'md5',
'value': p_value[1:-1],
'to_ids': True
})
malware_sample['md5'] = p_value[1:-1]
elif 'file:hashes.' in p_type:
_, h = p_type.split('.')
h = h[1:-1]
attributes.append({
'type': h,
'object_relation': h,
'value': p_value[1:-1]
})
else:
try:
mapping = file_mapping[p_type]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': p_value[1:-1],
'to_ids': True
})
except KeyError:
if "x_misp_" in p_type:
attribute_type, relation = p_type.split(
"x_misp_")[1][:-1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': p_value[1:-1],
'to_ids': True
})
if 'data' in malware_sample:
value = "{}|{}".format(malware_sample['filename'],
malware_sample['md5'])
attributes.append({
'type':
'malware-sample',
'object_relation':
'malware-sample',
'value':
value,
'to_ids':
True,
'data':
io.BytesIO(malware_sample['data'].encode())
})
return attributes
def observable_pe(self, observable):
extension = observable['1']['extensions']['windows-pebinary-ext']
sections = extension['sections']
pe = MISPObject('pe')
pe_uuid = str(uuid.uuid4())
pe.uuid = pe_uuid
self.fill_object_attributes_observable(pe, pe_mapping, extension)
for section in sections:
pe_section = MISPObject('pe-section')
if 'hashes' in section:
for h_type, h_value in section['hashes'].items():
h_type = h_type.lower().replace('-', '')
pe_section.add_attribute(
**{
'type': h_type,
'object_relation': h_type,
'value': h_value,
'to_ids': False
})
self.fill_object_attributes_observable(pe_section,
pe_section_mapping, section)
section_uuid = str(uuid.uuid4())
pe_section.uuid = section_uuid
pe.add_reference(section_uuid, 'included-in')
self.misp_event.add_object(**pe_section)
self.misp_event.add_object(**pe)
return observable_file(observable), pe_uuid
@staticmethod
def fill_object_attributes_observable(misp_object, mapping_dict,
stix_object):
for stix_type, value in stix_object.items():
try:
mapping = mapping_dict[stix_type]
misp_object.add_attribute(
**{
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': value,
'to_ids': False
})
except KeyError:
if stix_type.startswith("x_misp_"):
attribute_type, relation = parse_custom_property(stix_type)
misp_object.add_attribute(
**{
'type': attribute_type,
'object_relation': relation[:-1],
'value': value,
'to_ids': False
})
def pattern_pe(self, pattern):
attributes = []
sections = defaultdict(dict)
pe = MISPObject('pe')
pe_uuid = str(uuid.uuid4())
pe.uuid = pe_uuid
for p in pattern:
p_type, p_value = p.split(' = ')
p_value = p_value[1:-1]
if ':extensions.' in p_type:
if '.sections[' in p_type:
p_type_list = p_type.split('.')
stix_type = "hashes.{}".format(
p_type_list[4]
[1:-1]) if '.hashes.' in p_type else p_type_list[3]
sections[p_type_list[2]][stix_type] = p_value
else:
stix_type = p_type.split('.')[-1]
try:
mapping = pe_mapping[stix_type]
pe.add_attribute(
**{
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': p_value,
'to_ids': True
})
except KeyError:
if stix_type.startswith("x_misp_"):
attribute_type, relation = parse_custom_property(
stix_type)
pe.add_attribute(
**{
'type': attribute_type,
'object_relation': relation[:-2],
'value': p_value,
'to_ids': False
})
else:
if 'file:hashes.' in p_type:
_, h = p_type.split('.')
h = h[1:-1]
attributes.append({
'type': h,
'object_relation': h,
'value': p_value,
'to_ids': True
})
else:
try:
mapping = file_mapping[p_type]
attributes.append({
'type':
mapping['type'],
'object_relation':
mapping['relation'],
'value':
p_value,
'to_ids':
True
})
except KeyError:
if "x_misp_" in p_type:
attribute_type, relation = p_type.split(
"x_misp_")[1][:-1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': p_value,
'to_ids': True
})
for _, section in sections.items():
pe_section = MISPObject('pe-section')
for stix_type, value in section.items():
if 'hashes.' in stix_type:
h_type = stix_type.split('.')[1]
pe_section.add_attribute(
**{
'type': h_type,
'object_relation': h_type,
'value': value,
'to_ids': True
})
else:
try:
mapping = pe_section_mapping[stix_type]
pe_section.add_attribute(
**{
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': value,
'to_ids': True
})
except KeyError:
if "x_misp_" in stix_type:
attribute_type, relation = stix_type.split(
"x_misp_")[1][:-1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': value,
'to_ids': True
})
section_uuid = str(uuid.uuid4())
pe_section.uuid = pe_uuid
pe.add_reference(section_uuid, 'included-in')
self.misp_event.add_object(**pe_section)
self.misp_event.add_object(**pe)
return attributes, pe_uuid
def buildExternalDict(self):
self.fetch_report()
for o in self.event:
object_type = o._type
if object_type in ('relationship', 'report'):
continue
if object_type in galaxy_types:
self.parse_external_galaxy(o)
elif object_type == 'vulnerability':
attribute = {'type': 'vulnerability', 'value': o.get('name')}
if 'description' in o:
attribute['comment'] = o.get('description')
self.misp_event.add_attribute(**attribute)
elif object_type == 'course-of-action':
self.parse_course_of_action(o)
elif object_type == 'indicator':
pattern = o.get('pattern')
self.parse_external_pattern(pattern)
attribute = {
'type': 'stix2-pattern',
'object_relation': 'stix2-pattern',
'value': pattern
}
misp_object = {
'name': 'stix2-pattern',
'meta-category': 'stix2-pattern',
'Attribute': [self.version_attribute, attribute]
}
self.misp_event.add_object(**misp_object)
def fetch_report(self):
reports = []
for o in self.event:
if o._type == 'report':
reports.append(o)
if len(reports) == 1:
self.report = reports[0]
self.parse_report()
def parse_external_galaxy(self, o):
galaxy = {'name': galaxy_types[o._type]}
if 'kill_chain_phases' in o:
galaxy['type'] = o['kill_chain_phases'][0].get('phase_name')
cluster = defaultdict(dict)
cluster['value'] = o.get('name')
cluster['description'] = o.get('description')
if 'aliases' in o:
aliases = []
for a in o.get('aliases'):
aliases.append(a)
cluster['meta']['synonyms'] = aliases
galaxy['GalaxyCluster'] = [cluster]
self.misp_event['Galaxy'].append(galaxy)
def parse_external_pattern(self, pattern):
if ' OR ' in pattern and ' AND ' not in pattern:
pattern = pattern.split('OR')
for p in pattern:
attribute = self.attribute_from_external_pattern(p)
self.misp_event.add_attribute(**attribute)
elif ' OR ' not in pattern and ' LIKE ' not in pattern:
pattern = pattern.split('AND')
if len(pattern) == 1:
attribute = self.attribute_from_external_pattern(pattern[0])
self.misp_event.add_attribute(**attribute)
@staticmethod
def attribute_from_external_pattern(pattern):
pattern_type, pattern_value = pattern.split(' = ')
pattern_type, pattern_value = pattern_type[1:].strip(
), pattern_value[1:-2].strip()
stix_type, value_type = pattern_type.split(':')
if 'hashes' in value_type and 'x509' not in stix_type:
h_type = value_type.split('.')[1]
return {'type': h_type, 'value': pattern_value}
else:
# Might cause some issues, need more examples to test
return {
'type':
external_pattern_mapping[stix_type][value_type].get('type'),
'value': pattern_value
}
def set_distribution(self):
for attribute in self.misp_event.attributes:
attribute.distribution = self.__attribute_distribution
for misp_object in self.misp_event.objects:
misp_object.distribution = self.__attribute_distribution
for attribute in misp_object.attributes:
attribute.distribution = self.__attribute_distribution
def saveFile(self):
eventDict = self.misp_event.to_json()
outputfile = '{}.stix2'.format(self.filename)
with open(outputfile, 'w') as f:
f.write(eventDict)
@staticmethod
def getTimestampfromDate(stix_date):
try:
return int(stix_date.timestamp())
except:
return int(
time.mktime(
time.strptime(
stix_date.split('+')[0], "%Y-%m-%d %H:%M:%S")))
@staticmethod
def get_misp_type(labels):
return labels[0].split('=')[1][1:-1]
@staticmethod
def get_misp_category(labels):
return labels[1].split('=')[1][1:-1]
@staticmethod
def parse_observable(observable, attribute_type):
return misp_types_mapping[attribute_type](observable, attribute_type)
@staticmethod
def parse_pattern(pattern):
if ' AND ' in pattern:
pattern_parts = pattern.split(' AND ')
if len(pattern_parts) == 3:
_, value1 = pattern_parts[2].split(' = ')
_, value2 = pattern_parts[0].split(' = ')
return '{}|{}'.format(value1[1:-2], value2[1:-1])
else:
_, value1 = pattern_parts[0].split(' = ')
_, value2 = pattern_parts[1].split(' = ')
if value1 in ("'ipv4-addr'", "'ipv6-addr'"):
return value2[1:-2]
return '{}|{}'.format(value1[1:-1], value2[1:-2])
else:
return pattern.split(' = ')[1][1:-2]
def parse_pattern_with_data(self, pattern):
if 'artifact:payload_bin' not in pattern:
return self.parse_pattern(pattern)
pattern_parts = pattern.split(' AND ')
if len(pattern_parts) == 3:
filename = pattern_parts[0].split(' = ')[1]
md5 = pattern_parts[1].split(' = ')[1]
return "{}|{}".format(
filename[1:-1],
md5[1:-1]), pattern_parts[2].split(' = ')[1][1:-2]
else:
return pattern_parts[0].split(
' = ')[1][1:-1], pattern_parts[1].split(' = ')[1][1:-2]
class AssemblyLineParser():
def __init__(self):
self.misp_event = MISPEvent()
self.results = {}
self.attribute = {'to_ids': True}
self._results_mapping = {
'NET_DOMAIN_NAME': 'domain',
'NET_FULL_URI': 'url',
'NET_IP': 'ip-dst'
}
self._file_mapping = {
'entropy': {
'type': 'float',
'object_relation': 'entropy'
},
'md5': {
'type': 'md5',
'object_relation': 'md5'
},
'mime': {
'type': 'mime-type',
'object_relation': 'mimetype'
},
'sha1': {
'type': 'sha1',
'object_relation': 'sha1'
},
'sha256': {
'type': 'sha256',
'object_relation': 'sha256'
},
'size': {
'type': 'size-in-bytes',
'object_relation': 'size-in-bytes'
},
'ssdeep': {
'type': 'ssdeep',
'object_relation': 'ssdeep'
}
}
def get_submission(self, attribute, client):
sid = attribute['value'].split('=')[-1]
try:
if not client.submission.is_completed(sid):
self.results[
'error'] = 'Submission not completed, please try again later.'
return
except Exception as e:
self.results[
'error'] = f'Something went wrong while trying to check if the submission in AssemblyLine is completed: {e.__str__()}'
return
try:
submission = client.submission.full(sid)
except Exception as e:
self.results[
'error'] = f"Something went wrong while getting the submission from AssemblyLine: {e.__str__()}"
return
self._parse_report(submission)
def finalize_results(self):
if 'error' in self.results:
return self.results
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object', 'Tag')
if (key in event and event[key])
}
return {'results': results}
def _create_attribute(self, result, attribute_type):
attribute = MISPAttribute()
attribute.from_dict(type=attribute_type,
value=result['value'],
**self.attribute)
if result['classification'] != 'UNCLASSIFIED':
attribute.add_tag(result['classification'].lower())
self.misp_event.add_attribute(**attribute)
return {
'referenced_uuid': attribute.uuid,
'relationship_type': '-'.join(result['context'].lower().split(' '))
}
def _create_file_object(self, file_info):
file_object = MISPObject('file')
filename_attribute = {'type': 'filename'}
filename_attribute.update(self.attribute)
if file_info['classification'] != "UNCLASSIFIED":
tag = {'Tag': [{'name': file_info['classification'].lower()}]}
filename_attribute.update(tag)
for feature, attribute in self._file_mapping.items():
attribute.update(tag)
file_object.add_attribute(value=file_info[feature],
**attribute)
return filename_attribute, file_object
for feature, attribute in self._file_mapping.items():
file_object.add_attribute(value=file_info[feature], **attribute)
return filename_attribute, file_object
@staticmethod
def _get_results(submission_results):
results = defaultdict(list)
for k, values in submission_results.items():
h = k.split('.')[0]
for t in values['result']['tags']:
if t['context'] is not None:
results[h].append(t)
return results
def _get_scores(self, file_tree):
scores = {}
for h, f in file_tree.items():
score = f['score']
if score > 0:
scores[h] = {'name': f['name'], 'score': score}
if f['children']:
scores.update(self._get_scores(f['children']))
return scores
def _parse_report(self, submission):
if submission['classification'] != 'UNCLASSIFIED':
self.misp_event.add_tag(submission['classification'].lower())
filtered_results = self._get_results(submission['results'])
scores = self._get_scores(submission['file_tree'])
for h, results in filtered_results.items():
if h in scores:
attribute, file_object = self._create_file_object(
submission['file_infos'][h])
print(file_object)
for filename in scores[h]['name']:
file_object.add_attribute('filename',
value=filename,
**attribute)
for reference in self._parse_results(results):
file_object.add_reference(**reference)
self.misp_event.add_object(**file_object)
def _parse_results(self, results):
references = []
for result in results:
try:
attribute_type = self._results_mapping[result['type']]
except KeyError:
continue
references.append(self._create_attribute(result, attribute_type))
return references
class TruSTARParser:
ENTITY_TYPE_MAPPINGS = {
'BITCOIN_ADDRESS': "btc",
'CIDR_BLOCK': "ip-src",
'CVE': "vulnerability",
'URL': "url",
'EMAIL_ADDRESS': "email-src",
'SOFTWARE': "filename",
'IP': "ip-src",
'MALWARE': "malware-type",
'MD5': "md5",
'REGISTRY_KEY': "regkey",
'SHA1': "sha1",
'SHA256': "sha256"
}
# Relevant fields from each TruSTAR endpoint
SUMMARY_FIELDS = ["severityLevel", "source", "score", "attributes"]
METADATA_FIELDS = ["sightings", "firstSeen", "lastSeen", "tags"]
REPORT_BASE_URL = "https://station.trustar.co/constellation/reports/{}"
CLIENT_METATAG = f"MISP-{pymisp.__version__}"
def __init__(self, attribute, config):
config['enclave_ids'] = config.get('enclave_ids',
"").strip().split(',')
config['client_metatag'] = self.CLIENT_METATAG
self.ts_client = TruStar(config=config)
self.misp_event = MISPEvent()
self.misp_attribute = MISPAttribute()
self.misp_attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.misp_attribute)
def get_results(self):
"""
Returns the MISP Event enriched with TruSTAR indicator summary data.
"""
try:
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object')
if (key in event and event[key])
}
return {'results': results}
except Exception as e:
misperrors[
'error'] += f" -- Encountered issue serializing enrichment data -- {e}"
return misperrors
def generate_trustar_link(self, entity_type, entity_value):
"""
Generates link to TruSTAR report of entity.
:param entity_type: <str> Type of entity.
:param entity_value: <str> Value of entity.
:return: <str> Link to indicator report in TruSTAR platform.
"""
report_id = b64encode(
quote(f"{entity_type}|{entity_value}").encode()).decode()
return self.REPORT_BASE_URL.format(report_id)
@staticmethod
def extract_tags(enrichment_report):
"""
Extracts tags from the enrichment report in order to add them
to the TruSTAR MISP Object. Removes tags from report to avoid
redundancy.
:param: <OrderedDict> Enrichment data.
:return: <list> List of tags.
"""
if enrichment_report and enrichment_report.get('tags'):
return [tag.get('name') for tag in enrichment_report.pop('tags')]
return None
def generate_enrichment_report(self, summary, metadata):
"""
Extracts desired fields from summary and metadata reports and
generates an enrichment report.
:param summary: <trustar.IndicatorSummary> Indicator summary report.
:param metadata: <trustar.Indicator> Indicator metadata report.
:return: <str> Enrichment report.
"""
# Preserve order of fields as they exist in SUMMARY_FIELDS and METADATA_FIELDS
enrichment_report = OrderedDict()
if summary:
summary_dict = summary.to_dict()
enrichment_report.update({
field: summary_dict[field]
for field in self.SUMMARY_FIELDS if summary_dict.get(field)
})
if metadata:
metadata_dict = metadata.to_dict()
enrichment_report.update({
field: metadata_dict[field]
for field in self.METADATA_FIELDS if metadata_dict.get(field)
})
return enrichment_report
def parse_indicator_summary(self, indicator, summary, metadata):
"""
Pulls enrichment data from the TruSTAR /indicators/summaries and /indicators/metadata endpoints
and creates a MISP trustar_report.
:param indicator: <str> Value of the attribute
:summary: <trustar.IndicatorSummary> Indicator summary response object.
:metadata: <trustar.Indicator> Indicator response object.
"""
# Verify that the indicator type is supported by TruSTAR
if summary and summary.indicator_type in self.ENTITY_TYPE_MAPPINGS:
indicator_type = summary.indicator_type
elif metadata and metadata.type in self.ENTITY_TYPE_MAPPINGS:
indicator_type = metadata.type
else:
misperrors['error'] += " -- Attribute not found or not supported"
raise Exception
try:
# Extract most relevant fields from indicator summary and metadata responses
enrichment_report = self.generate_enrichment_report(
summary, metadata)
tags = self.extract_tags(enrichment_report)
if enrichment_report:
# Create MISP trustar_report object and populate it with enrichment data
trustar_obj = MISPObject('trustar_report')
trustar_obj.add_attribute(
indicator_type,
attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type],
value=indicator)
trustar_obj.add_attribute("INDICATOR_SUMMARY",
attribute_type="text",
value=json.dumps(enrichment_report,
indent=4))
report_link = self.generate_trustar_link(
indicator_type, indicator)
trustar_obj.add_attribute("REPORT_LINK",
attribute_type="link",
value=report_link)
self.misp_event.add_object(**trustar_obj)
elif not tags:
# If enrichment report is empty and there are no tags, nothing to add to attribute
raise Exception("No relevant data found")
if tags:
for tag in tags:
self.misp_event.add_attribute_tag(tag, indicator)
except Exception as e:
misperrors[
'error'] += f" -- Error enriching attribute {indicator} -- {e}"
raise e
def handler(q=False):
if q is False:
return False
request = json.loads(q)
if not request.get('attribute') or not check_input_attribute(
request['attribute']):
return {
'error':
f'{standard_error_message}, which should contain at least a type, a value and an uuid.'
}
toquery = request['attribute']
if toquery['type'] not in mispattributes['input']:
return {'error': 'Unsupported attribute type.'}
bgpranking = BGPRanking()
value_toquery = int(
toquery['value'][2:]) if toquery['value'].startswith('AS') else int(
toquery['value'])
values = bgpranking.query(value_toquery,
date=(date.today() - timedelta(1)).isoformat())
if not values['response'] or not values['response']['asn_description']:
misperrors[
'error'] = 'There is no result about this ASN in BGP Ranking'
return misperrors
event = MISPEvent()
attribute = MISPAttribute()
attribute.from_dict(**toquery)
event.add_attribute(**attribute)
asn_object = MISPObject('asn')
asn_object.add_attribute(**{
'type': 'AS',
'object_relation': 'asn',
'value': values['meta']['asn']
})
description, country = values['response']['asn_description'].split(', ')
for relation, value in zip(('description', 'country'),
(description, country)):
asn_object.add_attribute(**{
'type': 'text',
'object_relation': relation,
'value': value
})
mapping = {
'address_family': {
'type': 'text',
'object_relation': 'address-family'
},
'date': {
'type': 'datetime',
'object_relation': 'date'
},
'position': {
'type': 'float',
'object_relation': 'position'
},
'rank': {
'type': 'float',
'object_relation': 'ranking'
}
}
bgp_object = MISPObject('bgp-ranking')
for feature in ('rank', 'position'):
bgp_attribute = {'value': values['response']['ranking'][feature]}
bgp_attribute.update(mapping[feature])
bgp_object.add_attribute(**bgp_attribute)
date_attribute = {
'value': datetime.strptime(values['meta']['date'], '%Y-%m-%d')
}
date_attribute.update(mapping['date'])
bgp_object.add_attribute(**date_attribute)
address_attribute = {'value': values['meta']['address_family']}
address_attribute.update(mapping['address_family'])
bgp_object.add_attribute(**address_attribute)
asn_object.add_reference(attribute.uuid, 'describes')
asn_object.add_reference(bgp_object.uuid, 'ranked-with')
event.add_object(asn_object)
event.add_object(bgp_object)
event = json.loads(event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object')}
return {'results': results}
class TestMISPEvent(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
def init_event(self):
self.mispevent.info = 'This is a test'
self.mispevent.distribution = 1
self.mispevent.threat_level_id = 1
self.mispevent.analysis = 1
self.mispevent.set_date("2017-12-31") # test the set date method
def test_simple(self):
with open('tests/mispevent_testfiles/simple.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event(self):
self.init_event()
self.mispevent.publish()
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_loadfile(self):
self.mispevent.load_file('tests/mispevent_testfiles/event.json')
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_tag(self):
self.init_event()
self.mispevent.add_tag('bar')
self.mispevent.add_tag(name='baz')
new_tag = MISPTag()
new_tag.from_dict(name='foo')
self.mispevent.add_tag(new_tag)
with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_attribute(self):
self.init_event()
self.mispevent.add_attribute('filename', 'bar.exe')
self.mispevent.add_attribute_tag('osint', 'bar.exe')
attr_tags = self.mispevent.get_attribute_tag('bar.exe')
self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint')
self.assertEqual(attr_tags[0].name, 'osint')
with open('tests/mispevent_testfiles/attribute.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
# Fake setting an attribute ID for testing
self.mispevent.attributes[0].id = 42
self.mispevent.delete_attribute(42)
with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_object_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}])
self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah')
self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(['filename']))
self.assertEqual(len(self.mispevent.objects[0].get_attributes_by_relation('filename')), 1)
self.mispevent.add_object(name='url', strict=True)
self.mispevent.objects[1].add_attribute('url', value='https://www.circl.lu')
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
self.mispevent.objects[0].add_reference('b', 'baz', comment='foo')
self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz')
with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168")
def test_object_level_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar')
self.mispevent.objects[0].add_tag('osint')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_malware(self):
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.init_event()
self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile)
attribute = self.mispevent.attributes[0]
self.assertEqual(attribute.malware_binary, pseudofile)
with open('tests/mispevent_testfiles/malware.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_malware(self):
self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json')
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.assertEqual(
self.mispevent.objects[0].get_attributes_by_relation('malware-sample')[0].malware_binary.read(),
pseudofile.read())
def test_sighting(self):
sighting = MISPSighting()
sighting.from_dict(value='1', type='bar', timestamp=11111111)
with open('tests/mispevent_testfiles/sighting.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_event(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
with open('tests/mispevent_testfiles/existing_event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes_existing(self):
self.mispevent.load_file('tests/mispevent_testfiles/shadow.json')
with open('tests/mispevent_testfiles/shadow.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes(self):
self.init_event()
self.mispevent.add_proposal(type='filename', value='baz.jpg')
self.mispevent.add_attribute('filename', 'bar.exe')
self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf')
with open('tests/mispevent_testfiles/proposals.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_default_attributes(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}])
self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent.objects[0].attributes[0])
self.mispevent.objects[1].add_attribute('filename', value='baz')
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_default_values(self):
self.init_event()
self.mispevent.add_object(name='whois', strict=True)
self.mispevent.objects[0].add_attribute('registrar', value='registar.example.com')
self.mispevent.objects[0].add_attribute('domain', value='domain.example.com')
self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com')
self.mispevent.objects[0].add_attribute('nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/def_param.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_not_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
def test_event_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.mispevent.info = 'blah'
self.assertTrue(self.mispevent.edited)
def test_event_tag_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.add_tag('foo')
self.assertTrue(self.mispevent.edited)
def test_event_attribute_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.mispevent.attributes[0].value = 'blah'
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertFalse(self.mispevent.attributes[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].tags[0].name = 'blah'
self.assertTrue(self.mispevent.attributes[0].tags[0].edited)
self.assertFalse(self.mispevent.attributes[0].tags[1].edited)
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited_second(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].add_tag(name='blah')
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].edited)
self.assertFalse(self.mispevent.objects[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited_tag(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].add_tag('blah')
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_by_id(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
misp_obj = self.mispevent.get_object_by_id(1556)
self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f')
def test_userdefined_object(self):
self.init_event()
self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles')
with self.assertRaises(InvalidMISPObject) as e:
# Fail on required
self.mispevent.to_json()
if sys.version_info >= (3, ):
self.assertEqual(e.exception.message, '{\'member3\'} are required.')
else:
# Python2 bullshit
self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.')
self.mispevent.objects[0].add_attribute('member3', value='foo')
with self.assertRaises(InvalidMISPObject) as e:
# Fail on requiredOneOf
self.mispevent.to_json()
self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2')
self.mispevent.objects[0].add_attribute('member1', value='bar')
self.mispevent.objects[0].add_attribute('member1', value='baz')
with self.assertRaises(InvalidMISPObject) as e:
# member1 is not a multiple
self.mispevent.to_json()
self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed')
self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2]
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
class SophosLabsApi():
def __init__(self, client_id, client_secret):
self.misp_event = MISPEvent()
self.client_id = client_id
self.client_secret = client_secret
self.authToken = f"{self.client_id}:{self.client_secret}"
self.baseurl = 'de.api.labs.sophos.com'
d = {'grant_type': 'client_credentials'}
h = {'Authorization': f"Basic {base64.b64encode(self.authToken.encode('UTF-8')).decode('ascii')}",\
'Content-Type': 'application/x-www-form-urlencoded'}
r = requests.post('https://api.labs.sophos.com/oauth2/token',
headers=h,
data=d)
if r.status_code == 200:
j = json.loads(r.text)
self.accessToken = j['access_token']
def get_result(self):
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object') if (key in event and event[key])
}
return {'results': results}
def hash_lookup(self, filehash):
sophos_object = MISPObject('SOPHOSLabs Intelix SHA256 Report')
h = {"Authorization": f"{self.accessToken}"}
r = requests.get(f"https://{self.baseurl}/lookup/files/v1/{filehash}",
headers=h)
if r.status_code == 200:
j = json.loads(r.text)
if 'reputationScore' in j:
sophos_object.add_attribute('Reputation Score',
type='text',
value=j['reputationScore'])
if 0 <= j['reputationScore'] <= 19:
sophos_object.add_attribute('Decision',
type='text',
value='This file is malicious')
if 20 <= j['reputationScore'] <= 29:
sophos_object.add_attribute(
'Decision',
type='text',
value='This file is potentially unwanted')
if 30 <= j['reputationScore'] <= 69:
sophos_object.add_attribute(
'Decision',
type='text',
value='This file is unknown and suspicious')
if 70 <= j['reputationScore'] <= 100:
sophos_object.add_attribute(
'Decision',
type='text',
value='This file is known good')
if 'detectionName' in j:
sophos_object.add_attribute('Detection Name',
type='text',
value=j['detectionName'])
else:
sophos_object.add_attribute(
'Detection Name',
type='text',
value='No name associated with this IoC')
self.misp_event.add_object(**sophos_object)
def ip_lookup(self, ip):
sophos_object = MISPObject('SOPHOSLabs Intelix IP Category Lookup')
h = {"Authorization": f"{self.accessToken}"}
r = requests.get(f"https://{self.baseurl}/lookup/ips/v1/{ip}",
headers=h)
if r.status_code == 200:
j = json.loads(r.text)
if 'category' in j:
for c in j['category']:
sophos_object.add_attribute('IP Address Categorisation',
type='text',
value=c)
else:
sophos_object.add_attribute(
'IP Address Categorisation',
type='text',
value='No category assocaited with IoC')
self.misp_event.add_object(**sophos_object)
def url_lookup(self, url):
sophos_object = MISPObject('SOPHOSLabs Intelix URL Lookup')
h = {"Authorization": f"{self.accessToken}"}
r = requests.get(
f"https://{self.baseurl}/lookup/urls/v1/{quote(url, safe='')}",
headers=h)
if r.status_code == 200:
j = json.loads(r.text)
if 'productivityCategory' in j:
sophos_object.add_attribute('URL Categorisation',
type='text',
value=j['productivityCategory'])
else:
sophos_object.add_attribute(
'URL Categorisation',
type='text',
value='No category assocaited with IoC')
if 'riskLevel' in j:
sophos_object.add_attribute('URL Risk Level',
type='text',
value=j['riskLevel'])
else:
sophos_object.add_attribute(
'URL Risk Level',
type='text',
value='No risk level associated with IoC')
if 'securityCategory' in j:
sophos_object.add_attribute('URL Security Category',
type='text',
value=j['securityCategory'])
else:
sophos_object.add_attribute(
'URL Security Category',
type='text',
value='No Security Category associated with IoC')
self.misp_event.add_object(**sophos_object)
class VulnerabilityParser():
def __init__(self, attribute, vulnerability):
self.attribute = attribute
self.vulnerability = vulnerability
self.misp_event = MISPEvent()
self.misp_event.add_attribute(**attribute)
self.references = defaultdict(list)
self.capec_features = ('id', 'name', 'summary', 'prerequisites',
'solutions')
self.vulnerability_mapping = {
'id': ('text', 'id'),
'summary': ('text', 'summary'),
'vulnerable_configuration_cpe_2_2':
('text', 'vulnerable_configuration'),
'Modified': ('datetime', 'modified'),
'Published': ('datetime', 'published'),
'references': ('link', 'references'),
'cvss': ('float', 'cvss-score')
}
self.weakness_mapping = {
'name': 'name',
'description_summary': 'description',
'status': 'status',
'weaknessabs': 'weakness-abs'
}
def get_result(self):
if self.references:
self.__build_references()
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object') if (key in event and event[key])
}
return {'results': results}
def parse_vulnerability_information(self):
vulnerability_object = MISPObject('vulnerability')
for feature in ('id', 'summary', 'Modified', 'cvss'):
value = self.vulnerability.get(feature)
if value:
attribute_type, relation = self.vulnerability_mapping[feature]
vulnerability_object.add_attribute(
relation, **{
'type': attribute_type,
'value': value
})
if 'Published' in self.vulnerability:
vulnerability_object.add_attribute(
'published', **{
'type': 'datetime',
'value': self.vulnerability['Published']
})
vulnerability_object.add_attribute(
'state', **{
'type': 'text',
'value': 'Published'
})
for feature in ('references', 'vulnerable_configuration_cpe_2_2'):
if feature in self.vulnerability:
attribute_type, relation = self.vulnerability_mapping[feature]
for value in self.vulnerability[feature]:
vulnerability_object.add_attribute(
relation, **{
'type': attribute_type,
'value': value
})
vulnerability_object.add_reference(self.attribute['uuid'],
'related-to')
self.misp_event.add_object(**vulnerability_object)
if 'cwe' in self.vulnerability and self.vulnerability[
'cwe'] != 'Unknown':
self.__parse_weakness(vulnerability_object.uuid)
if 'capec' in self.vulnerability:
self.__parse_capec(vulnerability_object.uuid)
def __build_references(self):
for object_uuid, references in self.references.items():
for misp_object in self.misp_event.objects:
if misp_object.uuid == object_uuid:
for reference in references:
misp_object.add_reference(**reference)
break
def __parse_capec(self, vulnerability_uuid):
attribute_type = 'text'
for capec in self.vulnerability['capec']:
capec_object = MISPObject('attack-pattern')
for feature in self.capec_features:
capec_object.add_attribute(
feature, **dict(type=attribute_type, value=capec[feature]))
for related_weakness in capec['related_weakness']:
attribute = dict(type='weakness',
value="CWE-{}".format(related_weakness))
capec_object.add_attribute('related-weakness', **attribute)
self.misp_event.add_object(**capec_object)
self.references[vulnerability_uuid].append(
dict(referenced_uuid=capec_object.uuid,
relationship_type='targeted-by'))
def __parse_weakness(self, vulnerability_uuid):
attribute_type = 'text'
cwe_string, cwe_id = self.vulnerability['cwe'].split('-')
cwes = requests.get(cveapi_url.replace('/cve/', '/cwe'))
if cwes.status_code == 200:
for cwe in cwes.json():
if cwe['id'] == cwe_id:
weakness_object = MISPObject('weakness')
weakness_object.add_attribute(
'id',
**dict(type=attribute_type,
value='-'.join([cwe_string, cwe_id])))
for feature, relation in self.weakness_mapping.items():
if cwe.get(feature):
weakness_object.add_attribute(
relation,
**dict(type=attribute_type,
value=cwe[feature]))
self.misp_event.add_object(**weakness_object)
self.references[vulnerability_uuid].append(
dict(referenced_uuid=weakness_object.uuid,
relationship_type='weakened-by'))
break
class Yeti():
def __init__(self, url, key, attribute):
self.misp_mapping = {
'Ip': 'ip-dst',
'Domain': 'domain',
'Hostname': 'hostname',
'Url': 'url',
'AutonomousSystem': 'AS',
'File': 'sha256'
}
self.yeti_client = pyeti.YetiApi(url=url, api_key=key)
self.attribute = attribute
self.misp_event = MISPEvent()
self.misp_event.add_attribute(**attribute)
def search(self, value):
obs = self.yeti_client.observable_search(value=value)
if obs:
return obs[0]
def get_neighboors(self, obs_id):
neighboors = self.yeti_client.neighbors_observables(obs_id)
if neighboors and 'objs' in neighboors:
links_by_id = {
link['dst']['id']: (link['description'], 'dst')
for link in neighboors['links'] if link['dst']['id'] != obs_id
}
links_by_id.update({
link['src']['id']: (link['description'], 'src')
for link in neighboors['links'] if link['src']['id'] != obs_id
})
for n in neighboors['objs']:
yield n, links_by_id[n['id']]
def parse_yeti_result(self):
obs = self.search(self.attribute['value'])
for obs_to_add, link in self.get_neighboors(obs['id']):
object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add)
if object_misp_domain_ip:
self.misp_event.add_object(object_misp_domain_ip)
continue
object_misp_url = self.__get_object_url(obs_to_add)
if object_misp_url:
self.misp_event.add_object(object_misp_url)
continue
if link[0] == 'NS record':
object_ns_record = self.__get_object_ns_record(
obs_to_add, link[1])
if object_ns_record:
self.misp_event.add_object(object_ns_record)
continue
self.__get_attribute(obs_to_add, link[0])
def get_result(self):
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object') if key in event
}
return results
def __get_attribute(self, obs_to_add, link):
try:
type_attr = self.misp_mapping[obs_to_add['type']]
value = None
if obs_to_add['type'] == 'File':
value = obs_to_add['value'].split(':')[1]
else:
value = obs_to_add['value']
attr = self.misp_event.add_attribute(value=value, type=type_attr)
attr.comment = '%s: %s' % (link, self.attribute['value'])
except KeyError:
logging.error('type not found %s' % obs_to_add['type'])
return
for t in obs_to_add['tags']:
self.misp_event.add_attribute_tag(t['name'], attr['uuid'])
def __get_object_domain_ip(self, obj_to_add):
if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname', 'domain']) or \
(obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')):
domain_ip_object = MISPObject('domain-ip')
domain_ip_object.add_attribute(self.__get_relation(obj_to_add),
obj_to_add['value'])
domain_ip_object.add_attribute(
self.__get_relation(self.attribute, is_yeti_object=False),
self.attribute['value'])
domain_ip_object.add_reference(self.attribute['uuid'],
'related_to')
return domain_ip_object
def __get_object_url(self, obj_to_add):
if (obj_to_add['type'] == 'Url' and self.attribute['type'] in [
'hostname', 'domain', 'ip-src', 'ip-dst'
]) or (obj_to_add['type'] in ('Hostname', 'Domain', 'Ip')
and self.attribute['type'] == 'url'):
url_object = MISPObject('url')
obj_relation = self.__get_relation(obj_to_add)
if obj_relation:
url_object.add_attribute(obj_relation, obj_to_add['value'])
obj_relation = self.__get_relation(self.attribute,
is_yeti_object=False)
if obj_relation:
url_object.add_attribute(obj_relation, self.attribute['value'])
url_object.add_reference(self.attribute['uuid'], 'related_to')
return url_object
def __get_object_ns_record(self, obj_to_add, link):
queried_domain = None
ns_domain = None
object_dns_record = MISPObject('dns-record')
if link == 'dst':
queried_domain = self.attribute['value']
ns_domain = obj_to_add['value']
elif link == 'src':
queried_domain = obj_to_add['value']
ns_domain = self.attribute['value']
if queried_domain and ns_domain:
object_dns_record.add_attribute('queried-domain', queried_domain)
object_dns_record.add_attribute('ns-record', ns_domain)
object_dns_record.add_reference(self.attribute['uuid'],
'related_to')
return object_dns_record
def __get_relation(self, obj, is_yeti_object=True):
if is_yeti_object:
type_attribute = self.misp_mapping[obj['type']]
else:
type_attribute = obj['type']
if type_attribute == 'ip-src' or type_attribute == 'ip-dst':
return 'ip'
elif 'domain' == type_attribute:
return 'domain'
elif 'hostname' == type_attribute:
return 'domain'
elif type_attribute == 'url':
return type_attribute
class RFEnricher:
"""Class for enriching an attribute with data from Recorded Future.
The enrichment data is returned as a custom MISP object.
"""
def __init__(self, attribute_props: dict):
self.event = MISPEvent()
self.enrichment_object = MISPObject("Recorded Future Enrichment")
description = ("An object containing the enriched attribute and "
"related entities from Recorded Future.")
self.enrichment_object.from_dict(**{
"meta-category": "misc",
"description": description,
"distribution": 0
})
# Create a copy of enriched attribute to add tags to
temp_attr = MISPAttribute()
temp_attr.from_dict(**attribute_props)
self.enriched_attribute = MISPAttribute()
self.enriched_attribute.from_dict(**{
"value": temp_attr.value,
"type": temp_attr.type,
"distribution": 0
})
self.related_attributes: List[Tuple[str, MISPAttribute]] = []
self.color_picker = RFColors()
self.galaxy_finder = GalaxyFinder()
# Mapping from MISP-type to RF-type
self.type_to_rf_category = {
"ip": "ip",
"ip-src": "ip",
"ip-dst": "ip",
"ip-src|port": "ip",
"ip-dst|port": "ip",
"domain": "domain",
"hostname": "domain",
"md5": "hash",
"sha1": "hash",
"sha256": "hash",
"uri": "url",
"url": "url",
"vulnerability": "vulnerability",
"weakness": "vulnerability",
}
# Related entities have 'Related' as part of the word and Links entities from RF
# portrayed as related attributes in MISP
self.related_attribute_types = [
"RelatedIpAddress",
"RelatedInternetDomainName",
"RelatedHash",
"RelatedEmailAddress",
"RelatedCyberVulnerability",
"IpAddress",
"InternetDomainName",
"Hash",
"EmailAddress",
"CyberVulnerability",
]
# Related entities have 'Related' as part of the word and and Links entities from RF portrayed as tags in MISP
self.galaxy_tag_types = [
"RelatedMalware",
"RelatedThreatActor",
"Threat Actor",
"MitreAttackIdentifier",
"Malware",
]
def enrich(self) -> None:
"""Run the enrichment."""
category = self.type_to_rf_category.get(self.enriched_attribute.type,
"")
enriched_attribute_value = self.enriched_attribute.value
# If enriched attribute has a port we need to remove that port
# since RF do not support enriching ip addresses with port
if self.enriched_attribute.type in ["ip-src|port", "ip-dst|port"]:
enriched_attribute_value = enriched_attribute_value.split("|")[0]
json_response = GLOBAL_REQUEST_HANDLER.rf_lookup(
category, enriched_attribute_value)
response = json.loads(json_response.content)
try:
# Add risk score and risk rules as tags to the enriched attribute
risk_score = response["data"]["risk"]["score"]
hex_color = self.color_picker.riskscore_color(risk_score)
tag_name = f'recorded-future:risk-score="{risk_score}"'
self.add_tag(tag_name, hex_color)
risk_criticality = response["data"]["risk"]["criticalityLabel"]
hex_color = self.color_picker.criticality_color(risk_criticality)
tag_name = f'recorded-future:criticality="{risk_criticality}"'
self.add_tag(tag_name, hex_color)
for evidence in response["data"]["risk"]["evidenceDetails"]:
risk_rule = evidence["rule"]
criticality = evidence["criticality"]
hex_color = self.color_picker.riskrule_color(criticality)
tag_name = f'recorded-future:risk-rule="{risk_rule}"'
self.add_tag(tag_name, hex_color)
links_data = response["data"].get("links", {}).get("hits")
# Check if we have error in links response. If yes, then user do not have right module enabled in token
links_access_error = response["data"].get("links", {}).get("error")
galaxy_tags = []
if not links_access_error:
for hit in links_data:
for section in hit["sections"]:
for sec_list in section["lists"]:
entity_type = sec_list["type"]["name"]
for entity in sec_list["entities"]:
if entity_type in self.galaxy_tag_types:
galaxy = self.galaxy_finder.find_galaxy_match(
entity["name"], entity_type)
if galaxy and galaxy not in galaxy_tags:
galaxy_tags.append(galaxy)
else:
self.add_attribute(entity["name"],
entity_type)
else:
# Retrieve related entities
for related_entity in response["data"]["relatedEntities"]:
related_type = related_entity["type"]
if related_type in self.related_attribute_types:
# Related entities returned as additional attributes
for related in related_entity["entities"]:
# filter those entities that have count bigger than 4, to reduce noise
# because there can be a huge list of related entities
if int(related["count"]) > 4:
indicator = related["entity"]["name"]
self.add_attribute(indicator, related_type)
elif related_type in self.galaxy_tag_types:
# Related entities added as galaxy-tags to the enriched attribute
galaxy_tags = []
for related in related_entity["entities"]:
# filter those entities that have count bigger than 4, to reduce noise
# because there can be a huge list of related entities
if int(related["count"]) > 4:
indicator = related["entity"]["name"]
galaxy = self.galaxy_finder.find_galaxy_match(
indicator, related_type)
# Handle deduplication of galaxy tags
if galaxy and galaxy not in galaxy_tags:
galaxy_tags.append(galaxy)
for galaxy in galaxy_tags:
self.add_tag(galaxy)
except KeyError:
misperrors[
"error"] = "Unexpected format in Recorded Future api response."
raise
def add_attribute(self, indicator: str, indicator_type: str) -> None:
"""Helper method for adding an indicator to the attribute list."""
out_type = self.get_output_type(indicator_type, indicator)
attribute = MISPAttribute()
attribute.from_dict(**{
"value": indicator,
"type": out_type,
"distribution": 0
})
self.related_attributes.append((indicator_type, attribute))
def add_tag(self, tag_name: str, hex_color: str = None) -> None:
"""Helper method for adding a tag to the enriched attribute."""
tag = MISPTag()
tag_properties = {"name": tag_name}
if hex_color:
tag_properties["colour"] = hex_color
tag.from_dict(**tag_properties)
self.enriched_attribute.add_tag(tag)
def get_output_type(self, related_type: str, indicator: str) -> str:
"""Helper method for translating a Recorded Future related type to a MISP output type."""
output_type = "text"
if related_type in ["RelatedIpAddress", "IpAddress"]:
output_type = "ip-dst"
elif related_type in [
"RelatedInternetDomainName", "InternetDomainName"
]:
output_type = "domain"
elif related_type in ["RelatedHash", "Hash"]:
hash_len = len(indicator)
if hash_len == 64:
output_type = "sha256"
elif hash_len == 40:
output_type = "sha1"
elif hash_len == 32:
output_type = "md5"
elif related_type in ["RelatedEmailAddress", "EmailAddress"]:
output_type = "email-src"
elif related_type in [
"RelatedCyberVulnerability", "CyberVulnerability"
]:
signature = indicator.split("-")[0]
if signature == "CVE":
output_type = "vulnerability"
elif signature == "CWE":
output_type = "weakness"
elif related_type == "MalwareSignature":
output_type = "malware-sample"
elif related_type == "Organization":
output_type = "target-org"
elif related_type == "Username":
output_type = "target-user"
return output_type
def get_results(self) -> dict:
"""Build and return the enrichment results."""
self.enrichment_object.add_attribute("Enriched attribute",
**self.enriched_attribute)
for related_type, attribute in self.related_attributes:
self.enrichment_object.add_attribute(related_type, **attribute)
self.event.add_object(**self.enrichment_object)
event = json.loads(self.event.to_json())
result = {key: event[key] for key in ["Object"] if key in event}
return {"results": result}
class APIVoidParser():
def __init__(self, attribute):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.url = 'https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&'
def get_results(self):
if hasattr(self, 'result'):
return self.result
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object')}
return {'results': results}
def parse_domain(self, apikey):
feature = 'dnslookup'
if requests.get(f'{self.url.format(feature, apikey)}stats').json(
)['credits_remained'] < 0.13:
self.result = {
'error':
'You do not have enough APIVoid credits to proceed your request.'
}
return
mapping = {
'A': 'resolution-of',
'MX': 'mail-server-of',
'NS': 'server-name-of'
}
dnslookup = requests.get(
f'{self.url.format(feature, apikey)}action=dns-any&host={self.attribute.value}'
).json()
for item in dnslookup['data']['records']['items']:
record_type = item['type']
try:
relationship = mapping[record_type]
except KeyError:
continue
self._handle_dns_record(item, record_type, relationship)
ssl = requests.get(
f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}'
).json()
self._parse_ssl_certificate(ssl['data']['certificate'])
def _handle_dns_record(self, item, record_type, relationship):
dns_record = MISPObject('dns-record')
dns_record.add_attribute('queried-domain',
type='domain',
value=item['host'])
attribute_type, feature = ('ip-dst',
'ip') if record_type == 'A' else ('domain',
'target')
dns_record.add_attribute(f'{record_type.lower()}-record',
type=attribute_type,
value=item[feature])
dns_record.add_reference(self.attribute.uuid, relationship)
self.misp_event.add_object(**dns_record)
def _parse_ssl_certificate(self, certificate):
x509 = MISPObject('x509')
fingerprint = 'x509-fingerprint-sha1'
x509.add_attribute(fingerprint,
type=fingerprint,
value=certificate['fingerprint'])
x509_mapping = {
'subject': {
'name': ('text', 'subject')
},
'issuer': {
'common_name': ('text', 'issuer')
},
'signature': {
'serial': ('text', 'serial-number')
},
'validity': {
'valid_from': ('datetime', 'validity-not-before'),
'valid_to': ('datetime', 'validity-not-after')
}
}
certificate = certificate['details']
for feature, subfeatures in x509_mapping.items():
for subfeature, mapping in subfeatures.items():
attribute_type, relation = mapping
x509.add_attribute(relation,
type=attribute_type,
value=certificate[feature][subfeature])
x509.add_reference(self.attribute.uuid, 'seen-by')
self.misp_event.add_object(**x509)
class VirusTotalParser(object):
def __init__(self, apikey, limit):
self.apikey = apikey
self.limit = limit
self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report"
self.misp_event = MISPEvent()
self.parsed_objects = {}
self.input_types_mapping = {
'ip-src': self.parse_ip,
'ip-dst': self.parse_ip,
'domain': self.parse_domain,
'hostname': self.parse_domain,
'md5': self.parse_hash,
'sha1': self.parse_hash,
'sha256': self.parse_hash,
'url': self.parse_url
}
self.proxies = None
def query_api(self, attribute):
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
return self.input_types_mapping[self.attribute.type](
self.attribute.value, recurse=True)
def get_result(self):
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object') if (key in event and event[key])
}
return {'results': results}
################################################################################
#### Main parsing functions #### # noqa
################################################################################
def parse_domain(self, domain, recurse=False):
req = requests.get(self.base_url.format('domain'),
params={
'apikey': self.apikey,
'domain': domain
},
proxies=self.proxies)
if req.status_code != 200:
return req.status_code
req = req.json()
hash_type = 'sha256'
whois = 'whois'
feature_types = {
'communicating': 'communicates-with',
'downloaded': 'downloaded-from',
'referrer': 'referring'
}
siblings = (self.parse_siblings(domain)
for domain in req['domain_siblings'])
uuid = self.parse_resolutions(
req['resolutions'],
req['subdomains'] if 'subdomains' in req else None, siblings)
for feature_type, relationship in feature_types.items():
for feature in ('undetected_{}_samples', 'detected_{}_samples'):
for sample in req.get(feature.format(feature_type),
[])[:self.limit]:
status_code = self.parse_hash(sample[hash_type], False,
uuid, relationship)
if status_code != 200:
return status_code
if req.get(whois):
whois_object = MISPObject(whois)
whois_object.add_attribute('text', type='text', value=req[whois])
self.misp_event.add_object(**whois_object)
return self.parse_related_urls(req, recurse, uuid)
def parse_hash(self, sample, recurse=False, uuid=None, relationship=None):
req = requests.get(self.base_url.format('file'),
params={
'apikey': self.apikey,
'resource': sample
},
proxies=self.proxies)
status_code = req.status_code
if req.status_code == 200:
req = req.json()
vt_uuid = self.parse_vt_object(req)
file_attributes = []
for hash_type in ('md5', 'sha1', 'sha256'):
if req.get(hash_type):
file_attributes.append({
'type': hash_type,
'object_relation': hash_type,
'value': req[hash_type]
})
if file_attributes:
file_object = MISPObject('file')
for attribute in file_attributes:
file_object.add_attribute(**attribute)
file_object.add_reference(vt_uuid, 'analyzed-with')
if uuid and relationship:
file_object.add_reference(uuid, relationship)
self.misp_event.add_object(**file_object)
return status_code
def parse_ip(self, ip, recurse=False):
req = requests.get(self.base_url.format('ip-address'),
params={
'apikey': self.apikey,
'ip': ip
},
proxies=self.proxies)
if req.status_code != 200:
return req.status_code
req = req.json()
if req.get('asn'):
asn_mapping = {
'network': ('ip-src', 'subnet-announced'),
'country': ('text', 'country')
}
asn_object = MISPObject('asn')
asn_object.add_attribute('asn', type='AS', value=req['asn'])
for key, value in asn_mapping.items():
if req.get(key):
attribute_type, relation = value
asn_object.add_attribute(relation,
type=attribute_type,
value=req[key])
self.misp_event.add_object(**asn_object)
uuid = self.parse_resolutions(
req['resolutions']) if req.get('resolutions') else None
return self.parse_related_urls(req, recurse, uuid)
def parse_url(self, url, recurse=False, uuid=None):
req = requests.get(self.base_url.format('url'),
params={
'apikey': self.apikey,
'resource': url
},
proxies=self.proxies)
status_code = req.status_code
if req.status_code == 200:
req = req.json()
vt_uuid = self.parse_vt_object(req)
if not recurse:
feature = 'url'
url_object = MISPObject(feature)
url_object.add_attribute(feature, type=feature, value=url)
url_object.add_reference(vt_uuid, 'analyzed-with')
if uuid:
url_object.add_reference(uuid, 'hosted-in')
self.misp_event.add_object(**url_object)
return status_code
################################################################################
#### Additional parsing functions #### # noqa
################################################################################
def parse_related_urls(self, query_result, recurse, uuid=None):
if recurse:
for feature in ('detected_urls', 'undetected_urls'):
if feature in query_result:
for url in query_result[feature]:
value = url['url'] if isinstance(url, dict) else url[0]
status_code = self.parse_url(value, False, uuid)
if status_code != 200:
return status_code
else:
for feature in ('detected_urls', 'undetected_urls'):
if feature in query_result:
for url in query_result[feature]:
value = url['url'] if isinstance(url, dict) else url[0]
self.misp_event.add_attribute('url', value)
return 200
def parse_resolutions(self, resolutions, subdomains=None, uuids=None):
domain_ip_object = MISPObject('domain-ip')
if self.attribute.type in ('domain', 'hostname'):
domain_ip_object.add_attribute('domain',
type='domain',
value=self.attribute.value)
attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address')
else:
domain_ip_object.add_attribute('ip',
type='ip-dst',
value=self.attribute.value)
attribute_type, relation, key = ('domain', 'domain', 'hostname')
for resolution in resolutions:
domain_ip_object.add_attribute(relation,
type=attribute_type,
value=resolution[key])
if subdomains:
for subdomain in subdomains:
attribute = MISPAttribute()
attribute.from_dict(**dict(type='domain', value=subdomain))
self.misp_event.add_attribute(**attribute)
domain_ip_object.add_reference(attribute.uuid, 'subdomain')
if uuids:
for uuid in uuids:
domain_ip_object.add_reference(uuid, 'sibling-of')
self.misp_event.add_object(**domain_ip_object)
return domain_ip_object.uuid
def parse_siblings(self, domain):
attribute = MISPAttribute()
attribute.from_dict(**dict(type='domain', value=domain))
self.misp_event.add_attribute(**attribute)
return attribute.uuid
def parse_vt_object(self, query_result):
if query_result['response_code'] == 1:
vt_object = MISPObject('virustotal-report')
vt_object.add_attribute('permalink',
type='link',
value=query_result['permalink'])
detection_ratio = '{}/{}'.format(query_result['positives'],
query_result['total'])
vt_object.add_attribute('detection-ratio',
type='text',
value=detection_ratio,
disable_correlation=True)
self.misp_event.add_object(**vt_object)
return vt_object.uuid
def set_proxy_settings(self, config: dict) -> dict:
"""Returns proxy settings in the requests format.
If no proxy settings are set, return None."""
proxies = None
host = config.get('proxy_host')
port = config.get('proxy_port')
username = config.get('proxy_username')
password = config.get('proxy_password')
if host:
if not port:
misperrors['error'] = 'The virustotal_proxy_host config is set, ' \
'please also set the virustotal_proxy_port.'
raise KeyError
parsed = urlparse(host)
if 'http' in parsed.scheme:
scheme = 'http'
else:
scheme = parsed.scheme
netloc = parsed.netloc
host = f'{netloc}:{port}'
if username:
if not password:
misperrors['error'] = 'The virustotal_proxy_username config is set, ' \
'please also set the virustotal_proxy_password.'
raise KeyError
auth = f'{username}:{password}'
host = auth + '@' + host
proxies = {
'http': f'{scheme}://{host}',
'https': f'{scheme}://{host}'
}
self.proxies = proxies
return True
