def _dump(self, event=None):
event_path = os.path.join(self.cur_path, 'misp_events')
if not os.path.exists(event_path):
os.makedirs(event_path)
if not event:
to_dump = __sessions__.current.misp_event.event
elif isinstance(event, MISPEvent):
to_dump = event
else:
to_dump = MISPEvent()
to_dump.load(event)
if to_dump.id:
filename = str(to_dump.id)
elif (__sessions__.is_attached_misp(True) and
__sessions__.current.misp_event.current_dump_file):
filename = __sessions__.current.misp_event.current_dump_file
else:
i = 1
while True:
filename = 'new_event_{}.json'.format(i)
if not os.path.exists(os.path.join(event_path, filename)):
break
i += 1
path = os.path.join(event_path, filename)
with open(path, 'w') as f:
f.write(to_dump.to_json())
self.log('success', '{} stored successfully.'.format(filename.rstrip('.json')))
return filename
def create_event(self):
if self.args.threat is not None:
# Dirty trick to keep consistency in the module: the threat level in the upload
# API can go from 0 import to 3 but it is 1 to 4 in the event mgmt API.
# It will be fixed in a near future, in the meantime, we do that:
self.args.threat += 1
if not self.args.info:
self.log('error', 'Info field is required for a new event')
info = ' '.join(self.args.info)
# Check if the following arguments have been set (and correctly set). If not, take the config values
self.args.distrib = self.distribution if self.args.distrib is None else self.args.distrib
self.args.sharing = self.sharinggroup if self.args.sharing is None else self.args.sharing
if self.args.sharing and self.args.distrib != 4:
self.args.sharing = None
self.log('info', "Sharing group can only be set if distribution is 4. Clearing set value")
misp_event = MISPEvent()
misp_event.set_all_values(info=info, distribution=self.args.distrib,
sharing_group_id=self.args.sharing, threat_level_id=self.args.threat,
analysis=self.args.analysis, date=self.args.date)
self._search_local_hashes(misp_event)
if self.offline_mode:
# New event created locally, no ID
__sessions__.current.misp_event.current_dump_file = self._dump()
__sessions__.current.misp_event.offline()
else:
misp_event = self.misp.add_event(json.dumps(misp_event, cls=EncodeUpdate))
if self._has_error_message(misp_event):
return
__sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode))
self._dump()
def load_openioc(openioc):
# Takes a opened file, or a string
if not has_bs4:
raise Exception('You need to install BeautifulSoup: pip install bs4')
misp_event = MISPEvent()
iocreport = BeautifulSoup(openioc, "html.parser")
# Set event fields
info = extract_field(iocreport, 'short_description')
if info:
misp_event.info = info
date = extract_field(iocreport, 'authored_date')
if date:
misp_event.set_date(date)
# Set special attributes
description = extract_field(iocreport, 'description')
if description:
if not misp_event.info:
misp_event.info = description
else:
misp_event.add_attribute('comment', description)
if not misp_event.info:
misp_event.info = 'OpenIOC import'
author = extract_field(iocreport, 'authored_by')
if author:
misp_event.add_attribute('comment', author)
misp_event = set_all_attributes(iocreport, misp_event)
return misp_event
def create_event(self):
if self.args.threat is not None:
# Dirty trick to keep consistency in the module: the threat level in the upload
# API can go from 0 import to 3 but it is 1 to 4 in the event mgmt API.
# It will be fixed in a near future, in the meantime, we do that:
self.args.threat += 1
if not self.args.info:
self.log('error', 'Info field is required for a new event')
info = ' '.join(self.args.info)
misp_event = MISPEvent()
misp_event.set_all_values(info=info, distribution=self.args.distrib,
threat_level_id=self.args.threat, analysis=self.args.analysis,
date=self.args.date)
self._search_local_hashes(misp_event)
if self.offline_mode:
# New event created locally, no ID
__sessions__.current.misp_event.current_dump_file = self._dump()
__sessions__.current.misp_event.offline()
else:
misp_event = self.misp.add_event(json.dumps(misp_event, cls=EncodeUpdate))
if self._has_error_message(misp_event):
return
__sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode))
self._dump()
def test_eventObject(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
misp_event = MISPEvent(pymisp.describe_types)
misp_event.load(open('tests/57c4445b-c548-4654-af0b-4be3950d210f.json', 'r').read())
json.dumps(misp_event, cls=EncodeUpdate)
json.dumps(misp_event, cls=EncodeFull)
def download(self):
if self.offline_mode:
self.log('error', 'Offline mode, unable to dodnload a sample')
return
ok = False
data = None
if self.args.hash:
ok, data = self.misp.download_samples(sample_hash=self.args.hash)
elif self.args.list is not None:
list_events = []
if len(self.args.list) == 0:
event_path = os.path.join(self.cur_path, 'misp_events')
for eid, path, title in self._get_local_events(event_path):
list_events.append(eid)
else:
list_events = self.args.list
all_data = []
for eid in list_events:
me = MISPEvent()
me.load(self.misp.get(eid))
ok, data = self.misp.download_samples(event_id=me.id)
if not ok:
self.log('error', data)
continue
if data:
all_data += data
data = all_data
else:
event_id = self._get_eventid()
if event_id is None:
return
ok, data = self.misp.download_samples(event_id=event_id)
if not ok:
self.log('error', data)
return
to_print = []
samples_path = os.path.join(self.cur_path, 'misp_samples')
for d in data:
eid, filename, payload = d
path = os.path.join(samples_path, eid, filename)
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
with open(path, 'wb') as f:
f.write(payload.getvalue())
to_print.append((eid, path))
if len(to_print) == 1:
self.log('success', 'The sample has been downloaded from Event {}'.format(to_print[0][0]))
event = self.misp.get(to_print[0][0])
if not self._has_error_message(event):
return __sessions__.new(to_print[0][1], MispEvent(event, self.offline_mode))
elif len(to_print) > 1:
self.log('success', 'The following files have been downloaded:')
self._display_tmp_files()
else:
self.log('warning', 'No samples available.')
def create_massive_dummy_events(misp, nbattribute):
event = MISPEvent()
event.info = 'massive dummy event'
event = misp.add_event(event)
print(event)
eventid = event.id
distribution = '0'
functions = [floodtxt, floodip, flooddomain, flooddomainip, floodemail, floodattachment]
for i in range(nbattribute):
choice = randint(0, 5)
if choice == 5:
floodattachment(misp, eventid, distribution, False, 'Payload delivery', '', event.info, event.analysis, event.threat_level_id)
else:
functions[choice](misp, event)
def from_remote(self, event_id):
from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
misp = PyMISP(misp_url, misp_key, misp_verifycert)
result = misp.get(event_id)
self.misp_event = MISPEvent()
self.misp_event.load(result)
def _change_event(self):
if self.offline_mode:
self._dump()
else:
if __sessions__.current.misp_event.event.id:
event = self.misp.update(__sessions__.current.misp_event.event)
else:
event = self.misp.add_event(__sessions__.current.misp_event.event)
if self._has_error_message(event):
return
try:
me = MISPEvent()
me.load(event)
self._check_add(me)
except Exception as e:
self.log('error', e)
def test_batch_OSINT_events(self):
# Test case ONLY for manual testing. Needs to download a full list of OSINT events !
if self.check_python_2():
self.assertTrue(True)
elif not manual_testing:
self.assertTrue(True)
else:
self.init_event()
file_nb = str(len(os.listdir(self.test_batch_folder)))
i = 0
t = time.time()
for curr_file in os.listdir(self.test_batch_folder):
self.mispevent = MISPEvent()
file_path = self.test_batch_folder + curr_file
print("Current file : " + file_path + " " + str(i) + " over " + file_nb)
i += 1
self.mispevent.load_file(file_path)
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
self.storage_folder_OSINT + curr_file + ".pdf")
print("Elapsed time : " + str(time.time() - t))
def test_batch_OSINT_with_config_events(self):
# Test case ONLY for manual testing. Needs to download a full list of OSINT events !
if self.check_python_2():
self.assertTrue(True)
elif not manual_testing:
self.assertTrue(True)
else:
self.init_event()
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
config[self.moduleconfig[2]] = True
config[self.moduleconfig[3]] = True
config[self.moduleconfig[4]] = True
config[self.moduleconfig[5]] = True
file_nb = str(len(os.listdir(self.test_batch_folder)))
i = 0
t = time.time()
for curr_file in os.listdir(self.test_batch_folder):
self.mispevent = MISPEvent()
file_path = self.test_batch_folder + curr_file
print("Current file : " + file_path + " " + str(i) + " over " + file_nb)
i += 1
self.mispevent.load_file(file_path)
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder_OSINT + curr_file + ".pdf")
print("Elapsed time : " + str(time.time() - t))
class MispEvent(object):
def __init__(self, event, offline=False):
if isinstance(event, MISPEvent):
self.event = event
else:
self.event = MISPEvent()
if isinstance(event, six.string_types) and os.path.exists(event):
self.event.load_file(event)
else:
self.event.load(event)
self.off = offline
if self.event.id:
self.current_dump_file = '{}.json'.format(self.event.id)
else:
self.current_dump_file = None
def online(self):
self.off = False
def offline(self):
self.off = True
def get_all_ips(self):
return [a.value for a in self.event.attributes if a.type in ['ip-dst', 'ip-src']]
def get_all_domains(self):
return [a.value for a in self.event.attributes if a.type in ['domain', 'hostname']]
def get_all_urls(self):
return [a.value for a in self.event.attributes if a.type == 'url']
def get_all_hashes(self):
event_hashes = []
sample_hashes = []
for a in self.event.attributes:
h = None
if a.type in ('md5', 'sha1', 'sha256'):
h = a.value
event_hashes.append(h)
elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'):
h = a.value.split('|')[1]
event_hashes.append(h)
elif a.type == 'malware-sample':
h = a.value.split('|')[1]
sample_hashes.append(h)
return event_hashes, sample_hashes
def __init__(self, event, offline=False):
if isinstance(event, MISPEvent):
self.event = event
else:
self.event = MISPEvent()
self.event.load(event)
self.off = offline
if self.event.id:
self.current_dump_file = '{}.json'.format(self.event.id)
else:
self.current_dump_file = None
def _search(self, query):
if self.offline_mode:
self.log('error', 'Offline mode, unable to search')
return
result = self.misp.search_all(query)
if self._has_error_message(result):
return
self.log('success', '{} matches on the following events:'.format(query))
for e in result['response']:
nb_samples = 0
nb_hashes = 0
me = MISPEvent()
me.load(e)
for a in me.attributes + [attribute for obj in me.objects for attribute in obj.attributes]:
if a.type == 'malware-sample':
nb_samples += 1
if a.type in ('md5', 'sha1', 'sha256', 'filename|md5', 'filename|sha1', 'filename|sha256'):
nb_hashes += 1
self.log('item', '{} ({} samples, {} hashes) - {}{}{}'.format(me.info, nb_samples, nb_hashes, self.url, '/events/view/', me.id))
def create_new_event():
me = MISPEvent()
me.info = "Fail2Ban blocking"
me.add_tag(args.tag)
start = datetime.now()
me.add_attribute('datetime', start.isoformat(), comment='Start Time')
return me
def _search_local_hashes(self, event, open_session=True):
local = []
samples_count = 0
if isinstance(event, MISPEvent):
misp_event = event
elif event.get('Event') is None:
self.log('error', event)
return
else:
misp_event = MISPEvent()
misp_event.load(event)
if not hasattr(misp_event, 'id'):
# The event doesn't exists upstream, breaking.
return
for a in misp_event.attributes + [attribute for obj in misp_event.objects for attribute in obj.attributes]:
row = None
if a.type == 'malware-sample':
samples_count += 1
if a.type in ('md5', 'sha1', 'sha256'):
row = Database().find(key=a.type, value=a.value)
elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'):
row = Database().find(key=a.type.split('|')[1], value=a.value.split('|')[1])
elif a.type == 'malware-sample':
row = Database().find(key='md5', value=a.value.split('|')[1])
if row:
local.append(row[0])
self.log('info', 'Event {} contains {} samples.'.format(misp_event.id, samples_count))
if not open_session:
return
shas = set([l.sha256 for l in local])
if len(shas) == 1:
__sessions__.new(get_sample_path(shas.pop()), MispEvent(misp_event, self.offline_mode))
elif len(shas) > 1:
self.log('success', 'The following samples are in this viper instance:')
__sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode))
for s in shas:
self.log('item', s)
else:
__sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode))
self.log('info', 'No known (in Viper) samples in that event.')
def __init__(self, event, offline=False):
if isinstance(event, MISPEvent):
self.event = event
else:
self.event = MISPEvent()
if isinstance(event, six.string_types) and os.path.exists(event):
self.event.load_file(event)
else:
self.event.load(event)
self.off = offline
if self.event.id:
self.current_dump_file = '{}.json'.format(self.event.id)
else:
self.current_dump_file = None
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
if not manual_testing:
self.root = "tests/"
else:
self.root = ""
self.test_folder = self.root + "reportlab_testfiles/"
self.test_batch_folder = self.root + "OSINT_output/"
self.storage_folder_OSINT = self.root + "OSINT_PDF/"
self.test_image_folder = self.root + "image_json/"
self.storage_folder = self.root + "reportlab_testoutputs/"
self.storage_image_folder = self.root + "reportlab_test_image_outputs/"
self.moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description",
"Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"]
def load_openioc(openioc):
if not has_bs4:
raise Exception('You need to install BeautifulSoup: pip install bs4')
misp_event = MISPEvent()
with open(openioc, "r") as ioc_file:
iocreport = BeautifulSoup(ioc_file, "lxml")
# Set event fields
info = extract_field(iocreport, 'short_description')
if info:
misp_event.info = info
date = extract_field(iocreport, 'authored_date')
if date:
misp_event.set_date(date)
# Set special attributes
description = extract_field(iocreport, 'description')
if description:
misp_event.add_attribute('comment', description)
author = extract_field(iocreport, 'authored_by')
if author:
misp_event.add_attribute('comment', author)
misp_event = set_all_attributes(iocreport, misp_event)
return misp_event
def store(self):
try:
event_path = os.path.join(self.cur_path, 'misp_events')
if not os.path.exists(event_path):
os.mkdir(event_path)
if self.args.list:
header = ['Event ID', 'Title']
rows = []
for eid, path, title in self._get_local_events(event_path):
rows.append((eid, title))
self.log('table', dict(header=header, rows=sorted(rows, key=lambda i: (int(i[0].split('_')[-1])))))
elif self.args.update:
if self.offline_mode:
self.log('error', 'Offline mode, cannot update locally stored events.')
return
for eid, path, title in self._get_local_events(event_path):
event = self.misp.get(eid)
with open(path, 'w') as f:
f.write(json.dumps(event))
self.log('success', '{} updated successfully.'.format(eid))
elif self.args.sync:
if self.offline_mode:
self.log('error', 'Offline mode, cannot synchronize locally stored events.')
return
for eid, path, title in self._get_local_events(event_path):
__sessions__.close()
event = MISPEvent()
event.load(path)
if 'new_event_' in path:
event = self.misp.add_event(json.dumps(event, cls=EncodeUpdate))
try:
self._dump(event)
os.remove(path)
except Exception as e:
self.log('error', 'Unable to create new event: {}.'.format(e))
else:
eid = event.id
try:
event = self.misp.update(event._json())
except Exception as e:
self.log('error', 'Unable to update event {}: {}.'.format(eid, e))
if self._has_error_message(event):
return
elif self.args.delete:
path = os.path.join(event_path, '{}.json'.format(self.args.delete))
if os.path.exists(path):
os.remove(path)
self.log('success', '{} removed successfully.'.format(self.args.delete))
else:
self.log('error', '{} does not exists.'.format(self.args.delete))
elif self.args.open:
filename = '{}.json'.format(self.args.open)
path = os.path.join(event_path, filename)
if os.path.exists(path):
try:
with open(path, 'r') as f:
e_json = json.load(f)
__sessions__.new(misp_event=MispEvent(e_json, self.offline_mode))
__sessions__.current.misp_event.current_dump_file = filename
except Exception as e:
self.log('error', 'Unable to open {}: {}'.format(path, e))
else:
self.log('error', '{} does not exists.'.format(self.args.open))
elif __sessions__.is_attached_misp():
self._dump()
except IOError as e:
self.log('error', e.strerror)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pymisp import ExpandedPyMISP, MISPEvent
from keys import misp_url, misp_key, misp_verifycert
import argparse
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Update a MISP event.")
parser.add_argument("-e",
"--event",
required=True,
help="Event ID to update.")
parser.add_argument("-i", "--input", required=True, help="Input file")
args = parser.parse_args()
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
me = MISPEvent()
me.load_file(args.input)
result = misp.update_event(args.event, me)
class Yeti():
def __init__(self, url, key, attribute):
self.misp_mapping = {
'Ip': 'ip-dst',
'Domain': 'domain',
'Hostname': 'hostname',
'Url': 'url',
'AutonomousSystem': 'AS',
'File': 'sha256'
}
self.yeti_client = pyeti.YetiApi(url=url, api_key=key)
self.attribute = attribute
self.misp_event = MISPEvent()
self.misp_event.add_attribute(**attribute)
def search(self, value):
obs = self.yeti_client.observable_search(value=value)
if obs:
return obs[0]
def get_neighboors(self, obs_id):
neighboors = self.yeti_client.neighbors_observables(obs_id)
if neighboors and 'objs' in neighboors:
links_by_id = {
link['dst']['id']: (link['description'], 'dst')
for link in neighboors['links'] if link['dst']['id'] != obs_id
}
links_by_id.update({
link['src']['id']: (link['description'], 'src')
for link in neighboors['links'] if link['src']['id'] != obs_id
})
for n in neighboors['objs']:
yield n, links_by_id[n['id']]
def parse_yeti_result(self):
obs = self.search(self.attribute['value'])
for obs_to_add, link in self.get_neighboors(obs['id']):
object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add)
if object_misp_domain_ip:
self.misp_event.add_object(object_misp_domain_ip)
continue
object_misp_url = self.__get_object_url(obs_to_add)
if object_misp_url:
self.misp_event.add_object(object_misp_url)
continue
if link[0] == 'NS record':
object_ns_record = self.__get_object_ns_record(
obs_to_add, link[1])
if object_ns_record:
self.misp_event.add_object(object_ns_record)
continue
self.__get_attribute(obs_to_add, link[0])
def get_result(self):
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object') if key in event
}
return results
def __get_attribute(self, obs_to_add, link):
try:
type_attr = self.misp_mapping[obs_to_add['type']]
value = None
if obs_to_add['type'] == 'File':
value = obs_to_add['value'].split(':')[1]
else:
value = obs_to_add['value']
attr = self.misp_event.add_attribute(value=value, type=type_attr)
attr.comment = '%s: %s' % (link, self.attribute['value'])
except KeyError:
logging.error('type not found %s' % obs_to_add['type'])
return
for t in obs_to_add['tags']:
self.misp_event.add_attribute_tag(t['name'], attr['uuid'])
def __get_object_domain_ip(self, obj_to_add):
if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname', 'domain']) or \
(obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')):
domain_ip_object = MISPObject('domain-ip')
domain_ip_object.add_attribute(self.__get_relation(obj_to_add),
obj_to_add['value'])
domain_ip_object.add_attribute(
self.__get_relation(self.attribute, is_yeti_object=False),
self.attribute['value'])
domain_ip_object.add_reference(self.attribute['uuid'],
'related_to')
return domain_ip_object
def __get_object_url(self, obj_to_add):
if (obj_to_add['type'] == 'Url' and self.attribute['type'] in [
'hostname', 'domain', 'ip-src', 'ip-dst'
]) or (obj_to_add['type'] in ('Hostname', 'Domain', 'Ip')
and self.attribute['type'] == 'url'):
url_object = MISPObject('url')
obj_relation = self.__get_relation(obj_to_add)
if obj_relation:
url_object.add_attribute(obj_relation, obj_to_add['value'])
obj_relation = self.__get_relation(self.attribute,
is_yeti_object=False)
if obj_relation:
url_object.add_attribute(obj_relation, self.attribute['value'])
url_object.add_reference(self.attribute['uuid'], 'related_to')
return url_object
def __get_object_ns_record(self, obj_to_add, link):
queried_domain = None
ns_domain = None
object_dns_record = MISPObject('dns-record')
if link == 'dst':
queried_domain = self.attribute['value']
ns_domain = obj_to_add['value']
elif link == 'src':
queried_domain = obj_to_add['value']
ns_domain = self.attribute['value']
if queried_domain and ns_domain:
object_dns_record.add_attribute('queried-domain', queried_domain)
object_dns_record.add_attribute('ns-record', ns_domain)
object_dns_record.add_reference(self.attribute['uuid'],
'related_to')
return object_dns_record
def __get_relation(self, obj, is_yeti_object=True):
if is_yeti_object:
type_attribute = self.misp_mapping[obj['type']]
else:
type_attribute = obj['type']
if type_attribute == 'ip-src' or type_attribute == 'ip-dst':
return 'ip'
elif 'domain' == type_attribute:
return 'domain'
elif 'hostname' == type_attribute:
return 'domain'
elif type_attribute == 'url':
return type_attribute
class RFEnricher:
"""Class for enriching an attribute with data from Recorded Future.
The enrichment data is returned as a custom MISP object.
"""
def __init__(self, attribute_props: dict):
self.event = MISPEvent()
self.enrichment_object = MISPObject("Recorded Future Enrichment")
description = ("An object containing the enriched attribute and "
"related entities from Recorded Future.")
self.enrichment_object.from_dict(**{
"meta-category": "misc",
"description": description,
"distribution": 0
})
# Create a copy of enriched attribute to add tags to
temp_attr = MISPAttribute()
temp_attr.from_dict(**attribute_props)
self.enriched_attribute = MISPAttribute()
self.enriched_attribute.from_dict(**{
"value": temp_attr.value,
"type": temp_attr.type,
"distribution": 0
})
self.related_attributes: List[Tuple[str, MISPAttribute]] = []
self.color_picker = RFColors()
self.galaxy_finder = GalaxyFinder()
# Mapping from MISP-type to RF-type
self.type_to_rf_category = {
"ip": "ip",
"ip-src": "ip",
"ip-dst": "ip",
"ip-src|port": "ip",
"ip-dst|port": "ip",
"domain": "domain",
"hostname": "domain",
"md5": "hash",
"sha1": "hash",
"sha256": "hash",
"uri": "url",
"url": "url",
"vulnerability": "vulnerability",
"weakness": "vulnerability",
}
# Related entities have 'Related' as part of the word and Links entities from RF
# portrayed as related attributes in MISP
self.related_attribute_types = [
"RelatedIpAddress",
"RelatedInternetDomainName",
"RelatedHash",
"RelatedEmailAddress",
"RelatedCyberVulnerability",
"IpAddress",
"InternetDomainName",
"Hash",
"EmailAddress",
"CyberVulnerability",
]
# Related entities have 'Related' as part of the word and and Links entities from RF portrayed as tags in MISP
self.galaxy_tag_types = [
"RelatedMalware",
"RelatedThreatActor",
"Threat Actor",
"MitreAttackIdentifier",
"Malware",
]
def enrich(self) -> None:
"""Run the enrichment."""
category = self.type_to_rf_category.get(self.enriched_attribute.type,
"")
enriched_attribute_value = self.enriched_attribute.value
# If enriched attribute has a port we need to remove that port
# since RF do not support enriching ip addresses with port
if self.enriched_attribute.type in ["ip-src|port", "ip-dst|port"]:
enriched_attribute_value = enriched_attribute_value.split("|")[0]
json_response = GLOBAL_REQUEST_HANDLER.rf_lookup(
category, enriched_attribute_value)
response = json.loads(json_response.content)
try:
# Add risk score and risk rules as tags to the enriched attribute
risk_score = response["data"]["risk"]["score"]
hex_color = self.color_picker.riskscore_color(risk_score)
tag_name = f'recorded-future:risk-score="{risk_score}"'
self.add_tag(tag_name, hex_color)
risk_criticality = response["data"]["risk"]["criticalityLabel"]
hex_color = self.color_picker.criticality_color(risk_criticality)
tag_name = f'recorded-future:criticality="{risk_criticality}"'
self.add_tag(tag_name, hex_color)
for evidence in response["data"]["risk"]["evidenceDetails"]:
risk_rule = evidence["rule"]
criticality = evidence["criticality"]
hex_color = self.color_picker.riskrule_color(criticality)
tag_name = f'recorded-future:risk-rule="{risk_rule}"'
self.add_tag(tag_name, hex_color)
links_data = response["data"].get("links", {}).get("hits")
# Check if we have error in links response. If yes, then user do not have right module enabled in token
links_access_error = response["data"].get("links", {}).get("error")
galaxy_tags = []
if not links_access_error:
for hit in links_data:
for section in hit["sections"]:
for sec_list in section["lists"]:
entity_type = sec_list["type"]["name"]
for entity in sec_list["entities"]:
if entity_type in self.galaxy_tag_types:
galaxy = self.galaxy_finder.find_galaxy_match(
entity["name"], entity_type)
if galaxy and galaxy not in galaxy_tags:
galaxy_tags.append(galaxy)
else:
self.add_attribute(entity["name"],
entity_type)
else:
# Retrieve related entities
for related_entity in response["data"]["relatedEntities"]:
related_type = related_entity["type"]
if related_type in self.related_attribute_types:
# Related entities returned as additional attributes
for related in related_entity["entities"]:
# filter those entities that have count bigger than 4, to reduce noise
# because there can be a huge list of related entities
if int(related["count"]) > 4:
indicator = related["entity"]["name"]
self.add_attribute(indicator, related_type)
elif related_type in self.galaxy_tag_types:
# Related entities added as galaxy-tags to the enriched attribute
galaxy_tags = []
for related in related_entity["entities"]:
# filter those entities that have count bigger than 4, to reduce noise
# because there can be a huge list of related entities
if int(related["count"]) > 4:
indicator = related["entity"]["name"]
galaxy = self.galaxy_finder.find_galaxy_match(
indicator, related_type)
# Handle deduplication of galaxy tags
if galaxy and galaxy not in galaxy_tags:
galaxy_tags.append(galaxy)
for galaxy in galaxy_tags:
self.add_tag(galaxy)
except KeyError:
misperrors[
"error"] = "Unexpected format in Recorded Future api response."
raise
def add_attribute(self, indicator: str, indicator_type: str) -> None:
"""Helper method for adding an indicator to the attribute list."""
out_type = self.get_output_type(indicator_type, indicator)
attribute = MISPAttribute()
attribute.from_dict(**{
"value": indicator,
"type": out_type,
"distribution": 0
})
self.related_attributes.append((indicator_type, attribute))
def add_tag(self, tag_name: str, hex_color: str = None) -> None:
"""Helper method for adding a tag to the enriched attribute."""
tag = MISPTag()
tag_properties = {"name": tag_name}
if hex_color:
tag_properties["colour"] = hex_color
tag.from_dict(**tag_properties)
self.enriched_attribute.add_tag(tag)
def get_output_type(self, related_type: str, indicator: str) -> str:
"""Helper method for translating a Recorded Future related type to a MISP output type."""
output_type = "text"
if related_type in ["RelatedIpAddress", "IpAddress"]:
output_type = "ip-dst"
elif related_type in [
"RelatedInternetDomainName", "InternetDomainName"
]:
output_type = "domain"
elif related_type in ["RelatedHash", "Hash"]:
hash_len = len(indicator)
if hash_len == 64:
output_type = "sha256"
elif hash_len == 40:
output_type = "sha1"
elif hash_len == 32:
output_type = "md5"
elif related_type in ["RelatedEmailAddress", "EmailAddress"]:
output_type = "email-src"
elif related_type in [
"RelatedCyberVulnerability", "CyberVulnerability"
]:
signature = indicator.split("-")[0]
if signature == "CVE":
output_type = "vulnerability"
elif signature == "CWE":
output_type = "weakness"
elif related_type == "MalwareSignature":
output_type = "malware-sample"
elif related_type == "Organization":
output_type = "target-org"
elif related_type == "Username":
output_type = "target-user"
return output_type
def get_results(self) -> dict:
"""Build and return the enrichment results."""
self.enrichment_object.add_attribute("Enriched attribute",
**self.enriched_attribute)
for related_type, attribute in self.related_attributes:
self.enrichment_object.add_attribute(related_type, **attribute)
self.event.add_object(**self.enrichment_object)
event = json.loads(self.event.to_json())
result = {key: event[key] for key in ["Object"] if key in event}
return {"results": result}
def handle_url(self, url, event: pymisp.MISPEvent):
"""Handle a single URL."""
event.add_attribute("url", str(url))
return event
def handle_domain(self, domain, event: pymisp.MISPEvent):
"""Handle a single domain."""
event.add_attribute("domain", str(domain))
return event
def import_all(self, stations_short_names, interval, data_type):
object_creator = getattr(self, f'{interval}_flask_{data_type}')
if data_type == 'co2':
base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/'
elif data_type in ['c13', 'o18']:
base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/'
for station in stations_short_names:
url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv'
infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations'
filepath = self.fetch(url)
if not filepath:
continue
if infofield in self.scrippts_meta:
event = MISPEvent()
event.load_file(str(self.output_dir / self.scrippts_meta[infofield]))
location = event.get_objects_by_name('geolocation')[0]
update = True
else:
event = MISPEvent()
event.uuid = str(uuid4())
event.info = infofield
event.Orgc = self.misp_org
event.add_tag(getattr(self, f'tag_{station}')())
location = getattr(self, f'geolocation_{station}')()
event.add_object(location)
event.add_attribute('link', f'https://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}')
update = False
with self.scrippts_meta_file.open('a') as f:
writer = csv.writer(f)
writer.writerow([infofield, f'{event.uuid}.json'])
object_creator(event, location, filepath, update)
if update:
# Bump the publish timestamp
event.publish_timestamp = datetime.datetime.timestamp(datetime.datetime.now())
feed_output = event.to_feed(with_meta=False)
with (self.output_dir / f'{event.uuid}.json').open('w') as f:
# json.dump(feed_output, f, indent=2, sort_keys=True) # For testing
json.dump(feed_output, f)
def from_file(self, path):
self.misp_event = MISPEvent()
self.misp_event.load_file(path)
def run(self, results):
"""Run analysis.
@return: MISP results dict.
"""
url = self.options.get("url", "")
apikey = self.options.get("apikey", "")
if not url or not apikey:
log.error("MISP URL or API key not configured.")
return
self.misp = PyMISP(url, apikey, False, "json")
self.threads = self.options.get("threads", "")
if not self.threads:
self.threads = 5
self.iocs = deque()
self.misper = dict()
try:
if self.options.get("upload_iocs", False) and results.get("malscore", 0) >= self.options.get("min_malscore", 0):
distribution = int(self.options.get("distribution", 0))
threat_level_id = int(self.options.get("threat_level_id", 4))
analysis = int(self.options.get("analysis", 0))
tag = self.options.get("tag") or "CAPEv2"
info = self.options.get("title", "")
upload_sample = self.options.get("upload_sample")
malfamily = ""
if results.get("detections", ""):
malfamily = results["detections"]
response = self.misp.search("attributes", value=results["target"]["file"]["sha256"], return_format="json", pythonify=True)
if response:
event = self.misp.get_event(response[0].event_id, pythonify=True)
else:
event = MISPEvent()
event.distribution = distribution
event.threat_level_id = threat_level_id
event.analysis = analysis
event.info = "{} {} - {}".format(info, malfamily, results.get("info", {}).get("id"))
event = self.misp.add_event(event, pythonify=True)
# Add a specific tag to flag Cuckoo's event
if tag:
self.misp.tag(event, tag)
# malpedia galaxy
if malpedia_json:
self.malpedia(results, event, malfamily)
# ToDo?
self.signature(results, event)
self.sample_hashes(results, event)
self.all_network(results, event)
self.dropped_files(results, event)
if upload_sample:
target = results.get("target", {})
f = target.get("file", {})
if target.get("category") == "file" and f:
with open(f["path"], "rb") as f:
event.add_attribute(
"malware-sample",
value=os.path.basename(f["path"]),
data=BytesIO(f.read()),
expand="binary",
comment="Sample run",
)
if results.get("target", {}).get("url", "") and results["target"]["url"] not in whitelist:
event.add_attribute("url", results["target"]["url"])
# ToDo migth be outdated!
# if self.options.get("ids_files", False) and "suricata" in results.keys():
# for surifile in results["suricata"]["files"]:
# if "file_info" in surifile.keys():
# self.misper["iocs"].append({"md5": surifile["file_info"]["md5"]})
# self.misper["iocs"].append({"sha1": surifile["file_info"]["sha1"]})
# self.misper["iocs"].append({"sha256": surifile["file_info"]["sha256"]})
if self.options.get("mutexes", False) and "behavior" in results and "summary" in results["behavior"]:
if "mutexes" in results.get("behavior", {}).get("summary", {}):
for mutex in results["behavior"]["summary"]["mutexes"]:
if mutex not in whitelist:
event.add_attribute("mutex", mutex)
if self.options.get("registry", False) and "behavior" in results and "summary" in results["behavior"]:
if "read_keys" in results["behavior"].get("summary", {}):
for regkey in results["behavior"]["summary"]["read_keys"]:
event.add_attribute("regkey", regkey)
event.run_expansions()
self.misp.update_event(event)
# Make event public
if self.options.get("published", True):
self.misp.publish(event)
except Exception as e:
log.error("Failed to generate JSON report: %s" % e, exc_info=True)
def create_misp_event(misp_instance, isight_report_instance, event_tags):
# No MISP event for this iSight report ID exists yet.
# Alas, create a new MISP event.
# Convert the publication date of the iSight report into a datetime object.
if isight_report_instance.publishDate:
date = datetime.datetime.fromtimestamp(
isight_report_instance.publishDate)
else:
# If iSight doesn't provide a date, use today's date.
date = datetime.datetime.now(datetime.timezone.utc)
# Create a MISP event from the FireEye iSight report with the following parameters.
event = MISPEvent()
event.distribution = 1 # This community only
if isight_report_instance.riskRating == 'CRITICAL' or isight_report_instance.riskRating == 'Critical':
event.threat_level_id = 1 # High
elif isight_report_instance.riskRating == 'HIGH' or isight_report_instance.riskRating == 'High':
event.threat_level_id = 1 # High
elif isight_report_instance.riskRating == 'MEDIUM' or isight_report_instance.riskRating == 'Medium':
event.threat_level_id = 2 # Medium
elif isight_report_instance.riskRating == 'LOW' or isight_report_instance.riskRating == 'Low':
event.threat_level_id = 3 # Low
else:
event.threat_level_id = 4 # Unknown
event.analysis = 2 # Completed
event.info = "iSIGHT: " + isight_report_instance.title
event.date = date
# Push the event to the MISP server.
my_event = misp_instance.add_event(event, pythonify=True)
PySight_settings.logger.debug('Created MISP event %s for iSight report %s',
event, isight_report_instance.reportId)
# Add the event ID to the global list of newly created events.
global new_events
new_events.append(my_event['id'])
# Add default tags to the event.
if event_tags:
for event_tag in event_tags:
misp_instance.tag(my_event, event_tag)
# Use some iSight ThreatScapes for event tagging. Reports can have multiple ThreatScapes.
if 'Cyber Espionage' in isight_report_instance.ThreatScape:
# VERIS distinguishes between external, internal or partner actors. This difference is not yet implemented in
# MISP. External would be most likely.
#misp_instance.tag(my_event, 'veris:actor:external:motive="Espionage"')
misp_instance.tag(my_event, 'veris:actor:motive="Espionage"')
if 'Hacktivism' in isight_report_instance.ThreatScape:
misp_instance.tag(my_event, 'veris:actor:external:variety="Activist"')
if 'Critical Infrastructure' in isight_report_instance.ThreatScape:
misp_instance.tag(my_event, 'basf:technology="OT"')
if 'Cyber Physical' in isight_report_instance.ThreatScape:
misp_instance.tag(my_event, 'basf:technology="OT"')
if 'Cyber Crime' in isight_report_instance.ThreatScape:
misp_instance.tag(my_event,
'veris:actor:external:variety="Organized crime"')
# Add the iSight report ID and web link as attributes.
if isight_report_instance.reportId:
misp_instance.add_attribute(my_event, {
'category': 'External analysis',
'type': 'text',
'to_ids': False,
'value': isight_report_instance.reportId
},
pythonify=True)
if isight_report_instance.webLink:
misp_instance.add_attribute(my_event, {
'category': 'External analysis',
'type': 'link',
'to_ids': False,
'value': isight_report_instance.webLink
},
pythonify=True)
# Put the ThreatScape into an Attribution attribute, but disable correlation.
if isight_report_instance.ThreatScape:
misp_instance.add_attribute(my_event, {
'category': 'Attribution',
'type': 'text',
'to_ids': False,
'value': isight_report_instance.ThreatScape,
'disable_correlation': True
},
pythonify=True)
# Add specific attributes from this iSight report.
update_misp_event(misp_instance, my_event, isight_report_instance)
def parse_response(censys_output, attribute):
misp_event = MISPEvent()
misp_event.add_attribute(**attribute)
# Generic fields (for IP/Websites)
if "autonomous_system" in censys_output:
cen_as = censys_output['autonomous_system']
asn_object = MISPObject('asn')
asn_object.add_attribute('asn', value=cen_as["asn"])
asn_object.add_attribute('description', value=cen_as['name'])
asn_object.add_attribute('subnet-announced',
value=cen_as['routed_prefix'])
asn_object.add_attribute('country', value=cen_as['country_code'])
asn_object.add_reference(attribute.uuid, 'associated-to')
misp_event.add_object(**asn_object)
if "ip" in censys_output and "ports" in censys_output:
ip_object = MISPObject('ip-port')
ip_object.add_attribute('ip', value=censys_output['ip'])
for p in censys_output['ports']:
ip_object.add_attribute('dst-port', value=p)
ip_object.add_reference(attribute.uuid, 'associated-to')
misp_event.add_object(**ip_object)
# We explore all ports to find https or ssh services
for k in censys_output.keys():
if not isinstance(censys_output[k], dict):
continue
if 'https' in censys_output[k]:
try:
cert = censys_output[k]['https']['tls']['certificate']
cert_obj = get_certificate_object(cert, attribute)
misp_event.add_object(**cert_obj)
except KeyError:
print("Error !")
if 'ssh' in censys_output[k]:
try:
cert = censys_output[k]['ssh']['v2']['server_host_key']
# TODO enable once the type is merged
# misp_event.add_attribute(type='hasshserver-sha256', value=cert['fingerprint_sha256'])
except KeyError:
pass
# Info from certificate query
if "parsed" in censys_output:
cert_obj = get_certificate_object(censys_output, attribute)
misp_event.add_object(**cert_obj)
# Location can be present for IP/Websites results
if "location" in censys_output:
loc_obj = MISPObject('geolocation')
loc = censys_output['location']
loc_obj.add_attribute('latitude', value=loc['latitude'])
loc_obj.add_attribute('longitude', value=loc['longitude'])
if 'city' in loc:
loc_obj.add_attribute('city', value=loc['city'])
loc_obj.add_attribute('country', value=loc['country'])
if 'postal_code' in loc:
loc_obj.add_attribute('zipcode', value=loc['postal_code'])
if 'province' in loc:
loc_obj.add_attribute('region', value=loc['province'])
loc_obj.add_reference(attribute.uuid, 'associated-to')
misp_event.add_object(**loc_obj)
event = json.loads(misp_event.to_json())
return {'Object': event['Object'], 'Attribute': event['Attribute']}
def __init__(self):
super(MISP, self).__init__()
self.cur_path = __project__.get_path()
self.parser.add_argument("--url", help='URL of the MISP instance')
self.parser.add_argument("--off", action='store_true', help='Use offline (can only work on pre-downloaded events)')
self.parser.add_argument("--on", action='store_true', help='Switch to online mode')
self.parser.add_argument("-k", "--key", help='Your key on the MISP instance')
self.parser.add_argument("-v", "--verify", action='store_false', help='Disable certificate verification (for self-signed)')
subparsers = self.parser.add_subparsers(dest='subname')
# ##### Upload sample to MISP #####
parser_up = subparsers.add_parser('upload', help='Send malware sample to MISP.',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent('''
Distribution levels:
* 0: Your organisation only
* 1: This community only
* 2: Connected communities
* 3: All communities
* 5: Inherit
Sample categories:
* 0: Payload delivery
* 1: Artifacts dropped
* 2: Payload installation
* 3: External analysis
Analysis levels:
* 0: Initial
* 1: Ongoing
* 2: Completed
Threat levels:
* 0: High
* 1: Medium
* 2: Low
* 3: Undefined
'''))
parser_up.add_argument("-e", "--event", type=int, help="Event ID to update. If None, and you're not connected to a MISP event a new one is created.")
parser_up.add_argument("-d", "--distrib", type=int, choices=[0, 1, 2, 3, 5], help="Distribution of the attributes for the new event.")
parser_up.add_argument("-s", "--sharing", type=int, help="Sharing group ID when distribution is set to 4.")
parser_up.add_argument("-ids", action='store_true', help="Is eligible for automatically creating IDS signatures.")
parser_up.add_argument("-c", "--categ", type=int, choices=[0, 1, 2, 3], default=1, help="Category of the samples.")
parser_up.add_argument("-i", "--info", nargs='+', help="Event info field of a new event.")
parser_up.add_argument("-o", "--comment", nargs='+', help="Comment associated to the sample.")
parser_up.add_argument("-a", "--analysis", type=int, choices=[0, 1, 2], help="Analysis level a new event.")
parser_up.add_argument("-t", "--threat", type=int, choices=[0, 1, 2, 3], help="Threat level of a new event.")
# ##### Download samples from event #####
parser_down = subparsers.add_parser('download', help='Download malware samples from MISP.')
group = parser_down.add_mutually_exclusive_group()
group.add_argument("-e", "--event", type=int, help="Download all the samples related to this event ID.")
group.add_argument("-l", "--list", nargs='*', help="Download all the samples related to a list of events. Empty list to download all the samples of all the events stored in the current project.") # noqa
group.add_argument("--hash", help="Download the sample related to this hash (only MD5).")
# ##### Search in MISP #####
parser_search = subparsers.add_parser('search', help='Search in all the attributes.')
parser_search.add_argument("query", nargs='*', help="String to search (if empty, search the hashes of the current file).")
# ##### Check hashes on VT #####
parser_checkhashes = subparsers.add_parser('check_hashes', help='Crosscheck hashes on VT.')
parser_checkhashes.add_argument("event", nargs='?', default=None, type=int, help="Lookup all the hashes of an event on VT.")
parser_checkhashes.add_argument("-p", "--populate", action='store_true', help="Automatically populate event with hashes found on VT.")
# ##### Download Yara rules #####
parser_checkhashes = subparsers.add_parser('yara', help='Get YARA rules of an event.')
parser_checkhashes.add_argument("event", nargs='?', default=None, type=int, help="Download the yara rules of that event.")
# ##### Get Events #####
parser_pull = subparsers.add_parser('pull', help='Initialize the session with an existing MISP event.')
parser_pull.add_argument("event", nargs='+', type=int, help="(List of) Event(s) ID.")
# ##### Create an Event #####
parser_create_event = subparsers.add_parser('create_event', help='Create a new event on MISP and initialize the session with it.',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent('''
Distribution levels:
* 0: Your organisation only
* 1: This community only
* 2: Connected communities
* 3: All communities
* 4: Sharing group
Sharing Group:
* #: ID of sharing group
Analysis levels:
* 0: Initial
* 1: Ongoing
* 2: Completed
Threat levels:
* 0: High
* 1: Medium
* 2: Low
* 3: Undefined
'''))
parser_create_event.add_argument("-d", "--distrib", type=int, choices=[0, 1, 2, 3, 4], help="Distribution of the attributes for the new event.")
parser_create_event.add_argument("-s", "--sharing", type=int, help="Sharing group ID when distribution is set to 4.")
parser_create_event.add_argument("-t", "--threat", type=int, choices=[0, 1, 2, 3], help="Threat level of a new event.")
parser_create_event.add_argument("-a", "--analysis", type=int, choices=[0, 1, 2], help="Analysis level a new event.")
parser_create_event.add_argument("-i", "--info", required=True, nargs='+', help="Event info field of a new event.")
parser_create_event.add_argument("--date", help="Date of the event. (Default: today).")
# ##### Add Hashes #####
h = subparsers.add_parser("add_hashes", help="If no parameters, add all the hashes of the current session.")
h.add_argument("-f", "--filename", help="Filename")
h.add_argument("-m", "--md5", help="MD5")
h.add_argument("-s", "--sha1", help="SHA1")
h.add_argument("-a", "--sha256", help="SHA256")
# ##### Add attributes #####
parser_add = subparsers.add_parser('add', help='Add attributes to an existing MISP event.')
subparsers_add = parser_add.add_subparsers(dest='add')
# Hashes
# Generic add
temp_me = MISPEvent()
if hasattr(temp_me, "types"):
known_types = temp_me.types
else:
# New API
known_types = temp_me.get_known_types()
for t in known_types:
sp = subparsers_add.add_parser(t, help="Add {} to the event.".format(t))
sp.add_argument(t, nargs='+')
# ##### Show attributes #####
subparsers.add_parser('show', help='Show attributes to an existing MISP event.')
# ##### Open file #####
o = subparsers.add_parser('open', help='Open a sample from the temp directory.')
ox = o.add_mutually_exclusive_group(required=True)
ox.add_argument("-l", "--list", action='store_true', help="List available files")
ox.add_argument("-d", "--delete", help="Delete temporary files (use 'all' to remove all the local samples or an Event ID to only remove the associated samples)")
ox.add_argument("sid", nargs='?', type=int, help='Sample ID to open (from the list option).')
# ##### Publish an event #####
subparsers.add_parser('publish', help='Publish an existing MISP event.')
# ##### Show version #####
subparsers.add_parser('version', help='Returns the version of the MISP instance.')
# Store
s = subparsers.add_parser('store', help='Store the current MISP event in the current project.')
s.add_argument("-l", "--list", action='store_true', help="List stored MISP events")
s.add_argument("-u", "--update", action='store_true', help="Update all stored MISP events")
s.add_argument("-s", "--sync", action='store_true', help="Sync all MISP Events with the remote MISP instance")
s.add_argument("-d", "--delete", type=int, help="Delete a stored MISP event")
s.add_argument("-o", "--open", help="Open a stored MISP event")
# Tags
s = subparsers.add_parser('tag', help='Tag managment using MISP taxonomies.')
s.add_argument("-l", "--list", action='store_true', help="List Existing taxonomies.")
s.add_argument("-d", "--details", help="Display all values of a taxonomy.")
s.add_argument("-s", "--search", help="Search all tags matching a value.")
s.add_argument("-e", "--event", help="Add tag to the current event.")
s.add_argument("-a", "--attribute", nargs='+', help="Add tag to an attribute of the current event. Syntax: <identifier for the attribute> <machinetag>")
# Galaxies
s = subparsers.add_parser('galaxies', help='Use misp-galaxy with PyMISPGalaxies.')
s.add_argument("-l", "--list", action='store_true', help="List existing galaxies.")
s.add_argument("-d", "--details", help="Display all values of a galaxy.")
s.add_argument("-v", "--cluster-value", nargs='+', help="Display all details of a cluster value.")
s.add_argument("-s", "--search", nargs='+', help="Search all galaxies matching a value.")
# Admin
s = subparsers.add_parser('admin', help='Administration options.')
admin_parser = s.add_subparsers(dest='admin')
# Organisation
org = admin_parser.add_parser('org', help="Organisation managment.")
subparsers_org = org.add_subparsers(dest='org')
# Get
display = subparsers_org.add_parser('display', help="Display an organisation.")
display.add_argument('id', help='ID of the organisation to display. Use "local" to display all local organisations, "external" for all remote organisations, and "all", for both.')
# Search
search = subparsers_org.add_parser('search', help="Search an organisation by name.")
search.add_argument('name', help='(Partial) name of the organisation.')
search.add_argument('-t', '--type', default='local', choices=['local', 'external', 'all'],
help='Use "local" to search in all local organisations, "external" for remote organisations, and "all", for both.')
# Add
add_org = subparsers_org.add_parser('add', help="Add an organisation.")
add_org.add_argument('name', help='Organisation name.')
add_org.add_argument('-u', '--uuid', default=None, help='UUID of the organisation.')
add_org.add_argument('-d', '--description', default=[], nargs='+', help='Description of the organisation.')
add_org.add_argument('-t', '--type', default=[], nargs='+', help='Type of the organisation.')
add_org.add_argument('-n', '--nationality', default=None, help='Nationality of the organisation.')
add_org.add_argument('-s', '--sector', default=[], nargs='+', help='Sector of the organisation.')
add_org.add_argument('-c', '--contacts', default=[], nargs='+', help='Contact point(s) in the organisation.')
add_org.add_argument('--not-local', default=True, action='store_false', help='**Not** a local organisation.')
# Delete
delete = subparsers_org.add_parser('delete', help="Delete an organisation.")
delete.add_argument('id', help='ID of the organisation to delete.')
# Edit
edit = subparsers_org.add_parser('edit', help="Edit an organisation.")
edit.add_argument('id', help='ID of the organisation to edit.')
edit.add_argument('-n', '--name', help='Organisation name.')
edit.add_argument('-u', '--uuid', help='UUID of the organisation.')
edit.add_argument('-d', '--description', default=[], nargs='+', help='Description of the organisation.')
edit.add_argument('-t', '--type', default=[], nargs='+', help='Type of the organisation.')
edit.add_argument('--nationality', help='Nationality of the organisation.')
edit.add_argument('-s', '--sector', default=[], nargs='+', help='Sector of the organisation.')
edit.add_argument('-c', '--contacts', default=[], nargs='+', help='Contact point(s) in the organisation.')
edit.add_argument('--not-local', default=True, action='store_false', help='**Not** a local organisation.')
# User
user = admin_parser.add_parser('user', help="User managment.")
subparsers_user = user.add_subparsers(dest='user')
# Get
display = subparsers_user.add_parser('display', help="Display a user.")
display.add_argument('id', help='ID of the user to display. Use "all" to display all users.')
# Search
search = subparsers_user.add_parser('search', help="Search a user by email.")
search.add_argument('name', help='(Partial) email of the user.')
# Add
add_usr = subparsers_user.add_parser('add', help="Add a user.")
add_usr.add_argument('email', help='User email address.')
add_usr.add_argument('-o', '--org-id', default=None, help='Organisation ID of the user.')
add_usr.add_argument('-r', '--role-id', default=None, help='Role of the user')
add_usr.add_argument('-g', '--gpgkey', default=None, help='Path to the GPG public key export')
add_usr.add_argument('-c', '--change-pw', default=None, action='store_true', help='Force thanging the password after next login')
add_usr.add_argument('-t', '--termsaccepted', default=None, action='store_true', help='Set the TOC to accepted')
add_usr.add_argument('-p', '--password', default=None, help='Set a new password')
add_usr.add_argument('-d', '--disabled', default=None, action='store_true', help='Disable the account')
# Delete
delete = subparsers_user.add_parser('delete', help="Delete a user.")
delete.add_argument('id', help='ID of the user to delete.')
# Edit
edit = subparsers_user.add_parser('edit', help="Edit a user.")
edit.add_argument('id', help='ID of the user to edit.')
edit.add_argument('-e', '--email', help='User email address.')
edit.add_argument('-o', '--org-id', default=None, help='Organisation ID of the user.')
edit.add_argument('-r', '--role-id', default=None, help='Role of the user')
edit.add_argument('-g', '--gpgkey', default=None, help='Path to the GPG public key export')
edit.add_argument('-c', '--change-pw', default=None, action='store_true', help='Force thanging the password after next login')
edit.add_argument('-t', '--termsaccepted', default=None, action='store_true', help='Set the TOC to accepted')
edit.add_argument('-p', '--password', default=None, help='Set a new password')
edit.add_argument('-d', '--disabled', default=None, action='store_true', help='Disable the account')
# Role
role = admin_parser.add_parser('role', help="Role managment.")
subparsers_role = role.add_subparsers(dest='role')
# Get
display = subparsers_role.add_parser('display', help="Display all the roles.")
# Search
search = subparsers_role.add_parser('search', help="Search a role by name.")
search.add_argument('name', help='(Partial) name of the role.')
# Tags
t = admin_parser.add_parser('tag', help="Tag managment.")
subparsers_tag = t.add_subparsers(dest='tag')
# Get
display = subparsers_tag.add_parser('display', help="Display all the tags.")
# Search
search = subparsers_tag.add_parser('search', help="Search a tag by name.")
search.add_argument('name', help='(Partial) name of the tag.')
self.categories = {0: 'Payload delivery', 1: 'Artifacts dropped', 2: 'Payload installation', 3: 'External analysis'}
"The distribution setting used for the attributes and for the newly created event, if relevant. [0-3]."
)
parser.add_argument(
"-i",
"--info",
help="Used to populate the event info field if no event ID supplied.")
parser.add_argument(
"-a",
"--analysis",
type=int,
help=
"The analysis level of the newly created event, if applicable. [0-2]")
parser.add_argument(
"-t",
"--threat",
type=int,
help=
"The threat level ID of the newly created event, if applicable. [1-4]")
args = parser.parse_args()
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
event = MISPEvent()
event.distribution = args.distrib
event.threat_level_id = args.threat
event.analysis = args.analysis
event.info = args.info
event = misp.add_event(event, pythonify=True)
print(event)
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
def __init__(self):
self.misp_event = MISPEvent()
self.event = defaultdict(dict)
self.misp_event['Galaxy'] = []
def twitter_account(data):
"""
Do the things.
:param data:
:return:
"""
misp_event_id = data['misp_event_id']
twitter_post_id = data['twitter_post_id']
response_url = data['response_url']
try:
# Get the Twitter status.
status = twitter_get_account(twitter_post_id)
# Extract relevant values from the Twitter status.
microblog_data = transform_twitter_account(status._json)
# Load the microblog version from it's definition.json file.
with open("misp-objects/twitter-account/definition.json") as f:
microblog_definition = json.load(f)
f.close()
# Create the MISP mircroblog object.
# TODO: get the object definition from github
# misp_objects_path_custom searches the var path for {objectname}/definition.json to load the object definition
# This file needs to be updated when the upstream object is updated.
microblog = TwitterAccountObject(
parameters=microblog_data,
misp_objects_path_custom="misp-objects",
template_version=str(microblog_definition["version"]))
# Get the MISP event.
working_event = misp.get_event(misp_event_id,
extended=True,
pythonify=True)
# Get the Slackbot's MISP org ID.
user_profile = misp.get_user("me")
bot_org_id = user_profile["User"]["org_id"]
# If the bot org can update the MISP Event with the new microblog do so.
if str(bot_org_id) == str(working_event["org_id"]):
working_event.Object.append(microblog)
result = misp.update_event(working_event)
print(result)
else:
new_event = True
# If an extension exists for Slackbot objects use it.
if "extensionEvents" in working_event:
for k, event_extension in working_event[
"extensionEvents"].items():
if event_extension["Orgc"]["id"] == bot_org_id:
if event_extension[
"info"] == "Covid Slack: Disinfo Bot":
extension_event = misp.get_event(
event_extension["id"], pythonify=True)
extension_event.Object.append(microblog)
result = misp.update_event(extension_event)
print(result)
new_event = False
# Create a new extension to the parent event.
if new_event:
extended_event = MISPEvent()
extended_event.info = "Covid Slack: Disinfo Bot"
extended_event.extends_uuid = working_event["id"]
extended_event.Object.append(microblog)
result = misp.add_event(extended_event, pythonify=True)
print(result)
# Build slack response.
response = {'blocks': [], 'response_type': 'in_channel'}
response['blocks'].append({
'type': 'section',
'text': {
'type': 'mrkdwn',
'text': 'Twitter Status: {}'.format(twitter_post_id)
}
})
twitter_message = ""
if microblog_data.get('name'):
twitter_message += 'Username: {}\n'.format(microblog_data['name'])
if microblog_data.get('display-name'):
twitter_message += 'Display Name: {}\n'.format(
microblog_data['display-name'])
if microblog_data.get('verified'):
twitter_message += 'Verified Account: {}\n'.format(
microblog_data['verified'])
if microblog_data.get('description'):
twitter_message += 'Bio: {}\n'.format(
microblog_data['description'])
if len(microblog_data['hashtag']) > 0:
twitter_message += 'Hashtags:\n'
for hashtag in microblog_data['hashtag']:
twitter_message += '* {}\n'.format(hashtag)
if len(microblog_data['embedded-link']) > 0:
twitter_message += 'Embedded URLs:\n'
for url in microblog_data['embedded-link']:
twitter_message += '* {}\n'.format(url)
# Add Twitter message as block.
response['blocks'].append({
'type': 'section',
'text': {
'type': 'mrkdwn',
'text': twitter_message
}
})
requests.post(response_url, json=response)
except Exception:
logger.info(traceback.print_exc(file=sys.stdout))
message = "An error has occurred!"
resp = build_response(message, False)
requests.post(response_url, json=resp)
def submit_tf_update(misp: ExpandedPyMISP, attributes: list) -> MISPEvent:
"""
create/update abuse.ch MISP-Event and append the new attributes
"""
eventinfo = event_info_template.format(
datetime.now().strftime(info_dateformat))
# logging.debug(eventinfo)
events = misp.search(controller='events',
eventinfo=eventinfo,
org=1,
pythonify=True)
if events: # current event exists already
event = events[0]
else: # create a new event
event = MISPEvent()
event.distribution = event_distribution
event.threat_level_id = event_threat_level
event.analysis = 2
event.info = eventinfo
for tag in tagging:
event.add_tag(tag)
event = misp.add_event(event, pythonify=True)
for att in attributes:
event.add_attribute(**att)
event.published = autopublish
return misp.update_event(event)
def load_events_directory(self, directory):
self.events = []
for path in glob.glob(os.path.join(directory, '*.json')):
e = MISPEvent()
e.load(path)
self.import_event(e)
"reportname":"top-attacks",
"key":"#{API_KEY}" # put your Panorama api key here
}
# try with "..., verify=False)" if you get an SSL error
response = requests.request("GET", url, params=querystring)
resp_text = response.text
json_data = json.loads(json.dumps(xmltodict.parse(resp_text)))
# initialize and set MISPOrganisation
orgc = MISPOrganisation()
orgc.name = 'Palo Alto'
orgc.id = '#{ORGC_ID}' # organisation id
orgc.uuid = '#{ORGC_UUID}' # organisation uuid
# initialize and set MISPEvent()
event = MISPEvent()
event.Orgc = orgc
event.info = json_data['report']['result']['@name'] + " | " + json_data['report']['result']['@range']
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 0 # Optional, defaults to MISP.default_event_threat_level in MISP config
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
event.add_tag('firewall threats')
for threatid in json_data['report']['result']['entry']:
attribute = event.add_attribute('comment', threatid['threatid'])
attribute.comment = threatid['count']
misp.add_event(event.to_json())
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pymisp import ExpandedPyMISP, MISPEvent
from pymisp import MISPObject
from keys import misp_url, misp_key, misp_verifycert
from datetime import date
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
event = MISPEvent()
event.info = 'IoT malware' # Event Title
event.distribution = 1 # 0 = Your Organisation Only, 1 = Community
event.threat_level_id = 2 # 1 = High, 2 = Medium, 3 = Low
event.analysis = 2 # 0 (initial analysis), 1 (On-Going), 2 (Complete)
event.add_tag('malware_classification:malware-category="Botnet"')
event.add_tag('tlp:amber')
d = date.today()
event.set_date(d)
attribute_second = event.add_attribute('url',
'http://1.2.3.4/example',
disable_correlation=False,
comment="Botnet example text",
to_ids=False)
event = misp.add_event(event, pythonify=True)
# Publish event
class TestMISPEvent(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
def init_event(self):
self.mispevent.info = 'This is a test'
self.mispevent.distribution = 1
self.mispevent.threat_level_id = 1
self.mispevent.analysis = 1
self.mispevent.set_date("2017-12-31") # test the set date method
def test_simple(self):
with open('tests/mispevent_testfiles/simple.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event(self):
self.init_event()
self.mispevent.publish()
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_loadfile(self):
self.mispevent.load_file('tests/mispevent_testfiles/event.json')
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_tag(self):
self.init_event()
self.mispevent.add_tag('bar')
self.mispevent.add_tag(name='baz')
new_tag = MISPTag()
new_tag.from_dict(name='foo')
self.mispevent.add_tag(new_tag)
with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_attribute(self):
self.init_event()
a = self.mispevent.add_attribute('filename', 'bar.exe')
del a.uuid
a = self.mispevent.add_attribute_tag('osint', 'bar.exe')
attr_tags = self.mispevent.get_attribute_tag('bar.exe')
self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint')
self.assertEqual(attr_tags[0].name, 'osint')
with open('tests/mispevent_testfiles/attribute.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
# Fake setting an attribute ID for testing
self.mispevent.attributes[0].id = 42
self.mispevent.delete_attribute(42)
with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_object_tag(self):
self.mispevent.add_object(name='file', strict=True)
a = self.mispevent.objects[0].add_attribute('filename', value='')
self.assertEqual(a, None)
a = self.mispevent.objects[0].add_attribute('filename', value=None)
self.assertEqual(a, None)
a = self.mispevent.objects[0].add_attribute('filename',
value='bar',
Tag=[{
'name': 'blah'
}])
del a.uuid
self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name,
'blah')
self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(
['filename']))
self.assertEqual(
len(self.mispevent.objects[0].get_attributes_by_relation(
'filename')), 1)
self.mispevent.add_object(name='url', strict=True)
a = self.mispevent.objects[1].add_attribute(
'url', value='https://www.circl.lu')
del a.uuid
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
reference = self.mispevent.objects[0].add_reference(
self.mispevent.objects[1], 'baz', comment='foo')
del reference.uuid
self.assertEqual(
self.mispevent.objects[0].references[0].relationship_type, 'baz')
with open('tests/mispevent_testfiles/event_obj_attr_tag.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip(
"Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168"
)
def test_object_level_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar')
self.mispevent.objects[0].add_tag('osint')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_malware(self):
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.init_event()
a = self.mispevent.add_attribute('malware-sample',
'bar.exe',
data=pseudofile)
del a.uuid
attribute = self.mispevent.attributes[0]
self.assertEqual(attribute.malware_binary, pseudofile)
with open('tests/mispevent_testfiles/malware.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_malware(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/malware_exist.json')
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.assertEqual(
self.mispevent.objects[0].get_attributes_by_relation(
'malware-sample')[0].malware_binary.read(), pseudofile.read())
def test_sighting(self):
sighting = MISPSighting()
sighting.from_dict(value='1', type='bar', timestamp=11111111)
with open('tests/mispevent_testfiles/sighting.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(sighting.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_event(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
with open('tests/mispevent_testfiles/existing_event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes_existing(self):
self.mispevent.load_file('tests/mispevent_testfiles/shadow.json')
with open('tests/mispevent_testfiles/shadow.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip("Not supported on MISP.")
def test_shadow_attributes(self):
self.init_event()
p = self.mispevent.add_proposal(type='filename', value='baz.jpg')
del p.uuid
a = self.mispevent.add_attribute('filename', 'bar.exe')
del a.uuid
p = self.mispevent.attributes[0].add_proposal(type='filename',
value='bar.pdf')
del p.uuid
with open('tests/mispevent_testfiles/proposals.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_default_attributes(self):
self.mispevent.add_object(name='file', strict=True)
a = self.mispevent.objects[0].add_attribute('filename',
value='bar',
Tag=[{
'name': 'blah'
}])
del a.uuid
a = self.mispevent.objects[0].add_attribute('pattern-in-file',
value='baz')
self.assertEqual(a.category, 'Artifacts dropped')
del a.uuid
self.mispevent.add_object(name='file',
strict=False,
default_attributes_parameters=self.mispevent.
objects[0].attributes[0])
a = self.mispevent.objects[1].add_attribute('filename', value='baz')
del a.uuid
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
with open('tests/mispevent_testfiles/event_obj_def_param.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_default_values(self):
self.init_event()
self.mispevent.add_object(name='whois', strict=True)
a = self.mispevent.objects[0].add_attribute(
'registrar', value='registar.example.com')
del a.uuid
a = self.mispevent.objects[0].add_attribute('domain',
value='domain.example.com')
del a.uuid
a = self.mispevent.objects[0].add_attribute('nameserver',
value='ns1.example.com')
del a.uuid
a = self.mispevent.objects[0].add_attribute(
'nameserver',
value='ns2.example.com',
disable_correlation=False,
to_ids=True,
category='External analysis')
del a.uuid
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/def_param.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_not_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
def test_event_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.mispevent.info = 'blah'
self.assertTrue(self.mispevent.edited)
def test_event_tag_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.add_tag('foo')
self.assertTrue(self.mispevent.edited)
def test_event_attribute_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.mispevent.attributes[0].value = 'blah'
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertFalse(self.mispevent.attributes[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].tags[0].name = 'blah'
self.assertTrue(self.mispevent.attributes[0].tags[0].edited)
self.assertFalse(self.mispevent.attributes[0].tags[1].edited)
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited_second(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].add_tag(name='blah')
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].edited)
self.assertFalse(self.mispevent.objects[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited_tag(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].add_tag('blah')
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
with open('tests/mispevent_testfiles/existing_event_edited.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_by_id(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
misp_obj = self.mispevent.get_object_by_id(1556)
self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f')
def test_userdefined_object(self):
self.init_event()
self.mispevent.add_object(
name='test_object_template',
strict=True,
misp_objects_path_custom='tests/mispevent_testfiles')
with self.assertRaises(InvalidMISPObject) as e:
# Fail on required
self.mispevent.to_json(sort_keys=True, indent=2)
if sys.version_info >= (3, ):
self.assertEqual(e.exception.message,
'{\'member3\'} are required.')
else:
# Python2 bullshit
self.assertEqual(e.exception.message,
'set([u\'member3\']) are required.')
a = self.mispevent.objects[0].add_attribute('member3', value='foo')
del a.uuid
with self.assertRaises(InvalidMISPObject) as e:
# Fail on requiredOneOf
self.mispevent.to_json(sort_keys=True, indent=2)
self.assertEqual(
e.exception.message,
'At least one of the following attributes is required: member1, member2'
)
a = self.mispevent.objects[0].add_attribute('member1', value='bar')
del a.uuid
a = self.mispevent.objects[0].add_attribute('member1', value='baz')
del a.uuid
with self.assertRaises(InvalidMISPObject) as e:
# member1 is not a multiple
self.mispevent.to_json(sort_keys=True, indent=2)
self.assertEqual(e.exception.message,
'Multiple occurrences of member1 is not allowed')
self.mispevent.objects[0].attributes = self.mispevent.objects[
0].attributes[:2]
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def handle_ipaddress(self, ipaddress, event: pymisp.MISPEvent):
"""Handle a single IP address."""
event.add_attribute("ip-dst", str(ipaddress))
return event
def check_hashes(self):
if self.offline_mode:
self.log('error', 'Offline mode, unable to query VirusTotal')
return
event_id = self._get_eventid()
if event_id is None:
return
event = self.misp.get(event_id)
if self._has_error_message(event):
return
misp_event = MISPEvent()
misp_event.load(event)
hashes_to_expand = {}
hashes_expanded = [] # Thoses hashes are known and already processed
local_samples_hashes = []
partial_objects = {}
for o in misp_event.Object:
if o.name != 'file':
continue
if o.has_attributes_by_relation(['md5', 'sha1', 'sha256']):
# This object has all the hashes we care about
tmphashes = []
tmphashes += [h.value for h in o.get_attributes_by_relation('md5')]
tmphashes += [h.value for h in o.get_attributes_by_relation('sha1')]
tmphashes += [h.value for h in o.get_attributes_by_relation('sha256')]
# Make sure to query VT for the sha256, even if expanded locally
hashes_to_expand[o.get_attributes_by_relation('sha256')[0].value] = o.get_attributes_by_relation('sha256')[0]
if o.has_attributes_by_relation(['malware-sample']):
# ... and it has a malware sample
local_samples_hashes += tmphashes
hashes_expanded += tmphashes
elif o.has_attributes_by_relation(['malware-sample']):
# This object has a malware sample, but is missing hashes. We can expand locally.
# get the MD5 from the malware-sample attribute
malware_sample = o.get_attributes_by_relation('malware-sample')[0] # at most one sample/file object
local_samples_hashes.append(malware_sample.value.split('|')[1])
local_samples_hashes += [h.value for h in o.get_attributes_by_relation('md5')]
local_samples_hashes += [h.value for h in o.get_attributes_by_relation('sha1')]
local_samples_hashes += [h.value for h in o.get_attributes_by_relation('sha256')]
if self.args.populate:
# The object is missing hashes, keeping track of it for expansion if it isn't already done.
partial_objects[o.uuid] = malware_sample
else:
sha256 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('sha256')}
sha1 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('sha1')}
md5 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('md5')}
if sha256:
hashes_to_expand.update(sha256)
elif sha1:
hashes_to_expand.update(sha1)
elif md5:
hashes_to_expand.update(md5)
for ref_uuid, sample in partial_objects.items():
if sample.value.split('|')[1] in hashes_expanded:
# Already expanded in an other object
continue
new_obj, hashes = self._expand_local_sample(pseudofile=sample.malware_binary,
filename=sample.value.split('|')[0],
refobj=ref_uuid,
default_attributes_paramaters=sample)
misp_event.Object += new_obj
local_samples_hashes += hashes
# Make sure to query VT for the sha256, even if expanded locally
hashes_to_expand[hashes[0]] = sample
hashes_expanded += local_samples_hashes
for a in misp_event.attributes:
if a.type == 'malware-sample' and a.value.split('|')[1] not in hashes_expanded:
new_obj, hashes = self._expand_local_sample(pseudofile=a.malware_binary,
filename=a.value.split('|')[0],
default_attributes_paramaters=a)
misp_event.Object += new_obj
local_samples_hashes += hashes
# Make sure to query VT for the sha256, even if expanded locally
hashes_to_expand[hashes[0]] = a
elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'):
# We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway
fname, hashval = a.value.split('|')
hashes_to_expand[hashval] = a
elif a.type in ('md5', 'sha1', 'sha256'):
# We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway
hashes_to_expand[a.value] = a
unk_vt_hashes = []
if cfg.virustotal.virustotal_has_private_key is False:
quota = 4
timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)
hashes_expanded += local_samples_hashes
processed_on_vt = []
# Make sure to start getting reports for the longest possible hashes (reduce risks of collisions)
for to_expand in sorted(list(set(hashes_to_expand)), key=len):
if to_expand in processed_on_vt:
# Always run VT, once per sample
continue
original_attribute = hashes_to_expand[to_expand]
if original_attribute.get('object_id'):
original_object_id = original_attribute.get('object_id')
vt_object = self._make_VT_object(to_expand, original_attribute)
if not vt_object:
unk_vt_hashes.append(to_expand)
continue
result = vt_object.get_report()
md5 = result['md5']
sha1 = result['sha1']
sha256 = result['sha256']
processed_on_vt += [sha256, sha1, md5]
if all(h in local_samples_hashes for h in [md5, sha1, sha256]):
self.log('success', 'Sample available in MISP:')
else:
self.log('success', 'Sample available in VT:')
self.log('item', '{}\n\t{}\n\t{}\n\t{}'.format(result["permalink"], md5, sha1, sha256))
if self.args.populate:
if not all(h in hashes_expanded for h in [md5, sha1, sha256]):
# If all the "new" expanded hashes are in the hashes_expanded list, skip
file_object = MISPObject('file', default_attributes_paramaters=original_attribute)
file_object.add_attribute('md5', value=md5)
file_object.add_attribute('sha1', value=sha1)
file_object.add_attribute('sha256', value=sha256)
file_object.add_reference(vt_object.uuid, 'analysed-with')
misp_event.Object.append(file_object)
hashes_expanded += [md5, sha1, sha256]
else:
if not original_object_id or original_object_id == '0':
# Not an object, but the hashes are in an other object, skipping
continue
else:
# We already have a MISP object, adding the link to the new VT object
file_object = misp_event.get_object_by_id(original_object_id)
file_object.add_reference(vt_object.uuid, 'analysed-with')
misp_event.Object.append(vt_object)
if cfg.virustotal.virustotal_has_private_key is False:
if quota > 0:
quota -= 1
else:
waiting_time = (timeout - datetime.datetime.now()).seconds
if waiting_time > 0:
self.log('warning', 'No private API key, 4 queries/min is the limit. Waiting for {} seconds.'.format(waiting_time))
time.sleep(waiting_time)
quota = 4
timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)
if self.args.populate:
self._populate(misp_event)
if len(unk_vt_hashes) > 0:
self.log('error', 'Unknown on VT:')
for h in unk_vt_hashes:
self.log('item', '{}'.format(h))
def handle_yarasignature(self, yarasignature, event: pymisp.MISPEvent):
"""Handle a single YARA signature."""
event.add_attribute("yara", str(yarasignature))
return event
def create_event(misp):
event = MISPEvent()
event.distribution = 0
event.threat_level_id = 1
event.analysis = 0
return event
def __init__(self, attribute_props: dict):
self.event = MISPEvent()
self.enrichment_object = MISPObject("Recorded Future Enrichment")
description = ("An object containing the enriched attribute and "
"related entities from Recorded Future.")
self.enrichment_object.from_dict(**{
"meta-category": "misc",
"description": description,
"distribution": 0
})
# Create a copy of enriched attribute to add tags to
temp_attr = MISPAttribute()
temp_attr.from_dict(**attribute_props)
self.enriched_attribute = MISPAttribute()
self.enriched_attribute.from_dict(**{
"value": temp_attr.value,
"type": temp_attr.type,
"distribution": 0
})
self.related_attributes: List[Tuple[str, MISPAttribute]] = []
self.color_picker = RFColors()
self.galaxy_finder = GalaxyFinder()
# Mapping from MISP-type to RF-type
self.type_to_rf_category = {
"ip": "ip",
"ip-src": "ip",
"ip-dst": "ip",
"ip-src|port": "ip",
"ip-dst|port": "ip",
"domain": "domain",
"hostname": "domain",
"md5": "hash",
"sha1": "hash",
"sha256": "hash",
"uri": "url",
"url": "url",
"vulnerability": "vulnerability",
"weakness": "vulnerability",
}
# Related entities have 'Related' as part of the word and Links entities from RF
# portrayed as related attributes in MISP
self.related_attribute_types = [
"RelatedIpAddress",
"RelatedInternetDomainName",
"RelatedHash",
"RelatedEmailAddress",
"RelatedCyberVulnerability",
"IpAddress",
"InternetDomainName",
"Hash",
"EmailAddress",
"CyberVulnerability",
]
# Related entities have 'Related' as part of the word and and Links entities from RF portrayed as tags in MISP
self.galaxy_tag_types = [
"RelatedMalware",
"RelatedThreatActor",
"Threat Actor",
"MitreAttackIdentifier",
"Malware",
]
class VirusTotalParser(object):
def __init__(self, apikey, limit):
self.apikey = apikey
self.limit = limit
self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report"
self.misp_event = MISPEvent()
self.parsed_objects = {}
self.input_types_mapping = {
'ip-src': self.parse_ip,
'ip-dst': self.parse_ip,
'domain': self.parse_domain,
'hostname': self.parse_domain,
'md5': self.parse_hash,
'sha1': self.parse_hash,
'sha256': self.parse_hash,
'url': self.parse_url
}
self.proxies = None
def query_api(self, attribute):
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
return self.input_types_mapping[self.attribute.type](
self.attribute.value, recurse=True)
def get_result(self):
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object') if (key in event and event[key])
}
return {'results': results}
################################################################################
#### Main parsing functions #### # noqa
################################################################################
def parse_domain(self, domain, recurse=False):
req = requests.get(self.base_url.format('domain'),
params={
'apikey': self.apikey,
'domain': domain
},
proxies=self.proxies)
if req.status_code != 200:
return req.status_code
req = req.json()
hash_type = 'sha256'
whois = 'whois'
feature_types = {
'communicating': 'communicates-with',
'downloaded': 'downloaded-from',
'referrer': 'referring'
}
siblings = (self.parse_siblings(domain)
for domain in req['domain_siblings'])
uuid = self.parse_resolutions(
req['resolutions'],
req['subdomains'] if 'subdomains' in req else None, siblings)
for feature_type, relationship in feature_types.items():
for feature in ('undetected_{}_samples', 'detected_{}_samples'):
for sample in req.get(feature.format(feature_type),
[])[:self.limit]:
status_code = self.parse_hash(sample[hash_type], False,
uuid, relationship)
if status_code != 200:
return status_code
if req.get(whois):
whois_object = MISPObject(whois)
whois_object.add_attribute('text', type='text', value=req[whois])
self.misp_event.add_object(**whois_object)
return self.parse_related_urls(req, recurse, uuid)
def parse_hash(self, sample, recurse=False, uuid=None, relationship=None):
req = requests.get(self.base_url.format('file'),
params={
'apikey': self.apikey,
'resource': sample
},
proxies=self.proxies)
status_code = req.status_code
if req.status_code == 200:
req = req.json()
vt_uuid = self.parse_vt_object(req)
file_attributes = []
for hash_type in ('md5', 'sha1', 'sha256'):
if req.get(hash_type):
file_attributes.append({
'type': hash_type,
'object_relation': hash_type,
'value': req[hash_type]
})
if file_attributes:
file_object = MISPObject('file')
for attribute in file_attributes:
file_object.add_attribute(**attribute)
file_object.add_reference(vt_uuid, 'analyzed-with')
if uuid and relationship:
file_object.add_reference(uuid, relationship)
self.misp_event.add_object(**file_object)
return status_code
def parse_ip(self, ip, recurse=False):
req = requests.get(self.base_url.format('ip-address'),
params={
'apikey': self.apikey,
'ip': ip
},
proxies=self.proxies)
if req.status_code != 200:
return req.status_code
req = req.json()
if req.get('asn'):
asn_mapping = {
'network': ('ip-src', 'subnet-announced'),
'country': ('text', 'country')
}
asn_object = MISPObject('asn')
asn_object.add_attribute('asn', type='AS', value=req['asn'])
for key, value in asn_mapping.items():
if req.get(key):
attribute_type, relation = value
asn_object.add_attribute(relation,
type=attribute_type,
value=req[key])
self.misp_event.add_object(**asn_object)
uuid = self.parse_resolutions(
req['resolutions']) if req.get('resolutions') else None
return self.parse_related_urls(req, recurse, uuid)
def parse_url(self, url, recurse=False, uuid=None):
req = requests.get(self.base_url.format('url'),
params={
'apikey': self.apikey,
'resource': url
},
proxies=self.proxies)
status_code = req.status_code
if req.status_code == 200:
req = req.json()
vt_uuid = self.parse_vt_object(req)
if not recurse:
feature = 'url'
url_object = MISPObject(feature)
url_object.add_attribute(feature, type=feature, value=url)
url_object.add_reference(vt_uuid, 'analyzed-with')
if uuid:
url_object.add_reference(uuid, 'hosted-in')
self.misp_event.add_object(**url_object)
return status_code
################################################################################
#### Additional parsing functions #### # noqa
################################################################################
def parse_related_urls(self, query_result, recurse, uuid=None):
if recurse:
for feature in ('detected_urls', 'undetected_urls'):
if feature in query_result:
for url in query_result[feature]:
value = url['url'] if isinstance(url, dict) else url[0]
status_code = self.parse_url(value, False, uuid)
if status_code != 200:
return status_code
else:
for feature in ('detected_urls', 'undetected_urls'):
if feature in query_result:
for url in query_result[feature]:
value = url['url'] if isinstance(url, dict) else url[0]
self.misp_event.add_attribute('url', value)
return 200
def parse_resolutions(self, resolutions, subdomains=None, uuids=None):
domain_ip_object = MISPObject('domain-ip')
if self.attribute.type in ('domain', 'hostname'):
domain_ip_object.add_attribute('domain',
type='domain',
value=self.attribute.value)
attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address')
else:
domain_ip_object.add_attribute('ip',
type='ip-dst',
value=self.attribute.value)
attribute_type, relation, key = ('domain', 'domain', 'hostname')
for resolution in resolutions:
domain_ip_object.add_attribute(relation,
type=attribute_type,
value=resolution[key])
if subdomains:
for subdomain in subdomains:
attribute = MISPAttribute()
attribute.from_dict(**dict(type='domain', value=subdomain))
self.misp_event.add_attribute(**attribute)
domain_ip_object.add_reference(attribute.uuid, 'subdomain')
if uuids:
for uuid in uuids:
domain_ip_object.add_reference(uuid, 'sibling-of')
self.misp_event.add_object(**domain_ip_object)
return domain_ip_object.uuid
def parse_siblings(self, domain):
attribute = MISPAttribute()
attribute.from_dict(**dict(type='domain', value=domain))
self.misp_event.add_attribute(**attribute)
return attribute.uuid
def parse_vt_object(self, query_result):
if query_result['response_code'] == 1:
vt_object = MISPObject('virustotal-report')
vt_object.add_attribute('permalink',
type='link',
value=query_result['permalink'])
detection_ratio = '{}/{}'.format(query_result['positives'],
query_result['total'])
vt_object.add_attribute('detection-ratio',
type='text',
value=detection_ratio,
disable_correlation=True)
self.misp_event.add_object(**vt_object)
return vt_object.uuid
def set_proxy_settings(self, config: dict) -> dict:
"""Returns proxy settings in the requests format.
If no proxy settings are set, return None."""
proxies = None
host = config.get('proxy_host')
port = config.get('proxy_port')
username = config.get('proxy_username')
password = config.get('proxy_password')
if host:
if not port:
misperrors['error'] = 'The virustotal_proxy_host config is set, ' \
'please also set the virustotal_proxy_port.'
raise KeyError
parsed = urlparse(host)
if 'http' in parsed.scheme:
scheme = 'http'
else:
scheme = parsed.scheme
netloc = parsed.netloc
host = f'{netloc}:{port}'
if username:
if not password:
misperrors['error'] = 'The virustotal_proxy_username config is set, ' \
'please also set the virustotal_proxy_password.'
raise KeyError
auth = f'{username}:{password}'
host = auth + '@' + host
proxies = {
'http': f'{scheme}://{host}',
'https': f'{scheme}://{host}'
}
self.proxies = proxies
return True
def add_event(self):
try:
event = MISPEvent()
event.distribution = 0
# ATD Threat mapping to MISP Threat Level
atd_threat_level = self.query['Summary']['Verdict']['Severity']
if not atd_threat_level:
pass
else:
if atd_threat_level == '3':
event.threat_level_id = 1
elif atd_threat_level == '4':
event.threat_level_id = 2
elif atd_threat_level == '5':
event.threat_level_id = 3
else:
event.threat_level_id = 0
event.analysis = 0 # initial
event.info = "ATD Analysis Report - {0}".format(self.mainfile)
event.attributes = self.attributes
event.Tag = 'ATD:Report'
event = self.misp.add_event(event, pythonify=True)
self.evenid = event.id
print('SUCCESS: New MISP Event got created with ID: {}'.format(str(event.id)))
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print("ERROR: Error in {location}.{funct_name}() - line {line_no} : {error}"
.format(location=__name__, funct_name=sys._getframe().f_code.co_name, line_no=exc_tb.tb_lineno,
error=str(e)))
samp_details=json.loads(details_request.text)
return samp_details.get('data')[0] if (samp_details.get('query_status') == 'ok') else None
except:
print(details_request.content)
return None
# a dict of samples in the Event, keyed with <k>sha256 containing <v>MISP-Object Files
samples={}
# a cache of all the attributes in the event
attributes={}
#misp setup
pm = PyMISP(misp_url, misp_key, ssl=misp_check_cert)
misp_event=MISPEvent()
search=pm.search(controller='events', eventinfo=misp_event_name)
if ( len(search) == 1):
misp_event.load(search[0])
#load 'samples' dictionary from misp_event
for obj in misp_event.get('Object'):
if (obj.name == "file"):
existing_hash = obj.get_attributes_by_relation("sha256")[0]['value']
samples.update({existing_hash : obj})
#reload 'attributes' dictionary from misp_event
for attr in misp_event.get('Attribute'):
attributes.update({attr.value : attr})
else:
urlVap = "https://tap-api-v2.proofpoint.com/v2/people/vap?window=30" # Window can be 14, 30, and 90 Days
headers = {'Authorization': "Basic " + proofpoint_key}
responseVap = requests.request("GET", urlVap, headers=headers)
jsonDataVap = json.loads(responseVap.text)
for alert in jsonDataVap["users"]:
orgc = MISPOrganisation()
orgc.name = 'Proofpoint'
orgc.id = '#{ORGC.ID}' # organisation id
orgc.uuid = '#{ORGC.UUID}' # organisation uuid
# initialize and set MISPEvent()
event = MISPEvent()
event.Orgc = orgc
event.info = 'Very Attacked Person ' + jsonDataVap["interval"]
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 2 # setting this to 0 breaks the integration
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
totalVapUsers = event.add_attribute('counter',
jsonDataVap["totalVapUsers"],
comment="Total VAP Users")
averageAttackIndex = event.add_attribute('counter',
jsonDataVap["averageAttackIndex"],
comment="Average Attack Count")
vapAttackIndexThreshold = event.add_attribute(
class APIVoidParser():
def __init__(self, attribute):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.url = 'https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&'
def get_results(self):
if hasattr(self, 'result'):
return self.result
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object')}
return {'results': results}
def parse_domain(self, apikey):
feature = 'dnslookup'
if requests.get(f'{self.url.format(feature, apikey)}stats').json(
)['credits_remained'] < 0.13:
self.result = {
'error':
'You do not have enough APIVoid credits to proceed your request.'
}
return
mapping = {
'A': 'resolution-of',
'MX': 'mail-server-of',
'NS': 'server-name-of'
}
dnslookup = requests.get(
f'{self.url.format(feature, apikey)}action=dns-any&host={self.attribute.value}'
).json()
for item in dnslookup['data']['records']['items']:
record_type = item['type']
try:
relationship = mapping[record_type]
except KeyError:
continue
self._handle_dns_record(item, record_type, relationship)
ssl = requests.get(
f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}'
).json()
self._parse_ssl_certificate(ssl['data']['certificate'])
def _handle_dns_record(self, item, record_type, relationship):
dns_record = MISPObject('dns-record')
dns_record.add_attribute('queried-domain',
type='domain',
value=item['host'])
attribute_type, feature = ('ip-dst',
'ip') if record_type == 'A' else ('domain',
'target')
dns_record.add_attribute(f'{record_type.lower()}-record',
type=attribute_type,
value=item[feature])
dns_record.add_reference(self.attribute.uuid, relationship)
self.misp_event.add_object(**dns_record)
def _parse_ssl_certificate(self, certificate):
x509 = MISPObject('x509')
fingerprint = 'x509-fingerprint-sha1'
x509.add_attribute(fingerprint,
type=fingerprint,
value=certificate['fingerprint'])
x509_mapping = {
'subject': {
'name': ('text', 'subject')
},
'issuer': {
'common_name': ('text', 'issuer')
},
'signature': {
'serial': ('text', 'serial-number')
},
'validity': {
'valid_from': ('datetime', 'validity-not-before'),
'valid_to': ('datetime', 'validity-not-after')
}
}
certificate = certificate['details']
for feature, subfeatures in x509_mapping.items():
for subfeature, mapping in subfeatures.items():
attribute_type, relation = mapping
x509.add_attribute(relation,
type=attribute_type,
value=certificate[feature][subfeature])
x509.add_reference(self.attribute.uuid, 'seen-by')
self.misp_event.add_object(**x509)
class TestMISPEvent(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
def init_event(self):
self.mispevent.info = 'This is a test'
self.mispevent.distribution = 1
self.mispevent.threat_level_id = 1
self.mispevent.analysis = 1
self.mispevent.set_date("2017-12-31") # test the set date method
def test_simple(self):
with open('tests/mispevent_testfiles/simple.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event(self):
self.init_event()
self.mispevent.publish()
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_loadfile(self):
self.mispevent.load_file('tests/mispevent_testfiles/event.json')
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_tag(self):
self.init_event()
self.mispevent.add_tag('bar')
self.mispevent.add_tag(name='baz')
new_tag = MISPTag()
new_tag.from_dict(name='foo')
self.mispevent.add_tag(new_tag)
with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_attribute(self):
self.init_event()
self.mispevent.add_attribute('filename', 'bar.exe')
self.mispevent.add_attribute_tag('osint', 'bar.exe')
attr_tags = self.mispevent.get_attribute_tag('bar.exe')
self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint')
self.assertEqual(attr_tags[0].name, 'osint')
with open('tests/mispevent_testfiles/attribute.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
# Fake setting an attribute ID for testing
self.mispevent.attributes[0].id = 42
self.mispevent.delete_attribute(42)
with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_object_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}])
self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah')
self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(['filename']))
self.assertEqual(len(self.mispevent.objects[0].get_attributes_by_relation('filename')), 1)
self.mispevent.add_object(name='url', strict=True)
self.mispevent.objects[1].add_attribute('url', value='https://www.circl.lu')
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
self.mispevent.objects[0].add_reference('b', 'baz', comment='foo')
self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz')
with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168")
def test_object_level_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar')
self.mispevent.objects[0].add_tag('osint')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_malware(self):
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.init_event()
self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile)
attribute = self.mispevent.attributes[0]
self.assertEqual(attribute.malware_binary, pseudofile)
with open('tests/mispevent_testfiles/malware.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_malware(self):
self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json')
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.assertEqual(
self.mispevent.objects[0].get_attributes_by_relation('malware-sample')[0].malware_binary.read(),
pseudofile.read())
def test_sighting(self):
sighting = MISPSighting()
sighting.from_dict(value='1', type='bar', timestamp=11111111)
with open('tests/mispevent_testfiles/sighting.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_event(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
with open('tests/mispevent_testfiles/existing_event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes_existing(self):
self.mispevent.load_file('tests/mispevent_testfiles/shadow.json')
with open('tests/mispevent_testfiles/shadow.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes(self):
self.init_event()
self.mispevent.add_proposal(type='filename', value='baz.jpg')
self.mispevent.add_attribute('filename', 'bar.exe')
self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf')
with open('tests/mispevent_testfiles/proposals.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_default_attributes(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}])
self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent.objects[0].attributes[0])
self.mispevent.objects[1].add_attribute('filename', value='baz')
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_default_values(self):
self.init_event()
self.mispevent.add_object(name='whois', strict=True)
self.mispevent.objects[0].add_attribute('registrar', value='registar.example.com')
self.mispevent.objects[0].add_attribute('domain', value='domain.example.com')
self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com')
self.mispevent.objects[0].add_attribute('nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/def_param.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_not_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
def test_event_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.mispevent.info = 'blah'
self.assertTrue(self.mispevent.edited)
def test_event_tag_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.add_tag('foo')
self.assertTrue(self.mispevent.edited)
def test_event_attribute_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.mispevent.attributes[0].value = 'blah'
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertFalse(self.mispevent.attributes[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].tags[0].name = 'blah'
self.assertTrue(self.mispevent.attributes[0].tags[0].edited)
self.assertFalse(self.mispevent.attributes[0].tags[1].edited)
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited_second(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].add_tag(name='blah')
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].edited)
self.assertFalse(self.mispevent.objects[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited_tag(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].add_tag('blah')
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_by_id(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
misp_obj = self.mispevent.get_object_by_id(1556)
self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f')
def test_userdefined_object(self):
self.init_event()
self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles')
with self.assertRaises(InvalidMISPObject) as e:
# Fail on required
self.mispevent.to_json()
if sys.version_info >= (3, ):
self.assertEqual(e.exception.message, '{\'member3\'} are required.')
else:
# Python2 bullshit
self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.')
self.mispevent.objects[0].add_attribute('member3', value='foo')
with self.assertRaises(InvalidMISPObject) as e:
# Fail on requiredOneOf
self.mispevent.to_json()
self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2')
self.mispevent.objects[0].add_attribute('member1', value='bar')
self.mispevent.objects[0].add_attribute('member1', value='baz')
with self.assertRaises(InvalidMISPObject) as e:
# member1 is not a multiple
self.mispevent.to_json()
self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed')
self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2]
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def __init__(self, attribute):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.url = 'https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&'
class TestPDFExport(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
if not manual_testing:
self.root = "tests/"
else:
self.root = ""
self.test_folder = self.root + "reportlab_testfiles/"
self.test_batch_folder = self.root + "OSINT_output/"
self.storage_folder_OSINT = self.root + "OSINT_PDF/"
self.test_image_folder = self.root + "image_json/"
self.storage_folder = self.root + "reportlab_testoutputs/"
self.storage_image_folder = self.root + "reportlab_test_image_outputs/"
self.moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description",
"Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"]
def init_event(self):
self.mispevent.info = 'This is a test'
self.mispevent.distribution = 1
self.mispevent.threat_level_id = 1
self.mispevent.analysis = 1
self.mispevent.set_date("2017-12-31") # test the set date method
def check_python_2(self):
if sys.version_info.major < 3:
# we want Python2 test to pass
return True
def test_basic_event(self):
if self.check_python_2():
self.assertTrue(True)
else:
self.init_event()
reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
self.storage_folder + "basic_event.pdf")
def test_event(self):
if self.check_python_2():
self.assertTrue(True)
else:
self.init_event()
self.mispevent.load_file(self.test_folder + 'to_delete1.json')
reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
self.storage_folder + "normal_event.pdf")
def test_HTML_json(self):
if self.check_python_2():
self.assertTrue(True)
else:
self.init_event()
self.mispevent.load_file(self.test_folder + 'HTML_event.json')
reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
self.storage_folder + "HTML_event.pdf")
def test_long_json(self):
if self.check_python_2():
self.assertTrue(True)
else:
self.init_event()
self.mispevent.load_file(self.test_folder + 'long_event.json')
reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
self.storage_folder + "long_event.pdf")
# Issue report : "We are not smart enough" : https://pairlist2.pair.net/pipermail/reportlab-users/2010-May/009529.html
# Not nice but working solution exposed there: https://pairlist2.pair.net/pipermail/reportlab-users/2016-March/011525.html
def test_very_long_json(self):
if self.check_python_2():
self.assertTrue(True)
else:
self.init_event()
self.mispevent.load_file(self.test_folder + 'very_long_event.json')
reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
self.storage_folder + "very_long_event.pdf")
def test_full_config_json(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
self.init_event()
self.mispevent.load_file(self.test_folder + 'very_long_event.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "config_complete_event.pdf")
def test_partial_0_config_json(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
self.init_event()
self.mispevent.load_file(self.test_folder + 'very_long_event.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "config_partial_0_event.pdf")
def test_partial_1_config_json(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[1]] = "My Wonderful CERT"
self.init_event()
self.mispevent.load_file(self.test_folder + 'very_long_event.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "config_partial_1_event.pdf")
def test_image_json(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
self.init_event()
self.mispevent.load_file(self.test_folder + 'image_event.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "image_event.pdf")
def test_objects_1_json(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
self.init_event()
self.mispevent.load_file(self.test_folder + 'mainly_objects_1.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "mainly_objects_1.pdf")
def test_objects_2_json(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
self.init_event()
self.mispevent.load_file(self.test_folder + 'mainly_objects_2.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "mainly_objects_2.pdf")
def test_sightings_1_json(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
self.init_event()
self.mispevent.load_file(self.test_folder + 'sighting_1.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "sighting_1.pdf")
def test_sightings_2_json(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
self.init_event()
self.mispevent.load_file(self.test_folder + 'sighting_2.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "sighting_2.pdf")
def test_textual_json(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
config[self.moduleconfig[2]] = True
self.init_event()
self.mispevent.load_file(self.test_folder + 'very_long_event.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "textual.pdf")
def test_galaxy_1(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
config[self.moduleconfig[2]] = True
config[self.moduleconfig[3]] = True
self.init_event()
self.mispevent.load_file(self.test_folder + 'galaxy_1.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "galaxy_1.pdf")
def test_related_events(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
config[self.moduleconfig[2]] = True
config[self.moduleconfig[3]] = True
config[self.moduleconfig[4]] = True
self.init_event()
self.mispevent.load_file(self.test_folder + 'galaxy_1.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "related_events.pdf")
def test_related_events_too_simple(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
config[self.moduleconfig[2]] = True
config[self.moduleconfig[3]] = True
config[self.moduleconfig[4]] = True
self.init_event()
self.mispevent.load_file(self.test_folder + 'to_delete1.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "related_events_no_related.pdf")
def test_utf(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
config[self.moduleconfig[2]] = True
config[self.moduleconfig[3]] = True
config[self.moduleconfig[4]] = True
config[self.moduleconfig[5]] = True
self.init_event()
self.mispevent.load_file(self.test_folder + 'japanese_test.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "japanese_test.pdf")
def test_utf_heavy(self):
if self.check_python_2():
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
config[self.moduleconfig[2]] = True
config[self.moduleconfig[3]] = True
config[self.moduleconfig[4]] = True
config[self.moduleconfig[5]] = True
self.init_event()
self.mispevent.load_file(self.test_folder + 'japanese_test_heavy.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "japanese_test_heavy.pdf")
def test_utf_ArialUNI_custompath(self):
if self.check_python_2():
self.assertTrue(True)
elif not manual_testing:
self.assertTrue(True)
else:
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
config[self.moduleconfig[2]] = True
config[self.moduleconfig[3]] = True
config[self.moduleconfig[4]] = True
config[self.moduleconfig[5]] = True
config[self.moduleconfig[6]] = "/home/user/Desktop/PyMISP/pymisp/tools/pdf_fonts/arial-unicode-ms/ARIALUNI.TTF"
self.init_event()
self.mispevent.load_file(self.test_folder + 'japanese_test_heavy.json')
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder + "custom_path.pdf")
def test_batch_image_events(self):
# Test case ONLY for manual testing. Needs to download a full list of image events !
if self.check_python_2():
self.assertTrue(True)
elif not manual_testing:
self.assertTrue(True)
else:
self.init_event()
file_nb = str(len(os.listdir(self.test_image_folder)))
i = 0
t = time.time()
for curr_file in os.listdir(self.test_image_folder):
self.mispevent = MISPEvent()
file_path = self.test_image_folder + curr_file
print("Current file : " + file_path + " " + str(i) + " over " + file_nb)
i += 1
self.mispevent.load_file(file_path)
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
self.storage_image_folder + curr_file + ".pdf")
print("Elapsed time : " + str(time.time() - t))
# Local run : 73.061s for 102 files
def test_batch_OSINT_events(self):
# Test case ONLY for manual testing. Needs to download a full list of OSINT events !
if self.check_python_2():
self.assertTrue(True)
elif not manual_testing:
self.assertTrue(True)
else:
self.init_event()
file_nb = str(len(os.listdir(self.test_batch_folder)))
i = 0
t = time.time()
for curr_file in os.listdir(self.test_batch_folder):
self.mispevent = MISPEvent()
file_path = self.test_batch_folder + curr_file
print("Current file : " + file_path + " " + str(i) + " over " + file_nb)
i += 1
self.mispevent.load_file(file_path)
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
self.storage_folder_OSINT + curr_file + ".pdf")
print("Elapsed time : " + str(time.time() - t))
# Local run : 1958.930s for 1064 files
def test_batch_OSINT_with_config_events(self):
# Test case ONLY for manual testing. Needs to download a full list of OSINT events !
if self.check_python_2():
self.assertTrue(True)
elif not manual_testing:
self.assertTrue(True)
else:
self.init_event()
config = {}
config[self.moduleconfig[0]] = "http://localhost:8080"
config[self.moduleconfig[1]] = "My Wonderful CERT"
config[self.moduleconfig[2]] = True
config[self.moduleconfig[3]] = True
config[self.moduleconfig[4]] = True
config[self.moduleconfig[5]] = True
file_nb = str(len(os.listdir(self.test_batch_folder)))
i = 0
t = time.time()
for curr_file in os.listdir(self.test_batch_folder):
self.mispevent = MISPEvent()
file_path = self.test_batch_folder + curr_file
print("Current file : " + file_path + " " + str(i) + " over " + file_nb)
i += 1
self.mispevent.load_file(file_path)
reportlab_generator.register_value_to_file(
reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
self.storage_folder_OSINT + curr_file + ".pdf")
print("Elapsed time : " + str(time.time() - t))
def __init__(self):
super(MISP, self).__init__()
self.cur_path = __project__.get_path()
self.parser.add_argument("--url", help='URL of the MISP instance')
self.parser.add_argument(
"--off",
action='store_true',
help='Use offline (can only work on pre-downloaded events)')
self.parser.add_argument("--on",
action='store_true',
help='Switch to online mode')
self.parser.add_argument("-k",
"--key",
help='Your key on the MISP instance')
self.parser.add_argument(
"-v",
"--verify",
action='store_false',
help='Disable certificate verification (for self-signed)')
subparsers = self.parser.add_subparsers(dest='subname')
# ##### Upload sample to MISP #####
parser_up = subparsers.add_parser(
'upload',
help='Send malware sample to MISP.',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent('''
Distribution levels:
* 0: Your organisation only
* 1: This community only
* 2: Connected communities
* 3: All communities
* 5: Inherit
Sample categories:
* 0: Payload delivery
* 1: Artifacts dropped
* 2: Payload installation
* 3: External analysis
Analysis levels:
* 0: Initial
* 1: Ongoing
* 2: Completed
Threat levels:
* 0: High
* 1: Medium
* 2: Low
* 3: Undefined
'''))
parser_up.add_argument(
"-e",
"--event",
type=int,
help=
"Event ID to update. If None, and you're not connected to a MISP event a new one is created."
)
parser_up.add_argument(
"-d",
"--distrib",
type=int,
choices=[0, 1, 2, 3, 5],
help="Distribution of the attributes for the new event.")
parser_up.add_argument(
"-s",
"--sharing",
type=int,
help="Sharing group ID when distribution is set to 4.")
parser_up.add_argument(
"-ids",
action='store_true',
help="Is eligible for automatically creating IDS signatures.")
parser_up.add_argument("-c",
"--categ",
type=int,
choices=[0, 1, 2, 3],
default=1,
help="Category of the samples.")
parser_up.add_argument("-i",
"--info",
nargs='+',
help="Event info field of a new event.")
parser_up.add_argument("-o",
"--comment",
nargs='+',
help="Comment associated to the sample.")
parser_up.add_argument("-a",
"--analysis",
type=int,
choices=[0, 1, 2],
help="Analysis level a new event.")
parser_up.add_argument("-t",
"--threat",
type=int,
choices=[0, 1, 2, 3],
help="Threat level of a new event.")
# ##### Download samples from event #####
parser_down = subparsers.add_parser(
'download', help='Download malware samples from MISP.')
group = parser_down.add_mutually_exclusive_group()
group.add_argument(
"-e",
"--event",
type=int,
help="Download all the samples related to this event ID.")
group.add_argument(
"-l",
"--list",
nargs='*',
help=
"Download all the samples related to a list of events. Empty list to download all the samples of all the events stored in the current project."
) # noqa
group.add_argument(
"--hash",
help="Download the sample related to this hash (only MD5).")
# ##### Search in MISP #####
parser_search = subparsers.add_parser(
'search', help='Search in all the attributes.')
parser_search.add_argument(
"query",
nargs='*',
help=
"String to search (if empty, search the hashes of the current file)."
)
# ##### Check hashes on VT #####
parser_checkhashes = subparsers.add_parser(
'check_hashes', help='Crosscheck hashes on VT.')
parser_checkhashes.add_argument(
"event",
nargs='?',
default=None,
type=int,
help="Lookup all the hashes of an event on VT.")
parser_checkhashes.add_argument(
"-p",
"--populate",
action='store_true',
help="Automatically populate event with hashes found on VT.")
# ##### Download Yara rules #####
parser_checkhashes = subparsers.add_parser(
'yara', help='Get YARA rules of an event.')
parser_checkhashes.add_argument(
"event",
nargs='?',
default=None,
type=int,
help="Download the yara rules of that event.")
# ##### Get Events #####
parser_pull = subparsers.add_parser(
'pull', help='Initialize the session with an existing MISP event.')
parser_pull.add_argument("event",
nargs='+',
type=int,
help="(List of) Event(s) ID.")
# ##### Create an Event #####
parser_create_event = subparsers.add_parser(
'create_event',
help=
'Create a new event on MISP and initialize the session with it.',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent('''
Distribution levels:
* 0: Your organisation only
* 1: This community only
* 2: Connected communities
* 3: All communities
* 4: Sharing group
Sharing Group:
* #: ID of sharing group
Analysis levels:
* 0: Initial
* 1: Ongoing
* 2: Completed
Threat levels:
* 0: High
* 1: Medium
* 2: Low
* 3: Undefined
'''))
parser_create_event.add_argument(
"-d",
"--distrib",
type=int,
choices=[0, 1, 2, 3, 4],
help="Distribution of the attributes for the new event.")
parser_create_event.add_argument(
"-s",
"--sharing",
type=int,
help="Sharing group ID when distribution is set to 4.")
parser_create_event.add_argument("-t",
"--threat",
type=int,
choices=[0, 1, 2, 3],
help="Threat level of a new event.")
parser_create_event.add_argument("-a",
"--analysis",
type=int,
choices=[0, 1, 2],
help="Analysis level a new event.")
parser_create_event.add_argument(
"-i",
"--info",
required=True,
nargs='+',
help="Event info field of a new event.")
parser_create_event.add_argument(
"--date", help="Date of the event. (Default: today).")
# ##### Add Hashes #####
h = subparsers.add_parser(
"add_hashes",
help="If no parameters, add all the hashes of the current session."
)
h.add_argument("-f", "--filename", help="Filename")
h.add_argument("-m", "--md5", help="MD5")
h.add_argument("-s", "--sha1", help="SHA1")
h.add_argument("-a", "--sha256", help="SHA256")
# ##### Add attributes #####
parser_add = subparsers.add_parser(
'add', help='Add attributes to an existing MISP event.')
subparsers_add = parser_add.add_subparsers(dest='add')
# Hashes
# Generic add
temp_me = MISPEvent()
for t in sorted(temp_me.types):
sp = subparsers_add.add_parser(
t, help="Add {} to the event.".format(t))
sp.add_argument(t, nargs='+')
# ##### Show attributes #####
subparsers.add_parser(
'show', help='Show attributes to an existing MISP event.')
# ##### Open file #####
o = subparsers.add_parser(
'open', help='Open a sample from the temp directory.')
ox = o.add_mutually_exclusive_group(required=True)
ox.add_argument("-l",
"--list",
action='store_true',
help="List available files")
ox.add_argument(
"-d",
"--delete",
help=
"Delete temporary files (use 'all' to remove all the local samples or an Event ID to only remove the associated samples)"
)
ox.add_argument("sid",
nargs='?',
type=int,
help='Sample ID to open (from the list option).')
# ##### Publish an event #####
subparsers.add_parser('publish',
help='Publish an existing MISP event.')
# ##### Show version #####
subparsers.add_parser('version',
help='Returns the version of the MISP instance.')
# Store
s = subparsers.add_parser(
'store',
help='Store the current MISP event in the current project.')
s.add_argument("-l",
"--list",
action='store_true',
help="List stored MISP events")
s.add_argument("-u",
"--update",
action='store_true',
help="Update all stored MISP events")
s.add_argument(
"-s",
"--sync",
action='store_true',
help="Sync all MISP Events with the remote MISP instance")
s.add_argument("-d",
"--delete",
type=int,
help="Delete a stored MISP event")
s.add_argument("-o", "--open", help="Open a stored MISP event")
# Tags
s = subparsers.add_parser('tag',
help='Tag managment using MISP taxonomies.')
s.add_argument("-l",
"--list",
action='store_true',
help="List Existing taxonomies.")
s.add_argument("-d",
"--details",
help="Display all values of a taxonomy.")
s.add_argument("-s",
"--search",
help="Search all tags matching a value.")
s.add_argument("-e", "--event", help="Add tag to the current event.")
s.add_argument(
"-a",
"--attribute",
nargs='+',
help=
"Add tag to an attribute of the current event. Syntax: <identifier for the attribute> <machinetag>"
)
# Admin
s = subparsers.add_parser('admin', help='Administration options.')
admin_parser = s.add_subparsers(dest='admin')
# Organisation
org = admin_parser.add_parser('org', help="Organisation managment.")
subparsers_org = org.add_subparsers(dest='org')
# Get
display = subparsers_org.add_parser('display',
help="Display an organisation.")
display.add_argument(
'id',
help=
'ID of the organisation to display. Use "local" to display all local organisations, "external" for all remote organisations, and "all", for both.'
)
# Search
search = subparsers_org.add_parser(
'search', help="Search an organisation by name.")
search.add_argument('name', help='(Partial) name of the organisation.')
search.add_argument(
'-t',
'--type',
default='local',
choices=['local', 'external', 'all'],
help=
'Use "local" to search in all local organisations, "external" for remote organisations, and "all", for both.'
)
# Add
add_org = subparsers_org.add_parser('add', help="Add an organisation.")
add_org.add_argument('name', help='Organisation name.')
add_org.add_argument('-u',
'--uuid',
default=None,
help='UUID of the organisation.')
add_org.add_argument('-d',
'--description',
default=[],
nargs='+',
help='Description of the organisation.')
add_org.add_argument('-t',
'--type',
default=[],
nargs='+',
help='Type of the organisation.')
add_org.add_argument('-n',
'--nationality',
default=None,
help='Nationality of the organisation.')
add_org.add_argument('-s',
'--sector',
default=[],
nargs='+',
help='Sector of the organisation.')
add_org.add_argument('-c',
'--contacts',
default=[],
nargs='+',
help='Contact point(s) in the organisation.')
add_org.add_argument('--not-local',
default=True,
action='store_false',
help='**Not** a local organisation.')
# Delete
delete = subparsers_org.add_parser('delete',
help="Delete an organisation.")
delete.add_argument('id', help='ID of the organisation to delete.')
# Edit
edit = subparsers_org.add_parser('edit', help="Edit an organisation.")
edit.add_argument('id', help='ID of the organisation to edit.')
edit.add_argument('-n', '--name', help='Organisation name.')
edit.add_argument('-u', '--uuid', help='UUID of the organisation.')
edit.add_argument('-d',
'--description',
default=[],
nargs='+',
help='Description of the organisation.')
edit.add_argument('-t',
'--type',
default=[],
nargs='+',
help='Type of the organisation.')
edit.add_argument('--nationality',
help='Nationality of the organisation.')
edit.add_argument('-s',
'--sector',
default=[],
nargs='+',
help='Sector of the organisation.')
edit.add_argument('-c',
'--contacts',
default=[],
nargs='+',
help='Contact point(s) in the organisation.')
edit.add_argument('--not-local',
default=True,
action='store_false',
help='**Not** a local organisation.')
# User
user = admin_parser.add_parser('user', help="User managment.")
subparsers_user = user.add_subparsers(dest='user')
# Get
display = subparsers_user.add_parser('display', help="Display a user.")
display.add_argument(
'id',
help='ID of the user to display. Use "all" to display all users.')
# Search
search = subparsers_user.add_parser('search',
help="Search a user by email.")
search.add_argument('name', help='(Partial) email of the user.')
# Add
add_usr = subparsers_user.add_parser('add', help="Add a user.")
add_usr.add_argument('email', help='User email address.')
add_usr.add_argument('-o',
'--org-id',
default=None,
help='Organisation ID of the user.')
add_usr.add_argument('-r',
'--role-id',
default=None,
help='Role of the user')
add_usr.add_argument('-g',
'--gpgkey',
default=None,
help='Path to the GPG public key export')
add_usr.add_argument(
'-c',
'--change-pw',
default=None,
action='store_true',
help='Force thanging the password after next login')
add_usr.add_argument('-t',
'--termsaccepted',
default=None,
action='store_true',
help='Set the TOC to accepted')
add_usr.add_argument('-p',
'--password',
default=None,
help='Set a new password')
add_usr.add_argument('-d',
'--disabled',
default=None,
action='store_true',
help='Disable the account')
# Delete
delete = subparsers_user.add_parser('delete', help="Delete a user.")
delete.add_argument('id', help='ID of the user to delete.')
# Edit
edit = subparsers_user.add_parser('edit', help="Edit a user.")
edit.add_argument('id', help='ID of the user to edit.')
edit.add_argument('-e', '--email', help='User email address.')
edit.add_argument('-o',
'--org-id',
default=None,
help='Organisation ID of the user.')
edit.add_argument('-r',
'--role-id',
default=None,
help='Role of the user')
edit.add_argument('-g',
'--gpgkey',
default=None,
help='Path to the GPG public key export')
edit.add_argument('-c',
'--change-pw',
default=None,
action='store_true',
help='Force thanging the password after next login')
edit.add_argument('-t',
'--termsaccepted',
default=None,
action='store_true',
help='Set the TOC to accepted')
edit.add_argument('-p',
'--password',
default=None,
help='Set a new password')
edit.add_argument('-d',
'--disabled',
default=None,
action='store_true',
help='Disable the account')
# Role
role = admin_parser.add_parser('role', help="Role managment.")
subparsers_role = role.add_subparsers(dest='role')
# Get
display = subparsers_role.add_parser('display',
help="Display all the roles.")
# Search
search = subparsers_role.add_parser('search',
help="Search a role by name.")
search.add_argument('name', help='(Partial) name of the role.')
# Tags
t = admin_parser.add_parser('tag', help="Tag managment.")
subparsers_tag = t.add_subparsers(dest='tag')
# Get
display = subparsers_tag.add_parser('display',
help="Display all the tags.")
# Search
search = subparsers_tag.add_parser('search',
help="Search a tag by name.")
search.add_argument('name', help='(Partial) name of the tag.')
self.categories = {
0: 'Payload delivery',
1: 'Artifacts dropped',
2: 'Payload installation',
3: 'External analysis'
}
WHERE rel_cnt > 5
MATCH (m)-[r:has]->(n)
RETURN m, n LIMIT 200;
"""
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Get all the events matching a value.')
parser.add_argument("-s", "--search", required=True, help="String to search.")
parser.add_argument("--host", default='localhost:7474', help="Host where neo4j is running.")
parser.add_argument("-u", "--user", default='neo4j', help="User on neo4j.")
parser.add_argument("-p", "--password", default='neo4j', help="Password on neo4j.")
parser.add_argument("-d", "--deleteall", action="store_true", default=False, help="Delete all nodes from the database")
args = parser.parse_args()
neo4j = Neo4j(args.host, args.user, args.password)
if args.deleteall:
neo4j.del_all()
misp = PyMISP(misp_url, misp_key)
result = misp.search_all(args.search)
for json_event in result['response']:
if not json_event['Event']:
print(json_event)
continue
print('Importing', json_event['Event']['info'], json_event['Event']['id'])
try:
misp_event = MISPEvent()
misp_event.load(json_event)
neo4j.import_event(misp_event)
except:
print('broken')
class StixParser():
def __init__(self):
self.misp_event = MISPEvent()
self.event = defaultdict(dict)
self.misp_event['Galaxy'] = []
def loadEvent(self, args):
filename = os.path.join(os.path.dirname(args[0]), args[1])
with open(filename, 'r', encoding='utf-8') as f:
event = json.loads(f.read())
self.filename = filename
self.stix_version = 'stix {}'.format(event.get('spec_version'))
for o in event.get('objects'):
parsed_object = stix2.parse(o, allow_custom=True)
try:
object_type = parsed_object._type
except AttributeError:
object_type = parsed_object['type']
object_uuid = parsed_object['id'].split('--')[1]
if object_type.startswith('x-misp-object'):
object_type = 'x-misp-object'
self.event[object_type][object_uuid] = parsed_object
if not self.event:
print(
json.dumps({
'success': 0,
'message': 'There is no valid STIX object to import'
}))
sys.exit(1)
try:
event_distribution = args[2]
if not isinstance(event_distribution, int):
event_distribution = int(
event_distribution) if event_distribution.isdigit() else 5
except IndexError:
event_distribution = 5
try:
attribute_distribution = args[3]
if attribute_distribution != 'event' and not isinstance(
attribute_distribution, int):
attribute_distribution = int(
attribute_distribution) if attribute_distribution.isdigit(
) else 5
except IndexError:
attribute_distribution = 5
self.misp_event.distribution = event_distribution
self.__attribute_distribution = event_distribution if attribute_distribution == 'event' else attribute_distribution
self.load_mapping()
def load_mapping(self):
self.objects_mapping = {
'asn': {
'observable': observable_asn,
'pattern': pattern_asn
},
'domain-ip': {
'observable': observable_domain_ip,
'pattern': pattern_domain_ip
},
'email': {
'observable': self.observable_email,
'pattern': self.pattern_email
},
'file': {
'observable': observable_file,
'pattern': self.pattern_file
},
'ip-port': {
'observable': observable_ip_port,
'pattern': pattern_ip_port
},
'network-socket': {
'observable': observable_socket,
'pattern': pattern_socket
},
'process': {
'observable': observable_process,
'pattern': pattern_process
},
'registry-key': {
'observable': observable_regkey,
'pattern': pattern_regkey
},
'url': {
'observable': observable_url,
'pattern': pattern_url
},
'WindowsPEBinaryFile': {
'observable': self.observable_pe,
'pattern': self.pattern_pe
},
'x509': {
'observable': observable_x509,
'pattern': pattern_x509
}
}
self.object_from_refs = {
'course-of-action': self.parse_course_of_action,
'vulnerability': self.parse_vulnerability,
'x-misp-object': self.parse_custom
}
self.object_from_refs.update(
dict.fromkeys(list(galaxy_types.keys()), self.parse_galaxy))
self.object_from_refs.update(
dict.fromkeys(['indicator', 'observed-data'],
self.parse_usual_object))
def handler(self):
self.outputname = '{}.stix2'.format(self.filename)
if self.from_misp():
self.buildMispDict()
else:
self.version_attribute = {
'type': 'text',
'object_relation': 'version',
'value': self.stix_version
}
self.buildExternalDict()
self.set_distribution()
def from_misp(self):
for _, o in self.event['report'].items():
if o._type == 'report' and 'misp:tool="misp2stix2"' in o.get(
'labels'):
return True
return False
def buildMispDict(self):
report_attributes = defaultdict(list)
orgs = []
for _, report in self.event['report'].items():
org_uuid = report['created_by_ref'].split('--')[1]
if org_uuid not in orgs:
orgs.append(org_uuid)
report_name = report['name']
if report_name not in orgs:
report_attributes['name'].append(report_name)
if report.get('published'):
report_attributes['published'].append(report['published'])
if hasattr(report, 'labels'):
for l in report['labels']:
if l not in report_attributes['labels']:
report_attributes['labels'].append(l)
if hasattr(report, 'external_references'):
for e in report['external_references']:
self.add_link(e)
for ref in report['object_refs']:
object_type, uuid = ref.split('--')
if object_type == 'relationship':
continue
object2parse = self.event[object_type][uuid]
labels = object2parse.get('labels')
self.object_from_refs[object_type](object2parse, labels)
if len(orgs) == 1:
identity = self.event['identity'][orgs[0]]
self.misp_event['Org'] = {'name': identity['name']}
if len(report_attributes['published']) == 1:
self.misp_event.publish_timestamp = self.getTimestampfromDate(
report_attributes['published'][0])
if len(report_attributes['name']) == 1:
self.misp_event.info = report_attributes['name'][0]
else:
self.misp_event.info = "Imported from MISP import for STIX 2.0 script."
for l in report_attributes['labels']:
self.misp_event.add_tag(l)
def add_link(self, e):
link = {"type": "link"}
comment = e.get('source_name')
try:
comment = comment.split('url - ')[1]
except IndexError:
pass
if comment:
link['comment'] = comment
link['value'] = e.get('url')
self.misp_event.add_attribute(**link)
def parse_usual_object(self, o, labels):
if 'from_object' in labels:
self.parse_object(o, labels)
else:
self.parse_attribute(o, labels)
def parse_galaxy(self, o, labels):
galaxy_type = self.get_misp_type(labels)
tag = labels[1]
value = tag.split(':')[1].split('=')[1]
galaxy_description, cluster_description = o.get('description').split(
'|')
_, uuid = o.get('id').split('--')
galaxy = {
'type':
galaxy_type,
'name':
o.get('name'),
'description':
galaxy_description,
'GalaxyCluster': [{
'type': galaxy_type,
'value': value,
'tag_name': tag,
'description': cluster_description,
'uuid': uuid
}]
}
self.misp_event['Galaxy'].append(galaxy)
def parse_course_of_action(self, o, _):
misp_object = MISPObject('course-of-action')
if 'name' in o:
attribute = {
'type': 'text',
'object_relation': 'name',
'value': o.get('name')
}
misp_object.add_attribute(**attribute)
else:
return
if 'description' in o:
attribute = {
'type': 'text',
'object_relation': 'description',
'value': o.get('description')
}
misp_object.add_attribute(**attribute)
self.misp_event.add_object(**misp_object)
def parse_custom(self, o, labels):
if 'from_object' in labels:
self.parse_custom_object(o)
else:
self.parse_custom_attribute(o, labels)
def parse_custom_object(self, o):
name = o.get('type').split('x-misp-object-')[1]
timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp'))
category = o.get('category')
attributes = []
values = o.get('x_misp_values')
for v in values:
attribute_type, object_relation = v.split('_')
attribute = {
'type': attribute_type,
'value': values.get(v),
'object_relation': object_relation
}
attributes.append(attribute)
misp_object = {
'name': name,
'timestamp': timestamp,
'meta-category': category,
'Attribute': attributes
}
self.misp_event.add_object(**misp_object)
def parse_custom_attribute(self, o, labels):
attribute_type = o.get('type').split('x-misp-object-')[1]
if attribute_type not in misp_types:
attribute_type = attribute_type.replace('-', '|')
timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp'))
to_ids = bool(labels[1].split('=')[1])
value = o.get('x_misp_value')
category = self.get_misp_category(labels)
attribute = {
'type': attribute_type,
'timestamp': timestamp,
'to_ids': to_ids,
'value': value,
'category': category
}
self.misp_event.add_attribute(**attribute)
def parse_object(self, o, labels):
object_type = self.get_misp_type(labels)
name = 'file' if object_type == 'WindowsPEBinaryFile' else object_type
object_category = self.get_misp_category(labels)
stix_type = o._type
misp_object = MISPObject(name)
misp_object['meta-category'] = object_category
if stix_type == 'indicator':
pattern = o.get('pattern').replace('\\\\', '\\').split(' AND ')
pattern[0] = pattern[0][1:]
pattern[-1] = pattern[-1][:-1]
attributes = self.objects_mapping[object_type]['pattern'](pattern)
if stix_type == 'observed-data':
observable = o.get('objects')
attributes = self.objects_mapping[object_type]['observable'](
observable)
if isinstance(attributes, tuple):
attributes, pe_uuid = attributes
misp_object.add_reference(pe_uuid, 'included-in')
for attribute in attributes:
misp_object.add_attribute(**attribute)
misp_object.to_ids = bool(labels[1].split('=')[1])
self.misp_event.add_object(**misp_object)
def parse_attribute(self, o, labels):
attribute_type = self.get_misp_type(labels)
attribute_category = self.get_misp_category(labels)
attribute = {'type': attribute_type, 'category': attribute_category}
tags = [{'name': label} for label in labels[3:]]
if tags:
attribute['Tag'] = tags
stix_type = o._type
if stix_type == 'vulnerability':
value = o.get('name')
else:
if stix_type == 'indicator':
o_date = o.get('valid_from')
pattern = o.get('pattern').replace('\\\\', '\\')
value = self.parse_pattern_with_data(
pattern) if attribute_type in (
'malware-sample',
'attachment') else self.parse_pattern(pattern)
attribute['to_ids'] = True
else:
o_date = o.get('first_observed')
observable = o.get('objects')
try:
value = self.parse_observable(observable, attribute_type)
except:
print('{}: {}'.format(attribute_type, observable))
attribute['to_ids'] = False
attribute['timestamp'] = self.getTimestampfromDate(o_date)
if 'description' in o:
attribute['comment'] = o.get('description')
if isinstance(value, tuple):
value, data = value
attribute['data'] = io.BytesIO(data.encode())
attribute['value'] = value
self.misp_event.add_attribute(**attribute)
def parse_vulnerability(self, o, labels):
if len(labels) > 2:
self.parse_usual_object(o, labels)
else:
self.parse_galaxy(o, labels)
@staticmethod
def observable_email(observable):
attributes = []
addresses = {}
files = {}
for o_key, o_dict in observable.items():
part_type = o_dict._type
if part_type == 'email-addr':
addresses[o_key] = o_dict.get('value')
elif part_type == 'file':
files[o_key] = o_dict.get('name')
else:
message = dict(o_dict)
attributes.append({
'type': 'email-src',
'object_relation': 'from',
'value': addresses[message.pop('from_ref')],
'to_ids': False
})
for ref in ('to_refs', 'cc_refs'):
if ref in message:
for item in message.pop(ref):
mapping = email_mapping[ref]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': addresses[item],
'to_ids': False
})
if 'body_multipart' in message:
for f in message.pop('body_multipart'):
attributes.append({
'type': 'email-attachment',
'object_relation': 'attachment',
'value': files[f.get('body_raw_ref')],
'to_ids': False
})
for m_key, m_value in message.items():
if m_key == 'additional_header_fields':
for field_key, field_value in m_value.items():
mapping = email_mapping[field_key]
if field_key == 'Reply-To':
for rt in field_value:
attributes.append({
'type':
mapping['type'],
'object_relation':
mapping['relation'],
'value':
rt,
'to_ids':
False
})
else:
attributes.append({
'type':
mapping['type'],
'object_relation':
mapping['relation'],
'value':
field_value,
'to_ids':
False
})
else:
try:
mapping = email_mapping[m_key]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': m_value,
'to_ids': False
})
except KeyError:
if m_key.startswith("x_misp_attachment_"):
attribute_type, relation = m_key.split(
"x_misp_")[1].split("_")
attributes.append({
'type':
attribute_type,
'object_relation':
relation,
'to_ids':
False,
'value':
m_value['value'],
'data':
io.BytesIO(m_value['data'].encode())
})
elif "x_misp_" in m_key:
attribute_type, relation = m_key.split(
"x_misp_")[1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': m_value,
'to_ids': False
})
return attributes
@staticmethod
def pattern_email(pattern):
attributes = []
attachments = defaultdict(dict)
for p in pattern:
p_type, p_value = p.split(' = ')
try:
mapping = email_mapping[p_type]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': p_value[1:-1],
'to_ids': True
})
except KeyError:
if p_type.startswith("email-message:'x_misp_attachment_"):
relation, field = p_type.split('.')
relation = relation.split(':')[1][1:-1]
attachments[relation][field] = p_value[1:-1]
elif "x_misp_" in p_type:
attribute_type, relation = p_type.split(
"x_misp_")[1][:-1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': p_value[1:-1],
'to_ids': True
})
for a_key, a_dict in attachments.items():
_, _, attribute_type, relation = a_key.split('_')
attributes.append({
'type': attribute_type,
'object_relation': relation,
'to_ids': True,
'value': a_dict['value'],
'data': io.BytesIO(a_dict['data'].encode())
})
return attributes
@staticmethod
def pattern_file(pattern):
attributes = []
malware_sample = {}
for p in pattern:
p_type, p_value = p.split(' = ')
if p_type == 'artifact:payload_bin':
malware_sample['data'] = p_value
elif p_type in ("file:name", "file:hashes.'md5'"):
try:
mapping = file_mapping[p_type]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': p_value[1:-1],
'to_ids': True
})
malware_sample['filename'] = p_value[1:-1]
except KeyError:
attributes.append({
'type': 'md5',
'object_relation': 'md5',
'value': p_value[1:-1],
'to_ids': True
})
malware_sample['md5'] = p_value[1:-1]
elif 'file:hashes.' in p_type:
_, h = p_type.split('.')
h = h[1:-1]
attributes.append({
'type': h,
'object_relation': h,
'value': p_value[1:-1]
})
else:
try:
mapping = file_mapping[p_type]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': p_value[1:-1],
'to_ids': True
})
except KeyError:
if "x_misp_" in p_type:
attribute_type, relation = p_type.split(
"x_misp_")[1][:-1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': p_value[1:-1],
'to_ids': True
})
if 'data' in malware_sample:
value = "{}|{}".format(malware_sample['filename'],
malware_sample['md5'])
attributes.append({
'type':
'malware-sample',
'object_relation':
'malware-sample',
'value':
value,
'to_ids':
True,
'data':
io.BytesIO(malware_sample['data'].encode())
})
return attributes
def observable_pe(self, observable):
extension = observable['1']['extensions']['windows-pebinary-ext']
sections = extension['sections']
pe = MISPObject('pe')
pe_uuid = str(uuid.uuid4())
pe.uuid = pe_uuid
self.fill_object_attributes_observable(pe, pe_mapping, extension)
for section in sections:
pe_section = MISPObject('pe-section')
if 'hashes' in section:
for h_type, h_value in section['hashes'].items():
h_type = h_type.lower().replace('-', '')
pe_section.add_attribute(
**{
'type': h_type,
'object_relation': h_type,
'value': h_value,
'to_ids': False
})
self.fill_object_attributes_observable(pe_section,
pe_section_mapping, section)
section_uuid = str(uuid.uuid4())
pe_section.uuid = section_uuid
pe.add_reference(section_uuid, 'included-in')
self.misp_event.add_object(**pe_section)
self.misp_event.add_object(**pe)
return observable_file(observable), pe_uuid
@staticmethod
def fill_object_attributes_observable(misp_object, mapping_dict,
stix_object):
for stix_type, value in stix_object.items():
try:
mapping = mapping_dict[stix_type]
misp_object.add_attribute(
**{
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': value,
'to_ids': False
})
except KeyError:
if stix_type.startswith("x_misp_"):
attribute_type, relation = parse_custom_property(stix_type)
misp_object.add_attribute(
**{
'type': attribute_type,
'object_relation': relation[:-1],
'value': value,
'to_ids': False
})
def pattern_pe(self, pattern):
attributes = []
sections = defaultdict(dict)
pe = MISPObject('pe')
pe_uuid = str(uuid.uuid4())
pe.uuid = pe_uuid
for p in pattern:
p_type, p_value = p.split(' = ')
p_value = p_value[1:-1]
if ':extensions.' in p_type:
if '.sections[' in p_type:
p_type_list = p_type.split('.')
stix_type = "hashes.{}".format(
p_type_list[4]
[1:-1]) if '.hashes.' in p_type else p_type_list[3]
sections[p_type_list[2]][stix_type] = p_value
else:
stix_type = p_type.split('.')[-1]
try:
mapping = pe_mapping[stix_type]
pe.add_attribute(
**{
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': p_value,
'to_ids': True
})
except KeyError:
if stix_type.startswith("x_misp_"):
attribute_type, relation = parse_custom_property(
stix_type)
pe.add_attribute(
**{
'type': attribute_type,
'object_relation': relation[:-2],
'value': p_value,
'to_ids': False
})
else:
if 'file:hashes.' in p_type:
_, h = p_type.split('.')
h = h[1:-1]
attributes.append({
'type': h,
'object_relation': h,
'value': p_value,
'to_ids': True
})
else:
try:
mapping = file_mapping[p_type]
attributes.append({
'type':
mapping['type'],
'object_relation':
mapping['relation'],
'value':
p_value,
'to_ids':
True
})
except KeyError:
if "x_misp_" in p_type:
attribute_type, relation = p_type.split(
"x_misp_")[1][:-1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': p_value,
'to_ids': True
})
for _, section in sections.items():
pe_section = MISPObject('pe-section')
for stix_type, value in section.items():
if 'hashes.' in stix_type:
h_type = stix_type.split('.')[1]
pe_section.add_attribute(
**{
'type': h_type,
'object_relation': h_type,
'value': value,
'to_ids': True
})
else:
try:
mapping = pe_section_mapping[stix_type]
pe_section.add_attribute(
**{
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': value,
'to_ids': True
})
except KeyError:
if "x_misp_" in stix_type:
attribute_type, relation = stix_type.split(
"x_misp_")[1][:-1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': value,
'to_ids': True
})
section_uuid = str(uuid.uuid4())
pe_section.uuid = pe_uuid
pe.add_reference(section_uuid, 'included-in')
self.misp_event.add_object(**pe_section)
self.misp_event.add_object(**pe)
return attributes, pe_uuid
def buildExternalDict(self):
self.fetch_report()
for o in self.event:
object_type = o._type
if object_type in ('relationship', 'report'):
continue
if object_type in galaxy_types:
self.parse_external_galaxy(o)
elif object_type == 'vulnerability':
attribute = {'type': 'vulnerability', 'value': o.get('name')}
if 'description' in o:
attribute['comment'] = o.get('description')
self.misp_event.add_attribute(**attribute)
elif object_type == 'course-of-action':
self.parse_course_of_action(o)
elif object_type == 'indicator':
pattern = o.get('pattern')
self.parse_external_pattern(pattern)
attribute = {
'type': 'stix2-pattern',
'object_relation': 'stix2-pattern',
'value': pattern
}
misp_object = {
'name': 'stix2-pattern',
'meta-category': 'stix2-pattern',
'Attribute': [self.version_attribute, attribute]
}
self.misp_event.add_object(**misp_object)
def fetch_report(self):
reports = []
for o in self.event:
if o._type == 'report':
reports.append(o)
if len(reports) == 1:
self.report = reports[0]
self.parse_report()
def parse_external_galaxy(self, o):
galaxy = {'name': galaxy_types[o._type]}
if 'kill_chain_phases' in o:
galaxy['type'] = o['kill_chain_phases'][0].get('phase_name')
cluster = defaultdict(dict)
cluster['value'] = o.get('name')
cluster['description'] = o.get('description')
if 'aliases' in o:
aliases = []
for a in o.get('aliases'):
aliases.append(a)
cluster['meta']['synonyms'] = aliases
galaxy['GalaxyCluster'] = [cluster]
self.misp_event['Galaxy'].append(galaxy)
def parse_external_pattern(self, pattern):
if ' OR ' in pattern and ' AND ' not in pattern:
pattern = pattern.split('OR')
for p in pattern:
attribute = self.attribute_from_external_pattern(p)
self.misp_event.add_attribute(**attribute)
elif ' OR ' not in pattern and ' LIKE ' not in pattern:
pattern = pattern.split('AND')
if len(pattern) == 1:
attribute = self.attribute_from_external_pattern(pattern[0])
self.misp_event.add_attribute(**attribute)
@staticmethod
def attribute_from_external_pattern(pattern):
pattern_type, pattern_value = pattern.split(' = ')
pattern_type, pattern_value = pattern_type[1:].strip(
), pattern_value[1:-2].strip()
stix_type, value_type = pattern_type.split(':')
if 'hashes' in value_type and 'x509' not in stix_type:
h_type = value_type.split('.')[1]
return {'type': h_type, 'value': pattern_value}
# Might cause some issues, need more examples to test
return {
'type':
external_pattern_mapping[stix_type][value_type].get('type'),
'value': pattern_value
}
def set_distribution(self):
for attribute in self.misp_event.attributes:
attribute.distribution = self.__attribute_distribution
for misp_object in self.misp_event.objects:
misp_object.distribution = self.__attribute_distribution
for attribute in misp_object.attributes:
attribute.distribution = self.__attribute_distribution
def saveFile(self):
eventDict = self.misp_event.to_json()
outputfile = '{}.stix2'.format(self.filename)
with open(outputfile, 'w') as f:
f.write(eventDict)
@staticmethod
def getTimestampfromDate(stix_date):
try:
return int(stix_date.timestamp())
except AttributeError:
return int(
time.mktime(
time.strptime(
stix_date.split('+')[0], "%Y-%m-%d %H:%M:%S")))
@staticmethod
def get_misp_type(labels):
return labels[0].split('=')[1][1:-1]
@staticmethod
def get_misp_category(labels):
return labels[1].split('=')[1][1:-1]
@staticmethod
def parse_observable(observable, attribute_type):
return misp_types_mapping[attribute_type](observable, attribute_type)
@staticmethod
def parse_pattern(pattern):
if ' AND ' in pattern:
pattern_parts = pattern.split(' AND ')
if len(pattern_parts) == 3:
_, value1 = pattern_parts[2].split(' = ')
_, value2 = pattern_parts[0].split(' = ')
return '{}|{}'.format(value1[1:-2], value2[1:-1])
else:
_, value1 = pattern_parts[0].split(' = ')
_, value2 = pattern_parts[1].split(' = ')
if value1 in ("'ipv4-addr'", "'ipv6-addr'"):
return value2[1:-2]
return '{}|{}'.format(value1[1:-1], value2[1:-2])
else:
return pattern.split(' = ')[1][1:-2]
def parse_pattern_with_data(self, pattern):
if 'artifact:payload_bin' not in pattern:
return self.parse_pattern(pattern)
pattern_parts = pattern.split(' AND ')
if len(pattern_parts) == 3:
filename = pattern_parts[0].split(' = ')[1]
md5 = pattern_parts[1].split(' = ')[1]
return "{}|{}".format(
filename[1:-1],
md5[1:-1]), pattern_parts[2].split(' = ')[1][1:-2]
return pattern_parts[0].split(' = ')[1][1:-1], pattern_parts[1].split(
' = ')[1][1:-2]
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
def check_hashes(self):
if self.offline_mode:
self.log('error', 'Offline mode, unable to query VirusTotal')
return
event_id = self._get_eventid()
if event_id is None:
return
event = self.misp.get(event_id)
if self._has_error_message(event):
return
misp_event = MISPEvent()
misp_event.load(event)
event_hashes = []
sample_hashes = []
base_new_attributes = {}
for a in misp_event.attributes:
h = None
if a.type in ('md5', 'sha1', 'sha256'):
h = a.value
event_hashes.append(h)
elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256', 'malware-sample'):
h = a.value.split('|')[1]
event_hashes.append(h)
if h is not None:
base_new_attributes[h] = {"category": a.category,
"comment": '{} - Xchecked via VT: {}'.format(a.comment, h),
"to_ids": a.to_ids,
"Tag": a.Tag,
"distribution": a.distribution}
unk_vt_hashes = []
vt_request = {'apikey': cfg.virustotal.virustotal_key}
# Make sure to start getting reports for the longest possible hashes (reduce risks of collisions)
hashes_to_check = sorted(event_hashes, key=len)
original_attributes = len(misp_event.attributes)
if cfg.virustotal.virustotal_has_private_key is False:
quota = 4
timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)
while len(hashes_to_check) > 0:
vt_request['resource'] = hashes_to_check.pop()
try:
response = requests.post(cfg.misp.misp_vturl, data=vt_request, proxies=cfg.virustotal.proxies)
except requests.ConnectionError:
self.log('error', 'Failed to connect to VT for {}'.format(vt_request['resource']))
return
if response.status_code == 403:
self.log('error', 'This command requires virustotal API key')
self.log('error', 'Please check that your key have the right permissions')
return
try:
result = response.json()
except:
self.log('error', 'Unable to get the report of {}'.format(vt_request['resource']))
continue
if result['response_code'] == 1:
md5 = result['md5']
sha1 = result['sha1']
sha256 = result['sha256']
hashes_to_check = [eh for eh in hashes_to_check if eh not in (md5, sha1, sha256)]
link = [False, result['permalink']]
# Do not re-add a link
for a in misp_event.attributes:
if a.value == link[1]:
link[0] = True
if md5 in sample_hashes:
self.log('success', 'Sample available in MISP:')
else:
self.log('success', 'Sample available in VT:')
if self.args.populate:
misp_event = self._prepare_attributes(md5, sha1, sha256, link, base_new_attributes, event_hashes, sample_hashes, misp_event)
self.log('item', '{}\n\t{}\n\t{}\n\t{}'.format(link[1], md5, sha1, sha256))
if cfg.virustotal.virustotal_has_private_key is False:
if quota > 0:
quota -= 1
else:
waiting_time = (timeout - datetime.datetime.now()).seconds
if waiting_time > 0:
self.log('warning', 'No private API key, 4 queries/min is the limit. Waiting for {} seconds.'.format(waiting_time))
time.sleep(waiting_time)
quota = 4
timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)
else:
unk_vt_hashes.append(vt_request['resource'])
if self.args.populate:
self.__populate(misp_event, original_attributes)
if len(unk_vt_hashes) > 0:
self.log('error', 'Unknown on VT:')
for h in unk_vt_hashes:
self.log('item', '{}'.format(h))
def misp_send(self, strMISPEventID, strInput, strInfo, strUsername):
# Establish communication with MISP
# event = MISPEvent()
# event.info = 'Test event'
# event.analysis = 0
# event.distribution = 3
# event.threat_level_id = 2
# event.add_attribute('md5', '678ff97bf16d8e1c95679c4681834c41')
# #<add more attributes>
# self.misp.add_event(event)
# exit()
try:
objects = []
#get comments and tags from string input
str_comment, tags = self.get_comm_and_tags(strInput)
print(tags)
if tags == None:
self.misp_logger.info('Irate not in Tags: %s equals None' %
tags)
response = None
return response
#setup misp objects
mispobj_email = MISPObject(name="email")
mispobj_file = MISPObject(name="file")
mispobj_files = {}
mispobj_domainip = MISPObject(name="domain-ip")
url_no = 0
file_no = 0
mispobj_urls = {}
#process input
for line in strInput.splitlines():
if ("domain:" in line.lower()
): #Catch domain and add to domain/IP object
mispobj_domainip = MISPObject(name="domain-ip")
vals = line.split(":", 1)
mispobj_domainip.add_attribute("domain",
value=vals[1].strip(),
comment=str_comment)
objects.append(mispobj_domainip)
elif ("ip:" in line.lower()) or ("ip-dst:" in line.lower(
)) or ("ip-src:"
in line.lower()): #Catch IP and add to domain/IP object
if "domain:" in strInput.splitlines():
mispobj_domainip = MISPObject(name="domain-ip")
vals = line.split(":", 1)
mispobj_domainip.add_attribute("ip",
value=vals[1].strip(),
comment=str_comment)
objects.append(mispobj_domainip)
else:
mispobj_network_connection = MISPObject(
name="network-connection")
vals = line.split(":", 1)
if ("ip:" in line.lower()) or ("ip-dst:"
in line.lower()):
mispobj_network_connection.add_attribute(
"ip-dst",
type="ip-dst",
value=vals[1].strip(),
comment=str_comment)
else:
mispobj_network_connection.add_attribute(
"ip-src",
type="ip-src",
value=vals[1].strip(),
comment=str_comment)
objects.append(mispobj_network_connection)
elif ("source-email:"
in line.lower()) or ("email-source" in line.lower()) or (
"from:" in line.lower()
): #Catch email and add to email object
vals = line.split(":", 1)
mispobj_email.add_attribute("from",
value=vals[1].strip(),
comment=str_comment)
elif ("url:" in line.lower()) or (
('kit:' in line.lower() or ('creds:' in line.lower())) and
(('hxxp' in line.lower()) or ('http' in line.lower()))
): #Catch URL and add to URL object
vals = line.split(":", 1)
url = vals[1].strip()
url = refang(url)
parsed = urlparse(url)
mispobj_url = MISPObject(name="url")
mispobj_url.add_attribute("url",
value=parsed.geturl(),
category="Payload delivery",
comment=str_comment)
if parsed.hostname:
mispobj_url.add_attribute("host",
value=parsed.hostname,
comment=str_comment)
if parsed.scheme:
mispobj_url.add_attribute("scheme",
value=parsed.scheme,
comment=str_comment)
if parsed.port:
mispobj_url.add_attribute("port",
value=parsed.port,
comment=str_comment)
mispobj_urls[url_no] = mispobj_url
url_no += 1
#Catch different hashes and add to file object
elif ("sha1:" in line.lower()) or ("SHA1:" in line):
vals = line.split(":", 1)
mispobj_file.add_attribute("sha1",
value=vals[1].strip(),
comment=str_comment)
elif ("sha256:" in line.lower()) or ("SHA256:" in line):
vals = line.split(":", 1)
mispobj_file.add_attribute("sha256",
value=vals[1].strip(),
comment=str_comment)
elif ("md5:" in line.lower()) or ("MD5:" in line):
vals = line.split(":", 1)
mispobj_file.add_attribute("md5",
value=vals[1].strip(),
comment=str_comment)
elif (
"subject:" in line.lower()
): #or ("subject:" in line): #Catch subject and add to email object
self.misp_logger.info('adding subject')
vals = line.split(":", 1)
mispobj_email.add_attribute("subject",
value=vals[1].strip(),
comment=str_comment)
elif ("hash|filename:" in line.lower()
): #catch hash|filename pair and add to file object
vals = line.split(":", 1)
val = vals[1].split("|")
l_hash = val[0]
l_filename = val[1]
l_mispobj_file = MISPObject(name="file")
if len(re.findall(r"\b[a-fA-F\d]{32}\b", l_hash)) > 0:
l_mispobj_file.add_attribute("md5",
value=l_hash.strip(),
comment=str_comment)
l_mispobj_file.add_attribute("filename",
value=l_filename.strip(),
comment=str_comment)
mispobj_files[file_no] = l_mispobj_file
elif len(re.findall(r'\b[0-9a-f]{40}\b', l_hash)) > 0:
l_mispobj_file.add_attribute("sha1",
value=l_hash.strip(),
comment=str_comment)
l_mispobj_file.add_attribute("filename",
value=l_filename.strip(),
comment=str_comment)
mispobj_files[file_no] = l_mispobj_file
elif len(re.findall(r'\b[A-Fa-f0-9]{64}\b', l_hash)) > 0:
l_mispobj_file.add_attribute("sha256",
value=l_hash.strip(),
comment=str_comment)
l_mispobj_file.add_attribute("filename",
value=l_filename.strip(),
comment=str_comment)
mispobj_files[file_no] = l_mispobj_file
file_no += 1
#add all misp objects to List to be processed and submitted to MISP server as one.
if len(mispobj_file.attributes) > 0:
objects.append(mispobj_file)
if len(mispobj_email.attributes) > 0:
objects.append(mispobj_email)
for u_key, u_value in mispobj_urls.items():
if len(u_value.attributes) > 0:
objects.append(u_value)
for f_key, f_value in mispobj_files.items():
if len(f_value.attributes) > 0:
objects.append(f_value)
# Update timestamp and event
except Exception as e:
error = traceback.format_exc()
response = "Error occured when converting string to misp objects:\n %s" % error
self.misp_logger.error(response)
return response
if self.check_object_length(objects) != True:
self.misp_logger.error(
'Input from %s did not contain accepted tags.\n Input: \n%s' %
(strUsername, strInput))
return "Error in the tags you entered. Please see the guide for accepted tags."
try:
# self.misp_logger.error(dir(self.misp))
misp_event = MISPEvent()
misp_event.info = strInfo
misp_event.distribution = 0
misp_event.analysis = 2
misp_event.threat_level_id = 3
# event.add_attribute('md5', '678ff97bf16d8e1c95679c4681834c41')
#event = self.misp.new_event(info=strInfo, distribution='0', analysis='2', threat_level_id='3', published=False)
#misp_event = MISPEvent()
#misp_event.load(event)
add = self.misp.add_event(misp_event)
self.misp_logger.info("Added event %s" % add)
a, b = self.submit_to_misp(self.misp, misp_event, objects)
for tag in tags:
self.misp.tag(misp_event.uuid, tag)
#self.misp.add_internal_comment(misp_event.id, reference="Author: " + strUsername, comment=str_comment)
ccc = self.misp.publish(misp_event, alert=False)
self.misp_logger.info(ccc)
misp_event = self.misp.get_event(misp_event)
response = misp_event
#for response in misp_event:
if ('errors' in response and response['errors'] != None):
return ("Submission error: " + repr(response['errors']))
else:
if response['Event']['RelatedEvent']:
e_related = ""
for each in response['Event']['RelatedEvent']:
e_related = e_related + each['Event']['id'] + ", "
return "Created ID: " + str(
response['Event']
['id']) + "\nRelated Events: " + ''.join(e_related)
else:
return "Created ID: " + str(response['Event']['id'])
except Exception as e:
error = traceback.format_exc()
response = "Error occured when submitting to misp:\n %s" % error
self.misp_logger.error(response)
return response
class ReportGenerator():
def __init__(self, profile="daily_report"):
self.taxonomies = Taxonomies()
self.report = ''
profile_name = "profiles.{}".format(profile)
self.template = importlib.import_module(name=profile_name)
def from_remote(self, event_id):
from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
misp = PyMISP(misp_url, misp_key, misp_verifycert)
result = misp.get(event_id)
self.misp_event = MISPEvent()
self.misp_event.load(result)
def from_file(self, path):
self.misp_event = MISPEvent()
self.misp_event.load_file(path)
def attributes(self):
if not self.misp_event.attributes:
return ''
list_attributes = []
for attribute in self.misp_event.attributes:
if attribute.type in self.template.types_to_attach:
list_attributes.append("* {}".format(defang(attribute.value)))
for obj in self.misp_event.Object:
if obj.name in self.template.objects_to_attach:
for attribute in obj.Attribute:
if attribute.type in self.template.types_to_attach:
list_attributes.append("* {}".format(defang(attribute.value)))
return self.template.attributes.format(list_attributes="\n".join(list_attributes))
def _get_tag_info(self, machinetag):
return self.taxonomies.revert_machinetag(machinetag)
def report_headers(self):
content = {'org_name': 'name',
'date': date.today().isoformat()}
self.report += self.template.headers.format(**content)
def event_level_tags(self):
if not self.misp_event.Tag:
return ''
for tag in self.misp_event.Tag:
# Only look for TLP for now
if tag['name'].startswith('tlp'):
tax, predicate = self._get_tag_info(tag['name'])
return self.template.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded)
def title(self):
internal_id = ''
summary = ''
# Get internal refs for report
for obj in self.misp_event.Object:
if obj.name != 'report':
continue
for a in obj.Attribute:
if a.object_relation == 'case-number':
internal_id = a.value
if a.object_relation == 'summary':
summary = a.value
return self.template.title.format(internal_id=internal_id, title=self.misp_event.info,
summary=summary)
def asciidoc(self, lang='en'):
self.report += self.title()
self.report += self.event_level_tags()
self.report += self.attributes()
