def main():
parser = argparse.ArgumentParser(
description="Test of adding event to MISP")
parser.add_argument("mwdb_user", help="Mwdb username")
parser.add_argument("mwdb_pass", help="Mwdb password")
parser.add_argument("config", help="Config")
parser.add_argument("misp_url", help="Misp url")
parser.add_argument("misp_key", help="Misp key")
args = parser.parse_args()
mwdb = Malwarecage()
mwdb.login(args.mwdb_user, args.mwdb_pass)
try:
cfg = mwdb.query_config(args.config)
iocs = parse(cfg.family, cfg.cfg)
except FamilyNotSupportedYetError:
logging.info("Family %s not supported yet...", cfg.family)
return
if not iocs:
# Nothing actionable found - skip the config
return
event = MISPEvent()
event.add_tag(f"mwdb:family:{cfg.family}")
event.info = f"Malware configuration ({cfg.family})"
for o in iocs.to_misp():
event.add_object(o)
misp = PyMISP(args.misp_url, args.misp_key, False)
misp.add_event(event)
def import_all(self, stations_short_names, interval, data_type):
object_creator = getattr(self, f'{interval}_flask_{data_type}')
if data_type == 'co2':
base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/'
elif data_type in ['c13', 'o18']:
base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/'
for station in stations_short_names:
url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv'
infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations'
filepath = self.fetch(url)
if not filepath:
continue
update = True
event = self.get_existing_event_to_update(infofield)
if event:
location = event.get_objects_by_name('geolocation')[0]
if not event:
event = MISPEvent()
event.info = infofield
event.add_tag(getattr(self, f'tag_{station}')())
location = getattr(self, f'geolocation_{station}')()
event.add_object(location)
event.add_attribute('link', f'http://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}')
update = False
object_creator(event, location, filepath, update)
if update:
self.misp.update_event(event)
else:
self.misp.add_event(event)
def create_new_event():
me = MISPEvent()
me.info = "Fail2Ban blocking"
me.add_tag(args.tag)
start = datetime.now()
me.add_attribute('datetime', start.isoformat(), comment='Start Time')
return me
def create_complex_event(self):
event = MISPEvent()
event.info = 'Complex Event'
event.distribution = Distribution.all_communities
event.add_tag('tlp:white')
event.add_attribute('ip-src', '8.8.8.8')
event.add_attribute('ip-dst', '8.8.8.9')
event.add_attribute('domain', 'google.com')
event.add_attribute('md5', '3c656da41f4645f77e3ec3281b63dd43')
event.attributes[0].distribution = Distribution.your_organisation_only
event.attributes[1].distribution = Distribution.this_community_only
event.attributes[2].distribution = Distribution.connected_communities
event.attributes[0].add_tag('tlp:red')
event.attributes[1].add_tag('tlp:amber')
event.attributes[2].add_tag('tlp:green')
obj = MISPObject('file')
obj.distribution = Distribution.connected_communities
obj.add_attribute('filename', 'testfile')
obj.add_attribute('md5', '3c656da41f4645f77e3ec3281b63dd44')
obj.attributes[0].distribution = Distribution.your_organisation_only
event.add_object(obj)
return event
def create_new_event():
me = MISPEvent()
me.info = "Fail2Ban blocking"
me.add_tag(args.tag)
start = datetime.now()
me.add_attribute('datetime', start.isoformat(), comment='Start Time')
return me
def create_new_event(self, entry):
attribute = MISPAttribute()
attribute.type = "malware-sample"
attribute.value = entry["shasum"]
attribute.data = Path(entry["outfile"])
attribute.comment = "File uploaded to Cowrie ({})".format(entry["sensor"])
attribute.expand = "binary"
if "url" in entry:
attributeURL = MISPAttribute()
attributeURL.type = "url"
attributeURL.value = entry["url"]
attributeURL.to_ids = True
else:
attributeURL = MISPAttribute()
attributeURL.type = "text"
attributeURL.value = "External upload"
attributeIP = MISPAttribute()
attributeIP.type = "ip-src"
attributeIP.value = entry["src_ip"]
attributeDT = MISPAttribute()
attributeDT.type = "datetime"
attributeDT.value = entry["timestamp"]
event = MISPEvent()
event.info = "File uploaded to Cowrie ({})".format(entry["sensor"])
event.add_tag("tlp:white")
event.attributes = [attribute, attributeURL, attributeIP, attributeDT]
event.run_expansions()
if self.publish:
event.publish()
result = self.misp_api.add_event(event)
if self.debug:
log.msg(f"Event creation result: \n{result}")
def create_daily_event(self):
today = datetime.date.today()
# [0-3]
distribution = 0
info = "Daily AIL-leaks {}".format(today)
# [0-2]
analysis = 0
# [1-4]
threat = 3
published = False
org_id = None
orgc_id = None
sharing_group_id = None
date = None
event = MISPEvent()
event.distribution = distribution
event.info = info
event.analysis = analysis
event.threat = threat
event.published = published
event.add_tag('infoleak:output-format="ail-daily"')
existing_event = self.pymisp.add_event(event)
return existing_event
def submit_tf_update(misp: ExpandedPyMISP, attributes: list) -> MISPEvent:
"""
create/update abuse.ch MISP-Event and append the new attributes
"""
eventinfo = event_info_template.format(
datetime.now().strftime(info_dateformat))
# logging.debug(eventinfo)
events = misp.search(controller='events',
eventinfo=eventinfo,
org=1,
pythonify=True)
if events: # current event exists already
event = events[0]
else: # create a new event
event = MISPEvent()
event.distribution = event_distribution
event.threat_level_id = event_threat_level
event.analysis = 2
event.info = eventinfo
for tag in tagging:
event.add_tag(tag)
event = misp.add_event(event, pythonify=True)
for att in attributes:
event.add_attribute(**att)
event.published = autopublish
return misp.update_event(event)
def process(self, task: Task) -> None: # type: ignore
config = task.get_payload("config")
family = task.headers["family"]
dhash = config_dhash(config)
# Parse the config using iocextract library
iocs = parse(family, config)
if not iocs:
# Nothing actionable found - skip the config
return
# Upload structured data to MISP
event = MISPEvent()
event.uuid = str(uuid5(self.CONFIG_NAMESPACE, dhash))
event.add_tag(f"mwdb:family:{family}")
event.info = f"Malware configuration ({family})"
if self.mwdb_url is not None:
event.add_attribute("link", f"{self.mwdb_url}/config/{dhash}")
for o in iocs.to_misp():
event.add_object(o)
misp = ExpandedPyMISP(self.misp_url, self.misp_key, self.misp_verifycert)
misp.add_event(event)
def processEvent(self, event):
mevent = MISPEvent()
mevent.from_dict(Event=event)
changed = False
for attr in mevent.attributes:
if (attr["type"] == "ip-dst"
or attr["type"] == "ip-src") and attr["to_ids"]:
print("Removing IDS flag in event '{}' on attr '{}'".format(
mevent.id, attr["value"]))
changed = True
attr["to_ids"] = False
if changed:
mevent.add_tag(self.expiredTag)
res = self.misp.update_event(mevent.id, mevent)
def generate_event(self, modsecLog: ModsecLog) -> MISPEvent:
event = MISPEvent()
event.info = modsecLog.generateInfoLine()
[http_method, url, http_version] = modsecLog.getRequestLine()
event.add_attribute(type='ip-src|port', value=str(modsecLog.log['transaction']['remote_address']) + "|" + str(modsecLog.log['transaction']['remote_port']), comment="Attacker", pythonify=True)
event.add_attribute(type='ip-dst|port', value=str(modsecLog.log['transaction']['local_address']) + "|" + str(modsecLog.log['transaction']['local_port']), comment="Server", pythonify=True)
event.add_attribute(type='http-method', value=http_method, pythonify=True)
event.add_attribute(type='url', value=url, pythonify=True)
event.add_attribute(type='datetime', value=modsecLog.log['@timestamp'], pythonify=True)
event.add_attribute(type='other', value=modsecLog.log['audit_data']['producer'], comment="Producer", pythonify=True)
event.add_attribute(type='text', value=json.dumps(modsecLog.log, indent=2), comment="Json log", pythonify=True)
#event.add_attribute(type='vulnerability', value=json_log['transaction']['time'], pythonify=True)
for tag in self.tags:
event.add_tag(tag)
log.debug("elasticsearch data")
log.debug(json.dumps(modsecLog.log, indent=2))
return event
def make_new_event(misp):
LOGGER.info('Creating new fixed event...')
event = MISPEvent()
timestamp = datetime.utcnow()
event_date = timestamp.strftime('%Y-%m-%d')
event.info = MISP_EVENT_TITLE
event.analysis = Analysis.completed
event.distribution = Distribution.your_organisation_only
event.threat_level_id = ThreatLevel.low
event.add_tag('type:OSINT')
event.add_tag('tlp:white')
LOGGER.info('Saving event...')
time.sleep(1)
return misp.add_event(event, pythonify=True)
def environment(self):
first_event = MISPEvent()
first_event.info = 'First event - org only - low - completed'
first_event.distribution = Distribution.your_organisation_only
first_event.threat_level_id = ThreatLevel.low
first_event.analysis = Analysis.completed
first_event.set_date("2017-12-31")
first_event.add_attribute('text', str(uuid4()))
first_event.attributes[0].add_tag('admin_only')
first_event.attributes[0].add_tag('tlp:white___test')
first_event.add_attribute('text', str(uuid4()))
first_event.attributes[1].add_tag('unique___test')
second_event = MISPEvent()
second_event.info = 'Second event - org only - medium - ongoing'
second_event.distribution = Distribution.your_organisation_only
second_event.threat_level_id = ThreatLevel.medium
second_event.analysis = Analysis.ongoing
second_event.set_date("Aug 18 2018")
second_event.add_attribute('text', str(uuid4()))
second_event.attributes[0].add_tag('tlp:white___test')
second_event.add_attribute('ip-dst', '1.1.1.1')
# Same value as in first event.
second_event.add_attribute('text', first_event.attributes[0].value)
third_event = MISPEvent()
third_event.info = 'Third event - all orgs - high - initial'
third_event.distribution = Distribution.all_communities
third_event.threat_level_id = ThreatLevel.high
third_event.analysis = Analysis.initial
third_event.set_date("Jun 25 2018")
third_event.add_tag('tlp:white___test')
third_event.add_attribute('text', str(uuid4()))
third_event.attributes[0].add_tag('tlp:amber___test')
third_event.attributes[0].add_tag('foo_double___test')
third_event.add_attribute('ip-src', '8.8.8.8')
third_event.attributes[1].add_tag('tlp:amber___test')
third_event.add_attribute('ip-dst', '9.9.9.9')
# Create first and third event as admin
# usr won't be able to see the first one
first = self.admin_misp_connector.add_event(first_event)
third = self.admin_misp_connector.add_event(third_event)
# Create second event as user
second = self.user_misp_connector.add_event(second_event)
return first, second, third
def _create_event(self, artifact: Type[Artifact]) -> MISPEvent:
"""Create an event in MISP, return an Event object."""
event = MISPEvent()
event.info = self.event_info.format(source_name=artifact.source_name)
# Add tags.
for tag in self.tags:
event.add_tag(tag)
# Add references.
if artifact.reference_link != "":
event.add_attribute("link", artifact.reference_link)
if artifact.reference_text != "":
event.add_attribute("text", artifact.reference_text)
if artifact.source_name != "" and self.include_artifact_source_name:
event.add_attribute("other", f"source:{artifact.source_name}")
return event
def _create_new_event(self, org_uuid) -> MISPEvent:
""" Creates new MISP event. """
misp_event = MISPEvent()
# TODO turn on correct organization assignment
# misp_event.orgc = self.misp_inst.get_organisation(org_uuid, pythonify=True)
# completed
misp_event.analysis = 2
# low
misp_event.threat_level_id = 3
# TODO use sharing group instead
misp_event.distribution = 1
# misp_event.distribution = 4
# misp_event.sharing_group_id = 2
misp_event.uuid = self.uuid_generator.get_misp_event_uuid(org_uuid)
misp_event.add_tag("rsit:test")
misp_event.add_tag("tlp:amber")
misp_event.info = "CTI - IntelMQ feed"
return self.misp_inst.add_event(misp_event, pythonify=True)
def register_misp(misp: ExpandedPyMISP, misp_event_dict: dict) -> None:
""" MISPイベントデータを受け取り、MISPに登録する """
misp_event = MISPEvent()
misp_event.from_dict(**misp_event_dict)
threat_level_id = misp_event.get('threat_level_id')
threat_level = f'anyrun:threat_level:{threat_level_id}'
misp_event.add_tag('anyrun')
misp_event.add_tag(threat_level)
retry_count = 0
while True:
try:
# pymispをインスタンス化してイベント登録
event_data = misp.add_event(misp_event)
if event_data.get('errors'):
raise Exception(event_data['errors'])
# MISPに登録されるイベントIDを取得し、出力
event_id = event_data['Event']['id']
print(f'新規に登録されたEvent_ID: {event_id}')
return
except:
except_return = traceback.format_exc()
# インポート済みの場合
if const.DUPLICATE_EVENT_CONFIRM_WORD in except_return:
print('Importを行おうとしたイベントは既にMISPに登録されています')
return
# リトライ回数チェック
retry_count += 1
if retry_count >= const.RETRY_MAXIMUM_LIMIT:
raise
# インターバル処理
print('MISPへのイベントインポートをリトライします')
time.sleep(const.COMMAND_INTERVAL_TIME)
def create_event(FEED, type):
#'uuid', 'info', 'threat_level_id', 'analysis', 'timestamp','publish_timestamp', 'published', 'date', 'extends_uuid'}
event = MISPEvent()
event.info = f"[{datetime.now().date().isoformat()}] RST Cloud Daily {type} feed"
event.analysis = 2 # 0=initial; 1=ongoing; 2=completed
event.threat_level_id = 2 #1 = high ; 2 = medium; 3 = low; 4 = undefined
event.add_tag('tlp:white')
# add to the database and publish
event = misp.add_event(event)
if PUBLISH:
misp.publish(event)
# add attributes to the newly created event
for entry in FEED:
if type == 'Domain':
misp.add_attribute(
event, domain_attribute('Network activity', 'domain', entry))
if type == 'IP':
misp.add_attribute(
event, ip_attribute('Network activity', 'ip-dst', entry))
def inserta_misp(nombre_evento, full_tweet, fverbose):
#Instancio evento MISP
event = MISPEvent()
#Nombre del evento. Se cambiara por cada tweet recibido
event.info = nombre_evento # Required
#Valores por defecto
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 2 # Optional, defaults to MISP.default_event_threat_level in MISP config
event.analysis = 1 # Optional, defaults to 0 (initial analysis)
#Inserto el tweet completo
#event.add_attribute('External analysis', full_tweet)
event.add_attribute('text', full_tweet)
event.add_tag('tlp:white')
add_tweet_atributes(event, full_tweet, fverbose)
#Inserto el evento MISP
event = misp.add_event(event, pythonify=True)
def import_all(self, stations_short_names, interval, data_type):
object_creator = getattr(self, f'{interval}_flask_{data_type}')
if data_type == 'co2':
base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/'
elif data_type in ['c13', 'o18']:
base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/'
for station in stations_short_names:
url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv'
infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations'
filepath = self.fetch(url)
if not filepath:
continue
if infofield in self.scrippts_meta:
event = MISPEvent()
event.load_file(str(self.output_dir / self.scrippts_meta[infofield]))
location = event.get_objects_by_name('geolocation')[0]
update = True
else:
event = MISPEvent()
event.uuid = str(uuid4())
event.info = infofield
event.Orgc = self.misp_org
event.add_tag(getattr(self, f'tag_{station}')())
location = getattr(self, f'geolocation_{station}')()
event.add_object(location)
event.add_attribute('link', f'https://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}')
update = False
with self.scrippts_meta_file.open('a') as f:
writer = csv.writer(f)
writer.writerow([infofield, f'{event.uuid}.json'])
object_creator(event, location, filepath, update)
if update:
# Bump the publish timestamp
event.publish_timestamp = datetime.datetime.timestamp(datetime.datetime.now())
feed_output = event.to_feed(with_meta=False)
with (self.output_dir / f'{event.uuid}.json').open('w') as f:
# json.dump(feed_output, f, indent=2, sort_keys=True) # For testing
json.dump(feed_output, f)
def make_new_event(misp):
LOGGER.info('Creating new fixed event...')
event = MISPEvent()
event_date = datetime.now().strftime('%Y-%m-%d')
event_title = '{0} {1}'.format(MISP_EVENT_TITLE, event_date)
event.info = event_title
event.analysis = Analysis.completed
event.distribution = Distribution.your_organisation_only
event.threat_level_id = ThreatLevel.low
event.add_tag('type:OSINT')
event.add_tag('tlp:white')
LOGGER.info('Saving event...')
time.sleep(1)
try:
new_event = misp.add_event(event, pythonify=True)
return new_event
except Exception as ex:
LOGGER.error('Failed to make MISP event: {0}'.format(str(ex)))
return False
class TestMISPEvent(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
def init_event(self):
self.mispevent.info = 'This is a test'
self.mispevent.distribution = 1
self.mispevent.threat_level_id = 1
self.mispevent.analysis = 1
self.mispevent.set_date("2017-12-31") # test the set date method
def test_simple(self):
with open('tests/mispevent_testfiles/simple.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event(self):
self.init_event()
self.mispevent.publish()
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_loadfile(self):
self.mispevent.load_file('tests/mispevent_testfiles/event.json')
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_tag(self):
self.init_event()
self.mispevent.add_tag('bar')
self.mispevent.add_tag(name='baz')
new_tag = MISPTag()
new_tag.from_dict(name='foo')
self.mispevent.add_tag(new_tag)
with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_attribute(self):
self.init_event()
a = self.mispevent.add_attribute('filename', 'bar.exe')
del a.uuid
a = self.mispevent.add_attribute_tag('osint', 'bar.exe')
attr_tags = self.mispevent.get_attribute_tag('bar.exe')
self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint')
self.assertEqual(attr_tags[0].name, 'osint')
with open('tests/mispevent_testfiles/attribute.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
# Fake setting an attribute ID for testing
self.mispevent.attributes[0].id = 42
self.mispevent.delete_attribute(42)
with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_object_tag(self):
self.mispevent.add_object(name='file', strict=True)
a = self.mispevent.objects[0].add_attribute('filename', value='')
self.assertEqual(a, None)
a = self.mispevent.objects[0].add_attribute('filename', value=None)
self.assertEqual(a, None)
a = self.mispevent.objects[0].add_attribute('filename',
value='bar',
Tag=[{
'name': 'blah'
}])
del a.uuid
self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name,
'blah')
self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(
['filename']))
self.assertEqual(
len(self.mispevent.objects[0].get_attributes_by_relation(
'filename')), 1)
self.mispevent.add_object(name='url', strict=True)
a = self.mispevent.objects[1].add_attribute(
'url', value='https://www.circl.lu')
del a.uuid
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
reference = self.mispevent.objects[0].add_reference(
self.mispevent.objects[1], 'baz', comment='foo')
del reference.uuid
self.assertEqual(
self.mispevent.objects[0].references[0].relationship_type, 'baz')
with open('tests/mispevent_testfiles/event_obj_attr_tag.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip(
"Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168"
)
def test_object_level_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar')
self.mispevent.objects[0].add_tag('osint')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_malware(self):
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.init_event()
a = self.mispevent.add_attribute('malware-sample',
'bar.exe',
data=pseudofile)
del a.uuid
attribute = self.mispevent.attributes[0]
self.assertEqual(attribute.malware_binary, pseudofile)
with open('tests/mispevent_testfiles/malware.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_malware(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/malware_exist.json')
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.assertEqual(
self.mispevent.objects[0].get_attributes_by_relation(
'malware-sample')[0].malware_binary.read(), pseudofile.read())
def test_sighting(self):
sighting = MISPSighting()
sighting.from_dict(value='1', type='bar', timestamp=11111111)
with open('tests/mispevent_testfiles/sighting.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(sighting.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_event(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
with open('tests/mispevent_testfiles/existing_event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes_existing(self):
self.mispevent.load_file('tests/mispevent_testfiles/shadow.json')
with open('tests/mispevent_testfiles/shadow.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip("Not supported on MISP.")
def test_shadow_attributes(self):
self.init_event()
p = self.mispevent.add_proposal(type='filename', value='baz.jpg')
del p.uuid
a = self.mispevent.add_attribute('filename', 'bar.exe')
del a.uuid
p = self.mispevent.attributes[0].add_proposal(type='filename',
value='bar.pdf')
del p.uuid
with open('tests/mispevent_testfiles/proposals.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_default_attributes(self):
self.mispevent.add_object(name='file', strict=True)
a = self.mispevent.objects[0].add_attribute('filename',
value='bar',
Tag=[{
'name': 'blah'
}])
del a.uuid
a = self.mispevent.objects[0].add_attribute('pattern-in-file',
value='baz')
self.assertEqual(a.category, 'Artifacts dropped')
del a.uuid
self.mispevent.add_object(name='file',
strict=False,
default_attributes_parameters=self.mispevent.
objects[0].attributes[0])
a = self.mispevent.objects[1].add_attribute('filename', value='baz')
del a.uuid
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
with open('tests/mispevent_testfiles/event_obj_def_param.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_default_values(self):
self.init_event()
self.mispevent.add_object(name='whois', strict=True)
a = self.mispevent.objects[0].add_attribute(
'registrar', value='registar.example.com')
del a.uuid
a = self.mispevent.objects[0].add_attribute('domain',
value='domain.example.com')
del a.uuid
a = self.mispevent.objects[0].add_attribute('nameserver',
value='ns1.example.com')
del a.uuid
a = self.mispevent.objects[0].add_attribute(
'nameserver',
value='ns2.example.com',
disable_correlation=False,
to_ids=True,
category='External analysis')
del a.uuid
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/def_param.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_not_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
def test_event_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.mispevent.info = 'blah'
self.assertTrue(self.mispevent.edited)
def test_event_tag_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.add_tag('foo')
self.assertTrue(self.mispevent.edited)
def test_event_attribute_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.mispevent.attributes[0].value = 'blah'
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertFalse(self.mispevent.attributes[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].tags[0].name = 'blah'
self.assertTrue(self.mispevent.attributes[0].tags[0].edited)
self.assertFalse(self.mispevent.attributes[0].tags[1].edited)
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited_second(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].add_tag(name='blah')
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].edited)
self.assertFalse(self.mispevent.objects[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited_tag(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].add_tag('blah')
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
with open('tests/mispevent_testfiles/existing_event_edited.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_by_id(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
misp_obj = self.mispevent.get_object_by_id(1556)
self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f')
def test_userdefined_object(self):
self.init_event()
self.mispevent.add_object(
name='test_object_template',
strict=True,
misp_objects_path_custom='tests/mispevent_testfiles')
with self.assertRaises(InvalidMISPObject) as e:
# Fail on required
self.mispevent.to_json(sort_keys=True, indent=2)
if sys.version_info >= (3, ):
self.assertEqual(e.exception.message,
'{\'member3\'} are required.')
else:
# Python2 bullshit
self.assertEqual(e.exception.message,
'set([u\'member3\']) are required.')
a = self.mispevent.objects[0].add_attribute('member3', value='foo')
del a.uuid
with self.assertRaises(InvalidMISPObject) as e:
# Fail on requiredOneOf
self.mispevent.to_json(sort_keys=True, indent=2)
self.assertEqual(
e.exception.message,
'At least one of the following attributes is required: member1, member2'
)
a = self.mispevent.objects[0].add_attribute('member1', value='bar')
del a.uuid
a = self.mispevent.objects[0].add_attribute('member1', value='baz')
del a.uuid
with self.assertRaises(InvalidMISPObject) as e:
# member1 is not a multiple
self.mispevent.to_json(sort_keys=True, indent=2)
self.assertEqual(e.exception.message,
'Multiple occurrences of member1 is not allowed')
self.mispevent.objects[0].attributes = self.mispevent.objects[
0].attributes[:2]
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
class StixParser():
def __init__(self):
super(StixParser, self).__init__()
self.misp_event = MISPEvent()
self.misp_event['Galaxy'] = []
def load_data(self, filename, version, event, args):
self.filename = filename
self.stix_version = version
self.event = event
if args and args[0] is not None:
self.add_original_file(args[0])
try:
event_distribution = args[1]
if not isinstance(event_distribution, int):
event_distribution = int(
event_distribution) if event_distribution.isdigit() else 5
except IndexError:
event_distribution = 5
try:
attribute_distribution = args[2]
if attribute_distribution != 'event' and not isinstance(
attribute_distribution, int):
attribute_distribution = int(
attribute_distribution) if attribute_distribution.isdigit(
) else 5
except IndexError:
attribute_distribution = 5
self.misp_event.distribution = event_distribution
self._attribute_distribution = event_distribution if attribute_distribution == 'event' else attribute_distribution
def add_original_file(self, original_filename):
with open(self.filename, 'rb') as f:
sample = b64encode(f.read()).decode('utf-8')
original_file = MISPObject('original-imported-file')
original_file.add_attribute(
**{
'type': 'attachment',
'value': original_filename,
'object_relation': 'imported-sample',
'data': sample
})
original_file.add_attribute(
**{
'type': 'text',
'object_relation': 'format',
'value': self.stix_version
})
self.misp_event.add_object(**original_file)
def general_handler(self):
self.outputname = '{}.stix2'.format(self.filename)
self.buildMISPDict()
self.set_distribution()
def buildMISPDict(self):
report_attributes = defaultdict(set)
for _, report in self.event['report'].items():
report_attributes['orgs'].add(
report['created_by_ref'].split('--')[1])
report_attributes['name'].add(report['name'])
if report.get('published'):
report_attributes['published'].add(report['published'])
if 'labels' in report:
report_attributes['labels'].update(
[l for l in report['labels']])
if 'external_references' in report:
self.add_links(report['external_references'])
for ref in report['object_refs']:
if 'relationship' not in ref:
object_type, uuid = ref.split('--')
object2parse = self.event[object_type][uuid]
self.parsing_process(object2parse, object_type)
if len(report_attributes['orgs']) == 1:
identity = self.event['identity'][report_attributes['orgs'].pop()]
self.misp_event['Org'] = {'name': identity['name']}
if len(report_attributes['published']) == 1:
self.misp_event.publish_timestamp = self.getTimestampfromDate(
report_attributes['published'].pop())
if len(report_attributes['name']) == 1:
self.misp_event.info = report_attributes['name'].pop()
else:
self.misp_event.info = "Imported with MISP import script for {}.".format(
self.stix_version)
for l in report_attributes['labels']:
self.misp_event.add_tag(l)
def set_distribution(self):
for attribute in self.misp_event.attributes:
attribute.distribution = self._attribute_distribution
for misp_object in self.misp_event.objects:
misp_object.distribution = self._attribute_distribution
for attribute in misp_object.attributes:
attribute.distribution = self._attribute_distribution
def saveFile(self):
eventDict = self.misp_event.to_json()
outputfile = '{}.stix2'.format(self.filename)
with open(outputfile, 'w') as f:
f.write(eventDict)
def add_links(self, refs):
for e in refs:
link = {"type": "link"}
comment = e.get('source_name')
try:
comment = comment.split('url - ')[1]
except IndexError:
pass
if comment:
link['comment'] = comment
link['value'] = e.get('url')
self.misp_event.add_attribute(**link)
@staticmethod
def getTimestampfromDate(stix_date):
try:
return int(stix_date.timestamp())
except AttributeError:
return int(
time.mktime(
time.strptime(
stix_date.split('+')[0], "%Y-%m-%d %H:%M:%S")))
@staticmethod
def get_misp_type(labels):
return labels[0].split('=')[1][1:-1]
@staticmethod
def get_misp_category(labels):
return labels[1].split('=')[1][1:-1]
def parse_course_of_action(self, o):
misp_object = MISPObject('course-of-action')
if 'name' in o:
attribute = {
'type': 'text',
'object_relation': 'name',
'value': o.get('name')
}
misp_object.add_attribute(**attribute)
else:
return
if 'description' in o:
attribute = {
'type': 'text',
'object_relation': 'description',
'value': o.get('description')
}
misp_object.add_attribute(**attribute)
self.misp_event.add_object(**misp_object)
@staticmethod
def parse_observable(observable, attribute_type):
return misp_types_mapping[attribute_type](observable, attribute_type)
@staticmethod
def parse_pattern(pattern):
if ' AND ' in pattern:
pattern_parts = pattern.split(' AND ')
if len(pattern_parts) == 3:
_, value1 = pattern_parts[2].split(' = ')
_, value2 = pattern_parts[0].split(' = ')
return '{}|{}'.format(value1[1:-2], value2[1:-1])
else:
_, value1 = pattern_parts[0].split(' = ')
_, value2 = pattern_parts[1].split(' = ')
if value1 in ("'ipv4-addr'", "'ipv6-addr'"):
return value2[1:-2]
return '{}|{}'.format(value1[1:-1], value2[1:-2])
else:
return pattern.split(' = ')[1][1:-2]
def parse_pattern_with_data(self, pattern):
if 'artifact:payload_bin' not in pattern:
return self.parse_pattern(pattern)
pattern_parts = pattern.split(' AND ')
if len(pattern_parts) == 3:
filename = pattern_parts[0].split(' = ')[1]
md5 = pattern_parts[1].split(' = ')[1]
return "{}|{}".format(
filename[1:-1],
md5[1:-1]), pattern_parts[2].split(' = ')[1][1:-2]
return pattern_parts[0].split(' = ')[1][1:-1], pattern_parts[1].split(
' = ')[1][1:-2]
class AssemblyLineParser():
def __init__(self):
self.misp_event = MISPEvent()
self.results = {}
self.attribute = {'to_ids': True}
self._results_mapping = {
'NET_DOMAIN_NAME': 'domain',
'NET_FULL_URI': 'url',
'NET_IP': 'ip-dst'
}
self._file_mapping = {
'entropy': {
'type': 'float',
'object_relation': 'entropy'
},
'md5': {
'type': 'md5',
'object_relation': 'md5'
},
'mime': {
'type': 'mime-type',
'object_relation': 'mimetype'
},
'sha1': {
'type': 'sha1',
'object_relation': 'sha1'
},
'sha256': {
'type': 'sha256',
'object_relation': 'sha256'
},
'size': {
'type': 'size-in-bytes',
'object_relation': 'size-in-bytes'
},
'ssdeep': {
'type': 'ssdeep',
'object_relation': 'ssdeep'
}
}
def get_submission(self, attribute, client):
sid = attribute['value'].split('=')[-1]
try:
if not client.submission.is_completed(sid):
self.results[
'error'] = 'Submission not completed, please try again later.'
return
except Exception as e:
self.results[
'error'] = f'Something went wrong while trying to check if the submission in AssemblyLine is completed: {e.__str__()}'
return
try:
submission = client.submission.full(sid)
except Exception as e:
self.results[
'error'] = f"Something went wrong while getting the submission from AssemblyLine: {e.__str__()}"
return
self._parse_report(submission)
def finalize_results(self):
if 'error' in self.results:
return self.results
event = json.loads(self.misp_event.to_json())
results = {
key: event[key]
for key in ('Attribute', 'Object', 'Tag')
if (key in event and event[key])
}
return {'results': results}
def _create_attribute(self, result, attribute_type):
attribute = MISPAttribute()
attribute.from_dict(type=attribute_type,
value=result['value'],
**self.attribute)
if result['classification'] != 'UNCLASSIFIED':
attribute.add_tag(result['classification'].lower())
self.misp_event.add_attribute(**attribute)
return {
'referenced_uuid': attribute.uuid,
'relationship_type': '-'.join(result['context'].lower().split(' '))
}
def _create_file_object(self, file_info):
file_object = MISPObject('file')
filename_attribute = {'type': 'filename'}
filename_attribute.update(self.attribute)
if file_info['classification'] != "UNCLASSIFIED":
tag = {'Tag': [{'name': file_info['classification'].lower()}]}
filename_attribute.update(tag)
for feature, attribute in self._file_mapping.items():
attribute.update(tag)
file_object.add_attribute(value=file_info[feature],
**attribute)
return filename_attribute, file_object
for feature, attribute in self._file_mapping.items():
file_object.add_attribute(value=file_info[feature], **attribute)
return filename_attribute, file_object
@staticmethod
def _get_results(submission_results):
results = defaultdict(list)
for k, values in submission_results.items():
h = k.split('.')[0]
for t in values['result']['tags']:
if t['context'] is not None:
results[h].append(t)
return results
def _get_scores(self, file_tree):
scores = {}
for h, f in file_tree.items():
score = f['score']
if score > 0:
scores[h] = {'name': f['name'], 'score': score}
if f['children']:
scores.update(self._get_scores(f['children']))
return scores
def _parse_report(self, submission):
if submission['classification'] != 'UNCLASSIFIED':
self.misp_event.add_tag(submission['classification'].lower())
filtered_results = self._get_results(submission['results'])
scores = self._get_scores(submission['file_tree'])
for h, results in filtered_results.items():
if h in scores:
attribute, file_object = self._create_file_object(
submission['file_infos'][h])
print(file_object)
for filename in scores[h]['name']:
file_object.add_attribute('filename',
value=filename,
**attribute)
for reference in self._parse_results(results):
file_object.add_reference(**reference)
self.misp_event.add_object(**file_object)
def _parse_results(self, results):
references = []
for result in results:
try:
attribute_type = self._results_mapping[result['type']]
except KeyError:
continue
references.append(self._create_attribute(result, attribute_type))
return references
class TestMISPEvent(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
def init_event(self):
self.mispevent.info = 'This is a test'
self.mispevent.distribution = 1
self.mispevent.threat_level_id = 1
self.mispevent.analysis = 1
self.mispevent.set_date("2017-12-31") # test the set date method
def test_simple(self):
with open('tests/mispevent_testfiles/simple.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event(self):
self.init_event()
self.mispevent.publish()
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_loadfile(self):
self.mispevent.load_file('tests/mispevent_testfiles/event.json')
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_tag(self):
self.init_event()
self.mispevent.add_tag('bar')
self.mispevent.add_tag(name='baz')
new_tag = MISPTag()
new_tag.from_dict(name='foo')
self.mispevent.add_tag(new_tag)
with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_attribute(self):
self.init_event()
self.mispevent.add_attribute('filename', 'bar.exe')
self.mispevent.add_attribute_tag('osint', 'bar.exe')
attr_tags = self.mispevent.get_attribute_tag('bar.exe')
self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint')
self.assertEqual(attr_tags[0].name, 'osint')
with open('tests/mispevent_testfiles/attribute.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
# Fake setting an attribute ID for testing
self.mispevent.attributes[0].id = 42
self.mispevent.delete_attribute(42)
with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_object_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}])
self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah')
self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(['filename']))
self.assertEqual(len(self.mispevent.objects[0].get_attributes_by_relation('filename')), 1)
self.mispevent.add_object(name='url', strict=True)
self.mispevent.objects[1].add_attribute('url', value='https://www.circl.lu')
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
self.mispevent.objects[0].add_reference('b', 'baz', comment='foo')
self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz')
with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168")
def test_object_level_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar')
self.mispevent.objects[0].add_tag('osint')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_malware(self):
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.init_event()
self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile)
attribute = self.mispevent.attributes[0]
self.assertEqual(attribute.malware_binary, pseudofile)
with open('tests/mispevent_testfiles/malware.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_malware(self):
self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json')
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.assertEqual(
self.mispevent.objects[0].get_attributes_by_relation('malware-sample')[0].malware_binary.read(),
pseudofile.read())
def test_sighting(self):
sighting = MISPSighting()
sighting.from_dict(value='1', type='bar', timestamp=11111111)
with open('tests/mispevent_testfiles/sighting.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_event(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
with open('tests/mispevent_testfiles/existing_event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes_existing(self):
self.mispevent.load_file('tests/mispevent_testfiles/shadow.json')
with open('tests/mispevent_testfiles/shadow.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes(self):
self.init_event()
self.mispevent.add_proposal(type='filename', value='baz.jpg')
self.mispevent.add_attribute('filename', 'bar.exe')
self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf')
with open('tests/mispevent_testfiles/proposals.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_default_attributes(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}])
self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent.objects[0].attributes[0])
self.mispevent.objects[1].add_attribute('filename', value='baz')
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_default_values(self):
self.init_event()
self.mispevent.add_object(name='whois', strict=True)
self.mispevent.objects[0].add_attribute('registrar', value='registar.example.com')
self.mispevent.objects[0].add_attribute('domain', value='domain.example.com')
self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com')
self.mispevent.objects[0].add_attribute('nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/def_param.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_not_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
def test_event_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.mispevent.info = 'blah'
self.assertTrue(self.mispevent.edited)
def test_event_tag_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.add_tag('foo')
self.assertTrue(self.mispevent.edited)
def test_event_attribute_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.mispevent.attributes[0].value = 'blah'
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertFalse(self.mispevent.attributes[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].tags[0].name = 'blah'
self.assertTrue(self.mispevent.attributes[0].tags[0].edited)
self.assertFalse(self.mispevent.attributes[0].tags[1].edited)
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited_second(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].add_tag(name='blah')
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].edited)
self.assertFalse(self.mispevent.objects[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited_tag(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].add_tag('blah')
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_by_id(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
misp_obj = self.mispevent.get_object_by_id(1556)
self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f')
def test_userdefined_object(self):
self.init_event()
self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles')
with self.assertRaises(InvalidMISPObject) as e:
# Fail on required
self.mispevent.to_json()
if sys.version_info >= (3, ):
self.assertEqual(e.exception.message, '{\'member3\'} are required.')
else:
# Python2 bullshit
self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.')
self.mispevent.objects[0].add_attribute('member3', value='foo')
with self.assertRaises(InvalidMISPObject) as e:
# Fail on requiredOneOf
self.mispevent.to_json()
self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2')
self.mispevent.objects[0].add_attribute('member1', value='bar')
self.mispevent.objects[0].add_attribute('member1', value='baz')
with self.assertRaises(InvalidMISPObject) as e:
# member1 is not a multiple
self.mispevent.to_json()
self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed')
self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2]
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
class Mail2MISP():
def __init__(self,
misp_url,
misp_key,
verifycert,
config,
offline=False,
urlsonly=False):
self.offline = offline
if not self.offline:
self.misp = ExpandedPyMISP(misp_url,
misp_key,
verifycert,
debug=config.debug)
self.config = config
self.urlsonly = urlsonly
if not hasattr(self.config, 'enable_dns'):
setattr(self.config, 'enable_dns', True)
if self.urlsonly is False:
setattr(self.config, 'enable_dns', False)
self.debug = self.config.debug
self.config_from_email_body = {}
# Init Faup
self.f = Faup()
self.sightings_to_add = []
def load_email(self, pseudofile):
self.pseudofile = pseudofile
self.original_mail = message_from_bytes(self.pseudofile.getvalue(),
policy=policy.default)
self.subject = self.original_mail.get('Subject')
try:
self.sender = self.original_mail.get('From')
except:
self.sender = "<unknown sender>"
# Remove words from subject
for removeword in self.config.removelist:
self.subject = re.sub(removeword, "", self.subject).strip()
# Initialize the MISP event
self.misp_event = MISPEvent()
self.misp_event.info = f'{self.config.email_subject_prefix} - {self.subject}'
self.misp_event.distribution = self.config.default_distribution
self.misp_event.threat_level_id = self.config.default_threat_level
self.misp_event.analysis = self.config.default_analysis
def sighting(self, value, source):
if self.offline:
raise Exception('The script is running in offline mode, ')
'''Add a sighting'''
s = MISPSighting()
s.from_dict(value=value, source=source)
self.misp.add_sighting(s)
def _find_inline_forward(self):
'''Does the body contains a forwarded email?'''
for identifier in self.config.forward_identifiers:
if identifier in self.clean_email_body:
self.clean_email_body, fw_email = self.clean_email_body.split(
identifier)
return self.forwarded_email(
pseudofile=BytesIO(fw_email.encode()))
def _find_attached_forward(self):
forwarded_emails = []
for attachment in self.original_mail.iter_attachments():
attachment_content = attachment.get_content()
# Search for email forwarded as attachment
# I could have more than one, attaching everything.
if isinstance(attachment_content, message.EmailMessage):
forwarded_emails.append(
self.forwarded_email(
pseudofile=BytesIO(attachment_content.as_bytes())))
else:
if isinstance(attachment_content, str):
attachment_content = attachment_content.encode()
filename = attachment.get_filename()
if not filename:
filename = 'missing_filename'
if self.config_from_email_body.get(
'attachment'
) == self.config.m2m_benign_attachment_keyword:
# Attach sane file
self.misp_event.add_attribute(
'attachment',
value=filename,
data=BytesIO(attachment_content))
else:
f_object, main_object, sections = make_binary_objects(
pseudofile=BytesIO(attachment_content),
filename=filename,
standalone=False)
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
[
self.misp_event.add_object(section)
for section in sections
]
return forwarded_emails
def email_from_spamtrap(self):
'''The email comes from a spamtrap and should be attached as-is.'''
raw_body = self.original_mail.get_body(preferencelist=('html',
'plain'))
if raw_body:
self.clean_email_body = html.unescape(
raw_body.get_payload(decode=True).decode(
'utf8', 'surrogateescape'))
else:
self.clean_email_body = ''
return self.forwarded_email(self.pseudofile)
def forwarded_email(self, pseudofile: BytesIO):
'''Extracts all possible indicators out of an email and create a MISP event out of it.
* Gets all relevant Headers
* Attach the body
* Create MISP file objects (uses lief if possible)
* Set all references
'''
email_object = EMailObject(pseudofile=pseudofile,
attach_original_mail=True,
standalone=False)
if email_object.attachments:
# Create file objects for the attachments
for attachment_name, attachment in email_object.attachments:
if not attachment_name:
attachment_name = 'NameMissing.txt'
if self.config_from_email_body.get(
'attachment'
) == self.config.m2m_benign_attachment_keyword:
a = self.misp_event.add_attribute('attachment',
value=attachment_name,
data=attachment)
email_object.add_reference(a.uuid, 'related-to',
'Email attachment')
else:
f_object, main_object, sections = make_binary_objects(
pseudofile=attachment,
filename=attachment_name,
standalone=False)
if self.config.vt_key:
try:
vt_object = VTReportObject(
self.config.vt_key,
f_object.get_attributes_by_relation(
'sha256')[0].value,
standalone=False)
self.misp_event.add_object(vt_object)
f_object.add_reference(vt_object.uuid,
'analysed-with')
except InvalidMISPObject as e:
print(e)
pass
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
for section in sections:
self.misp_event.add_object(section)
email_object.add_reference(f_object.uuid, 'related-to',
'Email attachment')
self.process_body_iocs(email_object)
if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get(
'attach_original_mail'):
self.misp_event.add_object(email_object)
return email_object
def process_email_body(self):
mail_as_bytes = self.original_mail.get_body(
preferencelist=('html', 'plain')).get_payload(decode=True)
if mail_as_bytes:
self.clean_email_body = html.unescape(
mail_as_bytes.decode('utf8', 'surrogateescape'))
# Check if there are config lines in the body & convert them to a python dictionary:
# <config.body_config_prefix>:<key>:<value> => {<key>: <value>}
self.config_from_email_body = {
k.strip(): v.strip()
for k, v in re.findall(
f'{self.config.body_config_prefix}:(.*):(.*)',
self.clean_email_body)
}
if self.config_from_email_body:
# ... remove the config lines from the body
self.clean_email_body = re.sub(
rf'^{self.config.body_config_prefix}.*\n?',
'',
html.unescape(
self.original_mail.get_body(
preferencelist=('html', 'plain')).get_payload(
decode=True).decode('utf8',
'surrogateescape')),
flags=re.MULTILINE)
# Check if autopublish key is present and valid
if self.config_from_email_body.get(
'm2mkey') == self.config.m2m_key:
if self.config_from_email_body.get('distribution') is not None:
self.misp_event.distribution = self.config_from_email_body.get(
'distribution')
if self.config_from_email_body.get('threat_level') is not None:
self.misp_event.threat_level_id = self.config_from_email_body.get(
'threat_level')
if self.config_from_email_body.get('analysis') is not None:
self.misp_event.analysis = self.config_from_email_body.get(
'analysis')
if self.config_from_email_body.get('publish'):
self.misp_event.publish()
self._find_inline_forward()
else:
self.clean_email_body = ''
self._find_attached_forward()
def process_body_iocs(self, email_object=None):
if email_object:
body = html.unescape(
email_object.email.get_body(
preferencelist=('html',
'plain')).get_payload(decode=True).decode(
'utf8', 'surrogateescape'))
else:
body = self.clean_email_body
# Cleanup body content
# Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message
for ignoreline in self.config.ignorelist:
body = re.sub(rf'^{ignoreline}.*\n?', '', body, flags=re.MULTILINE)
# Remove everything after the stopword from the body
body = body.split(self.config.stopword, 1)[0]
# Add tags to the event if keywords are found in the mail
for tag in self.config.tlptags:
for alternativetag in self.config.tlptags[tag]:
if alternativetag in body.lower():
self.misp_event.add_tag(tag)
# Prepare extraction of IOCs
# Refang email data
body = refang(body)
# Extract and add hashes
contains_hash = False
for h in set(re.findall(hashmarker.MD5_REGEX, body)):
contains_hash = True
attribute = self.misp_event.add_attribute(
'md5', h, enforceWarninglist=self.config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sightings_to_add.append((h, self.config.sighting_source))
for h in set(re.findall(hashmarker.SHA1_REGEX, body)):
contains_hash = True
attribute = self.misp_event.add_attribute(
'sha1', h, enforceWarninglist=self.config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sightings_to_add.append((h, self.config.sighting_source))
for h in set(re.findall(hashmarker.SHA256_REGEX, body)):
contains_hash = True
attribute = self.misp_event.add_attribute(
'sha256', h, enforceWarninglist=self.config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sightings_to_add.append((h, self.config.sighting_source))
if contains_hash:
[
self.misp_event.add_tag(tag)
for tag in self.config.hash_only_tags
]
# # Extract network IOCs
urllist = []
urllist += re.findall(urlmarker.WEB_URL_REGEX, body)
urllist += re.findall(urlmarker.IP_REGEX, body)
if self.debug:
syslog.syslog(str(urllist))
hostname_processed = []
# Add IOCs and expanded information to MISP
for entry in set(urllist):
ids_flag = True
self.f.decode(entry)
domainname = self.f.get_domain()
if domainname in self.config.excludelist:
# Ignore the entry
continue
hostname = self.f.get_host()
scheme = self.f.get_scheme()
if scheme:
scheme = scheme
resource_path = self.f.get_resource_path()
if resource_path:
resource_path = resource_path
if self.debug:
syslog.syslog(domainname)
if domainname in self.config.internallist and self.urlsonly is False: # Add link to internal reference unless in urlsonly mode
attribute = self.misp_event.add_attribute(
'link',
entry,
category='Internal reference',
to_ids=False,
enforceWarninglist=False)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
elif domainname in self.config.externallist or self.urlsonly is False: # External analysis
attribute = self.misp_event.add_attribute(
'link',
entry,
category='External analysis',
to_ids=False,
enforceWarninglist=False)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
elif domainname in self.config.externallist or self.urlsonly: # External analysis
if self.urlsonly:
comment = self.subject + " (from: " + self.sender + ")"
else:
comment = ""
attribute = self.misp.add_attribute(
self.urlsonly, {
"type": 'link',
"value": entry,
"category": 'External analysis',
"to_ids": False,
"comment": comment
})
for tag in self.config.tlptags:
for alternativetag in self.config.tlptags[tag]:
if alternativetag in self.subject.lower():
self.misp.tag(attribute["uuid"], tag)
new_subject = comment.replace(alternativetag, '')
self.misp.change_comment(attribute["uuid"],
new_subject)
else: # The URL is probably an indicator.
comment = ""
if (domainname in self.config.noidsflaglist) or (
hostname in self.config.noidsflaglist):
ids_flag = False
comment = "Known host (mostly for connectivity test or IP lookup)"
if self.debug:
syslog.syslog(str(entry))
if scheme:
if is_ip(hostname):
attribute = self.misp_event.add_attribute(
'url',
entry,
to_ids=False,
enforceWarninglist=self.config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid,
'contains')
else:
if resource_path: # URL has path, ignore warning list
attribute = self.misp_event.add_attribute(
'url',
entry,
to_ids=ids_flag,
enforceWarninglist=False,
comment=comment)
if email_object:
email_object.add_reference(
attribute.uuid, 'contains')
else: # URL has no path
attribute = self.misp_event.add_attribute(
'url',
entry,
to_ids=ids_flag,
enforceWarninglist=self.config.
enforcewarninglist,
comment=comment)
if email_object:
email_object.add_reference(
attribute.uuid, 'contains')
if self.config.sighting:
self.sightings_to_add.append(
(entry, self.config.sighting_source))
if hostname in hostname_processed:
# Hostname already processed.
continue
hostname_processed.append(hostname)
if self.config.sighting:
self.sightings_to_add.append(
(hostname, self.config.sighting_source))
if self.debug:
syslog.syslog(hostname)
comment = ''
port = self.f.get_port()
if port:
port = port
comment = f'on port: {port}'
if is_ip(hostname):
attribute = self.misp_event.add_attribute(
'ip-dst',
hostname,
to_ids=ids_flag,
enforceWarninglist=self.config.enforcewarninglist,
comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else:
related_ips = []
if HAS_DNS and self.config.enable_dns:
try:
syslog.syslog(hostname)
for rdata in dns.resolver.query(hostname, 'A'):
if self.debug:
syslog.syslog(str(rdata))
related_ips.append(rdata.to_text())
except Exception as e:
if self.debug:
syslog.syslog(str(e))
if related_ips:
hip = MISPObject(name='ip-port')
hip.add_attribute(
'hostname',
value=hostname,
to_ids=ids_flag,
enforceWarninglist=self.config.enforcewarninglist,
comment=comment)
for ip in set(related_ips):
hip.add_attribute('ip',
type='ip-dst',
value=ip,
to_ids=False,
enforceWarninglist=self.config.
enforcewarninglist)
self.misp_event.add_object(hip)
if email_object:
email_object.add_reference(hip.uuid, 'contains')
else:
if self.urlsonly is False:
attribute = self.misp_event.add_attribute(
'hostname',
value=hostname,
to_ids=ids_flag,
enforceWarninglist=self.config.
enforcewarninglist,
comment=comment)
if email_object:
email_object.add_reference(attribute.uuid,
'contains')
def add_event(self):
'''Add event on the remote MISP instance.'''
# Add additional tags depending on others
tags = []
for tag in [t.name for t in self.misp_event.tags]:
if self.config.dependingtags.get(tag):
tags += self.config.dependingtags.get(tag)
# Add additional tags according to configuration
for malware in self.config.malwaretags:
if malware.lower() in self.subject.lower():
tags += self.config.malwaretags.get(malware)
if tags:
[self.misp_event.add_tag(tag) for tag in tags]
has_tlp_tag = False
for tag in [t.name for t in self.misp_event.tags]:
if tag.lower().startswith('tlp'):
has_tlp_tag = True
if not has_tlp_tag:
self.misp_event.add_tag(self.config.tlptag_default)
if self.offline:
return self.misp_event.to_json()
event = self.misp.add_event(self.misp_event, pythonify=True)
if self.config.sighting:
for value, source in self.sightings_to_add:
self.sighting(value, source)
return event
customerUserId = event.add_attribute(
'other',
alert["identity"]["customerUserId"],
comment="Customer User Id")
if alert["identity"]["department"] is not None:
department = event.add_attribute(alert['other',
"identity"]["department"],
comment="Department")
if alert["identity"]["location"] is not None:
location = event.add_attribute('other',
alert["identity"]["location"],
comment="Location")
if alert["identity"]["name"] is not None:
name = event.add_attribute('target-user',
alert["identity"]["name"],
comment="Name")
if alert["identity"]["title"] is not None:
title = event.add_attribute('other',
alert["identity"]["title"],
comment="Title")
event.add_tag("VAP")
misp.add_event(event.to_json())
# -*- coding: utf-8 -*-
from pymisp import ExpandedPyMISP, MISPEvent
from pymisp import MISPObject
from keys import misp_url, misp_key, misp_verifycert
from datetime import date
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
event = MISPEvent()
event.info = 'IoT malware' # Event Title
event.distribution = 1 # 0 = Your Organisation Only, 1 = Community
event.threat_level_id = 2 # 1 = High, 2 = Medium, 3 = Low
event.analysis = 2 # 0 (initial analysis), 1 (On-Going), 2 (Complete)
event.add_tag('malware_classification:malware-category="Botnet"')
event.add_tag('tlp:amber')
d = date.today()
event.set_date(d)
attribute_second = event.add_attribute('url',
'http://1.2.3.4/example',
disable_correlation=False,
comment="Botnet example text",
to_ids=False)
event = misp.add_event(event, pythonify=True)
# Publish event
event.publish()
# initialize and set MISPOrganisation()
orgc = MISPOrganisation()
orgc.name = 'RH-ISAC'
orgc.id = '#{ORGC_ID}' # organisation id
orgc.uuid = '#{ORGC_UUID}' # organisation uuid
# initialize and set MISPEvent()
event = MISPEvent()
event.Orgc = orgc
event.info = report.title
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 2 # Optional, defaults to MISP.default_event_threat_level in MISP config
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
# get tags for report
for tag in tru.get_enclave_tags(report.id):
event.add_tag(tag.name)
# get indicators for report
for indicator in tru.get_indicators_for_report(report.id):
# map trustar indicator type to MISP format
indicator_type = {
"MD5": "md5",
"SHA1": "sha1",
"SHA256": "sha256",
"SOFTWARE": "filename",
"URL": "link",
"EMAIL_ADDRESS": "email-src",
"IP": "ip-dst",
"MALWARE": "malware-type",
"CIDR_BLOCK": "ip-src",
if make_feed:
org = MISPOrganisation()
org.name = 'CIRCL'
org.uuid = "55f6ea5e-2c60-40e5-964f-47a8950d210f"
else:
from covid_key import url, key
misp = PyMISP(url, key)
for p in path.glob('*_json/current_china.json'):
d = parse(p.parent.name[:-5])
event = MISPEvent()
event.info = f"[{d.isoformat()}] DXY COVID-19 live report"
event.date = d
event.distribution = 3
event.add_tag('tlp:white')
if make_feed:
event.orgc = org
else:
e = misp.search(eventinfo=event.info, metadata=True, pythonify=True)
if e:
# Already added.
continue
event.add_attribute('attachment',
p.name,
data=BytesIO(p.open('rb').read()))
with p.open() as f:
data = json.load(f)
for province in data:
obj_province = event.add_object(name='covid19-dxy-live-province',
standalone=False)
"reportname":"top-attacks",
"key":"#{API_KEY}" # put your Panorama api key here
}
# try with "..., verify=False)" if you get an SSL error
response = requests.request("GET", url, params=querystring)
resp_text = response.text
json_data = json.loads(json.dumps(xmltodict.parse(resp_text)))
# initialize and set MISPOrganisation
orgc = MISPOrganisation()
orgc.name = 'Palo Alto'
orgc.id = '#{ORGC_ID}' # organisation id
orgc.uuid = '#{ORGC_UUID}' # organisation uuid
# initialize and set MISPEvent()
event = MISPEvent()
event.Orgc = orgc
event.info = json_data['report']['result']['@name'] + " | " + json_data['report']['result']['@range']
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 0 # Optional, defaults to MISP.default_event_threat_level in MISP config
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
event.add_tag('firewall threats')
for threatid in json_data['report']['result']['entry']:
attribute = event.add_attribute('comment', threatid['threatid'])
attribute.comment = threatid['count']
misp.add_event(event.to_json())
class StixParser():
def __init__(self):
self.misp_event = MISPEvent()
self.event = []
self.misp_event['Galaxy'] = []
def loadEvent(self, args):
try:
filename = os.path.join(os.path.dirname(args[0]), args[1])
tempFile = open(filename, 'r', encoding='utf-8')
self.filename = filename
event = json.loads(tempFile.read())
self.stix_version = 'stix {}'.format(event.get('spec_version'))
for o in event.get('objects'):
try:
try:
self.event.append(stix2.parse(o, allow_custom=True))
except:
self.parse_custom(o)
except:
pass
if not self.event:
print(
json.dumps({
'success':
0,
'message':
'There is no valid STIX object to import'
}))
sys.exit(1)
try:
event_distribution = args[2]
if not isinstance(event_distribution, int):
event_distribution = int(
event_distribution) if event_distribution.isdigit(
) else 5
except:
event_distribution = 5
try:
attribute_distribution = args[3]
if attribute_distribution != 'event' and not isinstance(
attribute_distribution, int):
attribute_distribution = int(
attribute_distribution
) if attribute_distribution.isdigit() else 5
except:
attribute_distribution = 5
self.misp_event.distribution = event_distribution
self.__attribute_distribution = event_distribution if attribute_distribution == 'event' else attribute_distribution
self.load_mapping()
except:
print(
json.dumps({
'success': 0,
'message': 'The STIX file could not be read'
}))
sys.exit(1)
def parse_custom(self, obj):
custom_object_type = obj.pop('type')
labels = obj['labels']
try:
@stix2.CustomObject(
custom_object_type,
[('id', stix2.properties.StringProperty(required=True)),
('x_misp_timestamp',
stix2.properties.StringProperty(required=True)),
('labels', stix2.properties.ListProperty(labels,
required=True)),
('x_misp_value',
stix2.properties.StringProperty(required=True)),
('created_by_ref',
stix2.properties.StringProperty(required=True)),
('x_misp_comment', stix2.properties.StringProperty()),
('x_misp_category', stix2.properties.StringProperty())])
class Custom(object):
def __init__(self, **kwargs):
return
custom = Custom(**obj)
except:
@stix2.CustomObject(
custom_object_type,
[('id', stix2.properties.StringProperty(required=True)),
('x_misp_timestamp',
stix2.properties.StringProperty(required=True)),
('labels', stix2.properties.ListProperty(labels,
required=True)),
('x_misp_values',
stix2.properties.DictionaryProperty(required=True)),
('created_by_ref',
stix2.properties.StringProperty(required=True)),
('x_misp_comment', stix2.properties.StringProperty()),
('x_misp_category', stix2.properties.StringProperty())])
class Custom(object):
def __init__(self, **kwargs):
return
custom = Custom(**obj)
self.event.append(stix2.parse(custom))
def load_mapping(self):
self.objects_mapping = {
'asn': {
'observable': observable_asn,
'pattern': pattern_asn
},
'domain-ip': {
'observable': observable_domain_ip,
'pattern': pattern_domain_ip
},
'email': {
'observable': self.observable_email,
'pattern': self.pattern_email
},
'file': {
'observable': observable_file,
'pattern': self.pattern_file
},
'ip-port': {
'observable': observable_ip_port,
'pattern': pattern_ip_port
},
'network-socket': {
'observable': observable_socket,
'pattern': pattern_socket
},
'process': {
'observable': observable_process,
'pattern': pattern_process
},
'registry-key': {
'observable': observable_regkey,
'pattern': pattern_regkey
},
'url': {
'observable': observable_url,
'pattern': pattern_url
},
'WindowsPEBinaryFile': {
'observable': self.observable_pe,
'pattern': self.pattern_pe
},
'x509': {
'observable': observable_x509,
'pattern': pattern_x509
}
}
def handler(self):
self.outputname = '{}.stix2'.format(self.filename)
if self.from_misp():
self.buildMispDict()
else:
self.version_attribute = {
'type': 'text',
'object_relation': 'version',
'value': self.stix_version
}
self.buildExternalDict()
self.set_distribution()
def from_misp(self):
for o in self.event:
if o._type == 'report' and 'misp:tool="misp2stix2"' in o.get(
'labels'):
index = self.event.index(o)
self.report = self.event.pop(index)
return True
return False
def buildMispDict(self):
self.parse_identity()
self.parse_report()
for o in self.event:
try:
object_type = o._type
except:
object_type = o['type']
labels = o.get('labels')
if object_type in galaxy_types:
self.parse_galaxy(o, labels)
elif object_type == 'course-of-action':
self.parse_course_of_action(o)
elif 'x-misp-object' in object_type:
if 'from_object' in labels:
self.parse_custom_object(o)
else:
self.parse_custom_attribute(o, labels)
else:
if 'from_object' in labels:
self.parse_object(o, labels)
else:
self.parse_attribute(o, labels)
def parse_identity(self):
identity = self.event.pop(0)
org = {'name': identity.get('name')}
self.misp_event['Org'] = org
def parse_report(self):
report = self.report
self.misp_event.info = report.get('name')
if report.get('published'):
self.misp_event.publish_timestamp = self.getTimestampfromDate(
report.get('published'))
if hasattr(report, 'labels'):
labels = report['labels']
for l in labels:
self.misp_event.add_tag(l)
if hasattr(report, 'external_references'):
ext_refs = report['external_references']
for e in ext_refs:
link = {"type": "link"}
comment = e.get('source_name')
try:
comment = comment.split('url - ')[1]
except:
pass
if comment:
link['comment'] = comment
link['value'] = e.get('url')
self.misp_event.add_attribute(**link)
def parse_galaxy(self, o, labels):
galaxy_type = self.get_misp_type(labels)
tag = labels[1]
value = tag.split(':')[1].split('=')[1]
galaxy_description, cluster_description = o.get('description').split(
'|')
galaxy = {
'type':
galaxy_type,
'name':
o.get('name'),
'description':
galaxy_description,
'GalaxyCluster': [{
'type': galaxy_type,
'value': value,
'tag_name': tag,
'description': cluster_description
}]
}
self.misp_event['Galaxy'].append(galaxy)
def parse_course_of_action(self, o):
misp_object = MISPObject('course-of-action')
if 'name' in o:
attribute = {
'type': 'text',
'object_relation': 'name',
'value': o.get('name')
}
misp_object.add_attribute(**attribute)
else:
return
if 'description' in o:
attribute = {
'type': 'text',
'object_relation': 'description',
'value': o.get('description')
}
misp_object.add_attribute(**attribute)
self.misp_event.add_object(**misp_object)
def parse_custom_object(self, o):
name = o.get('type').split('x-misp-object-')[1]
timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp'))
category = o.get('category')
attributes = []
values = o.get('x_misp_values')
for v in values:
attribute_type, object_relation = v.split('_')
attribute = {
'type': attribute_type,
'value': values.get(v),
'object_relation': object_relation
}
attributes.append(attribute)
misp_object = {
'name': name,
'timestamp': timestamp,
'meta-category': category,
'Attribute': attributes
}
self.misp_event.add_object(**misp_object)
def parse_custom_attribute(self, o, labels):
attribute_type = o.get('type').split('x-misp-object-')[1]
if attribute_type not in misp_types:
attribute_type = attribute_type.replace('-', '|')
timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp'))
to_ids = bool(labels[1].split('=')[1])
value = o.get('x_misp_value')
category = self.get_misp_category(labels)
attribute = {
'type': attribute_type,
'timestamp': timestamp,
'to_ids': to_ids,
'value': value,
'category': category
}
self.misp_event.add_attribute(**attribute)
def parse_object(self, o, labels):
object_type = self.get_misp_type(labels)
name = 'file' if object_type == 'WindowsPEBinaryFile' else object_type
object_category = self.get_misp_category(labels)
stix_type = o._type
misp_object = MISPObject(name)
misp_object['meta-category'] = object_category
if stix_type == 'indicator':
pattern = o.get('pattern').replace('\\\\', '\\').split(' AND ')
pattern[0] = pattern[0][1:]
pattern[-1] = pattern[-1][:-1]
attributes = self.objects_mapping[object_type]['pattern'](pattern)
if stix_type == 'observed-data':
observable = o.get('objects')
attributes = self.objects_mapping[object_type]['observable'](
observable)
if isinstance(attributes, tuple):
attributes, pe_uuid = attributes
misp_object.add_reference(pe_uuid, 'included-in')
for attribute in attributes:
misp_object.add_attribute(**attribute)
misp_object.to_ids = bool(labels[1].split('=')[1])
self.misp_event.add_object(**misp_object)
def parse_attribute(self, o, labels):
attribute_type = self.get_misp_type(labels)
attribute_category = self.get_misp_category(labels)
attribute = {'type': attribute_type, 'category': attribute_category}
stix_type = o._type
if stix_type == 'vulnerability':
value = o.get('name')
else:
if stix_type == 'indicator':
o_date = o.get('valid_from')
pattern = o.get('pattern').replace('\\\\', '\\')
value = self.parse_pattern_with_data(
pattern) if attribute_type in (
'malware-sample',
'attachment') else self.parse_pattern(pattern)
attribute['to_ids'] = True
else:
o_date = o.get('first_observed')
observable = o.get('objects')
try:
value = self.parse_observable(observable, attribute_type)
except:
print('{}: {}'.format(attribute_type, observable))
attribute['to_ids'] = False
attribute['timestamp'] = self.getTimestampfromDate(o_date)
if 'description' in o:
attribute['comment'] = o.get('description')
if isinstance(value, tuple):
value, data = value
attribute['data'] = io.BytesIO(data.encode())
attribute['value'] = value
self.misp_event.add_attribute(**attribute)
@staticmethod
def observable_email(observable):
attributes = []
addresses = {}
files = {}
for o_key, o_dict in observable.items():
part_type = o_dict._type
if part_type == 'email-addr':
addresses[o_key] = o_dict.get('value')
elif part_type == 'file':
files[o_key] = o_dict.get('name')
else:
message = dict(o_dict)
attributes.append({
'type': 'email-src',
'object_relation': 'from',
'value': addresses[message.pop('from_ref')],
'to_ids': False
})
for ref in ('to_refs', 'cc_refs'):
if ref in message:
for item in message.pop(ref):
mapping = email_mapping[ref]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': addresses[item],
'to_ids': False
})
if 'body_multipart' in message:
for f in message.pop('body_multipart'):
attributes.append({
'type': 'email-attachment',
'object_relation': 'attachment',
'value': files[f.get('body_raw_ref')],
'to_ids': False
})
for m_key, m_value in message.items():
if m_key == 'additional_header_fields':
for field_key, field_value in m_value.items():
mapping = email_mapping[field_key]
if field_key == 'Reply-To':
for rt in field_value:
attributes.append({
'type':
mapping['type'],
'object_relation':
mapping['relation'],
'value':
rt,
'to_ids':
False
})
else:
attributes.append({
'type':
mapping['type'],
'object_relation':
mapping['relation'],
'value':
field_value,
'to_ids':
False
})
else:
try:
mapping = email_mapping[m_key]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': m_value,
'to_ids': False
})
except:
if m_key.startswith("x_misp_attachment_"):
attribute_type, relation = m_key.split(
"x_misp_")[1].split("_")
attributes.append({
'type':
attribute_type,
'object_relation':
relation,
'to_ids':
False,
'value':
m_value['value'],
'data':
io.BytesIO(m_value['data'].encode())
})
elif "x_misp_" in m_key:
attribute_type, relation = m_key.split(
"x_misp_")[1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': m_value,
'to_ids': False
})
return attributes
@staticmethod
def pattern_email(pattern):
attributes = []
attachments = defaultdict(dict)
for p in pattern:
p_type, p_value = p.split(' = ')
try:
mapping = email_mapping[p_type]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': p_value[1:-1],
'to_ids': True
})
except KeyError:
if p_type.startswith("email-message:'x_misp_attachment_"):
relation, field = p_type.split('.')
relation = relation.split(':')[1][1:-1]
attachments[relation][field] = p_value[1:-1]
elif "x_misp_" in p_type:
attribute_type, relation = p_type.split(
"x_misp_")[1][:-1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': p_value[1:-1],
'to_ids': True
})
for a_key, a_dict in attachments.items():
_, _, attribute_type, relation = a_key.split('_')
attributes.append({
'type': attribute_type,
'object_relation': relation,
'to_ids': True,
'value': a_dict['value'],
'data': io.BytesIO(a_dict['data'].encode())
})
return attributes
@staticmethod
def pattern_file(pattern):
attributes = []
malware_sample = {}
for p in pattern:
p_type, p_value = p.split(' = ')
if p_type == 'artifact:payload_bin':
malware_sample['data'] = p_value
elif p_type in ("file:name", "file:hashes.'md5'"):
try:
mapping = file_mapping[p_type]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': p_value[1:-1],
'to_ids': True
})
malware_sample['filename'] = p_value[1:-1]
except KeyError:
attributes.append({
'type': 'md5',
'object_relation': 'md5',
'value': p_value[1:-1],
'to_ids': True
})
malware_sample['md5'] = p_value[1:-1]
elif 'file:hashes.' in p_type:
_, h = p_type.split('.')
h = h[1:-1]
attributes.append({
'type': h,
'object_relation': h,
'value': p_value[1:-1]
})
else:
try:
mapping = file_mapping[p_type]
attributes.append({
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': p_value[1:-1],
'to_ids': True
})
except KeyError:
if "x_misp_" in p_type:
attribute_type, relation = p_type.split(
"x_misp_")[1][:-1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': p_value[1:-1],
'to_ids': True
})
if 'data' in malware_sample:
value = "{}|{}".format(malware_sample['filename'],
malware_sample['md5'])
attributes.append({
'type':
'malware-sample',
'object_relation':
'malware-sample',
'value':
value,
'to_ids':
True,
'data':
io.BytesIO(malware_sample['data'].encode())
})
return attributes
def observable_pe(self, observable):
extension = observable['1']['extensions']['windows-pebinary-ext']
sections = extension['sections']
pe = MISPObject('pe')
pe_uuid = str(uuid.uuid4())
pe.uuid = pe_uuid
self.fill_object_attributes_observable(pe, pe_mapping, extension)
for section in sections:
pe_section = MISPObject('pe-section')
if 'hashes' in section:
for h_type, h_value in section['hashes'].items():
h_type = h_type.lower().replace('-', '')
pe_section.add_attribute(
**{
'type': h_type,
'object_relation': h_type,
'value': h_value,
'to_ids': False
})
self.fill_object_attributes_observable(pe_section,
pe_section_mapping, section)
section_uuid = str(uuid.uuid4())
pe_section.uuid = section_uuid
pe.add_reference(section_uuid, 'included-in')
self.misp_event.add_object(**pe_section)
self.misp_event.add_object(**pe)
return observable_file(observable), pe_uuid
@staticmethod
def fill_object_attributes_observable(misp_object, mapping_dict,
stix_object):
for stix_type, value in stix_object.items():
try:
mapping = mapping_dict[stix_type]
misp_object.add_attribute(
**{
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': value,
'to_ids': False
})
except KeyError:
if stix_type.startswith("x_misp_"):
attribute_type, relation = parse_custom_property(stix_type)
misp_object.add_attribute(
**{
'type': attribute_type,
'object_relation': relation[:-1],
'value': value,
'to_ids': False
})
def pattern_pe(self, pattern):
attributes = []
sections = defaultdict(dict)
pe = MISPObject('pe')
pe_uuid = str(uuid.uuid4())
pe.uuid = pe_uuid
for p in pattern:
p_type, p_value = p.split(' = ')
p_value = p_value[1:-1]
if ':extensions.' in p_type:
if '.sections[' in p_type:
p_type_list = p_type.split('.')
stix_type = "hashes.{}".format(
p_type_list[4]
[1:-1]) if '.hashes.' in p_type else p_type_list[3]
sections[p_type_list[2]][stix_type] = p_value
else:
stix_type = p_type.split('.')[-1]
try:
mapping = pe_mapping[stix_type]
pe.add_attribute(
**{
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': p_value,
'to_ids': True
})
except KeyError:
if stix_type.startswith("x_misp_"):
attribute_type, relation = parse_custom_property(
stix_type)
pe.add_attribute(
**{
'type': attribute_type,
'object_relation': relation[:-2],
'value': p_value,
'to_ids': False
})
else:
if 'file:hashes.' in p_type:
_, h = p_type.split('.')
h = h[1:-1]
attributes.append({
'type': h,
'object_relation': h,
'value': p_value,
'to_ids': True
})
else:
try:
mapping = file_mapping[p_type]
attributes.append({
'type':
mapping['type'],
'object_relation':
mapping['relation'],
'value':
p_value,
'to_ids':
True
})
except KeyError:
if "x_misp_" in p_type:
attribute_type, relation = p_type.split(
"x_misp_")[1][:-1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': p_value,
'to_ids': True
})
for _, section in sections.items():
pe_section = MISPObject('pe-section')
for stix_type, value in section.items():
if 'hashes.' in stix_type:
h_type = stix_type.split('.')[1]
pe_section.add_attribute(
**{
'type': h_type,
'object_relation': h_type,
'value': value,
'to_ids': True
})
else:
try:
mapping = pe_section_mapping[stix_type]
pe_section.add_attribute(
**{
'type': mapping['type'],
'object_relation': mapping['relation'],
'value': value,
'to_ids': True
})
except KeyError:
if "x_misp_" in stix_type:
attribute_type, relation = stix_type.split(
"x_misp_")[1][:-1].split("_")
attributes.append({
'type': attribute_type,
'object_relation': relation,
'value': value,
'to_ids': True
})
section_uuid = str(uuid.uuid4())
pe_section.uuid = pe_uuid
pe.add_reference(section_uuid, 'included-in')
self.misp_event.add_object(**pe_section)
self.misp_event.add_object(**pe)
return attributes, pe_uuid
def buildExternalDict(self):
self.fetch_report()
for o in self.event:
object_type = o._type
if object_type in ('relationship', 'report'):
continue
if object_type in galaxy_types:
self.parse_external_galaxy(o)
elif object_type == 'vulnerability':
attribute = {'type': 'vulnerability', 'value': o.get('name')}
if 'description' in o:
attribute['comment'] = o.get('description')
self.misp_event.add_attribute(**attribute)
elif object_type == 'course-of-action':
self.parse_course_of_action(o)
elif object_type == 'indicator':
pattern = o.get('pattern')
self.parse_external_pattern(pattern)
attribute = {
'type': 'stix2-pattern',
'object_relation': 'stix2-pattern',
'value': pattern
}
misp_object = {
'name': 'stix2-pattern',
'meta-category': 'stix2-pattern',
'Attribute': [self.version_attribute, attribute]
}
self.misp_event.add_object(**misp_object)
def fetch_report(self):
reports = []
for o in self.event:
if o._type == 'report':
reports.append(o)
if len(reports) == 1:
self.report = reports[0]
self.parse_report()
def parse_external_galaxy(self, o):
galaxy = {'name': galaxy_types[o._type]}
if 'kill_chain_phases' in o:
galaxy['type'] = o['kill_chain_phases'][0].get('phase_name')
cluster = defaultdict(dict)
cluster['value'] = o.get('name')
cluster['description'] = o.get('description')
if 'aliases' in o:
aliases = []
for a in o.get('aliases'):
aliases.append(a)
cluster['meta']['synonyms'] = aliases
galaxy['GalaxyCluster'] = [cluster]
self.misp_event['Galaxy'].append(galaxy)
def parse_external_pattern(self, pattern):
if ' OR ' in pattern and ' AND ' not in pattern:
pattern = pattern.split('OR')
for p in pattern:
attribute = self.attribute_from_external_pattern(p)
self.misp_event.add_attribute(**attribute)
elif ' OR ' not in pattern and ' LIKE ' not in pattern:
pattern = pattern.split('AND')
if len(pattern) == 1:
attribute = self.attribute_from_external_pattern(pattern[0])
self.misp_event.add_attribute(**attribute)
@staticmethod
def attribute_from_external_pattern(pattern):
pattern_type, pattern_value = pattern.split(' = ')
pattern_type, pattern_value = pattern_type[1:].strip(
), pattern_value[1:-2].strip()
stix_type, value_type = pattern_type.split(':')
if 'hashes' in value_type and 'x509' not in stix_type:
h_type = value_type.split('.')[1]
return {'type': h_type, 'value': pattern_value}
else:
# Might cause some issues, need more examples to test
return {
'type':
external_pattern_mapping[stix_type][value_type].get('type'),
'value': pattern_value
}
def set_distribution(self):
for attribute in self.misp_event.attributes:
attribute.distribution = self.__attribute_distribution
for misp_object in self.misp_event.objects:
misp_object.distribution = self.__attribute_distribution
for attribute in misp_object.attributes:
attribute.distribution = self.__attribute_distribution
def saveFile(self):
eventDict = self.misp_event.to_json()
outputfile = '{}.stix2'.format(self.filename)
with open(outputfile, 'w') as f:
f.write(eventDict)
@staticmethod
def getTimestampfromDate(stix_date):
try:
return int(stix_date.timestamp())
except:
return int(
time.mktime(
time.strptime(
stix_date.split('+')[0], "%Y-%m-%d %H:%M:%S")))
@staticmethod
def get_misp_type(labels):
return labels[0].split('=')[1][1:-1]
@staticmethod
def get_misp_category(labels):
return labels[1].split('=')[1][1:-1]
@staticmethod
def parse_observable(observable, attribute_type):
return misp_types_mapping[attribute_type](observable, attribute_type)
@staticmethod
def parse_pattern(pattern):
if ' AND ' in pattern:
pattern_parts = pattern.split(' AND ')
if len(pattern_parts) == 3:
_, value1 = pattern_parts[2].split(' = ')
_, value2 = pattern_parts[0].split(' = ')
return '{}|{}'.format(value1[1:-2], value2[1:-1])
else:
_, value1 = pattern_parts[0].split(' = ')
_, value2 = pattern_parts[1].split(' = ')
if value1 in ("'ipv4-addr'", "'ipv6-addr'"):
return value2[1:-2]
return '{}|{}'.format(value1[1:-1], value2[1:-2])
else:
return pattern.split(' = ')[1][1:-2]
def parse_pattern_with_data(self, pattern):
if 'artifact:payload_bin' not in pattern:
return self.parse_pattern(pattern)
pattern_parts = pattern.split(' AND ')
if len(pattern_parts) == 3:
filename = pattern_parts[0].split(' = ')[1]
md5 = pattern_parts[1].split(' = ')[1]
return "{}|{}".format(
filename[1:-1],
md5[1:-1]), pattern_parts[2].split(' = ')[1][1:-2]
else:
return pattern_parts[0].split(
' = ')[1][1:-1], pattern_parts[1].split(' = ')[1][1:-2]
def add_misp_fingerprint(self, fingerprint_json: dict):
"""
Upload fingerprint to MISP
:param fingerprint_json: fingerprint to upload
:return:
"""
LOGGER.info('Uploading the fingerprint to MISP')
start = time.time()
# Maximum number of source IPs to include
# 0 means all (no limit)
source_ips_limit = 10
# Possible dicts in each attack_vector of the fingerprint
# that will be added as comments (with the dict as value) to the event (not the ddos objects)
attack_vector_dicts = [
'ttl',
'tcp_flags',
'fragmentation_offset',
'ethernet_type',
'frame_len',
'dns_query_name',
'dns_query_type',
'ICMP type',
'ntp_requestcode',
'http_uri',
'http_method',
'http_user_agent',
]
# Possible fields in each attack_vector of the fingerprint
# that will be added as comments to the event (not the ddos objects)
attack_vector_fields = [
'service',
'fraction_of_attack',
'nr_flows',
'nr_packets',
'nr_megabytes',
'time_start',
'duration_seconds',
]
# Possible fields in the fingerprint
# that will be added as comments to the event
fingerprint_fields = [
'time_start',
'time_end',
'duration_seconds',
'total_flows',
'total_megabytes',
'total_packets',
'total_ips',
'avg_bps',
'avg_pps',
'avg_Bpp',
]
# Create the DDoSCH tag (returns existing one if already present)
ddosch_tag = self.get_misp_tag(self.ddosch_tag_name)
if ddosch_tag is None:
ddosch_tag = self.add_misp_tag(self.ddosch_tag_name,
self.ddosch_tag_colour)
LOGGER.debug(ddosch_tag)
# Retrieve (or create) the sharing group if specified
misp_sharing_group = None
if self.sharing_group:
misp_sharing_group = [
sh_grp for sh_grp in self.misp.sharing_groups(pythonify=True)
if sh_grp.name == self.sharing_group
]
if len(misp_sharing_group) == 0:
misp_sharing_group = self.misp.add_sharing_group(
{'name': self.sharing_group}, pythonify=True)
else:
misp_sharing_group = misp_sharing_group[0]
# Create an event to link everything to
LOGGER.debug('Creating a new event for the fingerprint')
event = MISPEvent()
event.info = fingerprint_json['key']
# TARGET
event.add_attribute(category='Network activity',
type='ip-dst',
value=fingerprint_json['target'],
comment='target')
# KEY
event.add_attribute(category='Network activity',
type='md5',
value=fingerprint_json['key'],
comment='attack key')
LOGGER.debug('Adding fingerprint fields')
for fp_field in fingerprint_fields:
if fp_field in fingerprint_json:
event.add_attribute(category='Network activity',
type='comment',
value=fingerprint_json[fp_field],
comment=fp_field)
# TAGS
if 'tags' in fingerprint_json:
LOGGER.debug('Adding fingerprint tags')
for tag in fingerprint_json['tags']:
event.add_tag(tag=tag)
event.add_tag(tag='validated')
if ddosch_tag is not None:
event.add_tag(tag=self.ddosch_tag_name)
# Add each attack vector as a MISP object to the MISP event
for v_i, attack_vector in enumerate(
fingerprint_json['attack_vectors']):
LOGGER.debug(f'Processing Attack Vector #{v_i}')
ddos_object = MISPObject(name='ddos')
# ATTACK VECTOR PROTOCOL
ddos_object.add_attribute('protocol',
attack_vector['protocol'],
comment=f'vector {v_i}')
for av_dict in attack_vector_dicts:
if av_dict in attack_vector and type(
attack_vector[av_dict]) == dict:
LOGGER.debug(f'Adding dict {av_dict}')
event.add_attribute(
category='Network activity',
type='comment',
value=json.dumps(attack_vector[av_dict]),
comment=f'vector {v_i} {av_dict} ({av_dict}:fraction)')
for av_field in attack_vector_fields:
if av_field in attack_vector and attack_vector[
av_field] is not None:
LOGGER.debug(f'Adding field {av_field}')
event.add_attribute(category='Network activity',
type='comment',
value=attack_vector[av_field],
comment=f'vector {v_i} {av_field}')
# ATTACK VECTOR SOURCE_PORT
if type(attack_vector['source_port']) == int:
LOGGER.debug('Adding source ports')
ddos_object.add_attribute('src-port',
attack_vector['source_port'],
comment=f'vector {v_i} src-port')
# ATTACK VECTOR DESTINATION PORTS
if type(attack_vector['destination_ports']) == dict:
LOGGER.debug('Adding destination ports')
for port in attack_vector['destination_ports'].keys():
ddos_object.add_attribute(
'dst-port',
int(port),
comment=f'vector {v_i} destination port '
f'(fraction:{attack_vector["destination_ports"][port]}'
)
# ATTACK VECTOR DNS
if 'dns_query_name' in attack_vector or 'dns_query_type' in attack_vector:
ddos_object.add_attribute(
'type',
'dns',
comment=f'vector {v_i} type of attack vector')
ddos_object.add_attribute(
'type',
'dns-amplification',
comment=f'vector {v_i} type of attack vector')
# ATTACK VECTOR ICMP
if 'ICMP type' in attack_vector:
ddos_object.add_attribute(
'type',
'icmp',
comment=f'vector {v_i} type of attack vector')
# ATTACK VECTOR NTP
if 'ntp_requestcode' in attack_vector:
ddos_object.add_attribute(
'type',
'ntp-amplification',
comment=f'vector {v_i} type of attack vector')
# ATTACK VECTOR SOURCE IPS
if 'source_ips' in attack_vector:
for i, src_ip in enumerate(attack_vector['source_ips'],
start=1):
ddos_object.add_attribute(
'ip-src', src_ip, comment=f'vector {v_i} source IP')
if i >= source_ips_limit > 0:
break
event.add_object(ddos_object, pythonify=True)
if misp_sharing_group:
self.misp.change_sharing_group_on_entity(event,
misp_sharing_group.id,
pythonify=True)
if self.publish:
event.publish()
event = self.misp.add_event(event, pythonify=True)
LOGGER.info(f'event: {event}')
LOGGER.debug('That took {} seconds'.format(time.time() - start))
