class TestMISPEvent(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
def init_event(self):
self.mispevent.info = 'This is a test'
self.mispevent.distribution = 1
self.mispevent.threat_level_id = 1
self.mispevent.analysis = 1
self.mispevent.set_date("2017-12-31") # test the set date method
def test_simple(self):
with open('tests/mispevent_testfiles/simple.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event(self):
self.init_event()
self.mispevent.publish()
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_loadfile(self):
self.mispevent.load_file('tests/mispevent_testfiles/event.json')
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_tag(self):
self.init_event()
self.mispevent.add_tag('bar')
self.mispevent.add_tag(name='baz')
new_tag = MISPTag()
new_tag.from_dict(name='foo')
self.mispevent.add_tag(new_tag)
with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_attribute(self):
self.init_event()
self.mispevent.add_attribute('filename', 'bar.exe')
self.mispevent.add_attribute_tag('osint', 'bar.exe')
attr_tags = self.mispevent.get_attribute_tag('bar.exe')
self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint')
self.assertEqual(attr_tags[0].name, 'osint')
with open('tests/mispevent_testfiles/attribute.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
# Fake setting an attribute ID for testing
self.mispevent.attributes[0].id = 42
self.mispevent.delete_attribute(42)
with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_object_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}])
self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah')
self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(['filename']))
self.assertEqual(len(self.mispevent.objects[0].get_attributes_by_relation('filename')), 1)
self.mispevent.add_object(name='url', strict=True)
self.mispevent.objects[1].add_attribute('url', value='https://www.circl.lu')
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
self.mispevent.objects[0].add_reference('b', 'baz', comment='foo')
self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz')
with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168")
def test_object_level_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar')
self.mispevent.objects[0].add_tag('osint')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_malware(self):
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.init_event()
self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile)
attribute = self.mispevent.attributes[0]
self.assertEqual(attribute.malware_binary, pseudofile)
with open('tests/mispevent_testfiles/malware.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_malware(self):
self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json')
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.assertEqual(
self.mispevent.objects[0].get_attributes_by_relation('malware-sample')[0].malware_binary.read(),
pseudofile.read())
def test_sighting(self):
sighting = MISPSighting()
sighting.from_dict(value='1', type='bar', timestamp=11111111)
with open('tests/mispevent_testfiles/sighting.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_event(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
with open('tests/mispevent_testfiles/existing_event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes_existing(self):
self.mispevent.load_file('tests/mispevent_testfiles/shadow.json')
with open('tests/mispevent_testfiles/shadow.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes(self):
self.init_event()
self.mispevent.add_proposal(type='filename', value='baz.jpg')
self.mispevent.add_attribute('filename', 'bar.exe')
self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf')
with open('tests/mispevent_testfiles/proposals.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_default_attributes(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}])
self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent.objects[0].attributes[0])
self.mispevent.objects[1].add_attribute('filename', value='baz')
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_default_values(self):
self.init_event()
self.mispevent.add_object(name='whois', strict=True)
self.mispevent.objects[0].add_attribute('registrar', value='registar.example.com')
self.mispevent.objects[0].add_attribute('domain', value='domain.example.com')
self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com')
self.mispevent.objects[0].add_attribute('nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/def_param.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_not_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
def test_event_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.mispevent.info = 'blah'
self.assertTrue(self.mispevent.edited)
def test_event_tag_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.add_tag('foo')
self.assertTrue(self.mispevent.edited)
def test_event_attribute_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.mispevent.attributes[0].value = 'blah'
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertFalse(self.mispevent.attributes[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].tags[0].name = 'blah'
self.assertTrue(self.mispevent.attributes[0].tags[0].edited)
self.assertFalse(self.mispevent.attributes[0].tags[1].edited)
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited_second(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].add_tag(name='blah')
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].edited)
self.assertFalse(self.mispevent.objects[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited_tag(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].add_tag('blah')
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_by_id(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
misp_obj = self.mispevent.get_object_by_id(1556)
self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f')
def test_userdefined_object(self):
self.init_event()
self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles')
with self.assertRaises(InvalidMISPObject) as e:
# Fail on required
self.mispevent.to_json()
if sys.version_info >= (3, ):
self.assertEqual(e.exception.message, '{\'member3\'} are required.')
else:
# Python2 bullshit
self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.')
self.mispevent.objects[0].add_attribute('member3', value='foo')
with self.assertRaises(InvalidMISPObject) as e:
# Fail on requiredOneOf
self.mispevent.to_json()
self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2')
self.mispevent.objects[0].add_attribute('member1', value='bar')
self.mispevent.objects[0].add_attribute('member1', value='baz')
with self.assertRaises(InvalidMISPObject) as e:
# member1 is not a multiple
self.mispevent.to_json()
self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed')
self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2]
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
class TestMISPEvent(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
def init_event(self):
self.mispevent.info = 'This is a test'
self.mispevent.distribution = 1
self.mispevent.threat_level_id = 1
self.mispevent.analysis = 1
self.mispevent.set_date("2017-12-31") # test the set date method
def test_simple(self):
with open('tests/mispevent_testfiles/simple.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event(self):
self.init_event()
self.mispevent.publish()
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_loadfile(self):
self.mispevent.load_file('tests/mispevent_testfiles/event.json')
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_tag(self):
self.init_event()
self.mispevent.add_tag('bar')
self.mispevent.add_tag(name='baz')
new_tag = MISPTag()
new_tag.from_dict(name='foo')
self.mispevent.add_tag(new_tag)
with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_attribute(self):
self.init_event()
a = self.mispevent.add_attribute('filename', 'bar.exe')
del a.uuid
a = self.mispevent.add_attribute_tag('osint', 'bar.exe')
attr_tags = self.mispevent.get_attribute_tag('bar.exe')
self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint')
self.assertEqual(attr_tags[0].name, 'osint')
with open('tests/mispevent_testfiles/attribute.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
# Fake setting an attribute ID for testing
self.mispevent.attributes[0].id = 42
self.mispevent.delete_attribute(42)
with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_object_tag(self):
self.mispevent.add_object(name='file', strict=True)
a = self.mispevent.objects[0].add_attribute('filename', value='')
self.assertEqual(a, None)
a = self.mispevent.objects[0].add_attribute('filename', value=None)
self.assertEqual(a, None)
a = self.mispevent.objects[0].add_attribute('filename',
value='bar',
Tag=[{
'name': 'blah'
}])
del a.uuid
self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name,
'blah')
self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(
['filename']))
self.assertEqual(
len(self.mispevent.objects[0].get_attributes_by_relation(
'filename')), 1)
self.mispevent.add_object(name='url', strict=True)
a = self.mispevent.objects[1].add_attribute(
'url', value='https://www.circl.lu')
del a.uuid
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
reference = self.mispevent.objects[0].add_reference(
self.mispevent.objects[1], 'baz', comment='foo')
del reference.uuid
self.assertEqual(
self.mispevent.objects[0].references[0].relationship_type, 'baz')
with open('tests/mispevent_testfiles/event_obj_attr_tag.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip(
"Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168"
)
def test_object_level_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar')
self.mispevent.objects[0].add_tag('osint')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_malware(self):
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.init_event()
a = self.mispevent.add_attribute('malware-sample',
'bar.exe',
data=pseudofile)
del a.uuid
attribute = self.mispevent.attributes[0]
self.assertEqual(attribute.malware_binary, pseudofile)
with open('tests/mispevent_testfiles/malware.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_malware(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/malware_exist.json')
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.assertEqual(
self.mispevent.objects[0].get_attributes_by_relation(
'malware-sample')[0].malware_binary.read(), pseudofile.read())
def test_sighting(self):
sighting = MISPSighting()
sighting.from_dict(value='1', type='bar', timestamp=11111111)
with open('tests/mispevent_testfiles/sighting.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(sighting.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_event(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
with open('tests/mispevent_testfiles/existing_event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes_existing(self):
self.mispevent.load_file('tests/mispevent_testfiles/shadow.json')
with open('tests/mispevent_testfiles/shadow.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip("Not supported on MISP.")
def test_shadow_attributes(self):
self.init_event()
p = self.mispevent.add_proposal(type='filename', value='baz.jpg')
del p.uuid
a = self.mispevent.add_attribute('filename', 'bar.exe')
del a.uuid
p = self.mispevent.attributes[0].add_proposal(type='filename',
value='bar.pdf')
del p.uuid
with open('tests/mispevent_testfiles/proposals.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_default_attributes(self):
self.mispevent.add_object(name='file', strict=True)
a = self.mispevent.objects[0].add_attribute('filename',
value='bar',
Tag=[{
'name': 'blah'
}])
del a.uuid
a = self.mispevent.objects[0].add_attribute('pattern-in-file',
value='baz')
self.assertEqual(a.category, 'Artifacts dropped')
del a.uuid
self.mispevent.add_object(name='file',
strict=False,
default_attributes_parameters=self.mispevent.
objects[0].attributes[0])
a = self.mispevent.objects[1].add_attribute('filename', value='baz')
del a.uuid
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
with open('tests/mispevent_testfiles/event_obj_def_param.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_default_values(self):
self.init_event()
self.mispevent.add_object(name='whois', strict=True)
a = self.mispevent.objects[0].add_attribute(
'registrar', value='registar.example.com')
del a.uuid
a = self.mispevent.objects[0].add_attribute('domain',
value='domain.example.com')
del a.uuid
a = self.mispevent.objects[0].add_attribute('nameserver',
value='ns1.example.com')
del a.uuid
a = self.mispevent.objects[0].add_attribute(
'nameserver',
value='ns2.example.com',
disable_correlation=False,
to_ids=True,
category='External analysis')
del a.uuid
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/def_param.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_not_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
def test_event_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.mispevent.info = 'blah'
self.assertTrue(self.mispevent.edited)
def test_event_tag_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.add_tag('foo')
self.assertTrue(self.mispevent.edited)
def test_event_attribute_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.mispevent.attributes[0].value = 'blah'
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertFalse(self.mispevent.attributes[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].tags[0].name = 'blah'
self.assertTrue(self.mispevent.attributes[0].tags[0].edited)
self.assertFalse(self.mispevent.attributes[0].tags[1].edited)
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited_second(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].add_tag(name='blah')
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].edited)
self.assertFalse(self.mispevent.objects[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited_tag(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].add_tag('blah')
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
with open('tests/mispevent_testfiles/existing_event_edited.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_by_id(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
misp_obj = self.mispevent.get_object_by_id(1556)
self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f')
def test_userdefined_object(self):
self.init_event()
self.mispevent.add_object(
name='test_object_template',
strict=True,
misp_objects_path_custom='tests/mispevent_testfiles')
with self.assertRaises(InvalidMISPObject) as e:
# Fail on required
self.mispevent.to_json(sort_keys=True, indent=2)
if sys.version_info >= (3, ):
self.assertEqual(e.exception.message,
'{\'member3\'} are required.')
else:
# Python2 bullshit
self.assertEqual(e.exception.message,
'set([u\'member3\']) are required.')
a = self.mispevent.objects[0].add_attribute('member3', value='foo')
del a.uuid
with self.assertRaises(InvalidMISPObject) as e:
# Fail on requiredOneOf
self.mispevent.to_json(sort_keys=True, indent=2)
self.assertEqual(
e.exception.message,
'At least one of the following attributes is required: member1, member2'
)
a = self.mispevent.objects[0].add_attribute('member1', value='bar')
del a.uuid
a = self.mispevent.objects[0].add_attribute('member1', value='baz')
del a.uuid
with self.assertRaises(InvalidMISPObject) as e:
# member1 is not a multiple
self.mispevent.to_json(sort_keys=True, indent=2)
self.assertEqual(e.exception.message,
'Multiple occurrences of member1 is not allowed')
self.mispevent.objects[0].attributes = self.mispevent.objects[
0].attributes[:2]
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def check_hashes(self):
if self.offline_mode:
self.log('error', 'Offline mode, unable to query VirusTotal')
return
event_id = self._get_eventid()
if event_id is None:
return
event = self.misp.get(event_id)
if self._has_error_message(event):
return
misp_event = MISPEvent()
misp_event.load(event)
hashes_to_expand = {}
hashes_expanded = [] # Thoses hashes are known and already processed
local_samples_hashes = []
partial_objects = {}
for o in misp_event.Object:
if o.name != 'file':
continue
if o.has_attributes_by_relation(['md5', 'sha1', 'sha256']):
# This object has all the hashes we care about
tmphashes = []
tmphashes += [h.value for h in o.get_attributes_by_relation('md5')]
tmphashes += [
h.value for h in o.get_attributes_by_relation('sha1')
]
tmphashes += [
h.value for h in o.get_attributes_by_relation('sha256')
]
# Make sure to query VT for the sha256, even if expanded locally
hashes_to_expand[o.get_attributes_by_relation(
'sha256')[0].value] = o.get_attributes_by_relation('sha256')[0]
if o.has_attributes_by_relation(['malware-sample']):
# ... and it has a malware sample
local_samples_hashes += tmphashes
hashes_expanded += tmphashes
elif o.has_attributes_by_relation(['malware-sample']):
# This object has a malware sample, but is missing hashes. We can expand locally.
# get the MD5 from the malware-sample attribute
malware_sample = o.get_attributes_by_relation('malware-sample')[
0] # at most one sample/file object
local_samples_hashes.append(malware_sample.value.split('|')[1])
local_samples_hashes += [
h.value for h in o.get_attributes_by_relation('md5')
]
local_samples_hashes += [
h.value for h in o.get_attributes_by_relation('sha1')
]
local_samples_hashes += [
h.value for h in o.get_attributes_by_relation('sha256')
]
if self.args.populate:
# The object is missing hashes, keeping track of it for expansion if it isn't already done.
partial_objects[o.uuid] = malware_sample
else:
sha256 = {
attribute.value: attribute
for attribute in o.get_attributes_by_relation('sha256')
}
sha1 = {
attribute.value: attribute
for attribute in o.get_attributes_by_relation('sha1')
}
md5 = {
attribute.value: attribute
for attribute in o.get_attributes_by_relation('md5')
}
if sha256:
hashes_to_expand.update(sha256)
elif sha1:
hashes_to_expand.update(sha1)
elif md5:
hashes_to_expand.update(md5)
for ref_uuid, sample in partial_objects.items():
if sample.value.split('|')[1] in hashes_expanded:
# Already expanded in an other object
continue
new_obj, hashes = self._expand_local_sample(
pseudofile=sample.malware_binary,
filename=sample.value.split('|')[0],
refobj=ref_uuid,
default_attributes_parameters=sample)
misp_event.Object += new_obj
local_samples_hashes += hashes
# Make sure to query VT for the sha256, even if expanded locally
hashes_to_expand[hashes[0]] = sample
hashes_expanded += local_samples_hashes
for a in misp_event.attributes:
if a.type == 'malware-sample' and a.value.split(
'|')[1] not in hashes_expanded:
new_obj, hashes = self._expand_local_sample(
pseudofile=a.malware_binary,
filename=a.value.split('|')[0],
default_attributes_parameters=a)
misp_event.Object += new_obj
local_samples_hashes += hashes
# Make sure to query VT for the sha256, even if expanded locally
hashes_to_expand[hashes[0]] = a
elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'):
# We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway
fname, hashval = a.value.split('|')
hashes_to_expand[hashval] = a
elif a.type in ('md5', 'sha1', 'sha256'):
# We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway
hashes_to_expand[a.value] = a
unk_vt_hashes = []
if cfg.virustotal.virustotal_has_private_key is False:
quota = 4
timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)
hashes_expanded += local_samples_hashes
processed_on_vt = []
# Make sure to start getting reports for the longest possible hashes (reduce risks of collisions)
for to_expand in sorted(list(set(hashes_to_expand)), key=len):
if to_expand in processed_on_vt:
# Always run VT, once per sample
continue
original_attribute = hashes_to_expand[to_expand]
if original_attribute.get('object_id'):
original_object_id = original_attribute.get('object_id')
vt_object = self._make_VT_object(to_expand, original_attribute)
if not vt_object:
unk_vt_hashes.append(to_expand)
continue
result = vt_object.get_report()
md5 = result['md5']
sha1 = result['sha1']
sha256 = result['sha256']
processed_on_vt += [sha256, sha1, md5]
if all(h in local_samples_hashes for h in [md5, sha1, sha256]):
self.log('success', 'Sample available in MISP:')
else:
self.log('success', 'Sample available in VT:')
self.log(
'item', '{}\n\t{}\n\t{}\n\t{}'.format(result["permalink"], md5,
sha1, sha256))
if self.args.populate:
if not all(h in hashes_expanded for h in [md5, sha1, sha256]):
# If all the "new" expanded hashes are in the hashes_expanded list, skip
file_object = MISPObject(
'file', default_attributes_parameters=original_attribute)
file_object.add_attribute('md5', value=md5)
file_object.add_attribute('sha1', value=sha1)
file_object.add_attribute('sha256', value=sha256)
file_object.add_reference(vt_object.uuid, 'analysed-with')
misp_event.Object.append(file_object)
hashes_expanded += [md5, sha1, sha256]
else:
if not original_object_id or original_object_id == '0':
# Not an object, but the hashes are in an other object, skipping
continue
else:
# We already have a MISP object, adding the link to the new VT object
file_object = misp_event.get_object_by_id(
original_object_id)
file_object.add_reference(vt_object.uuid, 'analysed-with')
misp_event.Object.append(vt_object)
if cfg.virustotal.virustotal_has_private_key is False:
if quota > 0:
quota -= 1
else:
waiting_time = (timeout - datetime.datetime.now()).seconds
if waiting_time > 0:
self.log(
'warning',
'No private API key, 4 queries/min is the limit. Waiting for {} seconds.'
.format(waiting_time))
time.sleep(waiting_time)
quota = 4
timeout = datetime.datetime.now() + datetime.timedelta(
minutes=1)
if self.args.populate:
self._populate(misp_event)
if len(unk_vt_hashes) > 0:
self.log('error', 'Unknown on VT:')
for h in unk_vt_hashes:
self.log('item', '{}'.format(h))
def check_hashes(self):
if self.offline_mode:
self.log('error', 'Offline mode, unable to query VirusTotal')
return
event_id = self._get_eventid()
if event_id is None:
return
event = self.misp.get(event_id)
if self._has_error_message(event):
return
misp_event = MISPEvent()
misp_event.load(event)
hashes_to_expand = {}
hashes_expanded = [] # Thoses hashes are known and already processed
local_samples_hashes = []
partial_objects = {}
for o in misp_event.Object:
if o.name != 'file':
continue
if o.has_attributes_by_relation(['md5', 'sha1', 'sha256']):
# This object has all the hashes we care about
tmphashes = []
tmphashes += [h.value for h in o.get_attributes_by_relation('md5')]
tmphashes += [h.value for h in o.get_attributes_by_relation('sha1')]
tmphashes += [h.value for h in o.get_attributes_by_relation('sha256')]
# Make sure to query VT for the sha256, even if expanded locally
hashes_to_expand[o.get_attributes_by_relation('sha256')[0].value] = o.get_attributes_by_relation('sha256')[0]
if o.has_attributes_by_relation(['malware-sample']):
# ... and it has a malware sample
local_samples_hashes += tmphashes
hashes_expanded += tmphashes
elif o.has_attributes_by_relation(['malware-sample']):
# This object has a malware sample, but is missing hashes. We can expand locally.
# get the MD5 from the malware-sample attribute
malware_sample = o.get_attributes_by_relation('malware-sample')[0] # at most one sample/file object
local_samples_hashes.append(malware_sample.value.split('|')[1])
local_samples_hashes += [h.value for h in o.get_attributes_by_relation('md5')]
local_samples_hashes += [h.value for h in o.get_attributes_by_relation('sha1')]
local_samples_hashes += [h.value for h in o.get_attributes_by_relation('sha256')]
if self.args.populate:
# The object is missing hashes, keeping track of it for expansion if it isn't already done.
partial_objects[o.uuid] = malware_sample
else:
sha256 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('sha256')}
sha1 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('sha1')}
md5 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('md5')}
if sha256:
hashes_to_expand.update(sha256)
elif sha1:
hashes_to_expand.update(sha1)
elif md5:
hashes_to_expand.update(md5)
for ref_uuid, sample in partial_objects.items():
if sample.value.split('|')[1] in hashes_expanded:
# Already expanded in an other object
continue
new_obj, hashes = self._expand_local_sample(pseudofile=sample.malware_binary,
filename=sample.value.split('|')[0],
refobj=ref_uuid,
default_attributes_paramaters=sample)
misp_event.Object += new_obj
local_samples_hashes += hashes
# Make sure to query VT for the sha256, even if expanded locally
hashes_to_expand[hashes[0]] = sample
hashes_expanded += local_samples_hashes
for a in misp_event.attributes:
if a.type == 'malware-sample' and a.value.split('|')[1] not in hashes_expanded:
new_obj, hashes = self._expand_local_sample(pseudofile=a.malware_binary,
filename=a.value.split('|')[0],
default_attributes_paramaters=a)
misp_event.Object += new_obj
local_samples_hashes += hashes
# Make sure to query VT for the sha256, even if expanded locally
hashes_to_expand[hashes[0]] = a
elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'):
# We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway
fname, hashval = a.value.split('|')
hashes_to_expand[hashval] = a
elif a.type in ('md5', 'sha1', 'sha256'):
# We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway
hashes_to_expand[a.value] = a
unk_vt_hashes = []
if cfg.virustotal.virustotal_has_private_key is False:
quota = 4
timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)
hashes_expanded += local_samples_hashes
processed_on_vt = []
# Make sure to start getting reports for the longest possible hashes (reduce risks of collisions)
for to_expand in sorted(list(set(hashes_to_expand)), key=len):
if to_expand in processed_on_vt:
# Always run VT, once per sample
continue
original_attribute = hashes_to_expand[to_expand]
if original_attribute.get('object_id'):
original_object_id = original_attribute.get('object_id')
vt_object = self._make_VT_object(to_expand, original_attribute)
if not vt_object:
unk_vt_hashes.append(to_expand)
continue
result = vt_object.get_report()
md5 = result['md5']
sha1 = result['sha1']
sha256 = result['sha256']
processed_on_vt += [sha256, sha1, md5]
if all(h in local_samples_hashes for h in [md5, sha1, sha256]):
self.log('success', 'Sample available in MISP:')
else:
self.log('success', 'Sample available in VT:')
self.log('item', '{}\n\t{}\n\t{}\n\t{}'.format(result["permalink"], md5, sha1, sha256))
if self.args.populate:
if not all(h in hashes_expanded for h in [md5, sha1, sha256]):
# If all the "new" expanded hashes are in the hashes_expanded list, skip
file_object = MISPObject('file', default_attributes_paramaters=original_attribute)
file_object.add_attribute('md5', value=md5)
file_object.add_attribute('sha1', value=sha1)
file_object.add_attribute('sha256', value=sha256)
file_object.add_reference(vt_object.uuid, 'analysed-with')
misp_event.Object.append(file_object)
hashes_expanded += [md5, sha1, sha256]
else:
if not original_object_id or original_object_id == '0':
# Not an object, but the hashes are in an other object, skipping
continue
else:
# We already have a MISP object, adding the link to the new VT object
file_object = misp_event.get_object_by_id(original_object_id)
file_object.add_reference(vt_object.uuid, 'analysed-with')
misp_event.Object.append(vt_object)
if cfg.virustotal.virustotal_has_private_key is False:
if quota > 0:
quota -= 1
else:
waiting_time = (timeout - datetime.datetime.now()).seconds
if waiting_time > 0:
self.log('warning', 'No private API key, 4 queries/min is the limit. Waiting for {} seconds.'.format(waiting_time))
time.sleep(waiting_time)
quota = 4
timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)
if self.args.populate:
self._populate(misp_event)
if len(unk_vt_hashes) > 0:
self.log('error', 'Unknown on VT:')
for h in unk_vt_hashes:
self.log('item', '{}'.format(h))
class TestMISPEvent(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
def init_event(self):
self.mispevent.info = 'This is a test'
self.mispevent.distribution = 1
self.mispevent.threat_level_id = 1
self.mispevent.analysis = 1
self.mispevent.set_date("2017-12-31") # test the set date method
def test_simple(self):
with open('tests/mispevent_testfiles/simple.json', 'r') as f:
ref_json = json.load(f)
del self.mispevent.uuid
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event(self):
self.init_event()
self.mispevent.publish()
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
del self.mispevent.uuid
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_loadfile(self):
self.mispevent.load_file('tests/mispevent_testfiles/event.json')
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
del self.mispevent.uuid
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_tag(self):
self.init_event()
self.mispevent.add_tag('bar')
self.mispevent.add_tag(name='baz')
new_tag = MISPTag()
new_tag.from_dict(name='foo')
self.mispevent.add_tag(new_tag)
with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
ref_json = json.load(f)
del self.mispevent.uuid
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_attribute(self):
self.init_event()
a = self.mispevent.add_attribute('filename', 'bar.exe')
del a.uuid
a = self.mispevent.add_attribute_tag('osint', 'bar.exe')
attr_tags = self.mispevent.get_attribute_tag('bar.exe')
self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint')
self.assertEqual(attr_tags[0].name, 'osint')
with open('tests/mispevent_testfiles/attribute.json', 'r') as f:
ref_json = json.load(f)
del self.mispevent.uuid
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
# Fake setting an attribute ID for testing
self.mispevent.attributes[0].id = 42
self.mispevent.delete_attribute(42)
with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_object_tag(self):
self.mispevent.add_object(name='file', strict=True)
a = self.mispevent.objects[0].add_attribute('filename', value='')
self.assertEqual(a, None)
a = self.mispevent.objects[0].add_attribute('filename', value=None)
self.assertEqual(a, None)
a = self.mispevent.objects[0].add_attribute('filename',
value='bar',
Tag=[{
'name': 'blah'
}])
del a.uuid
self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name,
'blah')
self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(
['filename']))
self.assertEqual(
len(self.mispevent.objects[0].get_attributes_by_relation(
'filename')), 1)
self.mispevent.add_object(name='url', strict=True)
a = self.mispevent.objects[1].add_attribute(
'url', value='https://www.circl.lu')
del a.uuid
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
reference = self.mispevent.objects[0].add_reference(
self.mispevent.objects[1], 'baz', comment='foo')
del reference.uuid
self.assertEqual(
self.mispevent.objects[0].references[0].relationship_type, 'baz')
with open('tests/mispevent_testfiles/event_obj_attr_tag.json',
'r') as f:
ref_json = json.load(f)
del self.mispevent.uuid
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip(
"Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168"
)
def test_object_level_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar')
self.mispevent.objects[0].add_tag('osint')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_malware(self):
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.init_event()
a = self.mispevent.add_attribute('malware-sample',
'bar.exe',
data=pseudofile)
del a.uuid
attribute = self.mispevent.attributes[0]
self.assertEqual(attribute.malware_binary, pseudofile)
with open('tests/mispevent_testfiles/malware.json', 'r') as f:
ref_json = json.load(f)
del self.mispevent.uuid
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_malware(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/malware_exist.json')
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.assertEqual(
self.mispevent.objects[0].get_attributes_by_relation(
'malware-sample')[0].malware_binary.read(), pseudofile.read())
def test_sighting(self):
sighting = MISPSighting()
sighting.from_dict(value='1', type='bar', timestamp=11111111)
with open('tests/mispevent_testfiles/sighting.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(sighting.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_event(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
with open('tests/mispevent_testfiles/existing_event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes_existing(self):
self.mispevent.load_file('tests/mispevent_testfiles/shadow.json')
with open('tests/mispevent_testfiles/shadow.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip("Not supported on MISP.")
def test_shadow_attributes(self):
self.init_event()
p = self.mispevent.add_proposal(type='filename', value='baz.jpg')
del p.uuid
a = self.mispevent.add_attribute('filename', 'bar.exe')
del a.uuid
p = self.mispevent.attributes[0].add_proposal(type='filename',
value='bar.pdf')
del p.uuid
with open('tests/mispevent_testfiles/proposals.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_default_attributes(self):
self.mispevent.add_object(name='file', strict=True)
a = self.mispevent.objects[0].add_attribute('filename',
value='bar',
Tag=[{
'name': 'blah'
}])
del a.uuid
a = self.mispevent.objects[0].add_attribute('pattern-in-file',
value='baz')
self.assertEqual(a.category, 'Artifacts dropped')
del a.uuid
self.mispevent.add_object(name='file',
strict=False,
default_attributes_parameters=self.mispevent.
objects[0].attributes[0])
a = self.mispevent.objects[1].add_attribute('filename', value='baz')
del a.uuid
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
with open('tests/mispevent_testfiles/event_obj_def_param.json',
'r') as f:
ref_json = json.load(f)
del self.mispevent.uuid
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_default_values(self):
self.init_event()
self.mispevent.add_object(name='whois', strict=True)
a = self.mispevent.objects[0].add_attribute(
'registrar', value='registar.example.com')
del a.uuid
a = self.mispevent.objects[0].add_attribute('domain',
value='domain.example.com')
del a.uuid
a = self.mispevent.objects[0].add_attribute('nameserver',
value='ns1.example.com')
del a.uuid
a = self.mispevent.objects[0].add_attribute(
'nameserver',
value='ns2.example.com',
disable_correlation=False,
to_ids=True,
category='External analysis')
del a.uuid
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/def_param.json', 'r') as f:
ref_json = json.load(f)
del self.mispevent.uuid
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_not_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
def test_event_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.mispevent.info = 'blah'
self.assertTrue(self.mispevent.edited)
def test_event_tag_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.add_tag('foo')
self.assertTrue(self.mispevent.edited)
def test_event_attribute_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.mispevent.attributes[0].value = 'blah'
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertFalse(self.mispevent.attributes[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].tags[0].name = 'blah'
self.assertTrue(self.mispevent.attributes[0].tags[0].edited)
self.assertFalse(self.mispevent.attributes[0].tags[1].edited)
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited_second(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].add_tag(name='blah')
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].edited)
self.assertFalse(self.mispevent.objects[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited_tag(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].add_tag('blah')
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
with open('tests/mispevent_testfiles/existing_event_edited.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_by_id(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
misp_obj = self.mispevent.get_object_by_id(1556)
self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f')
def test_userdefined_object(self):
self.init_event()
self.mispevent.add_object(
name='test_object_template',
strict=True,
misp_objects_path_custom='tests/mispevent_testfiles')
with self.assertRaises(InvalidMISPObject) as e:
# Fail on required
self.mispevent.to_json(sort_keys=True, indent=2)
if sys.version_info >= (3, ):
self.assertEqual(e.exception.message,
'{\'member3\'} are required.')
else:
# Python2 bullshit
self.assertEqual(e.exception.message,
'set([u\'member3\']) are required.')
a = self.mispevent.objects[0].add_attribute('member3', value='foo')
del a.uuid
with self.assertRaises(InvalidMISPObject) as e:
# Fail on requiredOneOf
self.mispevent.to_json(sort_keys=True, indent=2)
self.assertEqual(
e.exception.message,
'At least one of the following attributes is required: member1, member2'
)
a = self.mispevent.objects[0].add_attribute('member1', value='bar')
del a.uuid
a = self.mispevent.objects[0].add_attribute('member1', value='baz')
del a.uuid
with self.assertRaises(InvalidMISPObject) as e:
# member1 is not a multiple
self.mispevent.to_json(sort_keys=True, indent=2)
self.assertEqual(e.exception.message,
'Multiple occurrences of member1 is not allowed')
self.mispevent.objects[0].attributes = self.mispevent.objects[
0].attributes[:2]
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f:
ref_json = json.load(f)
del self.mispevent.uuid
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_first_last_seen(self):
me = MISPEvent()
me.info = 'Test First and Last Seen'
me.date = '2020.01.12'
self.assertEqual(me.date.day, 12)
me.add_attribute('ip-dst',
'8.8.8.8',
first_seen='06-21-1998',
last_seen=1580213607.469571)
self.assertEqual(me.attributes[0].first_seen.year, 1998)
self.assertEqual(me.attributes[0].last_seen.year, 2020)
now = datetime.now().astimezone()
me.attributes[0].last_seen = now
today = date.today()
me.attributes[0].first_seen = today
self.assertEqual(me.attributes[0].first_seen.year, today.year)
self.assertEqual(me.attributes[0].last_seen, now)
def test_feed(self):
me = MISPEvent()
me.info = 'Test feed'
org = MISPOrganisation()
org.name = 'TestOrg'
org.uuid = '123478'
me.Orgc = org
me.add_attribute('ip-dst', '8.8.8.8')
obj = me.add_object(name='file')
obj.add_attributes('filename', *['foo.exe', 'bar.exe'])
h = hashlib.new('md5')
h.update(b'8.8.8.8')
hash_attr_val = h.hexdigest()
feed = me.to_feed(with_meta=True)
self.assertEqual(feed['Event']['_hashes'][0], hash_attr_val)
self.assertEqual(feed['Event']['_manifest'][me.uuid]['info'],
'Test feed')
self.assertEqual(len(feed['Event']['Object'][0]['Attribute']), 2)
def test_object_templates(self):
me = MISPEvent()
for template in glob.glob(
str(me.misp_objects_path / '*' / 'definition.json')):
with open(template) as f:
t_json = json.load(f)
if 'requiredOneOf' in t_json:
obj_relations = set(t_json['attributes'].keys())
subset = set(
t_json['requiredOneOf']).issubset(obj_relations)
self.assertTrue(subset, f'{t_json["name"]}')
if 'required' in t_json:
obj_relations = set(t_json['attributes'].keys())
subset = set(t_json['required']).issubset(obj_relations)
self.assertTrue(subset, f'{t_json["name"]}')
for obj_relation, entry in t_json['attributes'].items():
self.assertTrue(
entry['misp-attribute'] in me.describe_types['types'])
if 'categories' in entry:
subset = set(entry['categories']).issubset(
me.describe_types['categories'])
self.assertTrue(subset,
f'{t_json["name"]} - {obj_relation}')
class TestMISPEvent(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
def init_event(self):
self.mispevent.info = 'This is a test'
self.mispevent.distribution = 1
self.mispevent.threat_level_id = 1
self.mispevent.analysis = 1
self.mispevent.set_date("2017-12-31") # test the set date method
def test_simple(self):
with open('tests/mispevent_testfiles/simple.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event(self):
self.init_event()
self.mispevent.publish()
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_loadfile(self):
self.mispevent.load_file('tests/mispevent_testfiles/event.json')
with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_tag(self):
self.init_event()
self.mispevent.add_tag('bar')
self.mispevent.add_tag(name='baz')
new_tag = MISPTag()
new_tag.from_dict(name='foo')
self.mispevent.add_tag(new_tag)
with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_attribute(self):
self.init_event()
self.mispevent.add_attribute('filename', 'bar.exe')
self.mispevent.add_attribute_tag('osint', 'bar.exe')
attr_tags = self.mispevent.get_attribute_tag('bar.exe')
self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint')
self.assertEqual(attr_tags[0].name, 'osint')
with open('tests/mispevent_testfiles/attribute.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
# Fake setting an attribute ID for testing
self.mispevent.attributes[0].id = 42
self.mispevent.delete_attribute(42)
with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_object_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename',
value='bar',
Tag=[{
'name': 'blah'
}])
self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name,
'blah')
self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(
['filename']))
self.assertEqual(
len(self.mispevent.objects[0].get_attributes_by_relation(
'filename')), 1)
self.mispevent.add_object(name='url', strict=True)
self.mispevent.objects[1].add_attribute('url',
value='https://www.circl.lu')
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
self.mispevent.objects[0].add_reference('b', 'baz', comment='foo')
self.assertEqual(
self.mispevent.objects[0].references[0].relationship_type, 'baz')
with open('tests/mispevent_testfiles/event_obj_attr_tag.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip(
"Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168"
)
def test_object_level_tag(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename', value='bar')
self.mispevent.objects[0].add_tag('osint')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_malware(self):
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.init_event()
self.mispevent.add_attribute('malware-sample',
'bar.exe',
data=pseudofile)
attribute = self.mispevent.attributes[0]
self.assertEqual(attribute.malware_binary, pseudofile)
with open('tests/mispevent_testfiles/malware.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_malware(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/malware_exist.json')
with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
pseudofile = BytesIO(f.read())
self.assertEqual(
self.mispevent.objects[0].get_attributes_by_relation(
'malware-sample')[0].malware_binary.read(), pseudofile.read())
def test_sighting(self):
sighting = MISPSighting()
sighting.from_dict(value='1', type='bar', timestamp=11111111)
with open('tests/mispevent_testfiles/sighting.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(sighting.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_event(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
with open('tests/mispevent_testfiles/existing_event.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes_existing(self):
self.mispevent.load_file('tests/mispevent_testfiles/shadow.json')
with open('tests/mispevent_testfiles/shadow.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes(self):
self.init_event()
self.mispevent.add_proposal(type='filename', value='baz.jpg')
self.mispevent.add_attribute('filename', 'bar.exe')
self.mispevent.attributes[0].add_proposal(type='filename',
value='bar.pdf')
with open('tests/mispevent_testfiles/proposals.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_default_attributes(self):
self.mispevent.add_object(name='file', strict=True)
self.mispevent.objects[0].add_attribute('filename',
value='bar',
Tag=[{
'name': 'blah'
}])
self.mispevent.add_object(name='file',
strict=False,
default_attributes_parameters=self.mispevent.
objects[0].attributes[0])
self.mispevent.objects[1].add_attribute('filename', value='baz')
self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b'
with open('tests/mispevent_testfiles/event_obj_def_param.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_default_values(self):
self.init_event()
self.mispevent.add_object(name='whois', strict=True)
self.mispevent.objects[0].add_attribute('registrar',
value='registar.example.com')
self.mispevent.objects[0].add_attribute('domain',
value='domain.example.com')
self.mispevent.objects[0].add_attribute('nameserver',
value='ns1.example.com')
self.mispevent.objects[0].add_attribute('nameserver',
value='ns2.example.com',
disable_correlation=False,
to_ids=True,
category='External analysis')
self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/def_param.json', 'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_not_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
def test_event_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.mispevent.info = 'blah'
self.assertTrue(self.mispevent.edited)
def test_event_tag_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.add_tag('foo')
self.assertTrue(self.mispevent.edited)
def test_event_attribute_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.mispevent.attributes[0].value = 'blah'
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertFalse(self.mispevent.attributes[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].tags[0].name = 'blah'
self.assertTrue(self.mispevent.attributes[0].tags[0].edited)
self.assertFalse(self.mispevent.attributes[0].tags[1].edited)
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_attribute_tag_edited_second(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.attributes[0].add_tag(name='blah')
self.assertTrue(self.mispevent.attributes[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].edited)
self.assertFalse(self.mispevent.objects[1].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].comment = 'blah'
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
def test_event_object_attribute_edited_tag(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
self.assertFalse(self.mispevent.edited)
self.mispevent.objects[0].attributes[0].add_tag('blah')
self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
self.assertTrue(self.mispevent.objects[0].edited)
self.assertTrue(self.mispevent.edited)
with open('tests/mispevent_testfiles/existing_event_edited.json',
'r') as f:
ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(),
json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_by_id(self):
self.mispevent.load_file(
'tests/mispevent_testfiles/existing_event.json')
misp_obj = self.mispevent.get_object_by_id(1556)
self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f')
