class TestMISPEvent(unittest.TestCase): def setUp(self): self.maxDiff = None self.mispevent = MISPEvent() def init_event(self): self.mispevent.info = 'This is a test' self.mispevent.distribution = 1 self.mispevent.threat_level_id = 1 self.mispevent.analysis = 1 self.mispevent.set_date("2017-12-31") # test the set date method def test_simple(self): with open('tests/mispevent_testfiles/simple.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event(self): self.init_event() self.mispevent.publish() with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_loadfile(self): self.mispevent.load_file('tests/mispevent_testfiles/event.json') with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_tag(self): self.init_event() self.mispevent.add_tag('bar') self.mispevent.add_tag(name='baz') new_tag = MISPTag() new_tag.from_dict(name='foo') self.mispevent.add_tag(new_tag) with open('tests/mispevent_testfiles/event_tags.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_attribute(self): self.init_event() self.mispevent.add_attribute('filename', 'bar.exe') self.mispevent.add_attribute_tag('osint', 'bar.exe') attr_tags = self.mispevent.get_attribute_tag('bar.exe') self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint') self.assertEqual(attr_tags[0].name, 'osint') with open('tests/mispevent_testfiles/attribute.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) # Fake setting an attribute ID for testing self.mispevent.attributes[0].id = 42 self.mispevent.delete_attribute(42) with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_object_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}]) self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah') self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(['filename'])) self.assertEqual(len(self.mispevent.objects[0].get_attributes_by_relation('filename')), 1) self.mispevent.add_object(name='url', strict=True) self.mispevent.objects[1].add_attribute('url', value='https://www.circl.lu') self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' self.mispevent.objects[0].add_reference('b', 'baz', comment='foo') self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz') with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168") def test_object_level_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar') self.mispevent.objects[0].add_tag('osint') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_malware(self): with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.init_event() self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile) attribute = self.mispevent.attributes[0] self.assertEqual(attribute.malware_binary, pseudofile) with open('tests/mispevent_testfiles/malware.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_malware(self): self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json') with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.assertEqual( self.mispevent.objects[0].get_attributes_by_relation('malware-sample')[0].malware_binary.read(), pseudofile.read()) def test_sighting(self): sighting = MISPSighting() sighting.from_dict(value='1', type='bar', timestamp=11111111) with open('tests/mispevent_testfiles/sighting.json', 'r') as f: ref_json = json.load(f) self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_event(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') with open('tests/mispevent_testfiles/existing_event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes_existing(self): self.mispevent.load_file('tests/mispevent_testfiles/shadow.json') with open('tests/mispevent_testfiles/shadow.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes(self): self.init_event() self.mispevent.add_proposal(type='filename', value='baz.jpg') self.mispevent.add_attribute('filename', 'bar.exe') self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf') with open('tests/mispevent_testfiles/proposals.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_default_attributes(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}]) self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent.objects[0].attributes[0]) self.mispevent.objects[1].add_attribute('filename', value='baz') self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_default_values(self): self.init_event() self.mispevent.add_object(name='whois', strict=True) self.mispevent.objects[0].add_attribute('registrar', value='registar.example.com') self.mispevent.objects[0].add_attribute('domain', value='domain.example.com') self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com') self.mispevent.objects[0].add_attribute('nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_not_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) def test_event_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.mispevent.info = 'blah' self.assertTrue(self.mispevent.edited) def test_event_tag_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.add_tag('foo') self.assertTrue(self.mispevent.edited) def test_event_attribute_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.mispevent.attributes[0].value = 'blah' self.assertTrue(self.mispevent.attributes[0].edited) self.assertFalse(self.mispevent.attributes[1].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].tags[0].name = 'blah' self.assertTrue(self.mispevent.attributes[0].tags[0].edited) self.assertFalse(self.mispevent.attributes[0].tags[1].edited) self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited_second(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].add_tag(name='blah') self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].edited) self.assertFalse(self.mispevent.objects[1].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited_tag(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].add_tag('blah') self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_by_id(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') misp_obj = self.mispevent.get_object_by_id(1556) self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f') def test_userdefined_object(self): self.init_event() self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles') with self.assertRaises(InvalidMISPObject) as e: # Fail on required self.mispevent.to_json() if sys.version_info >= (3, ): self.assertEqual(e.exception.message, '{\'member3\'} are required.') else: # Python2 bullshit self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.') self.mispevent.objects[0].add_attribute('member3', value='foo') with self.assertRaises(InvalidMISPObject) as e: # Fail on requiredOneOf self.mispevent.to_json() self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2') self.mispevent.objects[0].add_attribute('member1', value='bar') self.mispevent.objects[0].add_attribute('member1', value='baz') with self.assertRaises(InvalidMISPObject) as e: # member1 is not a multiple self.mispevent.to_json() self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed') self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2] self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
class TestMISPEvent(unittest.TestCase): def setUp(self): self.maxDiff = None self.mispevent = MISPEvent() def init_event(self): self.mispevent.info = 'This is a test' self.mispevent.distribution = 1 self.mispevent.threat_level_id = 1 self.mispevent.analysis = 1 self.mispevent.set_date("2017-12-31") # test the set date method def test_simple(self): with open('tests/mispevent_testfiles/simple.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event(self): self.init_event() self.mispevent.publish() with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_loadfile(self): self.mispevent.load_file('tests/mispevent_testfiles/event.json') with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_tag(self): self.init_event() self.mispevent.add_tag('bar') self.mispevent.add_tag(name='baz') new_tag = MISPTag() new_tag.from_dict(name='foo') self.mispevent.add_tag(new_tag) with open('tests/mispevent_testfiles/event_tags.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_attribute(self): self.init_event() a = self.mispevent.add_attribute('filename', 'bar.exe') del a.uuid a = self.mispevent.add_attribute_tag('osint', 'bar.exe') attr_tags = self.mispevent.get_attribute_tag('bar.exe') self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint') self.assertEqual(attr_tags[0].name, 'osint') with open('tests/mispevent_testfiles/attribute.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) # Fake setting an attribute ID for testing self.mispevent.attributes[0].id = 42 self.mispevent.delete_attribute(42) with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_object_tag(self): self.mispevent.add_object(name='file', strict=True) a = self.mispevent.objects[0].add_attribute('filename', value='') self.assertEqual(a, None) a = self.mispevent.objects[0].add_attribute('filename', value=None) self.assertEqual(a, None) a = self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{ 'name': 'blah' }]) del a.uuid self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah') self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation( ['filename'])) self.assertEqual( len(self.mispevent.objects[0].get_attributes_by_relation( 'filename')), 1) self.mispevent.add_object(name='url', strict=True) a = self.mispevent.objects[1].add_attribute( 'url', value='https://www.circl.lu') del a.uuid self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' reference = self.mispevent.objects[0].add_reference( self.mispevent.objects[1], 'baz', comment='foo') del reference.uuid self.assertEqual( self.mispevent.objects[0].references[0].relationship_type, 'baz') with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip( "Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168" ) def test_object_level_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar') self.mispevent.objects[0].add_tag('osint') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_malware(self): with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.init_event() a = self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile) del a.uuid attribute = self.mispevent.attributes[0] self.assertEqual(attribute.malware_binary, pseudofile) with open('tests/mispevent_testfiles/malware.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_malware(self): self.mispevent.load_file( 'tests/mispevent_testfiles/malware_exist.json') with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.assertEqual( self.mispevent.objects[0].get_attributes_by_relation( 'malware-sample')[0].malware_binary.read(), pseudofile.read()) def test_sighting(self): sighting = MISPSighting() sighting.from_dict(value='1', type='bar', timestamp=11111111) with open('tests/mispevent_testfiles/sighting.json', 'r') as f: ref_json = json.load(f) self.assertEqual(sighting.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_event(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') with open('tests/mispevent_testfiles/existing_event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes_existing(self): self.mispevent.load_file('tests/mispevent_testfiles/shadow.json') with open('tests/mispevent_testfiles/shadow.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip("Not supported on MISP.") def test_shadow_attributes(self): self.init_event() p = self.mispevent.add_proposal(type='filename', value='baz.jpg') del p.uuid a = self.mispevent.add_attribute('filename', 'bar.exe') del a.uuid p = self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf') del p.uuid with open('tests/mispevent_testfiles/proposals.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_default_attributes(self): self.mispevent.add_object(name='file', strict=True) a = self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{ 'name': 'blah' }]) del a.uuid a = self.mispevent.objects[0].add_attribute('pattern-in-file', value='baz') self.assertEqual(a.category, 'Artifacts dropped') del a.uuid self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent. objects[0].attributes[0]) a = self.mispevent.objects[1].add_attribute('filename', value='baz') del a.uuid self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_default_values(self): self.init_event() self.mispevent.add_object(name='whois', strict=True) a = self.mispevent.objects[0].add_attribute( 'registrar', value='registar.example.com') del a.uuid a = self.mispevent.objects[0].add_attribute('domain', value='domain.example.com') del a.uuid a = self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com') del a.uuid a = self.mispevent.objects[0].add_attribute( 'nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis') del a.uuid self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_not_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) def test_event_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.mispevent.info = 'blah' self.assertTrue(self.mispevent.edited) def test_event_tag_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.add_tag('foo') self.assertTrue(self.mispevent.edited) def test_event_attribute_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.mispevent.attributes[0].value = 'blah' self.assertTrue(self.mispevent.attributes[0].edited) self.assertFalse(self.mispevent.attributes[1].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].tags[0].name = 'blah' self.assertTrue(self.mispevent.attributes[0].tags[0].edited) self.assertFalse(self.mispevent.attributes[0].tags[1].edited) self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited_second(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].add_tag(name='blah') self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].edited) self.assertFalse(self.mispevent.objects[1].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited_tag(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].add_tag('blah') self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_by_id(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') misp_obj = self.mispevent.get_object_by_id(1556) self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f') def test_userdefined_object(self): self.init_event() self.mispevent.add_object( name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles') with self.assertRaises(InvalidMISPObject) as e: # Fail on required self.mispevent.to_json(sort_keys=True, indent=2) if sys.version_info >= (3, ): self.assertEqual(e.exception.message, '{\'member3\'} are required.') else: # Python2 bullshit self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.') a = self.mispevent.objects[0].add_attribute('member3', value='foo') del a.uuid with self.assertRaises(InvalidMISPObject) as e: # Fail on requiredOneOf self.mispevent.to_json(sort_keys=True, indent=2) self.assertEqual( e.exception.message, 'At least one of the following attributes is required: member1, member2' ) a = self.mispevent.objects[0].add_attribute('member1', value='bar') del a.uuid a = self.mispevent.objects[0].add_attribute('member1', value='baz') del a.uuid with self.assertRaises(InvalidMISPObject) as e: # member1 is not a multiple self.mispevent.to_json(sort_keys=True, indent=2) self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed') self.mispevent.objects[0].attributes = self.mispevent.objects[ 0].attributes[:2] self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def check_hashes(self): if self.offline_mode: self.log('error', 'Offline mode, unable to query VirusTotal') return event_id = self._get_eventid() if event_id is None: return event = self.misp.get(event_id) if self._has_error_message(event): return misp_event = MISPEvent() misp_event.load(event) hashes_to_expand = {} hashes_expanded = [] # Thoses hashes are known and already processed local_samples_hashes = [] partial_objects = {} for o in misp_event.Object: if o.name != 'file': continue if o.has_attributes_by_relation(['md5', 'sha1', 'sha256']): # This object has all the hashes we care about tmphashes = [] tmphashes += [h.value for h in o.get_attributes_by_relation('md5')] tmphashes += [ h.value for h in o.get_attributes_by_relation('sha1') ] tmphashes += [ h.value for h in o.get_attributes_by_relation('sha256') ] # Make sure to query VT for the sha256, even if expanded locally hashes_to_expand[o.get_attributes_by_relation( 'sha256')[0].value] = o.get_attributes_by_relation('sha256')[0] if o.has_attributes_by_relation(['malware-sample']): # ... and it has a malware sample local_samples_hashes += tmphashes hashes_expanded += tmphashes elif o.has_attributes_by_relation(['malware-sample']): # This object has a malware sample, but is missing hashes. We can expand locally. # get the MD5 from the malware-sample attribute malware_sample = o.get_attributes_by_relation('malware-sample')[ 0] # at most one sample/file object local_samples_hashes.append(malware_sample.value.split('|')[1]) local_samples_hashes += [ h.value for h in o.get_attributes_by_relation('md5') ] local_samples_hashes += [ h.value for h in o.get_attributes_by_relation('sha1') ] local_samples_hashes += [ h.value for h in o.get_attributes_by_relation('sha256') ] if self.args.populate: # The object is missing hashes, keeping track of it for expansion if it isn't already done. partial_objects[o.uuid] = malware_sample else: sha256 = { attribute.value: attribute for attribute in o.get_attributes_by_relation('sha256') } sha1 = { attribute.value: attribute for attribute in o.get_attributes_by_relation('sha1') } md5 = { attribute.value: attribute for attribute in o.get_attributes_by_relation('md5') } if sha256: hashes_to_expand.update(sha256) elif sha1: hashes_to_expand.update(sha1) elif md5: hashes_to_expand.update(md5) for ref_uuid, sample in partial_objects.items(): if sample.value.split('|')[1] in hashes_expanded: # Already expanded in an other object continue new_obj, hashes = self._expand_local_sample( pseudofile=sample.malware_binary, filename=sample.value.split('|')[0], refobj=ref_uuid, default_attributes_parameters=sample) misp_event.Object += new_obj local_samples_hashes += hashes # Make sure to query VT for the sha256, even if expanded locally hashes_to_expand[hashes[0]] = sample hashes_expanded += local_samples_hashes for a in misp_event.attributes: if a.type == 'malware-sample' and a.value.split( '|')[1] not in hashes_expanded: new_obj, hashes = self._expand_local_sample( pseudofile=a.malware_binary, filename=a.value.split('|')[0], default_attributes_parameters=a) misp_event.Object += new_obj local_samples_hashes += hashes # Make sure to query VT for the sha256, even if expanded locally hashes_to_expand[hashes[0]] = a elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'): # We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway fname, hashval = a.value.split('|') hashes_to_expand[hashval] = a elif a.type in ('md5', 'sha1', 'sha256'): # We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway hashes_to_expand[a.value] = a unk_vt_hashes = [] if cfg.virustotal.virustotal_has_private_key is False: quota = 4 timeout = datetime.datetime.now() + datetime.timedelta(minutes=1) hashes_expanded += local_samples_hashes processed_on_vt = [] # Make sure to start getting reports for the longest possible hashes (reduce risks of collisions) for to_expand in sorted(list(set(hashes_to_expand)), key=len): if to_expand in processed_on_vt: # Always run VT, once per sample continue original_attribute = hashes_to_expand[to_expand] if original_attribute.get('object_id'): original_object_id = original_attribute.get('object_id') vt_object = self._make_VT_object(to_expand, original_attribute) if not vt_object: unk_vt_hashes.append(to_expand) continue result = vt_object.get_report() md5 = result['md5'] sha1 = result['sha1'] sha256 = result['sha256'] processed_on_vt += [sha256, sha1, md5] if all(h in local_samples_hashes for h in [md5, sha1, sha256]): self.log('success', 'Sample available in MISP:') else: self.log('success', 'Sample available in VT:') self.log( 'item', '{}\n\t{}\n\t{}\n\t{}'.format(result["permalink"], md5, sha1, sha256)) if self.args.populate: if not all(h in hashes_expanded for h in [md5, sha1, sha256]): # If all the "new" expanded hashes are in the hashes_expanded list, skip file_object = MISPObject( 'file', default_attributes_parameters=original_attribute) file_object.add_attribute('md5', value=md5) file_object.add_attribute('sha1', value=sha1) file_object.add_attribute('sha256', value=sha256) file_object.add_reference(vt_object.uuid, 'analysed-with') misp_event.Object.append(file_object) hashes_expanded += [md5, sha1, sha256] else: if not original_object_id or original_object_id == '0': # Not an object, but the hashes are in an other object, skipping continue else: # We already have a MISP object, adding the link to the new VT object file_object = misp_event.get_object_by_id( original_object_id) file_object.add_reference(vt_object.uuid, 'analysed-with') misp_event.Object.append(vt_object) if cfg.virustotal.virustotal_has_private_key is False: if quota > 0: quota -= 1 else: waiting_time = (timeout - datetime.datetime.now()).seconds if waiting_time > 0: self.log( 'warning', 'No private API key, 4 queries/min is the limit. Waiting for {} seconds.' .format(waiting_time)) time.sleep(waiting_time) quota = 4 timeout = datetime.datetime.now() + datetime.timedelta( minutes=1) if self.args.populate: self._populate(misp_event) if len(unk_vt_hashes) > 0: self.log('error', 'Unknown on VT:') for h in unk_vt_hashes: self.log('item', '{}'.format(h))
def check_hashes(self): if self.offline_mode: self.log('error', 'Offline mode, unable to query VirusTotal') return event_id = self._get_eventid() if event_id is None: return event = self.misp.get(event_id) if self._has_error_message(event): return misp_event = MISPEvent() misp_event.load(event) hashes_to_expand = {} hashes_expanded = [] # Thoses hashes are known and already processed local_samples_hashes = [] partial_objects = {} for o in misp_event.Object: if o.name != 'file': continue if o.has_attributes_by_relation(['md5', 'sha1', 'sha256']): # This object has all the hashes we care about tmphashes = [] tmphashes += [h.value for h in o.get_attributes_by_relation('md5')] tmphashes += [h.value for h in o.get_attributes_by_relation('sha1')] tmphashes += [h.value for h in o.get_attributes_by_relation('sha256')] # Make sure to query VT for the sha256, even if expanded locally hashes_to_expand[o.get_attributes_by_relation('sha256')[0].value] = o.get_attributes_by_relation('sha256')[0] if o.has_attributes_by_relation(['malware-sample']): # ... and it has a malware sample local_samples_hashes += tmphashes hashes_expanded += tmphashes elif o.has_attributes_by_relation(['malware-sample']): # This object has a malware sample, but is missing hashes. We can expand locally. # get the MD5 from the malware-sample attribute malware_sample = o.get_attributes_by_relation('malware-sample')[0] # at most one sample/file object local_samples_hashes.append(malware_sample.value.split('|')[1]) local_samples_hashes += [h.value for h in o.get_attributes_by_relation('md5')] local_samples_hashes += [h.value for h in o.get_attributes_by_relation('sha1')] local_samples_hashes += [h.value for h in o.get_attributes_by_relation('sha256')] if self.args.populate: # The object is missing hashes, keeping track of it for expansion if it isn't already done. partial_objects[o.uuid] = malware_sample else: sha256 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('sha256')} sha1 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('sha1')} md5 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('md5')} if sha256: hashes_to_expand.update(sha256) elif sha1: hashes_to_expand.update(sha1) elif md5: hashes_to_expand.update(md5) for ref_uuid, sample in partial_objects.items(): if sample.value.split('|')[1] in hashes_expanded: # Already expanded in an other object continue new_obj, hashes = self._expand_local_sample(pseudofile=sample.malware_binary, filename=sample.value.split('|')[0], refobj=ref_uuid, default_attributes_paramaters=sample) misp_event.Object += new_obj local_samples_hashes += hashes # Make sure to query VT for the sha256, even if expanded locally hashes_to_expand[hashes[0]] = sample hashes_expanded += local_samples_hashes for a in misp_event.attributes: if a.type == 'malware-sample' and a.value.split('|')[1] not in hashes_expanded: new_obj, hashes = self._expand_local_sample(pseudofile=a.malware_binary, filename=a.value.split('|')[0], default_attributes_paramaters=a) misp_event.Object += new_obj local_samples_hashes += hashes # Make sure to query VT for the sha256, even if expanded locally hashes_to_expand[hashes[0]] = a elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'): # We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway fname, hashval = a.value.split('|') hashes_to_expand[hashval] = a elif a.type in ('md5', 'sha1', 'sha256'): # We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway hashes_to_expand[a.value] = a unk_vt_hashes = [] if cfg.virustotal.virustotal_has_private_key is False: quota = 4 timeout = datetime.datetime.now() + datetime.timedelta(minutes=1) hashes_expanded += local_samples_hashes processed_on_vt = [] # Make sure to start getting reports for the longest possible hashes (reduce risks of collisions) for to_expand in sorted(list(set(hashes_to_expand)), key=len): if to_expand in processed_on_vt: # Always run VT, once per sample continue original_attribute = hashes_to_expand[to_expand] if original_attribute.get('object_id'): original_object_id = original_attribute.get('object_id') vt_object = self._make_VT_object(to_expand, original_attribute) if not vt_object: unk_vt_hashes.append(to_expand) continue result = vt_object.get_report() md5 = result['md5'] sha1 = result['sha1'] sha256 = result['sha256'] processed_on_vt += [sha256, sha1, md5] if all(h in local_samples_hashes for h in [md5, sha1, sha256]): self.log('success', 'Sample available in MISP:') else: self.log('success', 'Sample available in VT:') self.log('item', '{}\n\t{}\n\t{}\n\t{}'.format(result["permalink"], md5, sha1, sha256)) if self.args.populate: if not all(h in hashes_expanded for h in [md5, sha1, sha256]): # If all the "new" expanded hashes are in the hashes_expanded list, skip file_object = MISPObject('file', default_attributes_paramaters=original_attribute) file_object.add_attribute('md5', value=md5) file_object.add_attribute('sha1', value=sha1) file_object.add_attribute('sha256', value=sha256) file_object.add_reference(vt_object.uuid, 'analysed-with') misp_event.Object.append(file_object) hashes_expanded += [md5, sha1, sha256] else: if not original_object_id or original_object_id == '0': # Not an object, but the hashes are in an other object, skipping continue else: # We already have a MISP object, adding the link to the new VT object file_object = misp_event.get_object_by_id(original_object_id) file_object.add_reference(vt_object.uuid, 'analysed-with') misp_event.Object.append(vt_object) if cfg.virustotal.virustotal_has_private_key is False: if quota > 0: quota -= 1 else: waiting_time = (timeout - datetime.datetime.now()).seconds if waiting_time > 0: self.log('warning', 'No private API key, 4 queries/min is the limit. Waiting for {} seconds.'.format(waiting_time)) time.sleep(waiting_time) quota = 4 timeout = datetime.datetime.now() + datetime.timedelta(minutes=1) if self.args.populate: self._populate(misp_event) if len(unk_vt_hashes) > 0: self.log('error', 'Unknown on VT:') for h in unk_vt_hashes: self.log('item', '{}'.format(h))
class TestMISPEvent(unittest.TestCase): def setUp(self): self.maxDiff = None self.mispevent = MISPEvent() def init_event(self): self.mispevent.info = 'This is a test' self.mispevent.distribution = 1 self.mispevent.threat_level_id = 1 self.mispevent.analysis = 1 self.mispevent.set_date("2017-12-31") # test the set date method def test_simple(self): with open('tests/mispevent_testfiles/simple.json', 'r') as f: ref_json = json.load(f) del self.mispevent.uuid self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event(self): self.init_event() self.mispevent.publish() with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) del self.mispevent.uuid self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_loadfile(self): self.mispevent.load_file('tests/mispevent_testfiles/event.json') with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) del self.mispevent.uuid self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_tag(self): self.init_event() self.mispevent.add_tag('bar') self.mispevent.add_tag(name='baz') new_tag = MISPTag() new_tag.from_dict(name='foo') self.mispevent.add_tag(new_tag) with open('tests/mispevent_testfiles/event_tags.json', 'r') as f: ref_json = json.load(f) del self.mispevent.uuid self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_attribute(self): self.init_event() a = self.mispevent.add_attribute('filename', 'bar.exe') del a.uuid a = self.mispevent.add_attribute_tag('osint', 'bar.exe') attr_tags = self.mispevent.get_attribute_tag('bar.exe') self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint') self.assertEqual(attr_tags[0].name, 'osint') with open('tests/mispevent_testfiles/attribute.json', 'r') as f: ref_json = json.load(f) del self.mispevent.uuid self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) # Fake setting an attribute ID for testing self.mispevent.attributes[0].id = 42 self.mispevent.delete_attribute(42) with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_object_tag(self): self.mispevent.add_object(name='file', strict=True) a = self.mispevent.objects[0].add_attribute('filename', value='') self.assertEqual(a, None) a = self.mispevent.objects[0].add_attribute('filename', value=None) self.assertEqual(a, None) a = self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{ 'name': 'blah' }]) del a.uuid self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah') self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation( ['filename'])) self.assertEqual( len(self.mispevent.objects[0].get_attributes_by_relation( 'filename')), 1) self.mispevent.add_object(name='url', strict=True) a = self.mispevent.objects[1].add_attribute( 'url', value='https://www.circl.lu') del a.uuid self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' reference = self.mispevent.objects[0].add_reference( self.mispevent.objects[1], 'baz', comment='foo') del reference.uuid self.assertEqual( self.mispevent.objects[0].references[0].relationship_type, 'baz') with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: ref_json = json.load(f) del self.mispevent.uuid self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip( "Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168" ) def test_object_level_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar') self.mispevent.objects[0].add_tag('osint') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_malware(self): with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.init_event() a = self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile) del a.uuid attribute = self.mispevent.attributes[0] self.assertEqual(attribute.malware_binary, pseudofile) with open('tests/mispevent_testfiles/malware.json', 'r') as f: ref_json = json.load(f) del self.mispevent.uuid self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_malware(self): self.mispevent.load_file( 'tests/mispevent_testfiles/malware_exist.json') with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.assertEqual( self.mispevent.objects[0].get_attributes_by_relation( 'malware-sample')[0].malware_binary.read(), pseudofile.read()) def test_sighting(self): sighting = MISPSighting() sighting.from_dict(value='1', type='bar', timestamp=11111111) with open('tests/mispevent_testfiles/sighting.json', 'r') as f: ref_json = json.load(f) self.assertEqual(sighting.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_event(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') with open('tests/mispevent_testfiles/existing_event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes_existing(self): self.mispevent.load_file('tests/mispevent_testfiles/shadow.json') with open('tests/mispevent_testfiles/shadow.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip("Not supported on MISP.") def test_shadow_attributes(self): self.init_event() p = self.mispevent.add_proposal(type='filename', value='baz.jpg') del p.uuid a = self.mispevent.add_attribute('filename', 'bar.exe') del a.uuid p = self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf') del p.uuid with open('tests/mispevent_testfiles/proposals.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_default_attributes(self): self.mispevent.add_object(name='file', strict=True) a = self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{ 'name': 'blah' }]) del a.uuid a = self.mispevent.objects[0].add_attribute('pattern-in-file', value='baz') self.assertEqual(a.category, 'Artifacts dropped') del a.uuid self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent. objects[0].attributes[0]) a = self.mispevent.objects[1].add_attribute('filename', value='baz') del a.uuid self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f: ref_json = json.load(f) del self.mispevent.uuid self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_default_values(self): self.init_event() self.mispevent.add_object(name='whois', strict=True) a = self.mispevent.objects[0].add_attribute( 'registrar', value='registar.example.com') del a.uuid a = self.mispevent.objects[0].add_attribute('domain', value='domain.example.com') del a.uuid a = self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com') del a.uuid a = self.mispevent.objects[0].add_attribute( 'nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis') del a.uuid self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/def_param.json', 'r') as f: ref_json = json.load(f) del self.mispevent.uuid self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_not_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) def test_event_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.mispevent.info = 'blah' self.assertTrue(self.mispevent.edited) def test_event_tag_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.add_tag('foo') self.assertTrue(self.mispevent.edited) def test_event_attribute_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.mispevent.attributes[0].value = 'blah' self.assertTrue(self.mispevent.attributes[0].edited) self.assertFalse(self.mispevent.attributes[1].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].tags[0].name = 'blah' self.assertTrue(self.mispevent.attributes[0].tags[0].edited) self.assertFalse(self.mispevent.attributes[0].tags[1].edited) self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited_second(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].add_tag(name='blah') self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].edited) self.assertFalse(self.mispevent.objects[1].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited_tag(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].add_tag('blah') self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_by_id(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') misp_obj = self.mispevent.get_object_by_id(1556) self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f') def test_userdefined_object(self): self.init_event() self.mispevent.add_object( name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles') with self.assertRaises(InvalidMISPObject) as e: # Fail on required self.mispevent.to_json(sort_keys=True, indent=2) if sys.version_info >= (3, ): self.assertEqual(e.exception.message, '{\'member3\'} are required.') else: # Python2 bullshit self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.') a = self.mispevent.objects[0].add_attribute('member3', value='foo') del a.uuid with self.assertRaises(InvalidMISPObject) as e: # Fail on requiredOneOf self.mispevent.to_json(sort_keys=True, indent=2) self.assertEqual( e.exception.message, 'At least one of the following attributes is required: member1, member2' ) a = self.mispevent.objects[0].add_attribute('member1', value='bar') del a.uuid a = self.mispevent.objects[0].add_attribute('member1', value='baz') del a.uuid with self.assertRaises(InvalidMISPObject) as e: # member1 is not a multiple self.mispevent.to_json(sort_keys=True, indent=2) self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed') self.mispevent.objects[0].attributes = self.mispevent.objects[ 0].attributes[:2] self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f: ref_json = json.load(f) del self.mispevent.uuid self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_first_last_seen(self): me = MISPEvent() me.info = 'Test First and Last Seen' me.date = '2020.01.12' self.assertEqual(me.date.day, 12) me.add_attribute('ip-dst', '8.8.8.8', first_seen='06-21-1998', last_seen=1580213607.469571) self.assertEqual(me.attributes[0].first_seen.year, 1998) self.assertEqual(me.attributes[0].last_seen.year, 2020) now = datetime.now().astimezone() me.attributes[0].last_seen = now today = date.today() me.attributes[0].first_seen = today self.assertEqual(me.attributes[0].first_seen.year, today.year) self.assertEqual(me.attributes[0].last_seen, now) def test_feed(self): me = MISPEvent() me.info = 'Test feed' org = MISPOrganisation() org.name = 'TestOrg' org.uuid = '123478' me.Orgc = org me.add_attribute('ip-dst', '8.8.8.8') obj = me.add_object(name='file') obj.add_attributes('filename', *['foo.exe', 'bar.exe']) h = hashlib.new('md5') h.update(b'8.8.8.8') hash_attr_val = h.hexdigest() feed = me.to_feed(with_meta=True) self.assertEqual(feed['Event']['_hashes'][0], hash_attr_val) self.assertEqual(feed['Event']['_manifest'][me.uuid]['info'], 'Test feed') self.assertEqual(len(feed['Event']['Object'][0]['Attribute']), 2) def test_object_templates(self): me = MISPEvent() for template in glob.glob( str(me.misp_objects_path / '*' / 'definition.json')): with open(template) as f: t_json = json.load(f) if 'requiredOneOf' in t_json: obj_relations = set(t_json['attributes'].keys()) subset = set( t_json['requiredOneOf']).issubset(obj_relations) self.assertTrue(subset, f'{t_json["name"]}') if 'required' in t_json: obj_relations = set(t_json['attributes'].keys()) subset = set(t_json['required']).issubset(obj_relations) self.assertTrue(subset, f'{t_json["name"]}') for obj_relation, entry in t_json['attributes'].items(): self.assertTrue( entry['misp-attribute'] in me.describe_types['types']) if 'categories' in entry: subset = set(entry['categories']).issubset( me.describe_types['categories']) self.assertTrue(subset, f'{t_json["name"]} - {obj_relation}')
class TestMISPEvent(unittest.TestCase): def setUp(self): self.maxDiff = None self.mispevent = MISPEvent() def init_event(self): self.mispevent.info = 'This is a test' self.mispevent.distribution = 1 self.mispevent.threat_level_id = 1 self.mispevent.analysis = 1 self.mispevent.set_date("2017-12-31") # test the set date method def test_simple(self): with open('tests/mispevent_testfiles/simple.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event(self): self.init_event() self.mispevent.publish() with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_loadfile(self): self.mispevent.load_file('tests/mispevent_testfiles/event.json') with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_tag(self): self.init_event() self.mispevent.add_tag('bar') self.mispevent.add_tag(name='baz') new_tag = MISPTag() new_tag.from_dict(name='foo') self.mispevent.add_tag(new_tag) with open('tests/mispevent_testfiles/event_tags.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_attribute(self): self.init_event() self.mispevent.add_attribute('filename', 'bar.exe') self.mispevent.add_attribute_tag('osint', 'bar.exe') attr_tags = self.mispevent.get_attribute_tag('bar.exe') self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint') self.assertEqual(attr_tags[0].name, 'osint') with open('tests/mispevent_testfiles/attribute.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) # Fake setting an attribute ID for testing self.mispevent.attributes[0].id = 42 self.mispevent.delete_attribute(42) with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_object_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{ 'name': 'blah' }]) self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah') self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation( ['filename'])) self.assertEqual( len(self.mispevent.objects[0].get_attributes_by_relation( 'filename')), 1) self.mispevent.add_object(name='url', strict=True) self.mispevent.objects[1].add_attribute('url', value='https://www.circl.lu') self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' self.mispevent.objects[0].add_reference('b', 'baz', comment='foo') self.assertEqual( self.mispevent.objects[0].references[0].relationship_type, 'baz') with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip( "Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168" ) def test_object_level_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar') self.mispevent.objects[0].add_tag('osint') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_malware(self): with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.init_event() self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile) attribute = self.mispevent.attributes[0] self.assertEqual(attribute.malware_binary, pseudofile) with open('tests/mispevent_testfiles/malware.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_malware(self): self.mispevent.load_file( 'tests/mispevent_testfiles/malware_exist.json') with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.assertEqual( self.mispevent.objects[0].get_attributes_by_relation( 'malware-sample')[0].malware_binary.read(), pseudofile.read()) def test_sighting(self): sighting = MISPSighting() sighting.from_dict(value='1', type='bar', timestamp=11111111) with open('tests/mispevent_testfiles/sighting.json', 'r') as f: ref_json = json.load(f) self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_event(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') with open('tests/mispevent_testfiles/existing_event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes_existing(self): self.mispevent.load_file('tests/mispevent_testfiles/shadow.json') with open('tests/mispevent_testfiles/shadow.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes(self): self.init_event() self.mispevent.add_proposal(type='filename', value='baz.jpg') self.mispevent.add_attribute('filename', 'bar.exe') self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf') with open('tests/mispevent_testfiles/proposals.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_default_attributes(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{ 'name': 'blah' }]) self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent. objects[0].attributes[0]) self.mispevent.objects[1].add_attribute('filename', value='baz') self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_default_values(self): self.init_event() self.mispevent.add_object(name='whois', strict=True) self.mispevent.objects[0].add_attribute('registrar', value='registar.example.com') self.mispevent.objects[0].add_attribute('domain', value='domain.example.com') self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com') self.mispevent.objects[0].add_attribute('nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_not_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) def test_event_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.mispevent.info = 'blah' self.assertTrue(self.mispevent.edited) def test_event_tag_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.add_tag('foo') self.assertTrue(self.mispevent.edited) def test_event_attribute_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.mispevent.attributes[0].value = 'blah' self.assertTrue(self.mispevent.attributes[0].edited) self.assertFalse(self.mispevent.attributes[1].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].tags[0].name = 'blah' self.assertTrue(self.mispevent.attributes[0].tags[0].edited) self.assertFalse(self.mispevent.attributes[0].tags[1].edited) self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited_second(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].add_tag(name='blah') self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].edited) self.assertFalse(self.mispevent.objects[1].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited_tag(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].add_tag('blah') self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_by_id(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') misp_obj = self.mispevent.get_object_by_id(1556) self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f')