def test_filter(self): t = MailAttachments.withhashes(self.attachments) check_list = deque(maxlen=10) md5 = "1e38e543279912d98cbfdc7b275a415e" check_list.append(md5) self.assertIn("payload", t[0]) self.assertNotIn("is_filtered", t[0]) r = t.filter(check_list, hash_type="md5") self.assertNotIn("payload", t[0]) self.assertIn("is_filtered", t[0]) self.assertEqual(True, t[0]["is_filtered"]) self.assertIn(md5, r) check_list.extend(r) self.assertEqual(2, len(check_list)) # It should not fail t.run(filtercontenttypes=False, intelligence=False) t = MailAttachments.withhashes(self.attachments) check_list = deque(maxlen=10) md5 = "1e38e543279912d98cbfdc7b275a415f" check_list.append(md5) r = t.filter(check_list, hash_type="md5") self.assertIn("payload", t[0]) self.assertIn("is_filtered", t[0]) self.assertEqual(False, t[0]["is_filtered"]) self.assertNotIn(md5, r)
def test_popcontenttype(self): t = MailAttachments.withhashes(self.attachments) t(filtercontenttypes=False, intelligence=False) self.assertEqual(len(t), 1) t.popcontenttype("application/zip") self.assertEqual(len(t), 0) # Test path when attach is filtered t = MailAttachments.withhashes(self.attachments) t(filtercontenttypes=False, intelligence=False) check_list = deque(maxlen=10) md5 = "1e38e543279912d98cbfdc7b275a415e" check_list.append(md5) t.filter(check_list, hash_type="md5") t.popcontenttype("application/zip") self.assertEqual(len(t), 0) t = MailAttachments.withhashes(self.attachments) t(filtercontenttypes=False, intelligence=False) self.assertEqual(len(t[0]["files"]), 1) t.popcontenttype("application/x-dosexec") self.assertEqual(len(t[0]["files"]), 0) t = MailAttachments.withhashes(self.attachments) t(filtercontenttypes=False, intelligence=False) t.popcontenttype("application/fake") self.assertEqual(len(t), 1) self.assertEqual(len(t[0]["files"]), 1) t = MailAttachments.withhashes(self.attachments) with self.assertRaises(ContentTypeError): t.popcontenttype("application/zip")
def test_tika(self): """Test add Tika processing.""" from src.modules.attachments import tika # Complete parameters conf = { "enabled": True, "path_jar": OPTIONS["TIKA_APP_JAR"], "memory_allocation": None, "whitelist_cont_types": ["application/zip"] } attachments = MailAttachments.withhashes(self.attachments) attachments(intelligence=False, filtercontenttypes=False) tika(conf, attachments) for i in attachments: self.assertIn("tika", i) self.assertEqual(len(attachments[0]["tika"]), 2) self.assertEqual(int(attachments[0]["tika"][0]["Content-Length"]), attachments[0]["size"]) # tika disabled conf["enabled"] = False attachments = MailAttachments.withhashes(self.attachments) attachments(intelligence=False, filtercontenttypes=False) tika(conf, attachments) for i in attachments: self.assertNotIn("tika", i) conf["enabled"] = True # attachments without run() with self.assertRaises(KeyError): attachments = MailAttachments.withhashes(self.attachments) tika(conf, attachments) # attachments a key of conf with self.assertRaises(KeyError): conf_inner = { "enabled": True, "path_jar": OPTIONS["TIKA_APP_JAR"], "memory_allocation": None } attachments = MailAttachments.withhashes(self.attachments) tika(conf_inner, attachments)
def test_zemana(self): """Test add Zemana processing.""" from src.modules.attachments import zemana conf = { "enabled": True, "PartnerId": OPTIONS["ZEMANA_PARTNERID"], "UserId": OPTIONS["ZEMANA_USERID"], "ApiKey": OPTIONS["ZEMANA_APIKEY"], "useragent": "SpamScope" } attachments = MailAttachments.withhashes(self.attachments) attachments(intelligence=False, filtercontenttypes=False) zemana(conf, attachments) # Zemana analysis for i in attachments: self.assertIn("zemana", i) self.assertIn("type", i["zemana"]) self.assertIn("aa", i["zemana"]) self.assertIsInstance(i["zemana"]["aa"], list) for j in i["files"]: self.assertIn("zemana", j) self.assertIn("type", j["zemana"]) self.assertIn("aa", j["zemana"]) self.assertIsInstance(j["zemana"]["aa"], list)
def test_error_extract_rar(self): t = MailAttachments.withhashes(self.attachments_test_2) t.run(filtercontenttypes=False, intelligence=False) self.assertEqual(len(t), 2) for i in t: self.assertEqual(i["Content-Type"], "application/x-rar") self.assertIn("files", i)
def test_virustotal(self): """Test add VirusTotal processing.""" from src.modules.attachments import virustotal conf = {"enabled": True, "api_key": OPTIONS["VIRUSTOTAL_APIKEY"]} attachments = MailAttachments.withhashes(self.attachments) attachments(intelligence=False, filtercontenttypes=False) virustotal(conf, attachments) # VirusTotal analysis for i in attachments: self.assertIn('virustotal', i) self.assertEqual(i['virustotal']['response_code'], 200) self.assertEqual(i['virustotal']['results']['sha1'], '2a7cee8c214ac76ba6fdbc3031e73dbede95b803') self.assertIsInstance(i["virustotal"]["results"]["scans"], list) for j in i["files"]: self.assertIn('virustotal', j) self.assertEqual(j['virustotal']['response_code'], 200) self.assertEqual(j['virustotal']['results']['sha1'], 'ed2e480e7ba7e37f77a85efbca4058d8c5f92664') self.assertIsInstance(j["virustotal"]["results"]["scans"], list)
def test_post_processing(self): t = MailAttachments.withhashes(self.attachments_thug) parameters = { "tika": {"enabled": True, "path_jar": OPTIONS["TIKA_APP_JAR"], "memory_allocation": None}, "tika_whitelist_cont_types": ["application/zip"], "virustotal": {"enabled": True, "api_key": OPTIONS["VIRUSTOTAL_APIKEY"]}, "thug": {"enabled": True, "extensions": [".html", ".js", ".jse"], "user_agents": ["win7ie90", "winxpie80"], "referer": "http://www.google.com/"}, "zemana": {"enabled": True, "PartnerId": OPTIONS["ZEMANA_PARTNERID"], "UserId": OPTIONS["ZEMANA_USERID"], "ApiKey": OPTIONS["ZEMANA_APIKEY"], "useragent": "SpamScope"}} t.reload(**parameters) t.run(filtercontenttypes=False) for i in t: self.assertIn("tika", i) self.assertIn("virustotal", i) self.assertIn("zemana", i) self.assertNotIn("thug", i) for j in i.get("files", []): self.assertIn("virustotal", j) self.assertIn("zemana", j) self.assertIn("thug", j)
def test_filenamestext(self): t = MailAttachments.withhashes(self.attachments) t.run(filtercontenttypes=False, intelligence=False) text = t.filenamestext() self.assertIsNotNone(text) self.assertNotIsInstance(text, list) self.assertNotIsInstance(text, dict) self.assertIn("20160523_916527.jpg_.zip", text) self.assertIn("20160523_211439.jpg_.jpg.exe", text)
def test_reload(self): dummy = {"key1": "value1", "key2": "value2"} t = MailAttachments.withhashes(self.attachments) t.reload(**dummy) self.assertEqual(t.key1, "value1") self.assertEqual(t.key2, "value2") self.assertEqual(len(t), 1) t(filtercontenttypes=False, intelligence=False) t.removeall() self.assertEqual(len(t), 0)
def test_pophash(self): t = MailAttachments.withhashes(self.attachments) md5 = "1e38e543279912d98cbfdc7b275a415e" self.assertEqual(len(t), 1) with self.assertRaises(HashError): t.pophash("fake") t.pophash(md5) self.assertEqual(len(t), 0)
def test_error_base64(self): t = MailAttachments.withhashes(self.attachments_test_1) t.run(filtercontenttypes=False, intelligence=False) files = [] for i in t: f = write_payload(i["payload"], i["extension"], i["content_transfer_encoding"]) files.append(f) for i in files: os.remove(i)
def test_thug(self): """Test add Thug processing.""" from src.modules.attachments import thug # Complete parameters conf = { "enabled": True, "extensions": [".html", ".js", ".jse"], "user_agents": ["win7ie90", "winxpie80"], "referer": "http://www.google.com/" } attachments = MailAttachments.withhashes(self.attachments_thug) attachments(intelligence=False, filtercontenttypes=False) first_attachment = attachments[0] self.assertNotIn('thug', first_attachment) thug(conf, attachments) # Thug attachment thug_attachment = first_attachment['files'][0] self.assertIn('thug', thug_attachment) thug_analysis = thug_attachment['thug'] self.assertIsInstance(thug_analysis, list) self.assertEqual(len(thug_analysis), 2) first_thug_analysis = thug_analysis[0] self.assertIn('files', first_thug_analysis) self.assertIn('code', first_thug_analysis) self.assertIn('exploits', first_thug_analysis) self.assertIn('url', first_thug_analysis) self.assertIn('timestamp', first_thug_analysis) self.assertIn('locations', first_thug_analysis) self.assertIn('connections', first_thug_analysis) self.assertIn('logtype', first_thug_analysis) self.assertIn('behavior', first_thug_analysis) self.assertIn('thug', first_thug_analysis) self.assertIn('classifiers', first_thug_analysis) self.assertEqual(first_thug_analysis['files'][0]['sha1'], "e2835a38f50d287c65b0e53b4787d41095a3514f") self.assertEqual(first_thug_analysis['files'][0]['md5'], "b83c7ac97c22ce248b09f4388c130df0") self.assertEqual( first_thug_analysis['thug']['personality']['useragent'], 'win7ie90') self.assertEqual(first_thug_analysis['thug']['options']['referer'], 'http://www.google.com/')
def test_withhashes(self): t = MailAttachments.withhashes(self.attachments) self.assertIsInstance(t, MailAttachments) self.assertEqual(len(t), 1) for i in t: self.assertIn("md5", i) self.assertIn("sha1", i) self.assertIn("sha256", i) self.assertIn("sha512", i) self.assertIn("ssdeep", i) self.assertIn("filename", i) self.assertIn("payload", i) self.assertIn("mail_content_type", i) self.assertIn("content_transfer_encoding", i)
def setUp(self): self.f = utils.reformat_output p = mailparser.parse_from_file(mail) self.mail_obj = p.mail self.mail_obj['analisys_date'] = datetime.datetime.utcnow().isoformat() self.attachments = MailAttachments.withhashes(p.attachments) self.attachments.run() self.parameters = { 'elastic_index_mail': "spamscope_mails-", 'elastic_type_mail': "spamscope", 'elastic_index_attach': "spamscope_attachments-", 'elastic_type_attach': "spamscope"}
def test_tika_bug_incorrect_padding(self): """Test add Tika processing.""" from src.modules.attachments import tika # Complete parameters conf = {"enabled": True, "path_jar": OPTIONS["TIKA_APP_JAR"], "memory_allocation": None, "whitelist_cont_types": ["application/zip"]} p = mailparser.parse_from_file(mail_test_4) attachments = MailAttachments.withhashes(p.attachments) attachments(intelligence=False, filtercontenttypes=False) tika(conf, attachments) for i in attachments: self.assertIn("tika", i)
def test_filtercontenttypes(self): t = MailAttachments.withhashes(self.attachments) parameters = {"filter_cont_types": ["application/x-dosexec", "application/zip"]} t.reload(**parameters) self.assertIn("application/x-dosexec", t.filter_cont_types) self.assertIn("application/zip", t.filter_cont_types) self.assertEqual(len(t), 1) t(intelligence=False) self.assertEqual(len(t), 0) t.extend(self.attachments) parameters = {"filter_cont_types": ["application/x-dosexec"]} t.reload(**parameters) t(intelligence=False) self.assertEqual(len(t), 1) self.assertEqual(len(t[0]["files"]), 0)
def test_payloadstext(self): t = MailAttachments.withhashes(self.attachments_thug) t.run(filtercontenttypes=False, intelligence=False) text = t.payloadstext() self.assertIsNotNone(text) self.assertNotIsInstance(text, list) self.assertNotIsInstance(text, dict) self.assertIn("Windows", text) self.assertIn("setRequestHeader", text) check_list = deque(maxlen=10) md5 = "778cf2c48ab482d6134d4d12eb51192f" check_list.append(md5) self.assertIn("payload", t[0]) self.assertNotIn("is_filtered", t[0]) t.filter(check_list, hash_type="md5") self.assertNotIn("payload", t[0]) text = t.payloadstext() self.assertFalse(text)
def test_run(self): t = MailAttachments.withhashes(self.attachments) t(filtercontenttypes=False, intelligence=False) for i in t: self.assertIn("analisys_date", i) self.assertIn("extension", i) self.assertIn("size", i) self.assertIn("Content-Type", i) self.assertIn("is_archive", i) self.assertIn("files", i) self.assertEqual(i["extension"], ".zip") self.assertTrue(i["is_archive"]) self.assertEqual(len(i["files"]), 1) for j in i["files"]: self.assertIn("analisys_date", j) self.assertIn("filename", j) self.assertIn("extension", j) self.assertIn("size", j) self.assertIn("Content-Type", j) self.assertIn("payload", j) self.assertIn("md5", j)