def test_filter(self):
t = MailAttachments.withhashes(self.attachments)
check_list = deque(maxlen=10)
md5 = "1e38e543279912d98cbfdc7b275a415e"
check_list.append(md5)
self.assertIn("payload", t[0])
self.assertNotIn("is_filtered", t[0])
r = t.filter(check_list, hash_type="md5")
self.assertNotIn("payload", t[0])
self.assertIn("is_filtered", t[0])
self.assertEqual(True, t[0]["is_filtered"])
self.assertIn(md5, r)
check_list.extend(r)
self.assertEqual(2, len(check_list))
# It should not fail
t.run(filtercontenttypes=False, intelligence=False)
t = MailAttachments.withhashes(self.attachments)
check_list = deque(maxlen=10)
md5 = "1e38e543279912d98cbfdc7b275a415f"
check_list.append(md5)
r = t.filter(check_list, hash_type="md5")
self.assertIn("payload", t[0])
self.assertIn("is_filtered", t[0])
self.assertEqual(False, t[0]["is_filtered"])
self.assertNotIn(md5, r)
def test_popcontenttype(self):
t = MailAttachments.withhashes(self.attachments)
t(filtercontenttypes=False, intelligence=False)
self.assertEqual(len(t), 1)
t.popcontenttype("application/zip")
self.assertEqual(len(t), 0)
# Test path when attach is filtered
t = MailAttachments.withhashes(self.attachments)
t(filtercontenttypes=False, intelligence=False)
check_list = deque(maxlen=10)
md5 = "1e38e543279912d98cbfdc7b275a415e"
check_list.append(md5)
t.filter(check_list, hash_type="md5")
t.popcontenttype("application/zip")
self.assertEqual(len(t), 0)
t = MailAttachments.withhashes(self.attachments)
t(filtercontenttypes=False, intelligence=False)
self.assertEqual(len(t[0]["files"]), 1)
t.popcontenttype("application/x-dosexec")
self.assertEqual(len(t[0]["files"]), 0)
t = MailAttachments.withhashes(self.attachments)
t(filtercontenttypes=False, intelligence=False)
t.popcontenttype("application/fake")
self.assertEqual(len(t), 1)
self.assertEqual(len(t[0]["files"]), 1)
t = MailAttachments.withhashes(self.attachments)
with self.assertRaises(ContentTypeError):
t.popcontenttype("application/zip")
def test_tika(self):
"""Test add Tika processing."""
from src.modules.attachments import tika
# Complete parameters
conf = {
"enabled": True,
"path_jar": OPTIONS["TIKA_APP_JAR"],
"memory_allocation": None,
"whitelist_cont_types": ["application/zip"]
}
attachments = MailAttachments.withhashes(self.attachments)
attachments(intelligence=False, filtercontenttypes=False)
tika(conf, attachments)
for i in attachments:
self.assertIn("tika", i)
self.assertEqual(len(attachments[0]["tika"]), 2)
self.assertEqual(int(attachments[0]["tika"][0]["Content-Length"]),
attachments[0]["size"])
# tika disabled
conf["enabled"] = False
attachments = MailAttachments.withhashes(self.attachments)
attachments(intelligence=False, filtercontenttypes=False)
tika(conf, attachments)
for i in attachments:
self.assertNotIn("tika", i)
conf["enabled"] = True
# attachments without run()
with self.assertRaises(KeyError):
attachments = MailAttachments.withhashes(self.attachments)
tika(conf, attachments)
# attachments a key of conf
with self.assertRaises(KeyError):
conf_inner = {
"enabled": True,
"path_jar": OPTIONS["TIKA_APP_JAR"],
"memory_allocation": None
}
attachments = MailAttachments.withhashes(self.attachments)
tika(conf_inner, attachments)
def test_zemana(self):
"""Test add Zemana processing."""
from src.modules.attachments import zemana
conf = {
"enabled": True,
"PartnerId": OPTIONS["ZEMANA_PARTNERID"],
"UserId": OPTIONS["ZEMANA_USERID"],
"ApiKey": OPTIONS["ZEMANA_APIKEY"],
"useragent": "SpamScope"
}
attachments = MailAttachments.withhashes(self.attachments)
attachments(intelligence=False, filtercontenttypes=False)
zemana(conf, attachments)
# Zemana analysis
for i in attachments:
self.assertIn("zemana", i)
self.assertIn("type", i["zemana"])
self.assertIn("aa", i["zemana"])
self.assertIsInstance(i["zemana"]["aa"], list)
for j in i["files"]:
self.assertIn("zemana", j)
self.assertIn("type", j["zemana"])
self.assertIn("aa", j["zemana"])
self.assertIsInstance(j["zemana"]["aa"], list)
def test_error_extract_rar(self):
t = MailAttachments.withhashes(self.attachments_test_2)
t.run(filtercontenttypes=False, intelligence=False)
self.assertEqual(len(t), 2)
for i in t:
self.assertEqual(i["Content-Type"], "application/x-rar")
self.assertIn("files", i)
def test_virustotal(self):
"""Test add VirusTotal processing."""
from src.modules.attachments import virustotal
conf = {"enabled": True, "api_key": OPTIONS["VIRUSTOTAL_APIKEY"]}
attachments = MailAttachments.withhashes(self.attachments)
attachments(intelligence=False, filtercontenttypes=False)
virustotal(conf, attachments)
# VirusTotal analysis
for i in attachments:
self.assertIn('virustotal', i)
self.assertEqual(i['virustotal']['response_code'], 200)
self.assertEqual(i['virustotal']['results']['sha1'],
'2a7cee8c214ac76ba6fdbc3031e73dbede95b803')
self.assertIsInstance(i["virustotal"]["results"]["scans"], list)
for j in i["files"]:
self.assertIn('virustotal', j)
self.assertEqual(j['virustotal']['response_code'], 200)
self.assertEqual(j['virustotal']['results']['sha1'],
'ed2e480e7ba7e37f77a85efbca4058d8c5f92664')
self.assertIsInstance(j["virustotal"]["results"]["scans"],
list)
def test_post_processing(self):
t = MailAttachments.withhashes(self.attachments_thug)
parameters = {
"tika": {"enabled": True,
"path_jar": OPTIONS["TIKA_APP_JAR"],
"memory_allocation": None},
"tika_whitelist_cont_types": ["application/zip"],
"virustotal": {"enabled": True,
"api_key": OPTIONS["VIRUSTOTAL_APIKEY"]},
"thug": {"enabled": True,
"extensions": [".html", ".js", ".jse"],
"user_agents": ["win7ie90", "winxpie80"],
"referer": "http://www.google.com/"},
"zemana": {"enabled": True,
"PartnerId": OPTIONS["ZEMANA_PARTNERID"],
"UserId": OPTIONS["ZEMANA_USERID"],
"ApiKey": OPTIONS["ZEMANA_APIKEY"],
"useragent": "SpamScope"}}
t.reload(**parameters)
t.run(filtercontenttypes=False)
for i in t:
self.assertIn("tika", i)
self.assertIn("virustotal", i)
self.assertIn("zemana", i)
self.assertNotIn("thug", i)
for j in i.get("files", []):
self.assertIn("virustotal", j)
self.assertIn("zemana", j)
self.assertIn("thug", j)
def test_filenamestext(self):
t = MailAttachments.withhashes(self.attachments)
t.run(filtercontenttypes=False, intelligence=False)
text = t.filenamestext()
self.assertIsNotNone(text)
self.assertNotIsInstance(text, list)
self.assertNotIsInstance(text, dict)
self.assertIn("20160523_916527.jpg_.zip", text)
self.assertIn("20160523_211439.jpg_.jpg.exe", text)
def test_reload(self):
dummy = {"key1": "value1", "key2": "value2"}
t = MailAttachments.withhashes(self.attachments)
t.reload(**dummy)
self.assertEqual(t.key1, "value1")
self.assertEqual(t.key2, "value2")
self.assertEqual(len(t), 1)
t(filtercontenttypes=False, intelligence=False)
t.removeall()
self.assertEqual(len(t), 0)
def test_pophash(self):
t = MailAttachments.withhashes(self.attachments)
md5 = "1e38e543279912d98cbfdc7b275a415e"
self.assertEqual(len(t), 1)
with self.assertRaises(HashError):
t.pophash("fake")
t.pophash(md5)
self.assertEqual(len(t), 0)
def test_error_base64(self):
t = MailAttachments.withhashes(self.attachments_test_1)
t.run(filtercontenttypes=False, intelligence=False)
files = []
for i in t:
f = write_payload(i["payload"], i["extension"],
i["content_transfer_encoding"])
files.append(f)
for i in files:
os.remove(i)
def test_thug(self):
"""Test add Thug processing."""
from src.modules.attachments import thug
# Complete parameters
conf = {
"enabled": True,
"extensions": [".html", ".js", ".jse"],
"user_agents": ["win7ie90", "winxpie80"],
"referer": "http://www.google.com/"
}
attachments = MailAttachments.withhashes(self.attachments_thug)
attachments(intelligence=False, filtercontenttypes=False)
first_attachment = attachments[0]
self.assertNotIn('thug', first_attachment)
thug(conf, attachments)
# Thug attachment
thug_attachment = first_attachment['files'][0]
self.assertIn('thug', thug_attachment)
thug_analysis = thug_attachment['thug']
self.assertIsInstance(thug_analysis, list)
self.assertEqual(len(thug_analysis), 2)
first_thug_analysis = thug_analysis[0]
self.assertIn('files', first_thug_analysis)
self.assertIn('code', first_thug_analysis)
self.assertIn('exploits', first_thug_analysis)
self.assertIn('url', first_thug_analysis)
self.assertIn('timestamp', first_thug_analysis)
self.assertIn('locations', first_thug_analysis)
self.assertIn('connections', first_thug_analysis)
self.assertIn('logtype', first_thug_analysis)
self.assertIn('behavior', first_thug_analysis)
self.assertIn('thug', first_thug_analysis)
self.assertIn('classifiers', first_thug_analysis)
self.assertEqual(first_thug_analysis['files'][0]['sha1'],
"e2835a38f50d287c65b0e53b4787d41095a3514f")
self.assertEqual(first_thug_analysis['files'][0]['md5'],
"b83c7ac97c22ce248b09f4388c130df0")
self.assertEqual(
first_thug_analysis['thug']['personality']['useragent'],
'win7ie90')
self.assertEqual(first_thug_analysis['thug']['options']['referer'],
'http://www.google.com/')
def test_withhashes(self):
t = MailAttachments.withhashes(self.attachments)
self.assertIsInstance(t, MailAttachments)
self.assertEqual(len(t), 1)
for i in t:
self.assertIn("md5", i)
self.assertIn("sha1", i)
self.assertIn("sha256", i)
self.assertIn("sha512", i)
self.assertIn("ssdeep", i)
self.assertIn("filename", i)
self.assertIn("payload", i)
self.assertIn("mail_content_type", i)
self.assertIn("content_transfer_encoding", i)
def setUp(self):
self.f = utils.reformat_output
p = mailparser.parse_from_file(mail)
self.mail_obj = p.mail
self.mail_obj['analisys_date'] = datetime.datetime.utcnow().isoformat()
self.attachments = MailAttachments.withhashes(p.attachments)
self.attachments.run()
self.parameters = {
'elastic_index_mail': "spamscope_mails-",
'elastic_type_mail': "spamscope",
'elastic_index_attach': "spamscope_attachments-",
'elastic_type_attach': "spamscope"}
def test_tika_bug_incorrect_padding(self):
"""Test add Tika processing."""
from src.modules.attachments import tika
# Complete parameters
conf = {"enabled": True,
"path_jar": OPTIONS["TIKA_APP_JAR"],
"memory_allocation": None,
"whitelist_cont_types": ["application/zip"]}
p = mailparser.parse_from_file(mail_test_4)
attachments = MailAttachments.withhashes(p.attachments)
attachments(intelligence=False, filtercontenttypes=False)
tika(conf, attachments)
for i in attachments:
self.assertIn("tika", i)
def test_filtercontenttypes(self):
t = MailAttachments.withhashes(self.attachments)
parameters = {"filter_cont_types": ["application/x-dosexec",
"application/zip"]}
t.reload(**parameters)
self.assertIn("application/x-dosexec", t.filter_cont_types)
self.assertIn("application/zip", t.filter_cont_types)
self.assertEqual(len(t), 1)
t(intelligence=False)
self.assertEqual(len(t), 0)
t.extend(self.attachments)
parameters = {"filter_cont_types": ["application/x-dosexec"]}
t.reload(**parameters)
t(intelligence=False)
self.assertEqual(len(t), 1)
self.assertEqual(len(t[0]["files"]), 0)
def test_payloadstext(self):
t = MailAttachments.withhashes(self.attachments_thug)
t.run(filtercontenttypes=False, intelligence=False)
text = t.payloadstext()
self.assertIsNotNone(text)
self.assertNotIsInstance(text, list)
self.assertNotIsInstance(text, dict)
self.assertIn("Windows", text)
self.assertIn("setRequestHeader", text)
check_list = deque(maxlen=10)
md5 = "778cf2c48ab482d6134d4d12eb51192f"
check_list.append(md5)
self.assertIn("payload", t[0])
self.assertNotIn("is_filtered", t[0])
t.filter(check_list, hash_type="md5")
self.assertNotIn("payload", t[0])
text = t.payloadstext()
self.assertFalse(text)
def test_run(self):
t = MailAttachments.withhashes(self.attachments)
t(filtercontenttypes=False, intelligence=False)
for i in t:
self.assertIn("analisys_date", i)
self.assertIn("extension", i)
self.assertIn("size", i)
self.assertIn("Content-Type", i)
self.assertIn("is_archive", i)
self.assertIn("files", i)
self.assertEqual(i["extension"], ".zip")
self.assertTrue(i["is_archive"])
self.assertEqual(len(i["files"]), 1)
for j in i["files"]:
self.assertIn("analisys_date", j)
self.assertIn("filename", j)
self.assertIn("extension", j)
self.assertIn("size", j)
self.assertIn("Content-Type", j)
self.assertIn("payload", j)
self.assertIn("md5", j)
