def test_meta_data(self):
"""Test meta data analysis."""
# Parsing mail
p = MailParser()
p.parse_from_file(mail)
# Init parameters
new_attachments = []
s = sp.SampleParser()
t = sp.TikaAnalysis(jar="/opt/tika/tika-app-1.13.jar")
# Parsing sample
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
if s.result:
new_attachments.append(s.result)
# Tika analysis
for i in new_attachments:
t.add_meta_data(i)
self.assertNotIn('tika', i)
t = sp.TikaAnalysis(jar="/opt/tika/tika-app-1.13.jar",
valid_content_types=["application/zip"])
for i in new_attachments:
t.add_meta_data(i)
self.assertIn('tika', i)
def test_add_analysis(self):
"""Test add VirusTotal analysis."""
# Parsing mail
p = MailParser()
p.parse_from_file(mail)
# Init parameters
new_attachments = []
s = sp.SampleParser()
v = sp.VirusTotalAnalysis(api_key=API_KEY)
# Parsing sample
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
if s.result:
new_attachments.append(s.result)
# VirusTotal analysis
for i in new_attachments:
v.add_analysis(i)
self.assertIn('virustotal', i)
for j in i["files"]:
self.assertIn('virustotal', j)
def test_parser_sample(self):
"""Test for parse_sample."""
parser = sp.SampleParser()
with open(sample_zip, 'rb') as f:
data = f.read()
with open(sample_txt, 'rb') as f:
data_txt_base64 = f.read().encode('base64')
parser.parse_sample(data, "test.zip")
result = parser.result
md5_file = "d41d8cd98f00b204e9800998ecf8427e"
size_file = 0
size_zip = 166
self.assertIsInstance(result, dict)
self.assertEqual(result['size'], size_zip)
self.assertIsNotNone(result['files'])
self.assertIsInstance(result['files'], list)
self.assertEqual(len(result['files']), 1)
self.assertEqual(result['files'][0]['size'], size_file)
self.assertEqual(result['files'][0]['md5'], md5_file)
self.assertEqual(result['files'][0]['filename'], "test.txt")
self.assertEqual(result['files'][0]['payload'], data_txt_base64)
def setUp(self):
self.f = utils.reformat_output
p = MailParser()
p.parse_from_file(mail)
self.mail_obj = p.parsed_mail_obj
self.mail_obj['analisys_date'] = datetime.datetime.utcnow().isoformat()
s = sp.SampleParser(
tika_enabled=True,
tika_jar="/opt/tika/tika-app-1.14.jar",
tika_memory_allocation=None,
tika_valid_content_types=['application/zip'])
self.attachments = []
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
self.attachments.append(s.result)
self.parameters = {
'elastic_index_mail': "spamscope_mails-",
'elastic_type_mail': "spamscope",
'elastic_index_attach': "spamscope_attachments-",
'elastic_type_attach': "spamscope"}
def test_tika(self):
"""Test for tika."""
parser = sp.SampleParser(tika_enabled=True,
tika_jar="/opt/tika/tika-app-1.13.jar",
tika_valid_content_types=["application/zip"])
with open(sample_zip, 'rb') as f:
data = f.read()
parser.parse_sample(data, "test.zip")
result = parser.result
self.assertIn('tika', result)
self.assertIsInstance(result['tika'], list)
self.assertIn('test.txt', result['tika'][0]['X-TIKA:content'])
def setUp(self):
# Init
p = MailParser()
s = sp.SampleParser()
self.virustotal = sp.VirusTotalProcessing(api_key=API_KEY)
# Parsing mail
p.parse_from_file(mail)
self.attachments = []
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
self.attachments.append(s.result)
def test_fingerprints(self):
"""Test fingerprints functions."""
parser = sp.SampleParser()
with open(sample_zip, 'rb') as f:
data = f.read()
data_base64 = data.encode('base64')
sha1_archive = "8760ff1422cf2922b649b796cfb58bfb0ccf7801"
# Test fingerprints
result1 = parser.fingerprints(data)[1]
self.assertEqual(sha1_archive, result1)
result2 = parser.fingerprints_from_base64(data_base64)[1]
self.assertEqual(sha1_archive, result2)
self.assertEqual(result1, result2)
def setUp(self):
# Init
p = MailParser()
s = sp.SampleParser()
self.thug = sp.ThugProcessing(referer="http://www.google.com/",
extensions=[".js"],
user_agents=["win7ie90", "winxpie80"])
# Parsing mail
p.parse_from_file(mail)
self.attachments = []
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
self.attachments.append(s.result)
def setUp(self):
# Init
p = MailParser()
s = sp.SampleParser()
self.tika = sp.TikaProcessing(jar="/opt/tika/tika-app-1.14.jar",
valid_content_types=['application/zip'],
memory_allocation=None)
# Parsing mail
p.parse_from_file(mail)
self.attachments = []
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
self.attachments.append(s.result)
def test_virustotal(self):
# Parsing mail
p = MailParser()
p.parse_from_file(mail)
# Init parameters
s = sp.SampleParser(virustotal_enabled=True,
virustotal_api_key=API_KEY)
# Parsing sample
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
self.assertIn('virustotal', s.result)
for j in s.result["files"]:
self.assertIn('virustotal', j)
def test_complete(self):
"""Test with all functions enabled. """
# Init
p = MailParser()
s = sp.SampleParser(
blacklist_content_types=[],
thug_enabled=True,
tika_enabled=True,
virustotal_enabled=True,
tika_jar="/opt/tika/tika-app-1.14.jar",
tika_memory_allocation=None,
tika_valid_content_types=['application/zip'],
virustotal_api_key=API_KEY,
thug_referer="http://www.google.com/",
thug_extensions=[".js"],
thug_user_agents=["winxpie80"])
# Parsing mail
p.parse_from_file(mail)
attachments = []
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
attachments.append(s.result)
for i in attachments:
self.assertIn("tika", i)
self.assertIn("virustotal", i)
self.assertNotIn("thug", i)
for j in i['files']:
self.assertNotIn("tika", j)
self.assertIn("virustotal", j)
self.assertIn("thug", j)
def test_is_archive(self):
"""Test is_archive functions."""
parser = sp.SampleParser()
with open(sample_zip, 'rb') as f:
data = f.read()
data_base64 = data.encode('base64')
sha1_archive = "8760ff1422cf2922b649b796cfb58bfb0ccf7801"
# Test is_archive with write_sample=False
result1 = parser.is_archive(data)
self.assertEqual(True, result1)
result2 = parser.is_archive_from_base64(data_base64)
self.assertEqual(True, result2)
self.assertEqual(result1, result2)
# Test is_archive with write_sample=True
result = parser.is_archive(data, write_sample=True)
self.assertEqual(os.path.exists(result[1]), True)
# Sample on disk
with open(result[1], 'rb') as f:
data_new = f.read()
result = parser.fingerprints(data_new)
self.assertEqual(sha1_archive, result[1])
result = parser.is_archive_from_base64(data_base64, write_sample=True)
self.assertEqual(True, result[0])
self.assertEqual(os.path.exists(result[1]), True)
# Test is_archive with write_sample=True
result = parser.is_archive(data, write_sample=True)
self.assertIsInstance(result, tuple)