def test_meta_data(self):
"""Test meta data analysis."""
# Parsing mail
p = MailParser()
p.parse_from_file(mail)
# Init parameters
new_attachments = []
s = sp.SampleParser()
t = sp.TikaAnalysis(jar="/opt/tika/tika-app-1.13.jar")
# Parsing sample
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
if s.result:
new_attachments.append(s.result)
# Tika analysis
for i in new_attachments:
t.add_meta_data(i)
self.assertNotIn('tika', i)
t = sp.TikaAnalysis(jar="/opt/tika/tika-app-1.13.jar",
valid_content_types=["application/zip"])
for i in new_attachments:
t.add_meta_data(i)
self.assertIn('tika', i)
def setUp(self):
self.f = utils.reformat_output
p = MailParser()
p.parse_from_file(mail)
self.mail_obj = p.parsed_mail_obj
self.mail_obj['analisys_date'] = datetime.datetime.utcnow().isoformat()
s = sp.SampleParser(
tika_enabled=True,
tika_jar="/opt/tika/tika-app-1.14.jar",
tika_memory_allocation=None,
tika_valid_content_types=['application/zip'])
self.attachments = []
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
self.attachments.append(s.result)
self.parameters = {
'elastic_index_mail': "spamscope_mails-",
'elastic_type_mail': "spamscope",
'elastic_index_attach': "spamscope_attachments-",
'elastic_type_attach': "spamscope"}
def test_add_analysis(self):
"""Test add VirusTotal analysis."""
# Parsing mail
p = MailParser()
p.parse_from_file(mail)
# Init parameters
new_attachments = []
s = sp.SampleParser()
v = sp.VirusTotalAnalysis(api_key=API_KEY)
# Parsing sample
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
if s.result:
new_attachments.append(s.result)
# VirusTotal analysis
for i in new_attachments:
v.add_analysis(i)
self.assertIn('virustotal', i)
for j in i["files"]:
self.assertIn('virustotal', j)
def send_mail(self, mail: mailparser.MailParser) -> bool:
subject = mail.subject
sender_email_raw = mail.from_[0][1]
if len(self.whitelist) > 0 and sender_email_raw not in self.whitelist:
logger.info("Ignoring email from: %s", sender_email_raw)
return False
sender_email = " ".join(mail.from_[0])
title = "{} from {}".format(subject, sender_email)
logger.info("Sending mail: %s", title)
if mail.text_html:
text_html = "\n".join(mail.text_html)
f = io.BytesIO(text_html.encode())
f.name = title + ".html"
self.bot.send_document(
self.chat,
f,
caption=title,
)
elif mail.text_plain:
message = "\n".join(mail.text_plain)
self.bot.send_message(self.chat, split(message)[0])
if len(message) >= 2:
f = io.BytesIO("{}\n\n{}".format(title, message).encode())
f.name = title + ".txt"
self.bot.send_document(self.chat, f, caption=title)
if mail.attachments:
with tempfile.TemporaryDirectory() as tmp_dir:
mail.write_attachments(tmp_dir)
for file in os.listdir(tmp_dir):
self.bot.send_document(
self.chat, open(os.path.join(tmp_dir, file), "rb"))
return True
def list_last_emails(self, nb=20):
"""List last messages"""
res, data = self.connection.search(None, 'ALL')
if res != "OK":
logger.error("Unable to fetch mails")
return []
lst = []
for mail_id in data[0].split()[-nb:]:
res, flags_data = self.connection.fetch(mail_id, '(FLAGS)')
if res != "OK":
logger.error(
"Unable to fetch flags for mail {0}".format(mail_id))
continue
flags = imaplib.ParseFlags(flags_data[0])
mail = self.get_mail(mail_id)
if not mail:
continue
parser = Parser(mail)
parsed_orig_mail = MailParser(mail)
if isinstance(mail_id, bytes):
mail_id = mail_id.decode()
flags = [fl.decode() for fl in flags]
lst.append(u"{}: '{}', '{}', {}, ''{}".format(
mail_id, parsed_orig_mail.subject, parser.headers['Subject'],
parsed_orig_mail.headers['From'], flags))
logger.info(lst[-1])
return lst
def __init__(self, message):
"""
:type message: email.message.Message
"""
self.initial_message = message
self.message = self._extract_relevant_message(message)
self.parsed_message = MailParser(self.message)
def setUp(self):
self.f = utils.reformat_output
p = MailParser()
p.parse_from_file(mail)
self.mail_obj = p.parsed_mail_obj
self.mail_obj['analisys_date'] = datetime.datetime.utcnow().isoformat()
self.attachments = MailAttachments.withhashes(p.attachments_list)
self.attachments.run()
self.parameters = {
'elastic_index_mail': "spamscope_mails-",
'elastic_type_mail': "spamscope",
'elastic_index_attach': "spamscope_attachments-",
'elastic_type_attach': "spamscope"
}
def initialize(self, stormconf, context):
super(Tokenizer, self).initialize(stormconf, context)
self._parser = MailParser()
self._mails_analyzed = deque(maxlen=self.conf["maxlen_mails"])
self._attachments_analyzed = deque(
maxlen=self.conf["maxlen_attachments"])
self._load_filters()
def setUp(self):
# Init
p = MailParser()
s = sp.SampleParser()
self.virustotal = sp.VirusTotalProcessing(api_key=API_KEY)
# Parsing mail
p.parse_from_file(mail)
self.attachments = []
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
self.attachments.append(s.result)
def setUp(self):
# Init
p = MailParser()
s = sp.SampleParser()
self.thug = sp.ThugProcessing(referer="http://www.google.com/",
extensions=[".js"],
user_agents=["win7ie90", "winxpie80"])
# Parsing mail
p.parse_from_file(mail)
self.attachments = []
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
self.attachments.append(s.result)
def headers(self):
headers = {
"From": self.parsed_message.from_,
"To": self.parsed_message.to,
"Cc": self.parsed_message.cc,
"Subject": self.parsed_message.subject,
"Origin": self.origin,
}
if self.origin == 'Agent forward':
headers['Agent'] = MailParser(self.initial_message).from_
return headers
def setUp(self):
# Init
p = MailParser()
s = sp.SampleParser()
self.tika = sp.TikaProcessing(jar="/opt/tika/tika-app-1.14.jar",
valid_content_types=['application/zip'],
memory_allocation=None)
# Parsing mail
p.parse_from_file(mail)
self.attachments = []
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
self.attachments.append(s.result)
def test_virustotal(self):
# Parsing mail
p = MailParser()
p.parse_from_file(mail)
# Init parameters
s = sp.SampleParser(virustotal_enabled=True,
virustotal_api_key=API_KEY)
# Parsing sample
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
self.assertIn('virustotal', s.result)
for j in s.result["files"]:
self.assertIn('virustotal', j)
def test_complete(self):
"""Test with all functions enabled. """
# Init
p = MailParser()
s = sp.SampleParser(
blacklist_content_types=[],
thug_enabled=True,
tika_enabled=True,
virustotal_enabled=True,
tika_jar="/opt/tika/tika-app-1.14.jar",
tika_memory_allocation=None,
tika_valid_content_types=['application/zip'],
virustotal_api_key=API_KEY,
thug_referer="http://www.google.com/",
thug_extensions=[".js"],
thug_user_agents=["winxpie80"])
# Parsing mail
p.parse_from_file(mail)
attachments = []
for i in p.attachments_list:
s.parse_sample_from_base64(
data=i['payload'],
filename=i['filename'],
mail_content_type=i['mail_content_type'],
transfer_encoding=i['content_transfer_encoding'])
attachments.append(s.result)
for i in attachments:
self.assertIn("tika", i)
self.assertIn("virustotal", i)
self.assertNotIn("thug", i)
for j in i['files']:
self.assertNotIn("tika", j)
self.assertIn("virustotal", j)
self.assertIn("thug", j)
def send_email_telegram(bot: TeleBot, chat_id: str,
mail: mailparser.MailParser):
subject = mail.subject
from_ = " ".join(mail.from_[0])
mail_name = "{} от {}".format(subject, from_)
if mail.text_plain: # Если существует текстовая версия письма
message = "\n".join(mail.text_plain)
bot.send_message(
chat_id,
split(message)[0]) # Отправляем ее первые 4092 символа
# И высылаем файл письма целиком (если письмо содержит более 4092 символов)
if len(message) >= 2:
f = io.BytesIO("{}\n\n{}".format(mail_name, message).encode())
f.name = mail_name + ".txt"
bot.send_document(chat_id, f, caption="Полный текст письма")
if mail.text_html: # Если существует веб версия письма, то отправляем ее файлом
text_html = "\n".join(mail.text_html)
f = io.BytesIO(text_html.encode())
f.name = mail_name + ".html"
bot.send_document(
chat_id,
f,
caption="{} (веб версия письма)".format("{} от {}".format(
subject, from_)),
)
if mail.attachments: # Если есть вложения в письме, то отправляем и их.
with tempfile.TemporaryDirectory(
) as tmp_dir: # Создание временной директории для хранения вложений
mail.write_attachments(tmp_dir) # Сохраняем вложения
for file in os.listdir(tmp_dir):
bot.send_document(chat_id,
open(os.path.join(tmp_dir, file),
"rb")) # Отправляем вложения
def setUp(self):
# Init
p = MailParser()
p.parse_from_file(mail)
self.attachments = p.attachments_list
p.parse_from_file(mail_thug)
self.attachments_thug = p.attachments_list
def main():
args = get_args()
parser = MailParser()
if args.file:
parser.parse_from_file(args.file)
elif args.string:
parser.parse_from_string(args.string)
if args.json:
j = json.loads(parser.parsed_mail_json)
safe_print(json.dumps(j, ensure_ascii=False, indent=4))
if args.body:
# safe_print(parser.body)
safe_print(parser.body)
if args.headers:
safe_print(parser.headers)
if args.to:
safe_print(parser.to_)
if args.from_:
safe_print(parser.from_)
if args.subject:
safe_print(parser.subject)
if args.defects:
for i in parser.defects_category:
safe_print(i)
if args.anomalies:
for i in parser.anomalies:
safe_print(i)
if args.senderip:
r = parser.get_server_ipaddress(args.senderip)
if r:
safe_print(r)
else:
safe_print("Not Found")
if args.attachments:
for i in parser.attachments_list:
safe_print(json.dumps(i, ensure_ascii=False, indent=4))
def main():
args = get_args()
parser = MailParser()
if args.file:
parser.parse_from_file(args.file)
elif args.string:
parser.parse_from_string(args.string)
if args.json:
j = json.loads(parser.parsed_mail_json)
print(json.dumps(j, ensure_ascii=False, indent=4).encode('utf-8'))
if args.body:
print(parser.body.encode('utf-8'))
if args.headers:
print(parser.headers.encode('utf-8'))
if args.to:
print(parser.to_.encode('utf-8'))
if args.from_:
print(parser.from_.encode('utf-8'))
if args.subject:
print(parser.subject.encode('utf-8'))
if args.defects:
for i in parser.defects_category:
print(i.encode('utf-8'))
if args.anomalies:
for i in parser.anomalies:
print(i.encode('utf-8'))
if args.senderip:
r = parser.get_server_ipaddress(args.senderip)
if r:
print(r.encode('utf-8'))
else:
print("Not Found")
if args.attachments:
for i in parser.attachments_list:
print(json.dumps(i, ensure_ascii=False, indent=4).encode('utf-8'))
def get_words_from_mail(maildir, mail):
# clear corpus (and voca), because such a big data is not needed in forked processes.
corpus.clear()
# with open('/dev/tty', 'w') as f:
# print(f'>>> {mail.path}', file=f)
mail_parser = MailParser(maildir, mail)
mail_parser.read_binary()
mail_parser.parse()
words = set()
for txt in mail.texts:
if txt:
words |= tokenizer.get_words(txt)
return words
def daemon():
global parser, Config, logger
# Open configuration file
try:
with open('config.json', encoding='utf8') as config_file:
Config = json.load(config_file)
except:
logger.error('Unable to open config.json!\n\
If you haven\'t created a configuration file, please use the example configuration file as a guide.\n\
Otherwise, make sure the file permissions are set correctly.')
exit(1)
logger.debug('Daemon time!')
logger.info('Starting Canary Server sync daemon service...')
# setup MailParser
parser = MailParser()
# setup scheduler
s = sched.scheduler(time.time, time.sleep)
sync(s, Config['sync_interval'])
s.run()
label.append(strt[0])
for char in strt:
if(ord(char)==10):
if(i<length-1):
label.append(strt[i+1])
if(len(label)==41):
break
i = i + 1
print(i)
print(label)
parser = MailParser()
spam = ""
ham = ""
no_of_spam=0
no_of_ham=0
i = 0
for lab in label:
a = str(i)
if(i<10):
a = "00"+a
elif(i < 100):
a = "0" +a
rows = cursor.fetchall()
return rows
except Error as e:
#print(e)
logger.error(e)
finally:
cursor.close()
conn.close()
if __name__ == "__main__":
try:
content = sys.stdin.read()
parser = MailParser()
parser.parse_from_string(content)
except:
logger.error(traceback.format_exc())
sys.exit()
data = {}
sfrom = email.utils.parseaddr(parser.from_)
ssubject = parser.subject
if sfrom[0]:
sfrom = sfrom[0]
if '=?utf-8?' in sfrom or '=?UTF-8?' in sfrom:
sfrom = email.Header.decode_header(sfrom)[0][0]
else:
def textExtraction(f):
parser = MailParser()
raw_mail = parser.parse_from_file(f)
body = parser.body
return body
def parse_dir(dir):
source_top_directory = dir
files = getAllFiles(source_top_directory)
dest_top_directory = "./output/"
dest_directory = dest_top_directory + source_top_directory[1:]
try:
makedirs(dest_top_directory)
print("Output top directory check: Created")
except:
print("Output top directory check: Exists")
try:
makedirs(dest_directory)
print("Output inner directory check: Created")
except:
print("Output inner directory check: Exists")
from time import sleep
delay_time = 5
print("Parsing will start in", delay_time, "seconds")
sleep(delay_time)
parser = MailParser()
success = 0
fail = 0
i = 0
for file in files:
print("file to be parsed:")
print(file)
try:
parser.parse_from_file(file)
success += 1
i += 1
except:
print("file cannot be parsed")
output_file_path = "./PreProcessed" + file[1:]
try:
remove(output_file_path)
except:
print("already removed")
fail += 1
continue
mail_subject = str(parser.subject)
mail_from = str(parser.from_)
mail_body = str(parser.body)
# print(mail_body)
init_count = countWords(mail_body)
mail_body = removePunc(mail_body)
# print("\n\n-----Removed Punc -----\n",mail_body)
try:
mail_body_words = stemWords(mail_body)
except:
pass
# print("\n\n-----Stemmed-----\n",mail_body_words)
mail_body_words = removeStopWords(mail_body_words)
# print("\n\n-----Removed Stop Words-----\n",mail_body_words)
final_count = len(mail_body_words)
# print("\nPre-Processing Done.. Reduced words count from:",init_count,"to: ",final_count)
output_file_path = dest_directory + str(i)
print("Parsing Complete")
print(file, "parsed to output file: ", output_file_path, "\n\n")
of = open(output_file_path, 'w')
for word in mail_body_words:
try:
of.write(str(word) + " ")
except:
continue
of.close()
print("Parsed: ", success)
print("Failed: ", fail)
spam[scol1v[i].value] = scol2p[i].value
wb = load_workbook(filename='ham.xlsx', read_only=True)
ws = wb['Sheet1']
allcolumns = ws.columns
hcol1v = allcolumns[0]
hcol2p = allcolumns[1]
ham = {}
for i in list(range(len(hcol1v))):
ham[hcol1v[i].value] = hcol2p[i].value
parser = MailParser()
parser.parse_from_file("TRAINING/TRAIN_00549.eml")
body = parser.body
tokens = Tokenizer.freqdata(body)
PR_wi_S = 1
#for spam
for keys in tokens:
if keys in spam.keys():
if (float(spam.get(keys)) > 0):
PR_wi_S = PR_wi_S * (float(spam.get(keys)) * tokens.get(keys))
PR_wi_H = 1
def setUp(self):
# Init
self.parser = MailParser()
class TestMailParser(unittest.TestCase):
def setUp(self):
# Init
self.parser = MailParser()
def test_ipaddress(self):
self.parser.parse_from_file(mail_test_2)
trust = "smtp.customers.net"
ip = "217.76.210.112"
result = self.parser.get_server_ipaddress(trust)
self.assertEqual(result, ip)
trust = ""
result = self.parser.get_server_ipaddress(trust)
self.assertEqual(result, None)
trust = " "
result = self.parser.get_server_ipaddress(trust)
self.assertEqual(result, None)
def test_fingerprints_body(self):
self.parser.parse_from_file(mail_test_1)
md5, sha1, sha256, sha512 = fingerprints(
self.parser.body.encode("utf-8"))
self.assertEqual(md5, "1bbdb7dcf511113bbc0c1b214aeac392")
self.assertEqual(sha1, "ce9e62b50fa4e2168278880b14460b905b24eb4b")
self.assertEqual(sha256, ("1e9b96e3f1bc74702f9703391e8ba0715b849"
"7127a7ff857013ab33385898574"))
self.assertEqual(sha512, ("ad858f7b5ec5549e55650fd13df7683e403489"
"77522995851fb6b625ac54744cf3a4bf652784"
"dba971ef99afeec4e6caf2fdd10be72eabb730"
"c312ffbe1c4de3"))
def test_malformed_mail(self):
self.parser.parse_from_file(mail_malformed_3)
defects_category = self.parser.defects_category
self.assertIn("StartBoundaryNotFoundDefect", defects_category)
self.assertIn("MultipartInvariantViolationDefect", defects_category)
def test_type_error(self):
self.parser.parse_from_file(mail_test_5)
self.assertEqual(len(self.parser.attachments_list), 5)
for i in self.parser.attachments_list:
self.assertIsInstance(i["filename"], six.text_type)
def test_valid_mail(self):
with self.assertRaises(InvalidMail):
self.parser.parse_from_string("fake mail")
def test_valid_date_mail(self):
self.parser.parse_from_file(mail_test_1),
self.assertIn("mail_without_date", self.parser.anomalies)
def test_parsing_know_values(self):
self.parser.parse_from_file(mail_test_2)
trust = "smtp.customers.net"
self.assertEqual(False, self.parser.has_defects)
raw = "217.76.210.112"
result = self.parser.get_server_ipaddress(trust)
self.assertEqual(raw, result)
raw = "<*****@*****.**>"
result = self.parser.message_id
self.assertEqual(raw, result)
raw = "mporcile@server_mail.it"
result = self.parser.to_
self.assertEqual(raw, result)
raw = "<*****@*****.**>"
result = self.parser.from_
self.assertEqual(raw, result)
raw = "Bollettino Meteorologico del 29/11/2015"
result = self.parser.subject
self.assertEqual(raw, result)
result = self.parser.has_defects
self.assertEqual(False, result)
result = len(self.parser.attachments_list)
self.assertEqual(3, result)
raw = "Sun, 29 Nov 2015 09:45:18 +0100"
raw_utc = datetime.datetime(2015, 11, 29, 8, 45, 18, 0).isoformat()
result = self.parser.date_mail.isoformat()
self.assertEqual(raw_utc, result)
def test_types(self):
self.parser.parse_from_file(mail_test_2)
trust = "smtp.customers.net"
self.assertEqual(False, self.parser.has_defects)
result = self.parser.parsed_mail_obj
self.assertIsInstance(result, dict)
self.assertNotIn("defects", result)
self.assertNotIn("anomalies", result)
self.assertIn("has_defects", result)
self.assertIn("has_anomalies", result)
result = self.parser.get_server_ipaddress(trust)
self.assertIsInstance(result, six.text_type)
result = self.parser.parsed_mail_json
self.assertIsInstance(result, six.text_type)
result = self.parser.headers
self.assertIsInstance(result, six.text_type)
result = self.parser.body
self.assertIsInstance(result, six.text_type)
result = self.parser.date_mail
self.assertIsInstance(result, datetime.datetime)
result = self.parser.from_
self.assertIsInstance(result, six.text_type)
result = self.parser.to_
self.assertIsInstance(result, six.text_type)
result = self.parser.subject
self.assertIsInstance(result, six.text_type)
result = self.parser.message_id
self.assertIsInstance(result, six.text_type)
result = self.parser.attachments_list
self.assertIsInstance(result, list)
result = self.parser.date_mail
self.assertIsInstance(result, datetime.datetime)
result = self.parser.defects
self.assertIsInstance(result, list)
result = self.parser.anomalies
self.assertIsInstance(result, list)
def test_defects_anomalies(self):
self.parser.parse_from_file(mail_malformed_1)
self.assertEqual(True, self.parser.has_defects)
self.assertEqual(1, len(self.parser.defects))
self.assertEqual(1, len(self.parser.defects_category))
self.assertIn("defects", self.parser.parsed_mail_obj)
self.assertIn("StartBoundaryNotFoundDefect",
self.parser.defects_category)
self.assertIsInstance(self.parser.parsed_mail_json, six.text_type)
result = len(self.parser.attachments_list)
self.assertEqual(1, result)
self.parser.parse_from_file(mail_test_1)
if six.PY2:
self.assertEqual(False, self.parser.has_defects)
self.assertNotIn("defects", self.parser.parsed_mail_obj)
elif six.PY3:
self.assertEqual(True, self.parser.has_defects)
self.assertEqual(1, len(self.parser.defects))
self.assertEqual(1, len(self.parser.defects_category))
self.assertIn("defects", self.parser.parsed_mail_obj)
self.assertIn(
"CloseBoundaryNotFoundDefect", self.parser.defects_category)
self.assertEqual(True, self.parser.has_anomalies)
self.assertEqual(2, len(self.parser.anomalies))
self.assertIn("anomalies", self.parser.parsed_mail_obj)
self.assertIn("has_anomalies", self.parser.parsed_mail_obj)
def test_defects_bug(self):
self.parser.parse_from_file(mail_malformed_2)
self.assertEqual(True, self.parser.has_defects)
self.assertEqual(1, len(self.parser.defects))
self.assertEqual(1, len(self.parser.defects_category))
self.assertIn("defects", self.parser.parsed_mail_obj)
self.assertIn("StartBoundaryNotFoundDefect",
self.parser.defects_category)
self.assertIsInstance(self.parser.parsed_mail_json, six.text_type)
result = len(self.parser.attachments_list)
self.assertEqual(0, result)
def test_add_content_type(self):
self.parser.parse_from_file(mail_test_3)
self.assertEqual(False, self.parser.has_defects)
result = self.parser.parsed_mail_obj
self.assertEqual(len(result["attachments"]), 1)
self.assertIsInstance(
result["attachments"][0]["mail_content_type"], six.text_type)
self.assertIsInstance(
result["attachments"][0]["payload"], six.text_type)
self.assertEqual(
result["attachments"][0]["content_transfer_encoding"],
"quoted-printable")
def message_to_obj(msg):
msg_obj = MailParser(msg)
if msg_obj.mail.get("date"):
msg_obj.mail["date"] = msg_obj.date.isoformat()
return msg_obj.mail