def test_bom_encoded(self):
"""Test utf-8-sig encoded email"""
bom_email_object = EMailObject(
Path("tests/email_testfiles/mail_1_bom.eml"))
eml_email_object = EMailObject(
Path("tests/email_testfiles/mail_1.eml"))
self.assertIsInstance(bom_email_object.email, EmailMessage)
for file_name, file_content in bom_email_object.attachments:
self.assertIsInstance(file_name, str)
self.assertIsInstance(file_content, BytesIO)
self.assertEqual(
self._get_values(bom_email_object, "subject")[0],
self._get_values(eml_email_object, "subject")[0])
self.assertEqual(
self._get_values(bom_email_object, "to")[0],
self._get_values(eml_email_object, "to")[0])
self.assertEqual(
self._get_values(bom_email_object, "from")[0],
self._get_values(eml_email_object, "from")[0])
self.assertEqual(
self._get_values(bom_email_object, "from-display-name")[0],
self._get_values(eml_email_object, "from-display-name")[0])
self.assertEqual(len(self._get_values(bom_email_object, "email-body")),
1)
self.assertEqual(
self._get_values(bom_email_object, "received-header-ip"),
self._get_values(eml_email_object, "received-header-ip"))
def test_msg(self):
# Test result of eml converted to msg is the same
eml_email_object = EMailObject(
Path("tests/email_testfiles/mail_1.eml"))
email_object = EMailObject(Path("tests/email_testfiles/mail_1.msg"))
self.assertIsInstance(email_object.email, EmailMessage)
for file_name, file_content in email_object.attachments:
self.assertIsInstance(file_name, str)
self.assertIsInstance(file_content, BytesIO)
self.assertEqual(
self._get_values(email_object, "subject")[0],
self._get_values(eml_email_object, "subject")[0])
self.assertEqual(
self._get_values(email_object, "to")[0],
self._get_values(eml_email_object, "to")[0])
self.assertEqual(
self._get_values(email_object, "from")[0],
self._get_values(eml_email_object, "from")[0])
self.assertEqual(
self._get_values(email_object, "from-display-name")[0],
self._get_values(eml_email_object, "from-display-name")[0])
self.assertEqual(len(self._get_values(email_object, "email-body")), 2)
self.assertEqual(
self._get_values(email_object, "received-header-ip"),
self._get_values(eml_email_object, "received-header-ip"))
def test_mail_1_msg(self):
email_object = EMailObject(Path("tests/email_testfiles/mail_1.msg"))
self.assertEqual(self._get_values(email_object, "subject")[0],
"Newsletter Prüfung Personalwesen / Prüfung Eröffnungsbilanz")
self.assertIsInstance(email_object.email, EmailMessage)
for file_name, file_content in email_object.attachments:
self.assertIsInstance(file_name, str)
self.assertIsInstance(file_content, BytesIO)
def test_mail_multiple_to(self):
email_object = EMailObject(Path("tests/email_testfiles/mail_multiple_to.eml"))
to = self._get_values(email_object, "to")
to_display_name = self._get_values(email_object, "to-display-name")
self.assertEqual(to[0], "*****@*****.**")
self.assertEqual(to_display_name[0], "Novak, Jan")
self.assertEqual(to[1], "*****@*****.**")
self.assertEqual(to_display_name[1], "Marek, Jan")
def test_mail_1_headers_only(self):
email_object = EMailObject(Path("tests/email_testfiles/mail_1_headers_only.eml"))
self.assertEqual(self._get_values(email_object, "subject")[0], "письмо уведом-е")
self.assertEqual(self._get_values(email_object, "to")[0], "*****@*****.**")
self.assertEqual(self._get_values(email_object, "from")[0], "*****@*****.**")
self.assertEqual(len(self._get_values(email_object, "email-body")), 0)
self.assertIsInstance(email_object.email, EmailMessage)
self.assertEqual(len(email_object.attachments), 0)
def _does_not_fail(self, path, test_type="test"):
found_error = None
try:
EMailObject(path)
except Exception as _e:
found_error = _e
if found_error is not None:
self.fail(
'Error {} raised when parsing test email {} which tests against {}. It should not have raised an error.'
.format(type(found_error), path, test_type))
def test_random_binary_blob(self):
"""Email parser fails correctly on random binary blob."""
random_data = urandom(1024)
random_blob = BytesIO(random_data)
found_error = None
try:
broken_obj = EMailObject(pseudofile=random_data)
except Exception as _e:
found_error = _e
if not isinstance(found_error, InvalidMISPObject):
self.fail(
"Expected InvalidMISPObject when EmailObject receives completely unknown binary input data. But, did not get that exception."
)
try:
broken_obj = EMailObject(pseudofile=random_blob)
except Exception as _e:
found_error = _e
if not isinstance(found_error, PyMISPNotImplementedYet):
self.fail(
"Expected PyMISPNotImplementedYet when EmailObject receives completely unknown binary input data in a pseudofile. But, did not get that exception."
)
def forwarded_email(self, pseudofile: BytesIO):
'''Extracts all possible indicators out of an email and create a MISP event out of it.
* Gets all relevant Headers
* Attach the body
* Create MISP file objects (uses lief if possible)
* Set all references
'''
email_object = EMailObject(pseudofile=pseudofile,
attach_original_mail=True,
standalone=False)
if email_object.attachments:
# Create file objects for the attachments
for attachment_name, attachment in email_object.attachments:
if not (self.ignore_nullsize_attachments
and attachment.getbuffer().nbytes == 0):
if not attachment_name:
attachment_name = 'NameMissing.txt'
if self.config_from_email_body.get(
'attachment'
) == self.config.m2m_benign_attachment_keyword:
a = self.misp_event.add_attribute(
'attachment',
value=attachment_name,
data=attachment)
email_object.add_reference(a.uuid, 'related-to',
'Email attachment')
else:
f_object, main_object, sections = make_binary_objects(
pseudofile=attachment,
filename=attachment_name,
standalone=False)
if self.config.vt_key:
try:
vt_object = VTReportObject(
self.config.vt_key,
f_object.get_attributes_by_relation(
'sha256')[0].value,
standalone=False)
self.misp_event.add_object(vt_object)
f_object.add_reference(vt_object.uuid,
'analysed-with')
except InvalidMISPObject as e:
print(e)
pass
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
for section in sections:
self.misp_event.add_object(section)
email_object.add_reference(f_object.uuid, 'related-to',
'Email attachment')
self.process_body_iocs(email_object)
if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get(
'attach_original_mail'):
self.misp_event.add_object(email_object)
return email_object
def test_mail_1(self):
email_object = EMailObject(Path("tests/email_testfiles/mail_1.eml"))
self.assertEqual(self._get_values(email_object, "subject")[0], "письмо уведом-е")
self.assertEqual(self._get_values(email_object, "to")[0], "*****@*****.**")
self.assertEqual(self._get_values(email_object, "from")[0], "*****@*****.**")
self.assertEqual(self._get_values(email_object, "from-display-name")[0], "служба ФНС Даниил Суворов")
self.assertEqual(len(self._get_values(email_object, "email-body")), 1)
self.assertEqual(self._get_values(email_object, "received-header-ip")[0], "43.230.105.145")
self.assertEqual(self._get_values(email_object, "received-header-ip")[1], "2a01:111:f400:7e49::205")
self.assertIsInstance(email_object.email, EmailMessage)
for file_name, file_content in email_object.attachments:
self.assertIsInstance(file_name, str)
self.assertIsInstance(file_content, BytesIO)
def add_email_object():
entry_id = demisto.getArg('entry_id')
event_id = demisto.getArg('event_id')
email_path = demisto.getFilePath(entry_id).get('path')
obj = EMailObject(email_path)
add_object(event_id, obj)
import argparse
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description=
'Extract indicators out of binaries and add MISP objects to a MISP instance.'
)
parser.add_argument("-e",
"--event",
required=True,
help="Event ID to update.")
parser.add_argument("-p",
"--path",
required=True,
help="Path to process (expanded using glob).")
args = parser.parse_args()
pymisp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=True)
for f in glob.glob(args.path):
try:
eo = EMailObject(f)
except Exception:
traceback.print_exc()
continue
if eo:
response = pymisp.add_object(args.event, eo, pythonify=True)
for ref in eo.ObjectReference:
r = pymisp.add_object_reference(ref)
def handler(q=False):
if q is False:
return False
# Decode and parse email
request = json.loads(q)
# request data is always base 64 byte encoded
data = base64.b64decode(request["data"])
email_object = EMailObject(pseudofile=BytesIO(data),
attach_original_mail=True,
standalone=False)
# Check if we were given a configuration
config = request.get("config", {})
# Don't be picky about how the user chooses to say yes to these
acceptable_config_yes = ['y', 'yes', 'true', 't']
# Do we unzip attachments we find?
unzip = config.get("unzip_attachments", None)
if (unzip is not None and unzip.lower() in acceptable_config_yes):
unzip = True
# Do we try to find passwords for protected zip files?
zip_pass_crack = config.get("guess_zip_attachment_passwords", None)
if (zip_pass_crack is not None
and zip_pass_crack.lower() in acceptable_config_yes):
zip_pass_crack = True
password_list = get_zip_passwords(email_object.email)
# Do we extract URL's from the email.
extract_urls = config.get("extract_urls", None)
if (extract_urls is not None
and extract_urls.lower() in acceptable_config_yes):
extract_urls = True
file_objects = [] # All possible file objects
# Get Attachments
# Get file names of attachments
for attachment_name, attachment in email_object.attachments:
# Create file objects for the attachments
if not attachment_name:
attachment_name = 'NameMissing.txt'
temp_filename = Path(attachment_name)
zipped_files = [
"doc", "docx", "dot", "dotx", "xls", "xlsx", "xlm", "xla", "xlc",
"xlt", "xltx", "xlw", "ppt", "pptx", "pps", "ppsx", "pot", "potx",
"potx", "sldx", "odt", "ods", "odp", "odg", "odf", "fodt", "fods",
"fodp", "fodg", "ott", "uot"
]
# Attempt to unzip the attachment and return its files
if unzip and temp_filename.suffix[1:] not in zipped_files:
try:
unzip_attachement(attachment_name, attachment, email_object,
file_objects)
except RuntimeError: # File is encrypted with a password
if zip_pass_crack is True:
password = test_zip_passwords(attachment, password_list)
if password:
unzip_attachement(attachment_name, attachment,
email_object, file_objects, password)
else: # Inform the analyst that we could not crack password
f_object, main_object, sections = make_binary_objects(
pseudofile=attachment,
filename=attachment_name,
standalone=False)
f_object.comment = "Encrypted Zip: Password could not be cracked from message"
file_objects.append(f_object)
file_objects.append(main_object)
file_objects += sections
email_object.add_reference(f_object.uuid, 'includes',
'Email attachment')
except zipfile.BadZipFile: # Attachment is not a zipfile
# Just straight add the file
f_object, main_object, sections = make_binary_objects(
pseudofile=attachment,
filename=attachment_name,
standalone=False)
file_objects.append(f_object)
file_objects.append(main_object)
file_objects += sections
email_object.add_reference(f_object.uuid, 'includes',
'Email attachment')
else:
# Just straight add the file
f_object, main_object, sections = make_binary_objects(
pseudofile=attachment,
filename=attachment_name,
standalone=False)
file_objects.append(f_object)
file_objects.append(main_object)
file_objects += sections
email_object.add_reference(f_object.uuid, 'includes',
'Email attachment')
mail_body = email_object.email.get_body(preferencelist=('html', 'plain'))
if extract_urls:
if mail_body:
charset = mail_body.get_content_charset()
if mail_body.get_content_type() == 'text/html':
url_parser = HTMLURLParser()
url_parser.feed(
mail_body.get_payload(decode=True).decode(charset,
errors='ignore'))
urls = url_parser.urls
else:
urls = re.findall(
r'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+',
mail_body.get_payload(decode=True).decode(charset,
errors='ignore'))
for url in urls:
if not url:
continue
url_object = URLObject(url, standalone=False)
file_objects.append(url_object)
email_object.add_reference(url_object.uuid, 'includes',
'URL in email body')
objects = [email_object.to_json()]
if file_objects:
objects += [o.to_json() for o in file_objects if o]
r = {'results': {'Object': [json.loads(o) for o in objects]}}
return r