def __create_cybox_headers(self, msg):
""" Returns a CybOX EmailHeaderType object """
if self.__verbose_output:
sys.stderr.write("** parsing headers\n")
headers = EmailHeader()
if 'received' in self.headers:
headers.received_lines = self._parse_received_headers(msg)
if 'to' in self.headers:
headers.to = _get_email_recipients(msg['to'])
if msg['delivered-to'] and not headers.to:
headers.to = _get_email_recipients(msg['delivered-to'])
if 'cc' in self.headers:
headers.cc = _get_email_recipients(msg['cc'])
if 'bcc' in self.headers:
headers.bcc = _get_email_recipients(msg['bcc'])
if 'from' in self.headers:
headers.from_ = _get_single_email_address(msg['from'])
if 'sender' in self.headers:
headers.sender = _get_single_email_address(msg['sender'])
if 'reply-to' in self.headers:
headers.reply_to = _get_single_email_address(msg['reply-to'])
if 'subject' in self.headers:
headers.subject = String(msg['subject'])
if 'in-reply-to' in self.headers:
headers.in_reply_to = String(msg['in-reply-to'])
if 'errors-to' in self.headers:
headers.errors_to = String(msg['errors-to'])
if 'date' in self.headers:
headers.date = DateTime(msg['date'])
if 'message-id' in self.headers:
headers.message_id = String(msg['message-id'])
if 'boundary' in self.headers:
headers.boundary = String(msg['boundary'])
if 'content-type' in self.headers:
headers.content_type = String(msg['content-type'])
if 'mime-version' in self.headers:
headers.mime_version = String(msg['mime-version'])
if 'precedence' in self.headers:
headers.precedence = String(msg['precedence'])
if 'user-agent' in self.headers:
headers.user_agent = String(msg['user-agent'])
if 'x-mailer' in self.headers:
headers.x_mailer = String(msg['x-mailer'])
if 'x-originating-ip' in self.headers:
headers.x_originating_ip = Address(msg['x-originating-ip'],
Address.CAT_IPV4)
if 'x-priority' in self.headers and 'x-priority' in msg:
#Must be a digit - pull one out of anything that could be a string such as 3 (Normal)
import re
priority = ''
for p in re.findall(r'\d+',msg['x-priority']):
if p.isdigit():
priority = p
if priority:
headers.x_priority = String(priority)
return headers
def test_list_of_dates_in_email_header(self):
from cybox.objects.email_message_object import EmailHeader
hdr = EmailHeader()
hdr.date = [datetime.datetime.now(), datetime.datetime.now()]
hdr.date.condition = "Equals"
hdr_xml = hdr.to_xml()
self.assertTrue(b'EmailMessageObj:Date condition="Equals"' in hdr_xml)
self.assertTrue(b"##comma##" in hdr_xml)
def test_list_of_dates_in_email_header(self):
from cybox.objects.email_message_object import EmailHeader
hdr = EmailHeader()
hdr.date = [datetime.datetime.now(), datetime.datetime.now()]
hdr.date.condition = "Equals"
hdr_xml = hdr.to_xml()
self.assertTrue(b'EmailMessageObj:Date condition="Equals"' in hdr_xml)
self.assertTrue(b"##comma##" in hdr_xml)
def __create_cybox_headers(self, msg):
""" Returns a CybOX EmailHeaderType object """
if self.__verbose_output:
sys.stderr.write("** parsing headers\n")
headers = EmailHeader()
if 'received' in self.headers:
headers.received_lines = self._parse_received_headers(msg)
if 'to' in self.headers:
headers.to = _get_email_recipients(msg['to'])
if 'cc' in self.headers:
headers.cc = _get_email_recipients(msg['cc'])
if 'bcc' in self.headers:
headers.bcc = _get_email_recipients(msg['bcc'])
if 'from' in self.headers:
headers.from_ = _get_single_email_address(msg['from'])
if 'sender' in self.headers:
headers.sender = _get_single_email_address(msg['sender'])
if 'reply-to' in self.headers:
headers.reply_to = _get_single_email_address(msg['reply-to'])
if 'subject' in self.headers:
headers.subject = String(msg['subject'])
if 'in-reply-to' in self.headers:
headers.in_reply_to = String(msg['in-reply-to'])
if 'errors-to' in self.headers:
headers.errors_to = String(msg['errors-to'])
if 'date' in self.headers:
headers.date = DateTime(msg['date'])
if 'message-id' in self.headers:
headers.message_id = String(msg['message-id'])
if 'boundary' in self.headers:
headers.boundary = String(msg['boundary'])
if 'content-type' in self.headers:
headers.content_type = String(msg['content-type'])
if 'mime-version' in self.headers:
headers.mime_version = String(msg['mime-version'])
if 'precedence' in self.headers:
headers.precedence = String(msg['precedence'])
if 'user-agent' in self.headers:
headers.user_agent = String(msg['user-agent'])
if 'x-mailer' in self.headers:
headers.x_mailer = String(msg['x-mailer'])
if 'x-originating-ip' in self.headers:
headers.x_originating_ip = Address(msg['x-originating-ip'],
Address.CAT_IPV4)
if 'x-priority' in self.headers:
headers.x_priority = String(msg['x-priority'])
return headers
def test_creation(self):
header = EmailHeader()
self.assertEqual(None, header.subject)
header.subject = "Howdy"
self.assertEqual(String, type(header.subject))
self.assertEqual(None, header.date)
header.date = "2010-12-10T14:15:30+02:00"
self.assertEqual(DateTime, type(header.date))
self.assertEqual(datetime.datetime, type(header.date.value))
self.assertEqual(None, header.message_id)
header.message_id = "<*****@*****.**>"
self.assertEqual(String, type(header.message_id))
def test_creation(self):
header = EmailHeader()
self.assertEqual(None, header.subject)
header.subject = "Howdy"
self.assertEqual(String, type(header.subject))
self.assertEqual(None, header.date)
header.date = "2010-12-10T14:15:30+02:00"
self.assertEqual(DateTime, type(header.date))
self.assertEqual(datetime.datetime, type(header.date.value))
self.assertEqual(None, header.message_id)
header.message_id = "<*****@*****.**>"
self.assertEqual(String, type(header.message_id))
def __create_cybox_headers(self, msg):
""" Returns a CybOX EmailHeaderType object """
if self.__verbose_output:
sys.stderr.write("** parsing headers\n")
headers = EmailHeader()
if 'received' in self.headers:
lines = self._parse_received_headers(msg)
if lines:
headers.received_lines = lines
if 'to' in self.headers:
headers.to = _get_email_recipients(msg['to'])
if msg['delivered-to'] and not headers.to:
headers.to = _get_email_recipients(msg['delivered-to'])
if 'cc' in self.headers:
headers.cc = _get_email_recipients(msg['cc'])
if 'bcc' in self.headers:
headers.bcc = _get_email_recipients(msg['bcc'])
if 'from' in self.headers:
headers.from_ = _get_single_email_address(msg['from'])
if 'sender' in self.headers:
headers.sender = _get_single_email_address(msg['sender'])
if 'reply-to' in self.headers:
headers.reply_to = _get_single_email_address(msg['reply-to'])
if 'subject' in self.headers and 'subject' in msg:
headers.subject = String(msg['subject'])
if 'in-reply-to' in self.headers and 'in-reply-to' in msg:
headers.in_reply_to = String(msg['in-reply-to'])
if 'errors-to' in self.headers and 'errors-to' in msg:
headers.errors_to = String(msg['errors-to'])
if 'date' in self.headers and 'date' in msg:
headers.date = DateTime(msg['date'])
if 'message-id' in self.headers and 'message-id' in msg:
headers.message_id = String(msg['message-id'])
if 'boundary' in self.headers and 'boundary' in msg:
headers.boundary = String(msg['boundary'])
if 'content-type' in self.headers and 'content-type' in msg:
headers.content_type = String(msg['content-type'])
if 'mime-version' in self.headers and 'mime-version' in msg:
headers.mime_version = String(msg['mime-version'])
if 'precedence' in self.headers and 'precedence' in msg:
headers.precedence = String(msg['precedence'])
if 'user-agent' in self.headers and 'user-agent' in msg:
headers.user_agent = String(msg['user-agent'])
if 'x-mailer' in self.headers and 'x-mailer' in msg:
headers.x_mailer = String(msg['x-mailer'])
if 'x-originating-ip' in self.headers and msg['x-originating-ip']:
headers.x_originating_ip = Address(msg['x-originating-ip'],
Address.CAT_IPV4)
if 'x-priority' in self.headers and 'x-priority' in msg:
#Must be a digit - pull one out of anything that could be a string such as 3 (Normal)
import re
priority = ''
for p in re.findall(r'\d+',msg['x-priority']):
if p.isdigit():
priority = p
if priority:
headers.x_priority = String(priority)
return headers
def execute(self, device_info, extracted_data_dir_path):
original_app_path = '/data/data/com.android.email'
headers_db_rel_file_path = os.path.join('databases', 'EmailProvider.db')
bodies_db_rel_file_path = os.path.join('databases', 'EmailProviderBody.db')
original_headers_db_file_path = os.path.join(original_app_path, headers_db_rel_file_path)
original_bodies_db_file_path = os.path.join(original_app_path, bodies_db_rel_file_path)
headers_db_file_path = os.path.join(extracted_data_dir_path, headers_db_rel_file_path)
bodies_db_file_path = os.path.join(extracted_data_dir_path, bodies_db_rel_file_path)
source_objects = [
create_file_object(headers_db_file_path, original_headers_db_file_path),
create_file_object(bodies_db_file_path, original_bodies_db_file_path)
]
inspected_objects = {}
cursor, conn = execute_query(headers_db_file_path, 'SELECT * FROM message')
for row in cursor:
header = EmailHeader()
header.to = row['toList']
header.cc = row['ccList']
header.bcc = row['bccList']
header.from_ = row['fromList']
header.subject = row['subject']
header.in_reply_to = row['replyToList']
header.date = datetime.fromtimestamp(row['timeStamp'] / 1000) # Convert from milliseconds to seconds
header.message_id = row['messageId']
email = EmailMessage()
email.header = header
email.add_related(source_objects[0], ObjectRelationship.TERM_EXTRACTED_FROM, inline=False)
# Add the email to the inspected_objects dict using its _id value as key.
email_id = row['_id']
inspected_objects[email_id] = email
cursor.close()
conn.close()
# Add full raw body to emails.
cursor, conn = execute_query(bodies_db_file_path, 'SELECT _id, htmlContent, textContent FROM body')
for row in cursor:
email_id = row['_id']
email = inspected_objects.get(email_id)
if email is not None:
if row['htmlContent'] != '':
email.raw_body = row['htmlContent']
email.header.content_type = 'text/html'
else:
email.raw_body = row['textContent']
email.header.content_type = 'text/plain'
email.add_related(source_objects[1], ObjectRelationship.TERM_EXTRACTED_FROM, inline=False)
cursor.close()
# Add attachments to emails.
cursor, conn = execute_query(headers_db_file_path,
'SELECT messageKey, contentUri FROM attachment')
# Iteration over attachments
for row in cursor:
# Get current attachment email_id.
email_id = row['messageKey']
# Find email in inspected_objects.
email = inspected_objects.get(email_id)
# If email has non attachments, initialize them.
if email.attachments is None:
email.attachments = Attachments()
# Using contentUri, get attachment folder_prefix and file_name.
attachment_rel_path_dirs = re.search('.*//.*/(.*)/(.*)/.*', row['contentUri'])
# Group(1): contains attachment folder.
# Group(2): contains attachment file_name.
attachment_rel_file_path = os.path.join('databases', attachment_rel_path_dirs.group(1) + '.db_att',
attachment_rel_path_dirs.group(2))
# Build attachment absolute file path in extracted_data.
attachment_file_path = os.path.join(extracted_data_dir_path, attachment_rel_file_path)
# Build attachment original file_path in device.
original_attachment_file_path = os.path.join(original_app_path, attachment_rel_file_path)
# Create attachment source_file.
attachment = create_file_object(attachment_file_path, original_attachment_file_path)
# Add attachment to email's attachments.
email.attachments.append(attachment.parent.id_)
# Add relation between attachment and it's email.
attachment.add_related(email, ObjectRelationship.TERM_CONTAINED_WITHIN, inline=False)
source_objects.append(attachment)
cursor.close()
conn.close()
return inspected_objects.values(), source_objects
def cybox_object_email(obj):
e = EmailMessage()
e.raw_body = obj.raw_body
e.raw_header = obj.raw_header
# Links
e.links = Links()
for link in obj.links.all():
pass
# Attachments
e.attachments = Attachments()
attachment_objects = []
for att in obj.attachments.all():
for meta in att.file_meta.all():
fobj = cybox_object_file(att, meta)
e.attachments.append(fobj.parent.id_)
fobj.add_related(e, "Contained_Within", inline=False)
attachment_objects.append(fobj)
# construct header information
h = EmailHeader()
h.subject = obj.subject
h.date = obj.email_date
h.message_id = obj.message_id
h.content_type = obj.content_type
h.mime_version = obj.mime_version
h.user_agent = obj.user_agent
h.x_mailer = obj.x_mailer
# From
for from_ in obj.from_string.all():
from_address = EmailAddress(from_.sender)
from_address.is_spoofed = from_.is_spoofed
from_address.condition = from_.condition
h.from_ = from_address
# Sender
for sender in obj.sender.all():
sender_address = EmailAddress(sender.sender)
sender_address.is_spoofed = sender.is_spoofed
sender_address.condition = sender.condition
h.sender.add(sender_address)
# To
recipients = EmailRecipients()
for recipient in obj.recipients.all():
rec_address = EmailAddress(recipient.recipient)
rec_address.is_spoofed = recipient.is_spoofed
rec_address.condition = recipient.condition
recipients.append(rec_address)
h.to = recipients
# CC
recipients = EmailRecipients()
for recipient in obj.recipients_cc.all():
rec_address = EmailAddress(recipient.recipient)
rec_address.is_spoofed = recipient.is_spoofed
rec_address.condition = recipient.condition
recipients.append(rec_address)
h.cc = recipients
# BCC
recipients = EmailRecipients()
for recipient in obj.recipients_bcc.all():
rec_address = EmailAddress(recipient.recipient)
rec_address.is_spoofed = recipient.is_spoofed
rec_address.condition = recipient.condition
recipients.append(rec_address)
h.bcc = recipients
e.header = h
return e, attachment_objects
def cybox_object_email(obj):
e = EmailMessage()
e.raw_body = obj.raw_body
e.raw_header = obj.raw_header
# Links
e.links = Links()
for link in obj.links.all():
pass
# Attachments
e.attachments = Attachments()
attachment_objects = []
for att in obj.attachments.all():
for meta in att.file_meta.all():
fobj = cybox_object_file(att, meta)
e.attachments.append(fobj.parent.id_)
fobj.add_related(e, "Contained_Within", inline=False)
attachment_objects.append(fobj)
# construct header information
h = EmailHeader()
h.subject = obj.subject
h.date = obj.email_date
h.message_id = obj.message_id
h.content_type = obj.content_type
h.mime_version = obj.mime_version
h.user_agent = obj.user_agent
h.x_mailer = obj.x_mailer
# From
for from_ in obj.from_string.all():
from_address = EmailAddress(from_.sender)
from_address.is_spoofed = from_.is_spoofed
from_address.condition = from_.condition
h.from_ = from_address
# Sender
for sender in obj.sender.all():
sender_address = EmailAddress(sender.sender)
sender_address.is_spoofed = sender.is_spoofed
sender_address.condition = sender.condition
h.sender.add(sender_address)
# To
recipients = EmailRecipients()
for recipient in obj.recipients.all():
rec_address = EmailAddress(recipient.recipient)
rec_address.is_spoofed = recipient.is_spoofed
rec_address.condition = recipient.condition
recipients.append(rec_address)
h.to = recipients
# CC
recipients = EmailRecipients()
for recipient in obj.recipients_cc.all():
rec_address = EmailAddress(recipient.recipient)
rec_address.is_spoofed = recipient.is_spoofed
rec_address.condition = recipient.condition
recipients.append(rec_address)
h.cc = recipients
# BCC
recipients = EmailRecipients()
for recipient in obj.recipients_bcc.all():
rec_address = EmailAddress(recipient.recipient)
rec_address.is_spoofed = recipient.is_spoofed
rec_address.condition = recipient.condition
recipients.append(rec_address)
h.bcc = recipients
e.header = h
return e, attachment_objects