def process_readmessage(self):
result = {'type': 'Read', 'message': ''}
## We could get several messages in the same response:
root = self.parser.root
for message in root.search('message'):
result['message_id'] = message.find("mid").innerHTML()
try:
result['sent'] = Time.parse(
message.find("receiveddate").innerHTML())
except:
pass
result['subject'] = message.find("subject").innerHTML()
for tag, field in [('from', 'From'), ('to', 'To')]:
result[field] = self.parse_email_address(message, tag)
## now iterate over all the parts:
for part in message.search("part"):
## Usually text/html are the main body
try:
if not result['message'] and part.attributes[
'type'] == 'text':
text = part.find("text")
result['message'] = HTML.unquote(
HTML.decode_entity(text.innerHTML()))
except KeyError:
pass
self.insert_message(result, "webmail")
def process_readmessage(self):
result = {'type': 'Read', 'message':'' }
## We could get several messages in the same response:
root = self.parser.root
for message in root.search('message'):
result['message_id'] = message.find("mid").innerHTML()
try:
result['sent'] = Time.parse(message.find("receiveddate").innerHTML())
except: pass
result['subject'] = message.find("subject").innerHTML()
for tag,field in [('from','From'),
('to','To')]:
result[field] = self.parse_email_address(message, tag)
## now iterate over all the parts:
for part in message.search("part"):
## Usually text/html are the main body
try:
if not result['message'] and part.attributes['type'] == 'text':
text = part.find("text")
result['message'] = HTML.unquote(HTML.decode_entity(text.innerHTML()))
except KeyError: pass
self.insert_message(result, "webmail")
def parse_email_address(self, message, tag):
from_tag = message.find(tag)
if from_tag:
try:
name = from_tag.find("name").innerHTML()
except: name = ''
email = HTML.unquote(HTML.decode_entity(from_tag.find("email").innerHTML()))
return "%s <%s>" % (name, email)
def parse_email_address(self, message, tag):
from_tag = message.find(tag)
if from_tag:
try:
name = from_tag.find("name").innerHTML()
except:
name = ''
email = HTML.unquote(
HTML.decode_entity(from_tag.find("email").innerHTML()))
return "%s <%s>" % (name, email)
def process_message_yahoo1(self, result, header):
""" Handle Yahoo mail from old version (prior to 20080224) """
## Look through all its rows:
context = None
for td in header.search("td"):
if context:
for i in td:
if type(i) == str:
result[context] = HTML.unquote(
HTML.decode_entity(i))
break
context = None
data = td.innerHTML()
if data.lower().strip().startswith('from:'):
context = 'From'
elif data.lower().strip().startswith('to:'):
context = 'To'
elif data.lower().strip().startswith('date:'):
context = 'Sent'
elif data.lower().strip().startswith('subject:'):
context = 'Subject'
## Now the message:
msgbody = self.parser.root.find('div', {"class": "msgbody"})
if msgbody:
result['message'] = msgbody.innerHTML()
if 'Sent' in result:
#result['Sent'] = ColumnTypes.guess_date(result['Sent'])
result['sent'] = Time.parse(result['sent'],
case=self.case,
evidence_tz=None)
## Find the message id:
tag = header.find('input', dict(name='MsgId'))
if tag:
result['message_id'] = tag['value']
if len(result.keys()) > 3:
return self.insert_message(result, inode_template="y%s")
def process_message_yahoo1(self, result, header):
""" Handle Yahoo mail from old version (prior to 20080224) """
## Look through all its rows:
context = None
for td in header.search("td"):
if context:
for i in td:
if type(i)==str:
result[context] = HTML.unquote(HTML.decode_entity(i))
break
context = None
data = td.innerHTML()
if data.lower().strip().startswith('from:'):
context = 'From'
elif data.lower().strip().startswith('to:'):
context = 'To'
elif data.lower().strip().startswith('date:'):
context = 'Sent'
elif data.lower().strip().startswith('subject:'):
context = 'Subject'
## Now the message:
msgbody = self.parser.root.find('div', {"class":"msgbody"})
if msgbody:
result['message'] = msgbody.innerHTML()
if 'Sent' in result:
#result['Sent'] = ColumnTypes.guess_date(result['Sent'])
result['sent'] = Time.parse(result['sent'], case=self.case, evidence_tz=None)
## Find the message id:
tag = header.find('input', dict(name='MsgId'))
if tag:
result['message_id'] = tag['value']
if len(result.keys())>3:
return self.insert_message(result, inode_template = "y%s")
def make_link(self, url):
return urlnorm.normalize(HTML.unquote(url))
def process_readmessage(self,fd):
## This is what the message tree looks like (XML):
## <GetDisplayMessageResponse>
## <message>
## <header>
## <part>
## <part>
## <message>
## <message>
## Each message is a seperate message - therefore the same
## HTTP object might relay several messages.
root = self.parser.root
for message in root.search('message'):
result = {'type': 'Read', 'service':self.service }
result['message_id'] = message.find("mid").innerHTML()
## Messages are made unique using the message_id. This
## ensures that even if the same message was seen multiple
## times in the traffic, we only retain one copy of it.
message_urn = "/Webmail/%s/%s" % (self.service,
result['message_id'].replace("/","_"))
## Make sure we dont have duplicates of the same message -
## duplicates may occur in other connections, so we check
## the webmail table for the same yahoo message id
fsfd = FileSystem.DBFS(fd.case)
try:
if fsfd.lookup(path = message_urn):
continue
except RuntimeError:
pass
try:
result['sent'] = Time.parse(message.find("receiveddate").innerHTML())
except: pass
result['subject'] = HTML.unquote(HTML.decode_entity(
message.find("subject").innerHTML()))
for tag,field in [('from','From'),
('to','To')]:
result[field] = self.parse_email_address(message, tag)
message_fd = CacheManager.AFF4_MANAGER.create_cache_data(
fd.case, message_urn,
inherited = fd.urn)
message_fd.insert_to_table("webmail_messages",
result)
## now iterate over all the parts:
for part in message.search("part"):
## Parts are basically message attachments.
ct = part.attributes['type']
part_number = part.attributes['partid']
part_urn = "/".join((message_urn, part_number))
## Usually text/html are the main body
data = None
if "text" in ct:
text = part.find("text")
message_fd.write(HTML.unquote(HTML.decode_entity(text.innerHTML())))
elif "image" in ct:
message_fd.write(DB.expand("<b>%s</b><br><img src='%s'/>",(
self.make_link(part.attributes.get('filename','')),
self.make_link(part.attributes['thumbnailurl']))))
message_fd.close()
def make_link(self, url):
return urlnorm.normalize(HTML.unquote(url))
def process_readmessage(self, fd):
## This is what the message tree looks like (XML):
## <GetDisplayMessageResponse>
## <message>
## <header>
## <part>
## <part>
## <message>
## <message>
## Each message is a seperate message - therefore the same
## HTTP object might relay several messages.
root = self.parser.root
for message in root.search('message'):
result = {'type': 'Read', 'service': self.service}
result['message_id'] = message.find("mid").innerHTML()
## Messages are made unique using the message_id. This
## ensures that even if the same message was seen multiple
## times in the traffic, we only retain one copy of it.
message_urn = "/Webmail/%s/%s" % (
self.service, result['message_id'].replace("/", "_"))
## Make sure we dont have duplicates of the same message -
## duplicates may occur in other connections, so we check
## the webmail table for the same yahoo message id
fsfd = FileSystem.DBFS(fd.case)
try:
if fsfd.lookup(path=message_urn):
continue
except RuntimeError:
pass
try:
result['sent'] = Time.parse(
message.find("receiveddate").innerHTML())
except:
pass
result['subject'] = HTML.unquote(
HTML.decode_entity(message.find("subject").innerHTML()))
for tag, field in [('from', 'From'), ('to', 'To')]:
result[field] = self.parse_email_address(message, tag)
message_fd = CacheManager.AFF4_MANAGER.create_cache_data(
fd.case, message_urn, inherited=fd.urn)
message_fd.insert_to_table("webmail_messages", result)
## now iterate over all the parts:
for part in message.search("part"):
## Parts are basically message attachments.
ct = part.attributes['type']
part_number = part.attributes['partid']
part_urn = "/".join((message_urn, part_number))
## Usually text/html are the main body
data = None
if "text" in ct:
text = part.find("text")
message_fd.write(
HTML.unquote(HTML.decode_entity(text.innerHTML())))
elif "image" in ct:
message_fd.write(
DB.expand(
"<b>%s</b><br><img src='%s'/>",
(self.make_link(part.attributes.get(
'filename', '')),
self.make_link(part.attributes['thumbnailurl']))))
message_fd.close()
