def getMessagesFromUIDs(self,UIDs,folder,readOnly=False):
"""Returns the parsed message from specified UIDs.
Args:
UIDs: List of UIDs
folder: folder from which the UIDs came
readOnly: (optional) Whether or not to mark the messages as read.
Returns:
a dictionary containing the UID and message as a pair.
"""
#initialize the server
server = imapclient.IMAPClient(self.smtp, ssl=True)
server._MAXLINE = 10000000 #Allows the server to read large emails
server.login(self.email,self.paswd)
server.select_folder(folder,readOnly)
rawMessages = server.fetch(UIDs, ['BODY[]'])
messages=[]
for ID in UIDs:
message = pyzmail.PyzMessage.factory(rawMessages[ID][b'BODY[]'])
for mailpart in message.mailparts:
if mailpart.type.startswith('text/plain'): #Grab only the plain text parts
payload, used_charset=pyzmail.decode_text(mailpart.get_payload(), mailpart.charset, None)
messages.append(payload)
return dict(zip(UIDs,messages))
def add_submission_email(request, remote_ip, name, rev, submission_pk, message,
by, msgtype):
"""Add email to submission history"""
#in_reply_to = form.cleaned_data['in_reply_to']
# create Message
parts = pyzmail.parse.get_mail_parts(message)
body = ''
for part in parts:
if part.is_body == 'text/plain' and part.disposition == None:
payload, used_charset = pyzmail.decode_text(
part.get_payload(), part.charset, None)
body = body + payload + '\n'
msg = submit_message_from_message(message, body, by)
if (submission_pk != None):
# Must exist - we're adding a message to an existing submission
submission = Submission.objects.get(pk=submission_pk)
else:
# Must not exist
submissions = Submission.objects.filter(
name=name, rev=rev).exclude(state_id='cancel')
if submissions.count() > 0:
raise ValidationError("Submission {} already exists".format(name))
# create Submission using the name
try:
submission = Submission.objects.create(
state_id="waiting-for-draft",
remote_ip=remote_ip,
name=name,
rev=rev,
title=name,
note="",
submission_date=datetime.date.today(),
replaces="",
)
from ietf.submit.utils import create_submission_event, docevent_from_submission
desc = "Submission created for rev {} in response to email".format(
rev)
create_submission_event(request, submission, desc)
docevent_from_submission(request, submission, desc)
except Exception as e:
log("Exception: %s\n" % e)
raise
if msgtype == 'msgin':
rs = "Received"
else:
rs = "Sent"
desc = "{} message - manual post - {}-{}".format(rs, name, rev)
submission_email_event = SubmissionEmailEvent.objects.create(
desc=desc, submission=submission, msgtype=msgtype, by=by, message=msg)
#in_reply_to = in_reply_to
save_submission_email_attachments(submission_email_event, parts)
return submission, submission_email_event
def process_response_email(msg):
"""Saves an incoming message. msg=string. Message "To" field is expected to
be in the format ietf-submit+[identifier]@ietf.org. Expect to find a message with
a matching value in the reply_to field, associated to a submission.
Create a Message object for the incoming message and associate it to
the original message via new SubmissionEvent"""
message = email.message_from_string(msg)
to = message.get('To')
# exit if this isn't a response we're interested in (with plus addressing)
local, domain = get_base_submission_message_address().split('@')
if not re.match(r'^{}\+[a-zA-Z0-9_\-]{}@{}'.format(local, '{16}', domain),
to):
return None
try:
to_message = Message.objects.get(reply_to=to)
except Message.DoesNotExist:
log('Error finding matching message ({})'.format(to))
return None
try:
submission = to_message.manualevents.first().submission
except:
log('Error processing message ({})'.format(to))
return None
if not submission:
log('Error processing message - no submission ({})'.format(to))
return None
parts = pyzmail.parse.get_mail_parts(message)
body = ''
for part in parts:
if part.is_body == 'text/plain' and part.disposition == None:
payload, used_charset = pyzmail.decode_text(
part.get_payload(), part.charset, None)
body = body + payload + '\n'
by = Person.objects.get(name="(System)")
msg = submit_message_from_message(message, body, by)
desc = "Email: received message - manual post - {}-{}".format(
submission.name, submission.rev)
submission_email_event = SubmissionEmailEvent.objects.create(
submission=submission,
desc=desc,
msgtype='msgin',
by=by,
message=msg,
in_reply_to=to_message)
save_submission_email_attachments(submission_email_event, parts)
log(u"Received submission email from %s" % msg.frm)
return msg
def getTextMessages(self,newMessagesOnly=False,returnUID=False):
"""Grabs all recieved messages
Retrieves all text messsages. Only retrieves the body of the email.
Args:
newMessagesOnly: (optional) Only retrieve unread messages
returnUID: (option) Whether or not to return the UIDs
Returns:
Default:
Python Dictionary containing the sms_address and a list of messages as a pair.
Example
{'*****@*****.**': ['Hello','World'], '*****@*****.**': ['foo','bar']}
To get the message as a list just use exampleDict[sms_address]
returnUID=True:
Python Dictionary containing the sms_address and a dictionary of UID: message as a pair.
example:
{'*****@*****.**': {123: 'Hello', 456: 'World'}}
"""
#initialize the server
server = imapclient.IMAPClient(self.smtp, ssl=True)
server._MAXLINE = 10000000 #Allows the server to read large emails
server.login(self.email,self.paswd)
server.select_folder('INBOX',readonly=False)
if newMessagesOnly:
UIDs = server.search(['UNSEEN'])
else:
UIDs = server.search(['ALL'])
rawMessages = server.fetch(UIDs, ['BODY[]'])
messages={}
for ID in UIDs:
message = pyzmail.PyzMessage.factory(rawMessages[ID][b'BODY[]'])
address = message.get_address('from')[1]
if not address in messages:
if returnUID:
messages[address] = {} #initialize
else:
messages[address] = [] #initialize
for mailpart in message.mailparts:
if mailpart.type.startswith('text/plain'): #Grab only the plain text parts
payload, used_charset=pyzmail.decode_text(mailpart.get_payload(), mailpart.charset, None)
if returnUID:
messages[address][ID] = payload
else:
messages[address].append(payload)
server.logout()
return messages
def StoreEmail(self,subjectstr='bpdbjobs',deleteflag=False,folder='INBOX'):
logger.info('deleteflag is %r' % deleteflag)
if deleteflag == True:
self.__obj_imap.select_folder(folder,readonly=False)
else:
self.__obj_imap.select_folder(folder,readonly=True)
uids = self.__obj_imap.search('SUBJECT %s' % subjectstr)
messages = {}
filenames_saved = []
for uid in uids:
rawMessages = self.__obj_imap.fetch([uid],['BODY[]', 'FLAGS'])
messages[uid] = pyzmail.PyzMessage.factory(rawMessages[uid]['BODY[]'])
if messages == {}:
logger.info( 'No emails are found with %s' % subjectstr )
else:
logger.info( '%d emails found with %s' % (len(messages),subjectstr) )
for uid,msg in messages.iteritems():
logger.info('Processing email id %d with subbject: %s' % (uid,msg.get_subject()))
from_email = msg.get_addresses('from')
for index,mailpart in enumerate(msg.mailparts):
logger.debug('email %d index %d disposition %r' % (uid,index,mailpart.part.get("Content-Disposition", None)))
content_disposition = mailpart.part.get("Content-Disposition", None)
if content_disposition:
dispositions = content_disposition.strip().split(";")
else:
continue
if dispositions[0].lower() != "attachment":
continue
logger.debug('mailpart.type is %s',mailpart.type)
if mailpart.type.startswith('text/') or mailpart.type.startswith('application/octet-stream'):
payload, used_charset=pyzmail.decode_text(mailpart.get_payload(), mailpart.charset, None)
filename = self.file_save_path + mailpart.filename + '_uid' + str(uid) + '_' + time.strftime('-%Y%m%d-%H%M%S') + ".act"
if not os.path.exists(self.file_save_path):
os.makedirs(self.file_save_path)
open(filename,'w').write(mailpart.get_payload())
self.filelist_csv_save(from_email[0][1],self.username_email,uid,filename,'Not Proccessed')
logger.info("Saving attachment in email %d %s as " % (uid,filename))
if deleteflag == True:
logger.info( "Deleting mail with uid %d..." % uid )
self.__obj_imap.delete_messages(uid)
self.__obj_imap.expunge()
return
def __init__(self, raw_mail_lines):
msg_content = b'\r\n'.join(raw_mail_lines)
msg = PyzMessage.factory(msg_content)
self.subject = msg.get_subject()
self.sender = msg.get_address('from')
self.date = msg.get_decoded_header('date', '')
self.id = msg.get_decoded_header('message-id', '')
for mailpart in msg.mailparts:
if mailpart.is_body=='text/plain':
payload, used_charset=decode_text(mailpart.get_payload(), mailpart.charset, None)
self.charset = used_charset
self.text = payload
return
else:
self.text = None
def extract_message_text(msg):
text_plain = None
text_type = None
body_part = msg.text_part or msg.html_part
if body_part is not None:
body_content, body_charset = pyzmail.decode_text(
body_part.get_payload(),
body_part.charset,
None,
)
text_type = body_part.type
if body_part.type == 'text/html':
text_plain = html_to_plain(body_content)
else:
text_plain = body_content
return text_type, text_plain
def extract_message_text(msg):
text_plain = None
text_type = None
body_part = msg.text_part or msg.html_part
if body_part is not None:
body_content, body_charset = pyzmail.decode_text(
body_part.get_payload(),
body_part.charset,
None,
)
text_type = body_part.type
if body_part.type == 'text/html':
text_plain = html_to_plain(body_content)
else:
text_plain = body_content
return text_type, text_plain
def printemail(directory,name,filetype,size,fullname): #print all the email stats
print('directory' + directory + ' filename:' + name + '\n filetype:' + filetype + '\n filesize:' + i, size, i + 'kb',)
msg= pyzmail.PyzMessage.factory(open(fullname, 'rb'))
print ('Subject: %r' % (msg.get_subject(), ))
print ('From: %r' % (msg.get_address('from'), ))
print ('To: %r' % (msg.get_addresses('to'), ))
print ('Cc: %r' % (msg.get_addresses('cc'), ))
print ('BCc: %r' % (msg.get_addresses('bcc'), ))
print ('body and details:')
for mailpart in msg.mailparts:
print (
mailpart.filename,
mailpart.sanitized_filename,
mailpart.type,
mailpart.charset,
mailpart.part.get('Content-Description'),
len(mailpart.get_payload()) )
if mailpart.type.startswith('text/'):
# display first line of the text
payload, used_charset=pyzmail.decode_text(mailpart.get_payload(), mailpart.charset, None)
print ( payload.split('\\n')[0])
def extractMailFromFile(filename):
msg=pyzmail.PyzMessage.factory(open(filename, 'rb'))
text = ""
'''if len(msg.mailparts) > 1:
for mailpart in msg.mailparts:
if mailpart.is_body=='text/html':
#print mailpart.charset
payload, used_charset=pyzmail.decode_text(mailpart.get_payload(), mailpart.charset, None)
for line in payload.replace('\r',' ').split('\n'):
# omit lines starting with >, which is normally a quote for previous mail
if not line.startswith('>'):
text = text + line
text = strip_tags(text)
else:'''
for mailpart in msg.mailparts:
if mailpart.is_body=='text/plain':
payload, used_charset=pyzmail.decode_text(mailpart.get_payload(), mailpart.charset, None)
for line in payload.replace('\r',' ').split('\n'):
# omit lines starting with >, which is normally a quote for previous mail
if not line.startswith('>'):
text = text + line
return text
def save_submission_email_attachments(submission_email_event, parts):
for part in parts:
if part.disposition != 'attachment':
continue
if part.type == 'text/plain':
payload, used_charset = pyzmail.decode_text(
part.get_payload(), part.charset, None)
encoding = ""
else:
# Need a better approach - for the moment we'll just handle these
# and encode as base64
payload = base64.b64encode(part.get_payload())
encoding = "base64"
#name = submission_email_event.submission.name
MessageAttachment.objects.create(
message=submission_email_event.message,
content_type=part.type,
encoding=encoding,
filename=part.filename,
body=payload)
def msg_to_dict(msg):
"""
Convert a PyZmail message to a dictionary
@type msg: PyzMessage
@param msg: email to convert
@returns: {'Header': 'content'}
"""
# FIXME: any repeated header will be ignored
# Usually it is only 'Received' header
d = {}
if msg.text_part:
body = msg.text_part.get_payload()
charset = msg.text_part.charset
else:
body = msg.get_payload()
charset = msg.get_charset()
if charset:
charset = charset.lower()
i = charset.find('iso')
u = charset.find('utf')
if i > 0:
charset = charset[i:]
elif u > 0:
charset = charset[u:]
# Some old emails say it's ascii or unkown but in reality is not
# not use any charset not iso or utf
elif i != 0 and u != 0:
charset = None
for header in msg.keys():
value = msg.get_decoded_header(header)
value, _ = pyzmail.decode_text(value, charset, None)
value = value.encode('UTF-8')
header = header.replace('.', ',') # mongoDB don't likes '.' on keys
d[header] = value
attach = []
if type(body) == str:
body, _ = pyzmail.decode_text(body, charset, None)
body = body.encode('UTF-8')
# On attachments of emails sometimes it end up with a list of email.message
elif type(body) == list:
for part in body:
zmail = pyzmail.PyzMessage(part)
a = msg_to_dict(zmail)
attach.append(a)
body = attach[0]['Body']
d['Body'] = body
if len(msg.mailparts) > 1:
for mailpart in msg.mailparts:
zmail = pyzmail.PyzMessage(mailpart.part)
a = msg_to_dict(zmail)
attach.append(a)
if attach:
d['Attachments'] = attach
return d
if isinstance(ret, dict):
if ret:
print 'failed recipients:', ', '.join(ret.keys())
else:
print 'success'
else:
print 'error:', ret
'''
msg = pyzmail.PyzMessage.factory(payload)
print 'Subject: %r' % (msg.get_subject(), )
print 'From: %r' % (msg.get_address('from'), )
print 'To: %r' % (msg.get_addresses('to'), )
print 'Cc: %r' % (msg.get_addresses('cc'), )
for mailpart in msg.mailparts:
print ' %sfilename=%r alt_filename=%r type=%s charset=%s desc=%s size=%d' % ( \
'*'if mailpart.is_body else ' ', \
mailpart.filename, \
mailpart.sanitized_filename, \
mailpart.type, \
mailpart.charset, \
mailpart.part.get('Content-Description'), \
len(mailpart.get_payload()) )
if mailpart.type.startswith('text/'):
payload, used_charset = pyzmail.decode_text(mailpart.get_payload(),
mailpart.charset, None)
print ' >', payload
print 'Current time is : %s' % ctime()
def getTextMessagesFrom(self,sms_address,newMessagesOnly=False,returnUID=False):
"""Grabs all messages sent by the sms address.
Retrieves all text messsages sent by the specified sms address.
Only retrieves the body of the email.
Args:
sms_address: List of strings containing the specified address.
newMessagesOnly: (optional) Only retrieve unread messages
returnUID: (option) Whether or not to return the UIDs
Returns:
Default:
Python Dictionary containing the sms_address and a list of messages as a pair.
Example
{'*****@*****.**': ['Hello','World'], '*****@*****.**': ['foo','bar']}
To get the message as a list just use exampleDict[sms_address]
returnUID=True:
Python Dictionary containing the sms_address and a dictionary of UID: message as a pair.
example:
{'*****@*****.**': {123: 'Hello', 456: 'World'}}
"""
#initialize the server
server = imapclient.IMAPClient(self.smtp, ssl=True)
server._MAXLINE = 10000000 #Allows the server to read large emails
server.login(self.email,self.paswd)
server.select_folder('INBOX',readonly=False)
if type(sms_address)==str: #needs to be a list
sms_address = [sms_address]
messageList = []
UIDList = []
for address in sms_address:
if newMessagesOnly:
UIDs = server.search(['UNSEEN','FROM',address])
else:
UIDs = server.search(['FROM',address])
rawMessages = server.fetch(UIDs, ['BODY[]'])
messages=[]
for ID in UIDs:
message = pyzmail.PyzMessage.factory(rawMessages[ID][b'BODY[]'])
for mailpart in message.mailparts:
if mailpart.type.startswith('text/plain'): #Grab only the plain text parts
payload, used_charset=pyzmail.decode_text(mailpart.get_payload(), mailpart.charset, None)
messages.append(payload)
messageList.append(messages)
UIDs = dict(zip(UIDs,messages))
UIDList.append(UIDs)
server.logout()
if returnUID:
return dict(zip(sms_address,UIDList))
else:
return dict(zip(sms_address,messageList))
if ret:
print 'failed recipients:', ', '.join(ret.keys())
else:
print 'success'
else:
print 'error:', ret
'''
msg=pyzmail.PyzMessage.factory(payload)
print 'Subject: %r' % (msg.get_subject(), )
print 'From: %r' % (msg.get_address('from'), )
print 'To: %r' % (msg.get_addresses('to'), )
print 'Cc: %r' % (msg.get_addresses('cc'), )
for mailpart in msg.mailparts:
print ' %sfilename=%r alt_filename=%r type=%s charset=%s desc=%s size=%d' % ( \
'*'if mailpart.is_body else ' ', \
mailpart.filename, \
mailpart.sanitized_filename, \
mailpart.type, \
mailpart.charset, \
mailpart.part.get('Content-Description'), \
len(mailpart.get_payload()) )
if mailpart.type.startswith('text/'):
payload, used_charset=pyzmail.decode_text(mailpart.get_payload(), mailpart.charset, None)
print ' >', payload
print 'Current time is : %s' % ctime()
def get_forecast_data(self, file_name):
'''
This method is intended to be used to extract client specific data from the
IMAP mail server that was uploaded (by the SMTP socket) from the seven
day forecast model.
Parameters
----------
file_name : str
A string representing the name of the file for which the most recent
data is to be recieved. This file name must be concsitent with all
other uses of the file name in the pipeline.
Returns
-------
payload_df : pandas dataframe
A pandas dataframe containing all the forecasting timeseries data
extracted from the csv pulled from the IMAP server.
'''
# Extracting the contents of the inbox folder of the IMAP server:
self.select_folder('INBOX', readonly=True)
# Searching for message where the subject contains the client name:
uid_lst = self.search(['SUBJECT', f'{file_name}'])
# Creating dict of mail messages and contents from uid_lst:
messages = self.fetch(uid_lst, data=['BODY[]', 'FLAGS'])
# Declaring an empty dict to be populated with message header datetimes:
header_datetime_dict = {}
# Declaring an empty list to be populated with datetime objects:
datetime_lst = []
# Iterating through the message keys to search for the correct email:
for uid in messages:
# Initalizing the message as a pyzmail object to extract header:
header = pyzmail.PyzMessage.factory(
messages[uid][b'BODY[]']).get_subject()
# Processing the header for list information:
header_lst = header.split(';')
# Parses the header to ensure that this is the correct message:
if header_lst[
0] == file_name: # If the email contains client data:
date_val = datetime.datetime.strptime(header_lst[-1],
'%d/%m/%Y %H:%M')
# Adding datetime to the dict as key/value {uid: datetime}:
header_datetime_dict[date_val] = uid
# Adding datetime object to the list:
datetime_lst.append(date_val)
# Sorting date list to find the most recent datetime object:
most_recent = max(datetime_lst)
self.debug_print(
f'[MOST RECENT EMAIL DATE FOR {file_name}]: {most_recent}')
# Extracting the corresponding uid for the most recent client email:
recent_uid = header_datetime_dict[most_recent]
# Initalizing a message as a pyzmail object:
pyz_msg = pyzmail.PyzMessage.factory(messages[recent_uid][b'BODY[]'])
# Iterating through the list of pyzmail mailparts to extract the base64 payload:
for mailpart in pyz_msg.mailparts:
try:
# TODO: ADD Debug information statements about payload.
csv_string = (pyzmail.decode_text(mailpart.get_payload(),
mailpart.charset, None)[0])
except:
debug_print('![NO DATA PAYLOAD FOUND]!')
# Converting csv string to a pandas dataframe through StringIO:
csv_string_data = StringIO(csv_string)
# Converting formatted string to dataframe:
payload_df = pd.read_table(csv_string_data, sep=',')
# Formatting Dataframe:
payload_df.rename(columns={payload_df.columns[0]: 'Date'},
inplace=True)
payload_df.set_index('Date', inplace=True)
# Closing connection to IMAP server:
self.logout()
self.debug_print(
f'[DATA RETRIEVAL SUCESSFULL:] Closing connection to {self.mail_address}'
)
return payload_df
def StoreEmail(self,
subjectstr='bpdbjobs',
deleteflag=False,
folder='INBOX'):
logger.info('deleteflag is %r' % deleteflag)
if deleteflag == True:
self.__obj_imap.select_folder(folder, readonly=False)
else:
self.__obj_imap.select_folder(folder, readonly=True)
uids = self.__obj_imap.search('SUBJECT %s' % subjectstr)
messages = {}
filenames_saved = []
for uid in uids:
rawMessages = self.__obj_imap.fetch([uid], ['BODY[]', 'FLAGS'])
messages[uid] = pyzmail.PyzMessage.factory(
rawMessages[uid]['BODY[]'])
if messages == {}:
logger.info('No emails are found with %s' % subjectstr)
else:
logger.info('%d emails found with %s' %
(len(messages), subjectstr))
for uid, msg in messages.iteritems():
logger.info('Processing email id %d with subbject: %s' %
(uid, msg.get_subject()))
from_email = msg.get_addresses('from')
for index, mailpart in enumerate(msg.mailparts):
logger.debug('email %d index %d disposition %r' %
(uid, index,
mailpart.part.get("Content-Disposition", None)))
content_disposition = mailpart.part.get(
"Content-Disposition", None)
if content_disposition:
dispositions = content_disposition.strip().split(";")
else:
continue
if dispositions[0].lower() != "attachment":
continue
logger.debug('mailpart.type is %s', mailpart.type)
if mailpart.type.startswith(
'text/') or mailpart.type.startswith(
'application/octet-stream'):
payload, used_charset = pyzmail.decode_text(
mailpart.get_payload(), mailpart.charset, None)
filename = self.file_save_path + mailpart.filename + '_uid' + str(
uid) + '_' + time.strftime('-%Y%m%d-%H%M%S') + ".act"
if not os.path.exists(self.file_save_path):
os.makedirs(self.file_save_path)
open(filename, 'w').write(mailpart.get_payload())
self.filelist_csv_save(from_email[0][1],
self.username_email, uid, filename,
'Not Proccessed')
logger.info("Saving attachment in email %d %s as " %
(uid, filename))
if deleteflag == True:
logger.info("Deleting mail with uid %d..." % uid)
self.__obj_imap.delete_messages(uid)
self.__obj_imap.expunge()
return
