def OldGetEmailPackageObject(self, strip_mime_headers=True):
import email
text = self._GetMessageText()
try:
try:
msg = email.message_from_string(text)
except email.Errors.BoundaryError:
try:
msg = email.message_from_string(text + "\n\n")
except email.Errors.BoundaryError:
msg = None
except email.Errors.HeaderParseError:
msg = None
if msg is None:
butcher_pos = text.lower().find("\ncontent-type: ")
if butcher_pos < 0:
raise RuntimeError(
"email package croaked with a MIME related error, but "
"there appears to be no 'Content-Type' header")
butchered = text[:butcher_pos] + "\nSpamBayes-" + \
text[butcher_pos+1:] + "\n\n"
msg = email.message_from_string(butchered)
except:
print "FAILED to create email.message from: ", `text`
raise
if strip_mime_headers:
if msg.has_key('content-type'):
del msg['content-type']
if msg.has_key('content-transfer-encoding'):
del msg['content-transfer-encoding']
return msg
def find_existing_object(uid, type, user_rec, lock=False):
"""
Search user's private folders for the given object (by UID+type)
"""
global imap
lock_key = None
if lock:
lock_key = get_lock_key(user_rec, uid)
set_write_lock(lock_key)
event = None
for folder in list_user_folders(user_rec, type):
log.debug(_("Searching folder %r for %s %r") % (folder, type, uid), level=8)
imap.imap.m.select(imap.folder_utf7(folder))
typ, data = imap.imap.m.search(None, '(UNDELETED HEADER SUBJECT "%s")' % (uid))
for num in reversed(data[0].split()):
typ, data = imap.imap.m.fetch(num, '(RFC822)')
try:
if type == 'task':
event = todo_from_message(message_from_string(data[0][1]))
else:
event = event_from_message(message_from_string(data[0][1]))
setattr(event, '_imap_folder', folder)
setattr(event, '_lock_key', lock_key)
except Exception, e:
log.error(_("Failed to parse %s from message %s/%s: %s") % (type, folder, num, traceback.format_exc()))
continue
if event and event.uid == uid:
return event
def parse_headers(fp, _class=Message):
"""Parses only RFC2822 headers from a file pointer.
This code is taken directly from the Python 3 stdlib, adapted for
2to3. Returns a dictionary of unicode strings mapping to unicode
strings.
"""
headers = []
while True:
line = fp.readline()
headers.append(line)
if line in HEADER_NEWLINES:
break
hbytes = ''.encode('ascii').join(headers)
# It turns out that in Python 3, email.Parser requires Unicode.
# Unfortunately,in Python 2, email.Parser refuses to handle
# Unicode and returns an empty object. We have to make sure that
# parse_headers returns Unicode in both Python 2 and Python 3. The
# easiest way is to throw away the email.message.Message interface
# and just return a dict instead, which lets us massage the bytes
# into Unicode.
# iso-8559-1 encoding taken from http/client.py, where this
# function was stolen from.
E = 'iso-8859-1'
if sys.version_info[0] < 3:
items = list(email.message_from_string(hbytes).items())
return dict((k.decode(E), v.decode(E)) for k, v in items)
hstring = hbytes.decode(E)
return dict(email.message_from_string(hstring))
def getMessage(self, mailID, fullname):
typ, header = self.IMAP.fetch(mailID, "(UID BODY.PEEK[HEADER])")
typ, body = self.IMAP.fetch(mailID, "(UID BODY.PEEK[])")
for x in range(0, len(header), 2):
firstPart = header[x][0]
uidStart = firstPart.find("UID ") + 4
uidEnd = firstPart.find(" ", uidStart)
uid = int(firstPart[uidStart:uidEnd])
if uid > self.lastUIDs[fullname]: self.lastUIDs[fullname] = uid
print "UID %s in %s" % (uid, fullname)
#info = data[x][0]
msgFD = open("%s/%s/%s" % (self.username, fullname, uid), "w")
mssg = email.message_from_string(body[x][1])
hdrmsg = email.message_from_string(header[x][1])
jsonHeader = {};
for key in hdrmsg.keys():
jsonHeader[key] = hdrmsg[key]
message = {"headers":jsonHeader, "body":{"parts":{}}}
for part in mssg.walk():
maintype = part.get_content_maintype();
mtype = part.get_content_type();
# multipart are just containers, so we skip them
if maintype == 'multipart':
continue
if maintype == 'text':
message["body"]["parts"][mtype] = {"payload":part.get_payload()};
else: #TODO: handle attachments
print mtype
try:
json.dump(message, msgFD)
except:
message["body"] = {"error":"Message download failed!!!"};
json.dump(message, msgFD)
def test_bounce_message(self):
eq = self.assertEqual
unless = self.failUnless
msg = email.message_from_string("""\
To: [email protected]
From: [email protected]
Subject: and another thing
yadda yadda yadda
""", Message.Message)
self._mlist.BounceMessage(msg, {})
qmsg = email.message_from_string(self._readmsg())
unless(qmsg.is_multipart())
eq(len(qmsg.get_payload()), 2)
# The first payload is the details of the bounce action, and the
# second message is the message/rfc822 attachment of the original
# message.
msg1 = qmsg.get_payload(0)
eq(msg1.get_content_type(), 'text/plain')
eq(msg1.get_payload(), '[No bounce details are available]')
msg2 = qmsg.get_payload(1)
eq(msg2.get_content_type(), 'message/rfc822')
unless(msg2.is_multipart())
msg3 = msg2.get_payload(0)
eq(msg3.get_payload(), 'yadda yadda yadda\n')
def testSMTPMailFromHeaderSetup(self):
msg = message_from_string("""\
From: =?utf-8?q?Jos=C3=A9_Andr=C3=A9s?= <*****@*****.**>
To: =?utf-8?q?Ferran_Adri=C3=A0?= <*****@*****.**>
Subject: =?utf-8?q?=C2=BFEsferificaci=C3=B3n=3F?=
Date: Sun, 27 Aug 2006 17:00:00 +0200
Content-Type: text/html; charset="utf-8"
Content-Transfer-Encoding: base64
MIME-Version: 1.1
wqFVbiB0cnVjbyA8c3Ryb25nPmZhbnTDoXN0aWNvPC9zdHJvbmc+IQ=3D=3D
""")
mailhost = self._makeOne('MailHost')
mailhost.send(msg, mfrom='Foo Bar <*****@*****.**>')
out = message_from_string(mailhost.sent)
# We need to check if the 'From' header in message that we passed
# haven't changed.
self.failUnlessEqual(out['From'], msg['From'])
# We need to make sure that proper 'SMTP MAIL FROM' header was send.
self.failUnlessEqual(mailhost.mfrom, 'Foo Bar <*****@*****.**>')
# If 'From' header is missing in the message
# it should be added with 'MAIL FROM' SMTP Envelope header value.
del msg['From']
mailhost.send(msg, mfrom='Foo Bar <*****@*****.**>')
out = message_from_string(mailhost.sent)
self.failUnlessEqual(out['From'], 'Foo Bar <*****@*****.**>')
def cb_msg(md, type, tid, params, sbd):
"Message received callback"
msncb.cb_msg(md, type, tid, params, sbd)
email = tid.split(" ")[0]
if email not in usrs:
usrs[email] = users.RealUser(md, email, conf)
else:
usrs[email].updatemd(md)
msg = email_lib.message_from_string(params)
routine = msg.get_payload()
if re.search("text/plain", msg["Content-Type"]):
usrs[email].execcmd(routine)
elif re.search("text/x-msmsgsinvite", msg["Content-Type"]):
msg = email_lib.message_from_string(routine)
if msg["Invitation-Command"] == "INVITE" and msg["Application-GUID"] == "{5D3E02AB-6190-11d3-BBBB-00C04F795683}":
# it's incoming file transfer
fn = msg["Application-File"]
ft = users.FileTransfer(md, usrs[email], fn, buildpath((os.getcwd(), "received_files", email, fn)), msg["Application-FileSize"], msg["Invitation-Cookie"], 'receive')
ft.accept(msg["Connectivity"] == "N")
elif msg["Invitation-Command"] == "ACCEPT":
# some file transfer was accepted
users.FileTransfer.dispatchMsg(msg)
elif re.search("text/x-msmsgsprofile", msg["Content-Type"]):
conf.myip = msg["ClientIP"]
def test_send_email(self):
"""Test sending mail."""
self.mailhost.reset()
portal.send_email(
recipient="*****@*****.**",
sender="*****@*****.**",
subject="Trappist",
body=u"One for you Bob!",
)
self.assertEqual(len(self.mailhost.messages), 1)
msg = message_from_string(self.mailhost.messages[0])
self.assertEqual(msg['To'], '*****@*****.**')
self.assertEqual(msg['From'], '*****@*****.**')
self.assertEqual(msg['Subject'], '=?utf-8?q?Trappist?=')
self.assertEqual(msg.get_payload(), u'One for you Bob!')
self.mailhost.reset()
# When no sender is set, we take the portal properties.
portal.send_email(
recipient="*****@*****.**",
subject="Trappist",
body=u"One for you Bob!",
)
self.assertEqual(len(self.mailhost.messages), 1)
msg = message_from_string(self.mailhost.messages[0])
self.assertEqual(msg['From'], 'Portal Owner <*****@*****.**>')
def check(self, label='INBOX', senders=None):
"""
Arguments:
label
senders
"""
logging.debug('GMAIL: Checking %s for %s' % (label, senders))
_, count = self.imap.select(label)
_, stats = self.imap.status(label, "(UNSEEN)")
unread = int(stats[0].split()[2].strip(').,]'))
if not unread:
logging.debug('GMAIL: No new messages.')
return []
messages = []
if not senders:
senders = ['(UNSEEN)']
else:
senders = ['(FROM "%s" UNSEEN)' % s for s in senders]
for sender in senders:
_, ids = self.imap.search(None, sender)
ids = ids[0].split(' ')
for email_id in ids:
_, data = self.imap.fetch(email_id, '(UID RFC822)')
msg = email.message_from_string(data[0][1])
messages.append(email.message_from_string(data[0][1]))
logging.debug('GMAIL: Received %d messages.' % len(messages))
return messages
def testGetSize(self):
msg = email.message_from_string(spam1, _class=IMAPMessage)
correct_msg = email.message_from_string(spam1)
# Our messages are designed for transmittal, so have
# \r\n rather than \n as end-of-line.
self.assertEqual(msg.getSize(),
len(correct_msg.as_string().replace('\n', '\r\n')))
def test_get_mail_addresses(self):
to_message_object = email.message_from_string("To: John Doe <*****@*****.**>")
self.assertEqual([{'email': '*****@*****.**', 'name': u'John Doe'}], get_mail_addresses(to_message_object, 'to'))
from_message_object = email.message_from_string("From: John Smith <*****@*****.**>")
self.assertEqual([{'email': '*****@*****.**', 'name': u'John Smith'}], get_mail_addresses(from_message_object, 'from'))
def setUp(self):
# setup some test mails
self.msg_empty = MIMEText('')
here = os.path.dirname(__file__)
msg_txt = open(os.path.join(here, 'mails', 'ascii_7bit.txt'), 'r').read()
self.msg_ascii = email.message_from_string(msg_txt)
msg_txt = open(os.path.join(here, 'mails', 'latin1.txt'), 'r').read()
self.msg_latin1 = email.message_from_string(msg_txt)
msg_txt = open(os.path.join(here, 'mails', 'utf8.txt'), 'r').read()
self.msg_utf8 = email.message_from_string(msg_txt)
msg_txt = open(os.path.join(here, 'mails', 'attachment.txt'), 'r').read()
self.msg_attachment = email.message_from_string(msg_txt)
msg_txt = open(os.path.join(here, 'mails', 'fwd_attachment.txt'), 'r').read()
self.msg_fwd_attachment = email.message_from_string(msg_txt)
msg_txt = open(os.path.join(here, 'mails', 'nested_attachments.txt'), 'r').read()
self.msg_nested_attachments = email.message_from_string(msg_txt)
msg_txt = open(os.path.join(here, 'mails', 'nested_referenced_image_attachment.txt'),
'r').read()
self.nested_referenced_image_attachment = email.message_from_string(msg_txt)
msg_txt = open(os.path.join(here, 'mails', 'multiple_html_parts.txt'), 'r').read()
self.msg_multiple_html_parts = email.message_from_string(msg_txt)
msg_txt = open(os.path.join(here, 'mails', 'from_header_with_quotes.txt'), 'r').read()
self.from_header_with_quotes = email.message_from_string(msg_txt)
msg_txt = open(os.path.join(here, 'mails', 'encoded_word_without_lwsp.txt'), 'r').read()
self.encoded_word_without_lwsp = email.message_from_string(msg_txt)
def load(self):
'''Read the Message substance from the file'''
if self.loaded:
return
assert self.file_name is not None, \
"Must set filename before using FileMessage instances."
if options["globals", "verbose"]:
print('loading', self.file_name)
pn = self.pathname()
fp = gzip.open(pn, 'rb')
try:
self._msg = email.message_from_string(\
fp.read(), _class = self.message_class)
except IOError as e:
if str(e) == 'Not a gzipped file' or \
str(e) == 'Unknown compression method':
fp.close()
fp = open(self.pathname(), 'rb')
self._msg = email.message_from_string(\
fp.read(), _class = self.message_class)
fp.close()
else:
raise
else:
fp.close()
self.loaded = True
def parseEmail(data):
msg = email.message_from_string(data[0][1])
t = time.strptime(msg['Date'], '%a, %d %b %Y %H:%M:%S +0000')
msg_date = "%s-%s-%s" % (str(t.tm_year), str(t.tm_mon).zfill(2), str(t.tm_mday).zfill(2))
msg = email.message_from_string(data[0][1])
msg_from_name, msg_from_addr = email.utils.parseaddr(msg['From'])
return msg_from_addr, msg_date
def test_top(self):
"""
`top(which, howmuch)'
Retrieves the message header plus HOWMUCH lines of the message
after the header of message number WHICH. Result is in form
`(RESPONSE, ['line', ...], OCTETS)'.
The POP3 TOP command this method uses, unlike the RETR command,
doesn't set the message's seen flag; unfortunately, TOP is poorly
specified in the RFCs and is frequently broken in off-brand
servers. Test this method by hand against the POP3 servers you
will use before trusting it.
"""
list = self.o.list()
id, size = [x for x in [
(int(x[0]), int(x[1])) for x in [
x.split(' ') for x in list[1]]] if x[1] > 1000][0]
result = self.o.top(id, 10)
self.assertEquals(result[0], '+OK 10 lines of message %d' % id)
result = self.o.top(id, 0)
msg = message_from_string('\r\n'.join(result[1]))
self.assertEquals(msg.get_payload(), "")
result = self.o.top(id, 2)
msg = message_from_string('\r\n'.join(result[1]))
self.assertEquals(len(msg.get_payload().split('\r\n')), 3)
def testSendMessageObject(self):
# send will accept an email.Message.Message object directly
msg = message_from_string("""\
From: =?utf-8?q?Jos=C3=A9_Andr=C3=A9s?= <*****@*****.**>
To: =?utf-8?q?Ferran_Adri=C3=A0?= <*****@*****.**>
Subject: =?utf-8?q?=C2=BFEsferificaci=C3=B3n=3F?=
Date: Sun, 27 Aug 2006 17:00:00 +0200
Content-Type: text/html; charset="utf-8"
Content-Transfer-Encoding: base64
MIME-Version: 1.1
wqFVbiB0cnVjbyA8c3Ryb25nPmZhbnTDoXN0aWNvPC9zdHJvbmc+IQ=3D=3D
""")
mailhost = self._makeOne('MailHost')
mailhost.send(msg)
out = message_from_string(mailhost.sent)
self.failUnlessEqual(out.as_string(), msg.as_string())
# we can even alter a from and subject headers without affecting the
# original object
mailhost.send(msg,
mfrom='Foo Bar <*****@*****.**>', subject='Changed!')
out = message_from_string(mailhost.sent)
# We need to make sure we didn't mutate the message we were passed
self.failIfEqual(out.as_string(), msg.as_string())
self.failUnlessEqual(out['From'], 'Foo Bar <*****@*****.**>')
self.failUnlessEqual(msg['From'],
'=?utf-8?q?Jos=C3=A9_Andr=C3=A9s?= <*****@*****.**>')
# The subject is encoded with the body encoding since no
# explicit encoding was specified
self.failUnlessEqual(out['Subject'], '=?utf-8?q?Changed!?=')
self.failUnlessEqual(msg['Subject'],
'=?utf-8?q?=C2=BFEsferificaci=C3=B3n=3F?=')
def test_processReply(self):
# Make sure an unknown token in an older email is deleted
msg = email.message_from_string(self.dataFile('good_reply_past'))
result = (yield self.receiver.processReply(msg))
self.assertEquals(result, MailReceiver.UNKNOWN_TOKEN_OLD)
# Make sure an unknown token is not processed
msg = email.message_from_string(self.dataFile('good_reply_future'))
result = (yield self.receiver.processReply(msg))
self.assertEquals(result, MailReceiver.UNKNOWN_TOKEN)
# Make sure a known token *is* processed
txn = self.store.newTransaction()
yield txn.imipCreateToken(
"urn:x-uid:5A985493-EE2C-4665-94CF-4DFEA3A89500",
"mailto:[email protected]",
"1E71F9C8-AEDA-48EB-98D0-76E898F6BB5C",
token="d7cdf68d-8b73-4df1-ad3b-f08002fb285f"
)
yield txn.commit()
result = (yield self.receiver.processReply(msg))
self.assertEquals(result, MailReceiver.INJECTION_SUBMITTED)
yield JobItem.waitEmpty(self.store.newTransaction, reactor, 60)
def test_email(self):
"""Check that the registration email has been recieved and is in good shape."""
messages = ''
testPass = False
time.sleep(10)
m = imaplib.IMAP4_SSL(settings.EMAIL_IMAP_HOST)
m.login(settings.EMAIL_HOST_USER, settings.EMAIL_HOST_PASSWORD)
sel_rv, data = m.select(settings.EMAIL_IMAP_INBOX)
try:
if sel_rv == 'OK':
search_rv, data = m.search(
None, 'FROM', settings.DEFAULT_FROM_EMAIL, 'SUBJECT', '{0}'.format(self.emailHeader))
if search_rv == 'OK':
if len(data[0].split()) > 0:
for num in data[0].split():
fetch_rv, msgData = m.fetch(num, '(RFC822)')
if fetch_rv == 'OK':
# This check should do something to make sure
# that the email is unique
if self.emailCheck(str(email.message_from_string(msgData[0][1]))):
m.store(num, '+FLAGS', '\\DELETED')
m.expunge()
else:
messages += str(
email.message_from_string(msgData[0][1]))
else:
raise RuntimeError('Message not found in inbox')
else:
raise RuntimeError('Message Searching failed')
else:
raise RuntimeError(
'Inbox selection failed. Perhaps a different inbox is needed for settings.EMAIL_IMAP_INBOX in settings.py.')
finally:
m.close()
m.logout()
def visit_listing(p, listing):
number, size = listing.decode('ascii').split()
print('Message', number, '(size is', size, 'bytes):')
print()
response, lines, octets = p.top(number, 0)
document = '\n'.join( line.decode('ascii') for line in lines )
message = email.message_from_string(document)
for header in 'From', 'To', 'Subject', 'Date':
if header in message:
print(header + ':', message[header])
print()
print('Read this message [ny]?')
answer = input()
if answer.lower().startswith('y'):
response, lines, octets = p.retr(number)
document = '\n'.join( line.decode('ascii') for line in lines )
message = email.message_from_string(document)
print('-' * 72)
for part in message.walk():
if part.get_content_type() == 'text/plain':
print(part.get_payload())
print('-' * 72)
print()
print('Delete this message [ny]?')
answer = input()
if answer.lower().startswith('y'):
p.dele(number)
print('Deleted.')
def deserialize(self, graph, content, mimetype):
if type(content) == str or type(content) == unicode:
msg = email.message_from_string("Content-Type:" + mimetype + "\n" + content)
else:
msg = email.message_from_string("Content-Type:" + mimetype + "\n" + content.read())
named_parts = {}
unnamed_parts = []
for part in msg.walk():
if part.is_multipart():
continue
filename = part.get_filename(None)
if filename:
named_parts[filename] = part
else:
unnamed_parts.append(part)
rdf = [part for part in unnamed_parts if part.get_content_type() in self.serializers]
if len(rdf) == 0:
raise Exception("SADI With Attachments requires one unnamed RDF part.")
print "\n".join([str(x) for x in rdf])
print "\n".join([str(x) for x in named_parts.values()])
rdf_content = rdf[0].get_payload()
print "using this RDF content:"
print rdf_content
ser = self.serializers[rdf[0].get_content_type()]
ser.deserialize(graph, unicode(rdf_content), rdf[0].get_content_type())
graph.attachments.update(named_parts)
return frir.RDFGraphDigest
def __init__(self, content):
"""Parses headers, content, attachments from given raw message.
:param content: Raw message."""
if six.PY2:
self.mail = email.message_from_string(safe_encode(content))
else:
if isinstance(content, bytes):
self.mail = email.message_from_bytes(content)
else:
self.mail = email.message_from_string(content)
self.text_content = ''
self.html_content = ''
self.attachments = []
self.cid_map = {}
self.parse()
self.set_content_and_type()
self.set_subject()
self.set_from()
self.message_id = (self.mail.get('Message-ID') or "").strip(" <>")
if self.mail["Date"]:
try:
utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"]))
utc_dt = datetime.datetime.utcfromtimestamp(utc)
self.date = convert_utc_to_user_timezone(utc_dt).strftime('%Y-%m-%d %H:%M:%S')
except:
self.date = now()
else:
self.date = now()
if self.date > now():
self.date = now()
def testExecuteMultiRecipients(self):
dummyMailHost = self._setup_mockmail()
e = MailAction()
e.source = '*****@*****.**'
e.recipients = '[email protected],[email protected]'
e.message = 'Document created !'
ex = getMultiAdapter((self.folder, e, DummyEvent(self.folder.d1)),
IExecutable)
ex()
self.assertEqual(len(dummyMailHost.messages), 2)
# in py3 the order of mails is non-determininistic
# because sending iterates over a set of recipients
for msg in dummyMailHost.messages:
if '*****@*****.**' in msg:
mailSent1 = message_from_string(msg)
else:
mailSent2 = message_from_string(msg)
self.assertEqual('text/plain; charset="utf-8"',
mailSent1.get('Content-Type'))
self.assertEqual('*****@*****.**', mailSent1.get('To'))
self.assertEqual('*****@*****.**', mailSent1.get('From'))
self.assertEqual('Document created !',
mailSent1.get_payload())
self.assertEqual('text/plain; charset="utf-8"',
mailSent2.get('Content-Type'))
self.assertEqual('*****@*****.**', mailSent2.get('To'))
self.assertEqual('*****@*****.**', mailSent2.get('From'))
self.assertEqual('Document created !',
mailSent2.get_payload())
self._teardown_mockmail()
def _retrieve_mails(self, callback, get_msgs_type='(UNSEEN)', delete=False):
"""
Calls a callback function for new (unseen) mails from account
:param callback: Callback function for processing incoming emails
:param get_msgs_type: Expression for emails to get '(UNSEEN)' by default to get new emails
:param delete: Delete obtained emails in account (default False)
"""
# get all unread messages
status, response = self.connection.search(None, get_msgs_type)
unread_msg_nums = response[0].split()
mails = []
for e_id in unread_msg_nums:
_, response = self.connection.fetch(e_id, '(RFC822)')
if sys.version_info >= (3, 0):
email_message = email.message_from_string(response[0][1].decode('utf-8'))
else:
email_message = email.message_from_string(response[0][1])
mails.append(email_message)
# Post Process
for e_id in unread_msg_nums:
self.connection.store(e_id, '+FLAGS', '\Seen')
if delete:
self.connection.store(e_id, '+FLAGS', '\\Deleted')
if delete:
self.connection.expunge()
callback(mails)
def test_filter_train(self):
self.h.open('c')
self.h.h.bayes.learn(tokenize(good1), False)
self.h.h.bayes.learn(tokenize(spam1), True)
self.h.h.store()
result = email.message_from_string(self.h.filter_train(spam1))
self.assert_(result[options["Headers",
"classification_header_name"]].\
startswith(options["Headers", "header_spam_string"]))
self.assertEqual(self.h.h.bayes.nspam, 2)
result = email.message_from_string(self.h.filter_train(good1))
self.assert_(result[options["Headers",
"classification_header_name"]].\
startswith(options["Headers", "header_ham_string"]))
self.assertEqual(self.h.h.bayes.nham, 2)
def lostphotosfound(self):
"""The actual program, which fetchs the mails and all its parts attachments"""
messages = self._filter_messages()
for msg in messages:
try:
idfetched = self._server.fetch([msg], ['X-GM-MSGID'])
except:
raise Exception('Could not fetch the message ID, server did not respond')
msgid = str(idfetched[idfetched.keys()[0]]['X-GM-MSGID'])
# mail has been processed in the past, skip it
if self._use_index and msgid in self._index.keys():
print 'Skipping X-GM-MSDID %s' % (msgid)
continue
# if it hasn't, fetch it and iterate through its parts
msgdata = self._server.fetch([msg], ['RFC822'])
for data in msgdata:
try:
mail = message_from_string(msgdata[data]['RFC822'].encode('utf-8'))
except UnicodeDecodeError:
print("Warning: can't encode message data to UTF-8")
mail = message_from_string(msgdata[data]['RFC822'])
if mail.get_content_maintype() != 'multipart':
continue
# logging
header_from = _charset_decoder(mail['From'])
header_subject = _charset_decoder(mail['Subject'])
print '[%s]: %s' % (header_from, header_subject)
# use raw header, header_from sometimes excludes the email address
sender = email.utils.parseaddr(mail['From'])[1]
if not sender:
sender = 'unknown_sender'
for part in mail.walk():
# if it's only plain text, i.e. no images
if part.get_content_maintype() == 'multipart':
continue
# if no explicit attachments unless they're inline
if part.get('Content-Disposition') is None:
pass
# if non-graphic inline data
if 'image/' not in part.get_content_type():
continue
# only then we can save this mail part
self._save_part(part, mail, sender)
# all parts of mail processed, add it to the index
self._index[msgid] = msgid
self._cleanup()
def testGetBodyFile(self):
msg = email.message_from_string(spam1, _class=IMAPMessage)
correct_msg = email.message_from_string(spam1)
body = msg.getBodyFile()
# Our messages are designed for transmittal, so have
# \r\n rather than \n as end-of-line.
self.assertEqual(body.read().replace('\r\n', '\n'),
correct_msg.get_payload())
def testGetSize(self):
msg = email.message_from_string(spam1, _class=IMAPMessage)
correct_msg = email.message_from_string(spam1)
self.assertEqual(msg.getSize(),
len(correct_msg.as_string().replace('\n', '\r\n')))
def test_missing_date(self):
msg = message_from_string('SUBject: nothing')
info = models.get_email_info(msg)
self.assertNotIn('date', info)
msg = message_from_string('Whatever: Sat, 11 Jan 2014 00:41:13 +08000 (CST)')
info = models.get_email_info(msg)
self.assertNotIn('date', info)
def message_from_string(string):
if six.PY3:
if isinstance(string, six.binary_type):
return email.message_from_bytes(string, policy=_compat32_crlf)
return email.message_from_string(string, policy=_compat32_crlf)
return email.message_from_string(string)
def get_body_html(self, body_data):
try:
parsed_body = email.message_from_string(body_data).get_payload(1)
return parsed_body
except TypeError:
parsed_body = email.message_from_string(body_data).get_payload()
return parsed_body
except:
print "No Body exists"
def examine(self, suspect):
# check if someone wants to skip sa checks
if suspect.get_tag('SAPlugin.skip') is True:
self.logger.debug(
'%s Skipping SA Plugin (requested by previous plugin)' %
suspect.id)
suspect.set_tag('SAPlugin.skipreason',
'requested by previous plugin')
return DUNNO
runtimeconfig = DBConfig(self.config, suspect)
spamsize = suspect.size
maxsize = self.config.getint(self.section, 'maxsize')
spamheadername = self.config.get(self.section, 'spamheader')
if spamsize > maxsize:
self.logger.info('%s Size Skip, %s > %s' %
(suspect.id, spamsize, maxsize))
suspect.debug('Too big for spamchecks. %s > %s' %
(spamsize, maxsize))
prependheader = self.config.get('main', 'prependaddedheaders')
suspect.addheader(
"%sSA-SKIP" % prependheader,
'Too big for spamchecks. %s > %s' % (spamsize, maxsize))
suspect.set_tag('SAPlugin.skipreason', 'size skip')
return self.check_sql_blacklist(suspect)
forwardoriginal = self.config.getboolean(self.section,
'forwardoriginal')
if self.config.getboolean(self.section, 'scanoriginal'):
spam = suspect.get_original_source()
else:
spam = suspect.get_source()
# prepend temporary headers set by other plugins
tempheader = suspect.get_tag('SAPlugin.tempheader')
if tempheader is not None:
if type(tempheader) == list:
tempheader = "\r\n".join(tempheader)
tempheader = tempheader.strip()
if tempheader != '':
spam = tempheader + '\r\n' + spam
if forwardoriginal:
ret = self.safilter_report(spam, suspect.to_address)
if ret is None:
suspect.debug('SA report Scan failed - please check error log')
self.logger.error('%s SA report scan FAILED' % suspect.id)
suspect.addheader(
'%sSA-SKIP' %
self.config.get('main', 'prependaddedheaders'),
'SA scan failed')
suspect.set_tag('SAPlugin.skipreason', 'scan failed')
return self._problemcode()
isspam, spamscore, report = ret
suspect.tags['SAPlugin.report'] = report
else:
filtered = self.safilter(spam, suspect.to_address)
if filtered is None:
suspect.debug('SA Scan failed - please check error log')
self.logger.error('%s SA scan FAILED' % suspect.id)
suspect.addheader(
'%sSA-SKIP' %
self.config.get('main', 'prependaddedheaders'),
'SA scan failed')
suspect.set_tag('SAPlugin.skipreason', 'scan failed')
return self._problemcode()
else:
content = filtered
newmsgrep = email.message_from_string(content)
suspect.set_source(content)
isspam, spamscore = self._extract_spamstatus(
newmsgrep, spamheadername, suspect)
action = DUNNO
message = None
if isspam:
self.logger.debug('%s Message is spam' % suspect.id)
suspect.debug('Message is spam')
configaction = string_to_actioncode(
runtimeconfig.get(self.section, 'lowspamaction'), self.config)
if configaction is not None:
action = configaction
values = dict(spamscore=spamscore)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect,
values)
else:
self.logger.debug('%s Message is not spam' % suspect.id)
suspect.debug('Message is not spam')
suspect.tags['spam']['SpamAssassin'] = isspam
suspect.tags['highspam']['SpamAssassin'] = False
if spamscore is not None:
suspect.tags['SAPlugin.spamscore'] = spamscore
highspamlevel = runtimeconfig.getfloat(self.section,
'highspamlevel')
if spamscore >= highspamlevel:
suspect.tags['highspam']['SpamAssassin'] = True
configaction = string_to_actioncode(
runtimeconfig.get(self.section, 'highspamaction'),
self.config)
if configaction is not None:
action = configaction
return action, message
def parse_headers(self, data):
if "\r\n" not in data:
return {}
raw_headers = data.split("\r\n", 1)[1]
headers = email.message_from_string(raw_headers)
return dict((k.lower(), v) for k, v in six.iteritems(dict(headers)))
def main():
while (True):
print datetime.datetime.now().time()
detach = "."
if 'attachments' not in os.listdir("."):
print "Could not find attachments folder, so i made it."
os.mkdir('attachments')
try:
imapSession = imaplib.IMAP4_SSL('imap.gmail.com')
typ, account = imapSession.login('username', 'password')
imapSession.list()
if typ != "OK":
print "Error: Could not sign in, check creds"
else:
print "Logged in, checking labelname Emails"
imapSession.select("label to check")
(typ, data) = imapSession.search(None, '(UNSEEN)')
if (data[0] == ""):
print "No new mail"
else:
print "There is new mail"
for msgs in str(data[0]).split():
typ, msgInfo = imapSession.fetch(msgs, '(RFC822)')
if typ != "OK":
print "error fetching mail"
else:
msgBody = msgInfo[0][1]
mail = email.message_from_string(msgBody)
for part in mail.walk():
fileName = part.get_filename()
if bool(fileName):
filePath = os.path.join(
detach, 'attachments', fileName)
print "Downloading", fileName
if not os.path.isfile(filePath):
print fileName, "from", str(
mail['From'])
fp = open(filePath, 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
print "Downloaded", fileName
print "Now printing", filePath
os.startfile(filePath, "print")
time.sleep(15)
os.system(
'cmd.exe /c taskkill /f /im AcroRd32.exe'
)
print "Deleting file"
os.remove(filePath)
imapSession.close()
print "checking again..."
time.sleep(60)
#imapSession.logout()
except:
print sys.exc_info()[0:]
break
def ProcessEmail():
import imaplib
import email
import urllib.request
from os import getcwd
from os.path import join
from bs4 import BeautifulSoup as bs
import io
import os
#import credentials
mail = imaplib.IMAP4_SSL('imap.gmail.com')
# imaplib module implements connection based on IMAPv4 protocol
senderEmail = os.environ['SenderEmail']
senderPassword = os.environ['SenderPassword']
receiverEmail = os.environ['ReceiverEmail']
fromMail = os.environ['FromMail']
mail.login(senderEmail, senderPassword)
# In[19]:
mail.list() # Lists all labels in GMail
mail.select('inbox') # Connected to inbox.
# In[20]:
result, data = mail.uid('search',
"From " + fromMail + ' Subject "Fly Morning News"',
"ALL")
i = len(data[0].split())
latest_email_uid = data[0].split()[i - 1]
result, email_data = mail.uid('fetch', latest_email_uid, '(RFC822)')
raw_email = email_data[0][1]
raw_email_string = raw_email.decode('utf-8')
email_message = email.message_from_string(raw_email_string)
for part in email_message.walk():
if part.get_content_type() == "text/html": # ignore attachments/html
body = part.get_payload(decode=True)
text = body.decode("utf-8")
soup = bs(text, 'html.parser')
# In[21]:
import pandas as pd
l = []
for tick in soup.select('a[href*="/news.php?symbol"]'):
titles = tick.parent.parent.parent.select(
'a[href^="https://thefly.com/permalinks/entry.php"] span')
if (tick.text.find(",") == -1 and len(titles) > 0):
dict1 = {}
dict1["Tick"] = tick.text
dict1["News"] = titles[0].text
l.append(dict1)
flydf = pd.DataFrame(l)
flydf
# In[22]:
group_ticker = flydf.groupby("Tick").count()
group_ticker = group_ticker.reset_index()
group_ticker.head()
# In[23]:
sort_ticker = group_ticker.sort_values("News", ascending=False)
sort_ticker.head()
# In[24]:
merged_df = pd.merge(flydf, sort_ticker, on=["Tick"])
merged_df.head()
# In[25]:
final_tickerdf = merged_df.rename(columns={
"News_x": "TickerNews",
"News_y": "Count"
})
final_tickerdf
final_tickerdf = final_tickerdf.sort_values(["Count", "Tick"],
ascending=False)
final_tickerdf.head()
final_tickerdf.to_csv("today_flymorningnews.csv", index=False)
# In[26]:
final_tickerdf.head()
# In[27]:
targetraised_df = final_tickerdf[final_tickerdf["TickerNews"].str.contains(
"target raised")]
targetraised_df
targetraised_df.to_csv("tagetraised.csv", index=False)
targetlowered_df = final_tickerdf[
final_tickerdf["TickerNews"].str.contains("target lowered")]
targetlowered_df.to_csv("tagetlowered.csv", index=False)
targetupgraded_df = final_tickerdf[
final_tickerdf["TickerNews"].str.contains("upgraded")]
targetupgraded_df.to_csv("tagetupgraded.csv", index=False)
targetdowngraded_df = final_tickerdf[
final_tickerdf["TickerNews"].str.contains("downgraded")]
targetdowngraded_df.to_csv("tagetdowngraded.csv", index=False)
# In[28]:
targetupgraded_df.head()
# In[29]:
b = final_tickerdf[final_tickerdf["Count"] >= 3]
final_ticker_more_3_df = b.sort_values(["Count", "Tick"], ascending=False)
final_ticker_more_3_df.head()
final_ticker_more_3_df.to_csv("3_or_more_ticks.csv", index=False)
# In[30]:
final_ticker_more_3_df.head()
# In[31]:
c = final_ticker_more_3_df["Tick"].unique()
ticksdf = pd.DataFrame(c)
ticksdf
# ticksdf.to_csv("allticks.csv",index=False)
# In[32]:
targetraised_df_ticker = final_ticker_more_3_df[
final_ticker_more_3_df["TickerNews"].str.contains("target raised")]
targetraised_df_ticker_3more = targetraised_df_ticker[
targetraised_df_ticker["Count"] >= 3]
targetraised_df_ticker_3more
a = targetraised_df_ticker_3more["Tick"].unique()
df4 = pd.DataFrame(a)
df4.to_csv("tagetraised_3ormore.txt", index=False)
# a.to_csv("tagetraised_3ormore.txt",index=False)
df4
# In[33]:
targetraised_df_ticker = final_tickerdf[
final_tickerdf["TickerNews"].str.contains("upgraded")]
targetraised_df_ticker_3more = targetraised_df_ticker[
targetraised_df_ticker["Count"] >= 3]
c = targetraised_df_ticker_3more["Tick"].unique()
df5 = pd.DataFrame(c)
df5.to_csv("tagetupgraded_3ormore.txt", index=False)
# In[34]:
df5
# In[ ]:
import smtplib
from email.message import EmailMessage
import csv
import os
# In[ ]:
contacts = [receiverEmail]
msg = EmailMessage()
msg['Subject'] = "Fly Morning News"
msg['From'] = senderEmail
msg['To'] = ','.join(contacts)
msg.set_content(
"Please find the files attached. Review and get back to me if any questions. -Naga"
)
files = [
"tagetraised_3ormore.txt", "tagetupgraded_3ormore.txt",
"today_flymorningnews.csv"
]
for file in files:
with open(file, "rb") as f:
filedata = f.read()
filename = f.name
msg.add_attachment(filedata,
maintype="text/csv",
subtype="octet-stream",
filename=filename)
with smtplib.SMTP_SSL("smtp.gmail.com", 465) as smtp:
smtp.login(senderEmail, senderPassword)
smtp.send_message(msg)
def __init__(self, email_text):
self.email = message_from_string(email_text)
self.reply_text = EmailReplyParser.read(self.email.get_payload()).reply
# oldest = inbox_item_list[0]
# result2, email_data = mail.uid('fetch', oldest, 'RFC822')
# raw_email = email_data[0][1].decode('utf-8')
# email_message = email.message_from_string(raw_email)
# print(dir(email_message))
# print('')
# print(email_message['To'])
# payload = email_message.get_payload()
# print(payload)
# ~ WAY TO FIND ALL ELEMENTS ~
print("")
for element in inbox_item_list:
result2, email_data = mail.uid('fetch', element, '(RFC822)')
raw_email = email_data[0][1].decode('utf-8')
email_message = email.message_from_string(raw_email)
to_ = email_message['To']
from_ = email_message['From']
subject_ = email_message['Subject']
date_ = email_message['date']
counter = 1
for part in email_message.walk():
if part.get_content_maintype() == "multipart":
continue
filename = part.get_filename()
content_type = part.get_content_type()
if not filename:
ext = mimetypes.guess_extension(content_type)
if not ext:
ext = '.bin'
if 'text' in content_type:
swith = ['BEAMS', 'CABLES']
try:
os.mkdir(detach_dir)
except FileExistsError:
pass
#CURRENTLY IT DOWNLOADS ALL FILES STARTING WITH THE KEYWORD
#I CAN CHANGE IT TO DOWNLOAD ATTACHMENTS TILL A SPECIFIED DATE TOO
count = 1
for emailid in reversed(items):
if count < 3:
typ, data = m.fetch(emailid, "(RFC822)")
email_body = data[0][1].decode('utf-8')
mail = email.message_from_string(email_body)
print(emailid)
print("[" + mail["From"] + "] :" + mail["Subject"])
for part in mail.walk():
if part.get_content_maintype() == 'multipart':
#print("maintype is multipart")
continue
if part.get('Content-Disposition') is None:
#print("content disposition is None")
continue
filename = part.get_filename()
#print('x ',filename)
if not filename:
def _get_email_object(self, name):
copy = self._get_email_as_text(name)
if six.PY3:
return email.message_from_bytes(copy)
else:
return email.message_from_string(copy)
def fetch_email_body(num):
global email
text_list = []
#https://docs.python.org/2/library/email.message.html
# non multipart .get_payload return string multipart, get_payload return list of message
rv, data = M.fetch(num, '(RFC822)')
#rv, data = M.uid('fetch', num, '(RFC822)')
if rv != 'OK':
print "ERROR getting message body", num
return (False, [])
raw_email = data[0][1]
if not raw_email: # ceinture et bretelles
return []
#raw_email_string = raw_email.decode('utf-8') will decode payload as utf-8 anyway
msg = email.message_from_string(raw_email)
for part in msg.walk(): # recursive
print "========== getting parts for message: ", num
content_type = part.get_content_type()
main_type = part.get_content_maintype()
print "main type: %s , content_type: %s, is_multipart: %r" % (
main_type, content_type, part.is_multipart())
# ismultipart can be true for non "multipart" main type
charset = part.get_content_charset()
if charset is None:
print "unknown charset"
else:
print "charset: ", charset
#Optional decode is a flag indicating whether the payload should be decoded or not, according to the
#Content-Transfer-Encoding header. When True and the message is not a multipart, the payload will be decoded
#if this header value is quoted-printable or base64. If some other encoding is used,
#or Content-Transfer-Encoding header is missing, or if the payload has bogus base64 data, the payload
#is returned as-is (undecoded). If the message is a multipart and the decode flag is True, then None is returned.
#The default for decode is False.
#https://stackoverflow.com/questions/43824650/encoding-issue-decode-quoted-printable-string-in-python
# quotable printable string: unicode coded as ascii eg =AC=E9 .
if content_type == 'text/plain':
print "=======>>>> got a text plain payload "
# return Return the current payload, which will be a list of messages or a string is_multipart
payload = part.get_payload(
decode=True
) # string or list of messages. true toi decode quotable printable string. otherwize shouw up in text
print chardet.detect(payload)
if isinstance(payload, str):
print "payload str"
else:
print "payload not str"
# print type(payload) ERROR TypeError: 'str' object is not callable
# let ^^ 0xe2 80 90 U+2019 rigth single quote
if payload.find(' ') == -1: # check if base 64
print("base64 ignored")
text_list.append(u"ignored binary") # space otherwise says dot
else:
try:
text = payload.decode('utf-8')
print "text decoded as utf "
except Exception as e:
print "Exception decoding payload as uft8: ", str(e)
#soup= BeautifulSoup(payload)
#text = soup.get_text()
#html = markdown(payload)
#html = payload # to not use markdown. also normally not needed. this is not html
#soup = BeautifulSoup(html,"html.parser")
#text = ''.join(soup.findAll(text=True))
text_list.append(text)
else:
type = re.sub(r'/', " ", content_type)
text_list.append(u"ignored %s" % (type.split(' ')[1]))
return (True, text_list) # list of text/ascii payloads.
def parse_message(self, message, save_original=False):
"""Parses a string or email.message.Message representing an
RFC-2822 email, and returns a generic dict holding the
message details.
:param message: the message to parse
:type message: email.message.Message | string | unicode
:param bool save_original: whether the returned dict
should include an ``original`` entry with the base64
encoded source of the message.
:rtype: dict
:return: A dict with the following structure, where each
field may not be present if missing in original
message::
{ 'message-id': msg_id,
'subject': subject,
'from': from,
'to': to,
'cc': cc,
'headers' : { 'X-Mailer': mailer,
#.. all X- headers...
},
'subtype': msg_mime_subtype,
'body_text': plaintext_body
'body_html': html_body,
'attachments': [('file1', 'bytes'),
('file2', 'bytes') }
# ...
'original': source_of_email,
}
"""
msg_txt = message
if isinstance(message, str):
msg_txt = email.message_from_string(message)
# Warning: message_from_string doesn't always work correctly on unicode,
# we must use utf-8 strings here :-(
if isinstance(message, unicode):
message = message.encode('utf-8')
msg_txt = email.message_from_string(message)
message_id = msg_txt.get('message-id', False)
msg = {}
if save_original:
# save original, we need to be able to read the original email sometimes
msg['original'] = message.as_string() if isinstance(message, Message) \
else message
msg['original'] = base64.b64encode(
msg['original']) # binary fields are b64
if not message_id:
# Very unusual situation, be we should be fault-tolerant here
message_id = time.time()
msg_txt['message-id'] = message_id
_logger.info(
'Parsing Message without message-id, generating a random one: %s',
message_id)
fields = msg_txt.keys()
msg['id'] = message_id
msg['message-id'] = message_id
if 'Subject' in fields:
msg['subject'] = decode(msg_txt.get('Subject'))
if 'Content-Type' in fields:
msg['content-type'] = msg_txt.get('Content-Type')
if 'From' in fields:
msg['from'] = decode(msg_txt.get('From') or msg_txt.get_unixfrom())
if 'To' in fields:
msg['to'] = decode(msg_txt.get('To'))
if 'Delivered-To' in fields:
msg['to'] = decode(msg_txt.get('Delivered-To'))
if 'CC' in fields:
msg['cc'] = decode(msg_txt.get('CC'))
if 'Cc' in fields:
msg['cc'] = decode(msg_txt.get('Cc'))
if 'Reply-To' in fields:
msg['reply'] = decode(msg_txt.get('Reply-To'))
if 'Date' in fields:
try:
date_hdr = decode(msg_txt.get('Date'))
parsed_date = dateutil.parser.parse(date_hdr, fuzzy=True)
if parsed_date.utcoffset() is None:
# naive datetime, so we arbitrarily decide to make it
# UTC, there's no better choice. Should not happen,
# as RFC2822 requires timezone offset in Date headers.
stored_date = parsed_date.replace(tzinfo=pytz.utc)
else:
stored_date = parsed_date.astimezone(pytz.utc)
except Exception:
_logger.warning(
'Failed to parse Date header %r in incoming mail '
'with message-id %r, assuming current date/time.',
msg_txt.get('Date'), message_id)
stored_date = datetime.datetime.now()
msg['date'] = stored_date.strftime("%Y-%m-%d %H:%M:%S")
if 'Content-Transfer-Encoding' in fields:
msg['encoding'] = msg_txt.get('Content-Transfer-Encoding')
if 'References' in fields:
msg['references'] = msg_txt.get('References')
if 'In-Reply-To' in fields:
msg['in-reply-to'] = msg_txt.get('In-Reply-To')
msg['headers'] = {}
msg['subtype'] = 'plain'
for item in msg_txt.items():
if item[0].startswith('X-'):
msg['headers'].update({item[0]: item[1]})
if not msg_txt.is_multipart() or 'text/plain' in msg.get(
'content-type', ''):
encoding = msg_txt.get_content_charset()
body = tools.ustr(msg_txt.get_payload(decode=True),
encoding,
errors='replace')
if 'text/html' in msg.get('content-type', ''):
msg['body_html'] = body
msg['subtype'] = 'html'
body = tools.html2plaintext(body)
msg['body_text'] = tools.ustr(body, encoding, errors='replace')
attachments = []
if msg_txt.is_multipart() or 'multipart/alternative' in msg.get(
'content-type', ''):
body = ""
if 'multipart/alternative' in msg.get('content-type', ''):
msg['subtype'] = 'alternative'
else:
msg['subtype'] = 'mixed'
for part in msg_txt.walk():
if part.get_content_maintype() == 'multipart':
continue
encoding = part.get_content_charset()
filename = part.get_filename()
if part.get_content_maintype() == 'text':
content = part.get_payload(decode=True)
if filename:
attachments.append((filename, content))
content = tools.ustr(content, encoding, errors='replace')
if part.get_content_subtype() == 'html':
msg['body_html'] = content
msg['subtype'] = 'html' # html version prevails
body = tools.ustr(tools.html2plaintext(content))
body = body.replace(' ', '')
elif part.get_content_subtype() == 'plain':
body = content
elif part.get_content_maintype() in ('application', 'image'):
if filename:
attachments.append(
(filename, part.get_payload(decode=True)))
else:
res = part.get_payload(decode=True)
body += tools.ustr(res, encoding, errors='replace')
msg['body_text'] = body
msg['attachments'] = attachments
# for backwards compatibility:
msg['body'] = msg['body_text']
msg['sub_type'] = msg['subtype'] or 'plain'
return msg
def post(self, user, data):
try:
# msg分解
msg = email.message_from_string(data)
# Subject取得
try:
# content-type指定でdecode
(subject_str,
subject_type) = email.Header.decode_header(msg['Subject'])[0]
subject = subject_str.decode(subject_type)
except BaseException:
# 存在しない場合はそのまま使用
subject = msg['Subject']
from ctirs.models import Feed
# Feed作成
feed = Feed()
# Title は Subject
feed.title = subject
# TLP は UserのデフォルトTLP
feed.tlp = user.tlp
# Use
feed.user = user
# 本文/Attachement取得
attachements = []
payloads = msg.get_payload()
# 添付がある場合は list, ない場合はstr
if isinstance(payloads, str):
content_type = self.get_char_set_from_content_type(
msg['Content-Type'])
content_type = content_type.split(':')[0]
if content_type is not None:
body = payloads.decode(content_type)
else:
body = payloads
elif isinstance(payloads, list):
# bodyは payloads[0]
body_payload = payloads[0]
body = self.get_unicode_content(body_payload)
# それ以降はattachement
for index in range(1, len(payloads)):
payload = payloads[index]
attachements.append(payload)
# Sharing Rangeはall
feed.sharing_range_type = SHARING_RANGE_TYPE_KEY_ALL
# 一旦Feedを保存しSTIXを作成する
feed.save()
# 添付ファイル処理
from feeds.views import save_post, save_attach_file
for payload in attachements:
file_name = self.get_file_name(payload)
content = self.get_content(payload)
# content を file stream にする
import io
o = io.BytesIO(content)
attach_file = save_attach_file(file_name, o, feed.id)
feed.files.add(attach_file)
feed.save()
# POSTする
save_post(None, feed, body)
except BaseException:
import traceback
traceback.print_exc()
return
def parse_message():
return email.message_from_string(sys.stdin.read())
def handling_incoming_email(context):
mailbox = Mailbox.objects.create(
name='Temporary Mailbox'
)
email_message = email.message_from_string(context.text)
mailbox.process_incoming_message(email_message)
def discussion_notify(comment_on_object, variables={}):
portal = comment_on_object.portal_url.getPortalObject()
send_from_address = portal.portal_properties.email_from_address
send_from_name = portal.portal_properties.email_from_name
host = portal.plone_utils.getMailHost()
encoding = portal.plone_utils.getSiteEncoding()
envelope_from = send_from_address
mt = portal.portal_membership
if IDiscussionResponse.providedBy(comment_on_object):
owner = comment_on_object.Creator()
if owner:
member = mt.getMemberById(owner)
if member:
send_to_address = member.getProperty('email')
if send_to_address:
message_body = portal.discussion_reply_notify_template(
portal,
comment_on_object=comment_on_object,
send_from_address=send_from_address,
send_from_name=send_from_name,
send_to_address=send_to_address,
**variables)
subject = "New comment on " + comment_on_object.title_or_id(
)
message = message_from_string(
message_body.encode(encoding))
message.set_charset(encoding)
message['From'] = Header(envelope_from)
if PLONE4:
host.send(message,
send_to_address,
envelope_from,
subject=subject,
charset=encoding,
msg_type='text/plain')
else:
host.secureSend(message_body,
send_to_address,
envelope_from,
subject=subject,
subtype='plain',
charset=encoding,
debug=False,
From=envelope_from)
parents = comment_on_object.parentsInThread()
if not parents:
return
comment_on_object = parents[0]
owner = comment_on_object.Creator()
if owner:
member = mt.getMemberById(owner)
if member:
send_to_address = member.getProperty('email')
if send_to_address:
message_body = portal.discussion_notify_template(
portal,
comment_on_object=comment_on_object,
send_from_address=send_from_address,
send_from_name=send_from_name,
send_to_address=send_to_address,
**variables)
subject = "New comment on " + comment_on_object.title_or_id()
message = message_from_string(message_body.encode(encoding))
message.set_charset(encoding)
message['From'] = Header(envelope_from)
if PLONE4:
host.send(message,
send_to_address,
envelope_from,
subject=subject,
charset=encoding,
msg_type='text/plain')
else:
host.secureSend(message_body,
send_to_address,
envelope_from,
subject=subject,
subtype='plain',
charset=encoding,
debug=False,
From=envelope_from)
def view_html_message():
with open('/Users/rpetrello/.mail/temporary/cur-message', 'rb') as data:
msgid = data.read()
subprocess.check_call(['notmuch', 'new'])
fnames = subprocess.check_output(
['notmuch', 'search', '--output=files', '%s' % msgid]
)
msg = None
for line in fnames.splitlines():
try:
with open(line, 'rb') as data:
msg = email.message_from_string(data.read())
if msg:
break
except IOError:
continue
html_parts = []
subfiles = []
for part in msg.walk():
if part.is_multipart() or part.get_content_type() == 'message/rfc822':
continue
if part.get_content_subtype() == 'html':
html_parts.append(part)
# Save it to a file in the temp dir.
mime = part.get_content_type()
for k in part.keys():
if k.lower() == 'content-id':
content_id = part[k]
if content_id.startswith('<') and content_id.endswith('>'):
content_id = content_id[1:-1]
subfiles.append(
(content_id, mime, part.get_payload(decode=True))
)
break
buff = StringIO()
for part in html_parts:
payload = part.get_payload(decode=True)
# Substitute all the filenames for CIDs:
for content_id, mime, attachment in subfiles:
payload = payload.replace(
'cid:%s' % content_id,
'data:%s;base64,%s' % (mime, base64.b64encode(attachment))
)
buff.write(payload)
with tempfile.NamedTemporaryFile(suffix='.html') as f:
filename = f.name
with codecs.open(filename, 'w', encoding='utf-8') as f:
f.write('<html><head><meta charset="utf-8"/></head><body>')
f.write(buff.getvalue().decode('utf8', 'ignore'))
f.write('</body></html>')
f.flush()
subprocess.check_call(['open', '-a', '/Applications/Firefox.app/', f.name])
time.sleep(3)
def mailPassword(self, login, REQUEST, immediate=False):
""" Wrapper around mailPassword """
membership = getToolByName(self, 'portal_membership')
if not membership.checkPermission('Mail forgotten password', self):
raise Unauthorized(
_(u"Mailing forgotten passwords has been disabled."))
utils = getToolByName(self, 'plone_utils')
member = get_member_by_login_name(self, login, raise_exceptions=False)
if member is None:
raise ValueError(
_(u'The username you entered could not be found.'))
# assert that we can actually get an email address, otherwise
# the template will be made with a blank To:, this is bad
email = member.getProperty('email')
if not email:
raise ValueError(_(u'That user does not have an email address.'))
else:
# add the single email address
if not utils.validateSingleEmailAddress(email):
raise ValueError(_(u'The email address did not validate.'))
check, msg = _checkEmail(email)
if not check:
raise ValueError(msg)
# Rather than have the template try to use the mailhost, we will
# render the message ourselves and send it from here (where we
# don't need to worry about 'UseMailHost' permissions).
reset_tool = getToolByName(self, 'portal_password_reset')
reset = reset_tool.requestReset(member.getId())
encoding = getUtility(ISiteRoot).getProperty('email_charset', 'utf-8')
mail_text = self.mail_password_template(self,
REQUEST,
member=member,
reset=reset,
password=member.getPassword(),
charset=encoding)
# The mail headers are not properly encoded we need to extract
# them and let MailHost manage the encoding.
if isinstance(mail_text, unicode):
mail_text = mail_text.encode(encoding)
message_obj = message_from_string(mail_text.strip())
subject = message_obj['Subject']
m_to = message_obj['To']
m_from = message_obj['From']
host = getToolByName(self, 'MailHost')
try:
host.send(mail_text,
m_to,
m_from,
subject=subject,
charset=encoding,
immediate=immediate)
except SMTPRecipientsRefused:
# Don't disclose email address on failure
raise SMTPRecipientsRefused(
_(u'Recipient address rejected by server.'))
except SMTPException as e:
raise (e)
# return the rendered template "mail_password_response.pt"
# (in Products.PasswordResetTool)
return self.mail_password_response(self, REQUEST)
def process_response(response):
logger.debug("Processing Request Response...")
if response.status_code == 200:
try:
data = bytes("Content-Type: ", 'utf-8') + bytes(
response.headers['content-type'], 'utf-8') + bytes(
'\r\n\r\n', 'utf-8') + response.content
msg = email.message_from_bytes(data) # pylint: disable=no-member
except AttributeError:
data = "Content-Type: " + response.headers[
'content-type'] + '\r\n\r\n' + response.content
msg = email.message_from_string(data)
for payload in msg.get_payload():
if payload.get_content_type() == "application/json":
j = json.loads(payload.get_payload())
logger.debug("JSON String Returned: %s", json.dumps(j,
indent=2))
elif payload.get_content_type() == "audio/mpeg":
filename = tmp_path + payload.get('Content-ID').strip(
"<>") + ".mp3"
with open(filename, 'wb') as f:
f.write(payload.get_payload(decode=True))
else:
logger.debug("NEW CONTENT TYPE RETURNED: %s",
payload.get_content_type())
# Now process the response
if 'directives' in j['messageBody']:
if not j['messageBody']['directives']:
logger.debug("0 Directives received")
for directive in j['messageBody']['directives']:
if directive['namespace'] == 'SpeechSynthesizer':
if directive['name'] == 'speak':
player.play_speech(
mrl_fix("file://" + tmp_path + directive['payload']
['audioContent'].lstrip("cid:") + ".mp3"))
elif directive['namespace'] == 'SpeechRecognizer':
if directive['name'] == 'listen':
logger.debug(
"Further Input Expected, timeout in: %sms",
directive['payload']['timeoutIntervalInMillis'])
player.play_speech(resources_path + 'beep.wav')
timeout = directive['payload'][
'timeoutIntervalInMillis'] / 116
audio_stream = capture.silence_listener(timeout)
# now process the response
alexa_speech_recognizer(audio_stream)
elif directive['namespace'] == 'AudioPlayer':
if directive['name'] == 'play':
player.play_playlist(directive['payload'])
elif directive['namespace'] == "Speaker":
# speaker control such as volume
if directive['name'] == 'SetVolume':
vol_token = directive['payload']['volume']
type_token = directive['payload']['adjustmentType']
if (type_token == 'relative'):
volume = player.get_volume() + int(vol_token)
else:
volume = int(vol_token)
if (volume > MAX_VOLUME):
volume = MAX_VOLUME
elif (volume < MIN_VOLUME):
volume = MIN_VOLUME
player.set_volume(volume)
logger.debug("new volume = %s", volume)
# Additional Audio Iten
elif 'audioItem' in j['messageBody']:
player.play_playlist(j['messageBody'])
return
elif response.status_code == 204:
logger.debug("Request Response is null (This is OKAY!)")
else:
logger.info("(process_response Error) Status Code: %s",
response.status_code)
response.connection.close()
platform.indicate_failure()
def _getMessage(self):
return email.message_from_string(self.getData())
def get_redirect_url(self, *args, **kwargs):
pk=self.kwargs.get('pk')
mail = imaplib.IMAP4_SSL('imap.gmail.com')
user = self.request.user
compania = Compania.objects.get(pk=pk)
print(compania)
assert compania.correo_intercambio, "No hay correo"
assert compania.pass_correo_intercambio, "No hay password"
correo = compania.correo_intercambio.strip()
passw = compania.pass_correo_intercambio.strip()
print(correo,passw)
try:
mail.login(correo,passw)
except imaplib.IMAP4.error as e:
messages.error(self.request, str(e))
return reverse_lazy('intercambios:lista', kwargs={'pk':pk})
mail.list()
mail.select("inbox")
result, data = mail.search(None, "ALL")
id_list = data[0].split()
try:
last_email = Intercambio.objects.latest('codigo_email')
except Intercambio.DoesNotExist:
last_email = 0
last_email_code = int(id_list[-1].decode())
if last_email == 0:
local_last_email_code = last_email
else:
local_last_email_code = int(last_email.codigo_email)
if last_email_code > local_last_email_code:
new_elements = last_email_code - local_last_email_code
else:
messages.info(self.request, "No posee nuevos correos")
return reverse_lazy('intercambios:lista', kwargs={'pk':pk})
if new_elements == 1:
latest_emails = [id_list[-1]]
else:
latest_emails = id_list[-new_elements:]
for element in latest_emails:
result, email_data = mail.fetch(element, "(RFC822)")
raw_email = email_data[0][1]
raw_multipart = email.message_from_bytes(raw_email)
raw_email_string = raw_email.decode('utf-8')
email_message = email.message_from_string(raw_email_string)
attachment_count, attachments = self.get_attachment(raw_multipart)
remisor_name, remisor_email = self.get_remisor(str(email.header.make_header(email.header.decode_header(email_message['From']))))
Intercambio.objects.create(
codigo_email = element.decode(),
receptor = compania,
remisor = remisor_name,
email_remisor = remisor_email,
fecha_de_recepcion = self.get_received_time(str(email.header.make_header(email.header.decode_header(email_message['Received'])))),
cantidad_dte = attachment_count,
titulo = str(email.header.make_header(email.header.decode_header(email_message['Subject']))),
contenido = self.get_body(raw_multipart).decode('latin-1')
)
messages.success(self.request, "Cantidad de correos nuevos: {}".format(new_elements))
return reverse_lazy('intercambios:lista', kwargs={'pk':pk})
def get_email_object(self):
return email.message_from_string(self.body)
def fetch_email_headers(num):
global yahoo_mail
#print "\nFetching Email header: ", num
print num
# id_list[-1] latest
#https://tools.ietf.org/html/rfc3501#section-6.4.5
# update global yahoo_mail dict and return value to hook
# peek does not seen seen ?
rv, data = M.fetch(num, '(BODY.PEEK[HEADER])') # header only
#rv, data = M.uid('fetch', num, '(BODY.PEEK[HEADER])') # header only
if rv != 'OK':
print "ERROR getting message header", num
return (False, " ", " ", " ")
raw_email = data[0][1]
# data type list, lot of lines
#https://docs.python.org/2/library/email.parser.html
raw_email_string = raw_email.decode('utf-8')
msg = email.message_from_string(raw_email_string)
# msg type instance , From, received, .. all headers and bodies all headers, list of tuple
# print msg.items()
# decode is a tuple (subject, None or 'utf-8')
decode = email.header.decode_header(msg['Subject'])[0]
subject = decode[0]
# type str
#print subject, type(subject), decode[1]
# P2(P3) str(bytes) unicode(str)
if isinstance(subject, str):
#print "str type, decode as utf"
subject = subject.decode('utf-8')
#print "decoded" , type(subject)
decode = email.header.decode_header(msg['From'])[0]
from_add = decode[0]
# type str
#print from_add, type(from_add), decode[1]
if isinstance(from_add, str):
#print "str type, decode as utf"
from_add = from_add.decode('utf-8')
#print "decoded" , type(from_add)
to_add = msg['To']
#print 'Raw Date:', msg['Date'] # type str
# Now convert to local date-time
date_tuple = email.utils.parsedate_tz(msg['Date'])
# type tuple (2018, 11, 16, 11, 0, 45, 0, 1, -1, 3600)
if date_tuple:
local_date = datetime.datetime.fromtimestamp(
email.utils.mktime_tz(date_tuple))
#print "Local Date:", local_date.strftime("%a, %d %b %Y %H:%M:%S")
# local_date 2018-11-17 16:31:26
# Local Date: Sat, 17 Nov 2018 16:31:26
# dict, value is list vs tuple as we want to change it later. add text
#print "\nFetching Email flag: ", num
(flag, (dele, seen)) = fetch_email_flag(num)
if not flag:
print "error getting flag: ", num
# default false, false
#type, data = M.store(num, '+FLAGS', '\\Seen')
return (True, from_add, to_add, subject, (dele, seen))
count = 0
print(
"______________________________________________________________________\n")
print("Empieza la recuperación y comparación de message-id\n")
# Se recorre cada elemento de la lista ID's, no condundir con el 'message-id'
for id in msg_list:
# Se deodifica de bytes a utf-8
id_raw = id.decode()
# Se recupera el encabezado del correo
# Retorna un mensaje del estado de la petición y la información solicitada
typ2, msg_data = conn.fetch(id, 'BODY.PEEK[HEADER]')
# Se decodifica
raw_email = msg_data[0][1].decode('utf-8')
# Se convierte en un objeto 'Message'
email_obj = email.message_from_string(raw_email)
# Se selecciona el 'message-id' y se le quitan los '<' y '>' del principio y final
msg_id = email_obj['message-id'].strip('<>')
# Se compara la expresión regular con el ID recién guardado
matched = re.fullmatch(regex, msg_id)
if not matched:
count = count + 1
print(
"______________________________________________________________________________________________________________________"
)
print("Un mensaje no cumple con la expresión regular!!!")
print("Fecha: " + email_obj['date'].strip('<>'))
print("Message-ID: " + email_obj['message-id'].strip('<>'))
print(
"Recuerda revisar si cumple con alguna expresión regular anterior o si la fecha es cercana a la validez de la actual ER!"
)
mail = imaplib.IMAP4_SSL(SMTP_SERVER)
mail.login(FROM_EMAIL, FROM_PWD) # user + password
mail.select('inbox')
type, data = mail.search(None,
'(FROM "*****@*****.**" SUBJECT "fileTesting")')
mail_ids = data[0]
id_list = mail_ids.split()
first_email_id = int(id_list[0])
latest_email_id = int(id_list[-1])
typ, data = mail.fetch(id_list[-1], '(RFC822)')
for response_part in data:
if isinstance(response_part, tuple):
msg = email.message_from_string(response_part[1])
email_subject = msg['subject']
email_from = msg['from']
for part in msg.walk():
#find the attachment part
if part.get_content_maintype() == 'multipart': continue
if part.get('Content-Disposition') is None: continue
#save the attachment in the program directory
filename = part.get_filename()
fp = open(filename, 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
print '%s saved!' % filename
for i in range(latest_email_id, first_email_id, -1):
def handle(self, *args, **options):
##########
# 1. Save the email-attachment file to the django model (database)
##########
sticker_email_host = env('STICKER_EMAIL_HOST', '')
sticker_email_port = env('STICKER_EMAIL_PORT', '')
sticker_email_username = env('STICKER_EMAIL_USERNAME', '')
sticker_email_password = env('STICKER_EMAIL_PASSWORD', '')
if not sticker_email_host or not sticker_email_port or not sticker_email_username or not sticker_email_password:
raise Exception(
'Configure email settings at .env to process sticker responses'
)
context = ssl.create_default_context()
imapclient = imaplib.IMAP4_SSL(sticker_email_host,
sticker_email_port,
ssl_context=context)
# login
imapclient.login(sticker_email_username, sticker_email_password)
"""
Retrieve messages
"""
imapclient.select('INBOX') # Select mail box
typ, data = imapclient.search(None, "ALL") # data = [b"1 2 3 4 ..."]
datas = data[0].split()
fetch_num = 5000 # The number of messages to fetch
if (len(datas) - fetch_num) < 0:
fetch_num = len(datas)
# msg_list = []
for num in datas[len(datas) - fetch_num::]:
try:
##########
# 2. Save the attached files into the database
##########
typ, data = imapclient.fetch(num, '(RFC822)')
# typ, data = imapclient.fetch(num, '(BODY[HEADER.FIELDS (MESSAGE-ID)])')
raw_email = data[0][1]
# converts byte literal to string removing b''
raw_email_string = raw_email.decode('utf-8')
email_message = email.message_from_string(raw_email_string)
email_message_id = str(
make_header(decode_header(email_message["Message-ID"])))
if StickerPrintingResponseEmail.objects.filter(
email_message_id=email_message_id):
# This email has been saved in the database already. Skip this email
continue
email_subject = str(
make_header(decode_header(email_message["Subject"])))
email_from = email_message['From']
body = get_text(email_message)
email_body = body.decode()
email_date = email_message['Date']
sticker_printing_response_email = StickerPrintingResponseEmail.objects.create(
email_subject=email_subject,
email_from=email_from,
email_body=email_body,
email_date=email_date,
email_message_id=email_message_id,
)
# downloading attachments
for part in email_message.walk():
try:
# this part comes from the snipped I don't understand yet...
if part.get_content_maintype() == 'multipart':
continue
if part.get('Content-Disposition') is None:
continue
fileName = part.get_filename()
if bool(fileName) and fileName.lower().endswith(
'.xlsx'):
now = timezone.localtime(timezone.now())
# Create sticker_printing_response object. File is not saved yet
sticker_printing_response = StickerPrintingResponse.objects.create(
sticker_printing_response_email=
sticker_printing_response_email,
name=fileName,
)
# Load attachment file
my_bytes = part.get_payload(decode=True)
content_file = ContentFile(my_bytes)
# Save file
sticker_printing_response._file.save(
fileName, content_file)
except Exception as e:
logger.exception(
'Exception has been raised when importing .xlsx file'
)
continue
# imapclient.store(num, "+FLAGS", "\\Deleted")
imapclient.copy(num, "Archive")
imapclient.store(num, "+FLAGS", "\\Deleted")
except:
logger.exception(
'Exception has been raised when processing emails')
continue
imapclient.close()
imapclient.logout()
##########
# 3. Process xlsx file saved in django model
##########
updates, errors = process_sticker_printing_response()
cmd_name = __name__.split('.')[-1].replace('_', ' ').upper()
# error_count = len(errors) + len(error_filenames)
error_count = len(errors)
err_str = '<strong style="color: red;">Errors: {}</strong>'.format(
error_count
) if error_count else '<strong style="color: green;">Errors: 0</strong>'
msg = '<p>{} completed. {}. IDs updated: {}.</p>'.format(
cmd_name, err_str, updates)
logger.info(msg)
print(
msg) # will redirect to cron_tasks.log file, by the parent script
"""
The init just contain versioning data.
"""
from collections import namedtuple
# get the raw PKG-INFO data
from pkg_resources import get_distribution
pkgInfo = get_distribution('gronk').get_metadata('PKG-INFO')
# parse it using email.Parser
from email import message_from_string
msg = message_from_string(pkgInfo)
inftuple = namedtuple('gronkinfo','author version year')
info = inftuple(msg['Author'],msg['Version'],'2018')
def process_mailbox(self, M, search_email, curr_dir):
rv, data = M.search(None, '(OR (TO %s) (FROM %s))' % (search_email, search_email))
if rv != 'OK':
print "No messages found!"
return
for num in data[0].split():
rv, data = M.fetch(num, '(RFC822)')
if rv != 'OK':
print "ERROR getting message", num
return
localinfo = []
print '-----'
msg = email.message_from_string(data[0][1])
decode = email.header.decode_header(msg['Subject'])[0]
subject = unicode(decode[0])
print 'Message %s : %s' % (num, subject)
self.ui.listWidget.addItem(QListWidgetItem('Message %s : %s' % (num, subject)))
localinfo.append(num)
localinfo.append(subject)
sender = msg['from'].split()[-1]
sender = sender[1:]
sender = sender[:-1]
print 'Sender : ', sender
localinfo.append(sender)
#print 'Raw Date:', msg['Date']
for part in msg.walk():
if part.get_content_type() == 'text/plain':
print part.get_payload()
localinfo.append(part.get_payload())
# Kepping up to local time
date_tuple = email.utils.parsedate_tz(msg['Date'])
if date_tuple:
local_date = datetime.datetime.fromtimestamp(
email.utils.mktime_tz(date_tuple))
print "Dated : ", \
local_date.strftime("%a, %d %b %Y %H:%M:%S")
fcount = 0
for part in msg.walk():
if(part.get('Content-Disposition' ) is not None ) :
filename = part.get_filename()
print filename
final_path= os.path.join(curr_dir + filename)
if not os.path.isfile(final_path) :
fp = open(curr_dir+"/"+(filename), 'wb')
fp.write(part.get_payload(decode=True))
fcount += 1
fp.close()
localinfo.append(fcount)
global mailinfo
mailinfo.append(localinfo)
print '%d attachment(s) fetched' %fcount
print '-----\n\n'
mailinfo = [item for sublist in mailinfo for item in sublist]
print mailinfo
def parse_email(raw_email):
if isinstance(raw_email, binary_type):
raw_email = str_encode(raw_email, 'utf-8')
try:
email_message = email.message_from_string(raw_email)
except UnicodeEncodeError:
email_message = email.message_from_string(raw_email.encode('utf-8'))
maintype = email_message.get_content_maintype()
parsed_email = {}
parsed_email['raw_email'] = raw_email
body = {"plain": [], "html": []}
attachments = []
if maintype in ('multipart', 'image'):
logger.debug("Multipart message. Will process parts.")
for part in email_message.walk():
content_type = part.get_content_type()
part_maintype = part.get_content_maintype()
content_disposition = part.get('Content-Disposition', None)
if content_disposition or not part_maintype == "text":
content = part.get_payload(decode=True)
else:
content = decode_content(part)
is_inline = content_disposition is None \
or content_disposition == "inline"
if content_type == "text/plain" and is_inline:
body['plain'].append(content)
elif content_type == "text/html" and is_inline:
body['html'].append(content)
elif content_disposition:
attachment = parse_attachment(part)
if attachment:
attachments.append(attachment)
elif maintype == 'text':
payload = decode_content(email_message)
body['plain'].append(payload)
parsed_email['attachments'] = attachments
parsed_email['body'] = body
email_dict = dict(email_message.items())
parsed_email['sent_from'] = get_mail_addresses(email_message, 'from')
parsed_email['sent_to'] = get_mail_addresses(email_message, 'to')
parsed_email['cc'] = get_mail_addresses(email_message, 'cc')
parsed_email['bcc'] = get_mail_addresses(email_message, 'bcc')
value_headers_keys = ['subject', 'date', 'message-id']
key_value_header_keys = [
'received-spf', 'mime-version', 'x-spam-status', 'x-spam-score',
'content-type'
]
parsed_email['headers'] = []
for key, value in email_dict.items():
if key.lower() in value_headers_keys:
valid_key_name = key.lower().replace('-', '_')
parsed_email[valid_key_name] = decode_mail_header(value)
if key.lower() in key_value_header_keys:
parsed_email['headers'].append({'Name': key, 'Value': value})
if parsed_email.get('date'):
timetuple = email.utils.parsedate(parsed_email['date'])
parsed_date = datetime.fromtimestamp(time.mktime(timetuple)) \
if timetuple else None
parsed_email['parsed_date'] = parsed_date
logger.info("Downloaded and parsed mail '{}' with {} attachments".format(
parsed_email.get('subject'), len(parsed_email.get('attachments'))))
return Struct(**parsed_email)
def _handle_encrypted(original, message, session_keys=None):
"""Handle encrypted messages helper.
RFC 3156 is quite strict:
* exactly two messages
* the first is of type 'application/pgp-encrypted'
* the first contains 'Version: 1'
* the second is of type 'application/octet-stream'
* the second contains the encrypted and possibly signed data
:param original: The original top-level mail. This is required to attache
special headers to
:type original: :class:`email.message.Message`
:param message: The multipart/signed payload to verify
:type message: :class:`email.message.Message`
:param session_keys: a list OpenPGP session keys
:type session_keys: [str]
"""
malformed = False
ct = message.get_payload(0).get_content_type()
if ct != _APP_PGP_ENC:
malformed = 'expected Content-Type: {0}, got: {1}'.format(
_APP_PGP_ENC, ct)
want = 'application/octet-stream'
ct = message.get_payload(1).get_content_type()
if ct != want:
malformed = 'expected Content-Type: {0}, got: {1}'.format(want, ct)
if not malformed:
# This should be safe because PGP uses US-ASCII characters only
payload = message.get_payload(1).get_payload().encode('ascii')
try:
sigs, d = crypto.decrypt_verify(payload, session_keys)
except GPGProblem as e:
# signature verification failures end up here too if the combined
# method is used, currently this prevents the interpretation of the
# recovered plain text mail. maybe that's a feature.
malformed = str(e)
else:
n = decrypted_message_from_bytes(d, session_keys)
# add the decrypted message to message. note that n contains all
# the attachments, no need to walk over n here.
original.attach(n)
original.defects.extend(n.defects)
# there are two methods for both signed and encrypted data, one is
# called 'RFC 1847 Encapsulation' by RFC 3156, and one is the
# 'Combined method'.
if not sigs:
# 'RFC 1847 Encapsulation', the signature is a detached
# signature found in the recovered mime message of type
# multipart/signed.
if X_SIGNATURE_VALID_HEADER in n:
for k in (X_SIGNATURE_VALID_HEADER,
X_SIGNATURE_MESSAGE_HEADER):
original[k] = n[k]
else:
# 'Combined method', the signatures are returned by the
# decrypt_verify function.
# note that if we reached this point, we know the signatures
# are valid. if they were not valid, the else block of the
# current try would not have been executed
add_signature_headers(original, sigs, '')
if malformed:
msg = 'Malformed OpenPGP message: {0}'.format(malformed)
content = email.message_from_string(msg,
_class=email.message.EmailMessage,
policy=email.policy.SMTP)
content.set_charset('utf-8')
original.attach(content)
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
if len(args) > 1:
if len(args) < 4:
print('You must specify the from address, to address and body text'
' on the command line')
return 1
msg = compose_mail(args[1],
args[2],
args[3],
subject=opts.subject,
attachment=opts.attachment)
from_, to = args[1:3]
eto = [extract_email_address(x.strip()) for x in to.split(',')]
efrom = extract_email_address(from_)
else:
msg = sys.stdin.read()
from email import message_from_string
from email.utils import getaddresses
eml = message_from_string(msg)
tos = eml.get_all('to', [])
ccs = eml.get_all('cc', []) + eml.get_all('bcc', [])
eto = [x[1] for x in getaddresses(tos + ccs) if x[1]]
if not eto:
raise ValueError(
'Email from STDIN does not specify any recipients')
efrom = getaddresses(eml.get_all('from', []))
if not efrom:
raise ValueError('Email from STDIN does not specify a sender')
efrom = efrom[0][1]
outbox = None
if opts.outbox is not None:
outbox = os.path.abspath(os.path.expanduser(opts.outbox))
from mailbox import Maildir
outbox = Maildir(opts.outbox, factory=None)
if opts.fork:
if os.fork() != 0:
return 0
if isinstance(msg, str):
msg = msg.encode('utf-8')
try:
sendmail(msg,
efrom,
eto,
localhost=opts.localhost,
verbose=opts.verbose,
timeout=opts.timeout,
relay=opts.relay,
username=opts.username,
password=opts.password,
port=opts.port,
encryption=opts.encryption_method,
verify_server_cert=not opts.dont_verify_server_certificate,
cafile=opts.cafile)
except:
if outbox is not None:
outbox.add(msg)
outbox.close()
print('Delivery failed. Message saved to', opts.outbox)
raise
return 0
"""
Take in text and replace all proofpoint url defense urls.
with the original url
"""
pp_url_pattern = re.compile(
r'(https://urldefense\.(?:proofpoint\.)?com/v\d.*?(&e=$|\$))')
for match in pp_url_pattern.finditer(c):
unescaped_url = match.group(0)
url = unescape(unescaped_url)
decoded_url = decode_proofpoint(url)
c = c.replace(unescaped_url, decoded_url)
return c
# Get the message from stdin
message = email.message_from_string(sys.stdin.read())
# # this should deal with either single messages or flat multipart messages.
# # A more thorough approach would be needed for complex mime trees
# # ref. http://blog.magiksys.net/parsing-email-using-python-content
#
if message.is_multipart():
for part in message.walk():
content_transfer = part.__getitem__('Content-Transfer-Encoding')
charset = part.get_content_charset()
content_type = part.get_content_type()
if content_type in ['text/html', 'text/plain']:
_content = part.get_payload(decode=True).decode('iso-8859-1')
_payload = revert_ppurls(_content)
if content_transfer and content_transfer.lower() == 'base64':
part.set_payload(base64.encodebytes(_payload.encode('utf=8')))
