def getMoreConversations(self, serial, count):
with self._store.writeLock as transaction:
self.manager.updated.emit(serial, State.InProgress, "step 1")
thrids = store.ConversationsDefaultDict(self._store.snapshot.conversations)
while len(thrids.added) < count:
n = count - len(thrids.added)
for item in self._fetch_uids(n):
thrids[item['X-GM-THRID']].message_ids.append(item['UID'])
thrids[item['X-GM-THRID']].message_ids.sort()
n -= 1
if n > 0:
break
self.manager.updated.emit(serial, State.InProgress, "step 2")
# TODO: sort thrids.added by date
for i, thrid in enumerate(thrids.added):
uids = self._search_thrid(thrid)
for uid, raw_headers in self._fetch_headers((uids[0],uids[-1]), ('SUBJECT','FROM','DATE')):
headers = self.email_parser.parsestr(raw_headers)
message = transaction.messages[uid]
message.subject = unicode(make_header(decode_header(headers['subject']))).replace('\r\n ',' ')
message.sender = unicode(make_header(decode_header(headers['from'])))
message.timestamp = mktime_tz(parsedate_tz(headers['date']))
transaction.thrids[thrid].message_ids = uids
transaction.thrids[thrid].subject = transaction.messages[uids[0]].subject
transaction.thrids[thrid].date = transaction.messages[uids[-1]].timestamp
self.manager.progress.emit(serial, float(i+1) / len(thrids.added))
transaction.commit(block=True)
self.manager.updated.emit(serial, State.Successful, "done")
def get_data():
mails = []
# Create connection to MySQL db
try:
con = connections.connect_MySQLdb()
except Exception as e:
logger.error(str(e))
raise Exception
logger.info('Successful connection to MySQLdb')
cur = con.cursor()
# Get value sender from database
try:
cur.execute("SELECT mb_value FROM `mb_properties` WHERE mb_field='Sender'")
sender = cur.fetchone()[0]
cur.execute("SELECT mb_value FROM `mb_properties` WHERE mb_field='Attempts'")
attempt = cur.fetchone()[0]
# Get all necessary information from database
cur.execute("""
SELECT mail_id,
mail_to,
mail_subject,
mail_body,
sender_marker
FROM mb_mail_buffer
WHERE mb_mail_buffer.attempt < %s
AND mb_mail_buffer.date_send is NULL
""" % attempt)
rows = cur.fetchall()
except Exception as e:
logger.error(str(e))
raise Exception
logger.info('Information from database is obtained')
# Create email from obtained information
for row in rows:
msg = MIMEMultipart('related')
msg['Subject'] = make_header([(row[2], 'UTF-8')])
msg['Date'] = formatdate(localtime=True)
msg['Organization'] = make_header([('SAMSUNG', 'UTF-8')])
msg['X-Mailer'] = make_header([('MailSender', 'UTF-8')])
msg['Message-ID'] = make_msgid()
msg['From'] = make_header([(sender, 'UTF-8')])
msg['To'] = make_header([(row[1].strip(), 'UTF-8')])
msg.preamble = "This is a multi-part message in MIME format."
msg.epilogue = "End of message"
partbody = MIMEText(row[3], '', 'UTF-8')
msg.attach(partbody)
# Create a list with tags mail_id, email object and sender_marker
mails.append((row[0], msg))
cur.close()
con.close()
if mails:
logger.info('Created %s letters' % len(mails))
return mails
def send_to_admin(notsendmsg):
try:
con = connections.connect_MySQLdb()
cur = con.cursor()
cur.execute("SELECT mb_value FROM `mb_properties` WHERE mb_field='Sender'")
sender = cur.fetchone()[0]
# Get SMTP server address and port from database
cur.execute("SELECT mb_value FROM `mb_properties` WHERE mb_field = 'SMTP server'")
server = cur.fetchone()[0]
cur.execute("SELECT mb_value FROM `mb_properties` WHERE mb_field = 'SMTP port'")
port = int(cur.fetchone()[0])
# Get admin email address from database
cur.execute("SELECT mb_value FROM `mb_properties` WHERE mb_field='Admin email'")
admin = cur.fetchone()[0]
cur.close()
con.close()
except Exception as e:
logger.error(str(e))
raise Exception
# Create connection to SMTP server
try:
# ransfer function of the parameter 2 server, port
serversmtp = connections.connect_SMTP()
except Exception as e:
logger.error(str(e))
raise Exception
logger.info('Connected\n')
serversmtp.noop()
for msg in notsendmsg:
# Create email fo admin
email = MIMEMultipart('related')
email['Subject'] = make_header([('SRK DPIportal [Mail Sender] Impossible send email!', 'UTF-8')])
email['Date'] = formatdate(localtime=True)
email['Organization'] = make_header([('SAMSUNG', 'UTF-8')])
email['X-Mailer'] = make_header([('MailSender', 'UTF-8')])
email['Message-ID'] = make_msgid()
email['From'] = make_header([(sender, 'UTF-8')])
email['To'] = make_header([(admin, 'UTF-8')])
email.preamble = "This is a multi-part message in MIME format."
email.epilogue = "End of message"
partbody = MIMEText('Impossible send email to ' + msg[0] + '\n' + 'Error:\n' + msg[1], '', 'UTF-8')
email.attach(partbody)
try:
serversmtp.sendmail(str(email['From']),
str(email['To']).split('; '),
email.as_string())
logger.info('Mail was send to %s' % admin)
except Exception as e:
logger.error(e)
logger.info('Impossible send email to admin %s' % admin)
serversmtp.quit()
def _set_info(self, msg):
if self.charset == 'us-ascii':
msg['Subject'] = self.Subject
msg['From'] = self.From
else:
if isinstance(self.Subject, unicode):
subject = self.Subject
else:
subject = unicode(self.Subject, self.charset)
msg['Subject'] = str(make_header([(subject, self.charset)]))
if isinstance(self.From, unicode):
from_ = self.From
else:
from_ = unicode(self.From, self.charset)
msg['From'] = str(make_header([(from_, self.charset)]))
if isinstance(self.To, basestring):
msg['To'] = self.To
else:
self.To = list(self.To)
msg['To'] = ", ".join(self.To)
if self.RTo:
if isinstance(self.RTo, basestring):
msg.add_header('reply-to', self.RTo)
else:
self.RTo = list(self.RTo)
msg.add_header('reply-to', ", ".join(self.RTo))
if self.CC:
if isinstance(self.CC, basestring):
msg['CC'] = self.CC
else:
self.CC = list(self.CC)
msg['CC'] = ", ".join(self.CC)
if self.BCC:
if isinstance(self.BCC, basestring):
msg['BCC'] = self.BCC
else:
self.BCC = list(self.BCC)
msg['BCC'] = ", ".join(self.BCC)
if self.Headers:
for key, value in self.Headers.items():
msg[key] = str(value).encode(self.charset)
msg['Date'] = self.Date
def get(self, key):
pdf = db.get(key)
fname = u"Beeline %s %s.pdf" % (pdf.name, gen_date(pdf),)
fname = str(make_header([(fname.encode("utf-8"), "utf-8")]))
self.response.headers["Content-Type"] = "application/pdf; name=\"%s\"" % (fname,)
self.response.headers["Content-Disposition"] = "filename=\"%s\"" % (fname,)
self.response.out.write(pdf.blob)
def parse_message(addresses, message):
for what in ["from", "to"]:
field = message[what]
if not field:
continue
header = str(make_header(decode_header(field)))
parse_header(addresses, header)
def ch_oneline(headerstr):
# Decode header string in one line and convert into single charset.
# Return (string, cset) tuple as check for failure.
try:
d = decode_header(headerstr)
# At this point, we should rstrip() every string because some
# MUA deliberately add trailing spaces when composing return
# message.
d = [(s.rstrip(), c) for (s, c) in d]
# Find all charsets in the original header. We use 'utf-8' rather
# than using the first charset (in mailman 2.1.x) if multiple
# charsets are used.
csets = []
for (s, c) in d:
if c and c not in csets:
csets.append(c)
if len(csets) == 0:
cset = 'us-ascii'
elif len(csets) == 1:
cset = csets[0]
else:
cset = 'utf-8'
h = make_header(d)
ustr = unicode(h)
oneline = ''.join(ustr.splitlines())
return oneline.encode(cset, 'replace'), cset
except (LookupError, UnicodeError, ValueError, HeaderParseError):
# possibly charset problem. return with undecoded string in one line.
return ''.join(headerstr.splitlines()), 'us-ascii'
def _set_info(self, msg):
if hasattr(self.additional_headers, 'items'):
for ahkey, value in self.additional_headers.items():
msg.add_header(ahkey, value);
if self.charset == 'us-ascii':
msg['Subject'] = self.Subject
else:
subject = unicode(self.Subject, self.charset)
msg['Subject'] = str(make_header([(subject, self.charset)]))
msg['From'] = self.From
if isinstance(self.To, basestring):
msg['To'] = self.To
else:
self.To = list(self.To)
msg['To'] = ", ".join(self.To)
if self.CC:
if isinstance(self.CC, basestring):
msg['CC'] = self.CC
else:
self.CC = list(self.CC)
msg['CC'] = ", ".join(self.CC)
msg['Date'] = self.Date
def oneline(s, cset='us-ascii', in_unicode=False):
"""Decode a header string in one line and convert into specified charset.
:param s: The header string
:type s: string
:param cset: The character set (encoding) to use.
:type cset: string
:param in_unicode: Flag specifying whether to return the converted string
as a unicode (True) or an 8-bit string (False, the default).
:type in_unicode: bool
:return: The decoded header string. If an error occurs while converting
the input string, return the string undecoded, as an 8-bit string.
:rtype: string
"""
try:
h = make_header(decode_header(s))
ustr = h.__unicode__()
line = EMPTYSTRING.join(ustr.splitlines())
if in_unicode:
return line
else:
return line.encode(cset, 'replace')
except (LookupError, UnicodeError, ValueError, HeaderParseError):
# possibly charset problem. return with undecoded string in one line.
return EMPTYSTRING.join(s.splitlines())
def __decode_header(header):
"""Decode a qp-encoded e-mail header as per rfc2047"""
try:
words_enc = decode_header(header)
hobj = make_header(words_enc)
except Exception as ex:
raise CmdException('header decoding error: %s' % str(ex))
return text(hobj)
def __init__(self, fetched_email):
self._source = fetched_email
accepted_types = ["text/plain", "text/html"]
parsed = email.message_from_string(fetched_email)
self._msg = parsed
try:
self.To = decodeaddresses(parsed.get_all("To", []))
self.From = decodeaddresses(parsed.get_all("From", []))
self.Cc = decodeaddresses(parsed.get_all("Cc", []))
self.timestamp = mktime_tz(parsedate_tz(parsed["date"]))
self.Date = datetime.fromtimestamp(self.timestamp)
self.Subject = make_header(decode_header(parsed["subject"]))
self.Subject = unicode(self.Subject)
self.body = {}
self.filenames = []
self._files = {}
except:
print self._source
pass
if parsed.is_multipart():
counter = 1
for part in parsed.walk():
# multipart/* are just containers
if part.get_content_maintype() == "multipart":
continue
filename = part.get_filename()
if not filename:
if part.get_content_type() in accepted_types:
if part.get_content_type() not in self.body:
self.body[part.get_content_type()] = []
self.body[part.get_content_type()].append(part.get_payload(decode=True))
else:
ext = mimetypes.guess_extension(part.get_content_type())
if not ext:
# Use a generic bag-of-bits extension
ext = ".bin"
filename = "part-%03d%s" % (counter, ext)
counter += 1
if filename:
filename = unicode(make_header(decode_header(filename)))
self.filenames.append(filename)
self._files[filename] = part
else:
if parsed.get_content_type() in accepted_types:
self.body = parsed.get_payload(decode=True)
def decode_mime(m):
"""Substitute matching encoded_word with unicode equiv."""
h = decode_header(clean_spaces(m))
try:
u = smart_text(make_header(h))
except (LookupError, UnicodeDecodeError):
return m.group(0)
return u
def oneline(s):
"""Inspired by mailman.utilities.string.oneline"""
try:
h = make_header(decode_header(s))
ustr = h.__unicode__()
return ''.join(ustr.splitlines())
except (LookupError, UnicodeError, ValueError, HeaderParseError):
return ''.join(s.splitlines())
def decode_mime(m):
"""substitute matching encoded_word with unicode equiv.
"""
h = decode_header(clean_spaces(m))
try:
u = unicode(make_header(h))
except UnicodeDecodeError:
return m.group(0)
return u
def poll(self, host, mailbox, user, passwd, room, ssl=True):
"""Poll an IMAP mailbox"""
log.info("Polling {0}@{1}".format(user, host))
if 'seen' not in self.shelf.keys():
seen = []
else:
seen = self.shelf['seen']
if ssl:
M = imaplib.IMAP4_SSL(host)
else:
M = imaplib.IMAP4(host)
log.debug("IMAP LOGIN")
code,message = M.login(user, passwd)
log.debug("{0}: {1}".format(code, message))
log.debug("IMAP SELECT: {0}".format(mailbox))
M.select(mailbox)
log.debug("{0}: {1}".format(code, message))
log.debug("IMAP SEARCH")
if self._highest_uid is None:
search = '(SENTSINCE {})'.format((datetime.datetime.now() + datetime.timedelta(weeks=-1)).strftime('%d-%b-%Y'))
else:
search = '(UID {}:*)'.format(self._highest_uid)
typ, data = M.search(None, search)
log.debug("{0}: {1}".format(typ, data))
for num in data[0].split():
typ, data = M.fetch(num, '(RFC822.HEADER)')
raw_mail = data[0][1]
# raw_message is a bytestring which must be decoded to make it usable
mail = email.message_from_string(raw_mail.decode("utf-8", "ignore"))
if mail.get('Message-ID') not in seen:
log.debug("New message: {0}".format(mail.get('Message-ID')))
seen.append(mail.get('Message-ID'))
message = 'New email arrived'
for hdrname in ['From','To','Cc','Subject']:
value = mail.get(hdrname) or None
if value:
if PY2:
pairs = decode_header(value)
hdrvalue = ' '.join([ unicode(t[0], t[1] or 'ASCII') for t in pairs ])
message += "\n\t{}: {}".format(hdrname, hdrvalue)
else:
hi = make_header(decode_header(value))
message += "\n\t{}: {}".format(hdrname, str(hi))
self.send(room, message, message_type='groupchat')
else:
log.debug("Seen message: {0}".format(mail.get('Message-ID')))
self._highest_uid = num.decode('ascii')
M.close()
M.logout()
self.shelf['seen'] = seen
self.shelf.sync()
def oneline(s):
"""Inspired by mailman.utilities.string.oneline"""
try:
h = make_header(decode_header(s))
ustr = h.__unicode__()
return ''.join(ustr.splitlines())
except (LookupError, UnicodeError, ValueError, HeaderParseError):
# possibly charset problem. return with undecoded string in one line.
return ''.join(s.splitlines())
def set_headers(self, message):
charset = str(self.context.charset)
extra = list(self.context.get_headers())
for key, val, encode in itertools.chain(self._headers.values(), extra):
if encode:
message[key] = make_header(((val, charset),))
else:
message[key] = val
def decode_email(file):
# Prepare result
theMail = {
'attachment_list': [],
'body': '',
# Place all the email header in the headers dictionary in theMail
'headers': {}
}
# Get Message
msg = email.message_from_string(file)
# Back up original file
theMail['__original__'] = file
# Recode headers to UTF-8 if needed
for key, value in msg.items():
decoded_value_list = decode_header(value)
unicode_value = make_header(decoded_value_list)
new_value = unicode_value.__unicode__().encode('utf-8')
theMail['headers'][key.lower()] = new_value
# Filter mail addresses
for header in ('resent-to', 'resent-from', 'resent-cc', 'resent-sender',
'to', 'from', 'cc', 'sender', 'reply-to'):
header_field = theMail['headers'].get(header)
if header_field:
theMail['headers'][header] = parseaddr(header_field)[1]
# Get attachments
body_found = 0
for part in msg.walk():
content_type = part.get_content_type()
file_name = part.get_filename()
# multipart/* are just containers
# XXX Check if data is None ?
if content_type.startswith('multipart'):
continue
# message/rfc822 contains attached email message
# next 'part' will be the message itself
# so we ignore this one to avoid doubling
elif content_type == 'message/rfc822':
continue
elif content_type in ("text/plain", "text/html"):
charset = part.get_content_charset()
payload = part.get_payload(decode=True)
#LOG('CMFMailIn -> ',0,'charset: %s, payload: %s' % (charset,payload))
if charset:
payload = unicode(payload, charset).encode('utf-8')
if body_found:
# Keep the content type
theMail['attachment_list'].append((file_name,
content_type, payload))
else:
theMail['body'] = payload
body_found = 1
else:
payload = part.get_payload(decode=True)
# Keep the content type
theMail['attachment_list'].append((file_name, content_type,
payload))
return theMail
def decodeaddresses(addresslist):
alist = []
for realname, email_address in getaddresses(addresslist):
dh = []
for s, c in decode_header(realname):
if c:
c = c.replace("gb2312", "gb18030")
dh.append((s, c))
alist.append((unicode(make_header(dh)), email_address))
return alist
def format_email(e, list_name):
subject = e['Subject']
subject = re.sub(r"(=\?.*\?=)(?!$)", r"\1 ", subject)
subject = str(make_header(decode_header(subject)))
subject = subject.replace('['+list_name.capitalize()+'] ', '')
# message = '{}: {} * {} * {}'.format(list_name, obfuscate_address(e['From']), subject, strip_reply_lines(e))
message = '{}: {}: {}'.format(list_name, obfuscate_address(e['From']), subject)
return truncate(message)
def _write_headers(self, headers):
if headers:
for name in sorted(headers.keys()):
value = headers[name]
if isinstance(value, unicode):
value = str(header.make_header([(value, 'utf-8')]))
self.fileobj.write(name)
self.fileobj.write(': ')
self.fileobj.write(value)
self.fileobj.write(CRLF)
self.fileobj.write(CRLF)
def _update_from_text(self):
try:
self._raw = make_header([(self._text, self._charset)]).encode()
return
except:
pass
try:
self._raw = make_header([(self._text, "utf-8")]).encode()
self._charset = "utf-8"
return
except:
pass
try:
self._raw = make_header([(self._text.encode("utf-8", errors="ignore"), "utf-8")]).encode()
self._charset = "utf-8"
return
except:
raise
def _write_headers(self, headers):
if headers:
for name in sorted(headers.keys()):
value = headers[name]
if value.encode('ascii', 'ignore') != value.encode('utf-8'):
value = header.make_header([(value, 'utf-8')]).encode()
self.fileobj.write(name.encode('utf-8'))
self.fileobj.write(b': ')
self.fileobj.write(value.encode('utf-8'))
self.fileobj.write(CRLF)
self.fileobj.write(CRLF)
def _set_info(self, msg):
if self.charset == 'us-ascii':
msg['Subject'] = self.Subject
else:
subject = str(self.Subject, self.charset)
msg['Subject'] = str(make_header([(subject, self.charset)]))
msg['From'] = self.From
if isinstance(self.To, str):
msg['To'] = self.To
else:
msg['To'] = ", ".join(self.To)
def masseges(self):
msg = MIMEMultipart()
msg['Message-ID'] = make_msgid()
msg['Date'] = formatdate(localtime=True)
msg['From'] = make_header([('{}'.format(self.from_text), 'UTF-8'), ('<' + self.from_email + '>', 'us-ascii')])
msg['To'] = self.to
msg['Subject'] = self.subject
msg['User-Agent'] = 'Horde Application Framework 5'
msg['Content-Disposition'] = "inline"
msg['Content-Transfer-Encoding'] = "8bit"
msg.attach(MIMEText(self.message_text, _subtype='html', _charset='utf-8'))
return(msg)
def mixed_charsets(subject, prefix, prefix_pattern, ws):
list_charset = 'us-ascii'
chunks = decode_header(subject.encode())
if len(chunks) == 0:
subject_text = '(no subject)'
chunks = [(prefix, list_charset),
(subject_text, list_charset),
]
return make_header(chunks, continuation_ws=ws)
# Only search the first chunk for Re and existing prefix.
chunk_text, chunk_charset = chunks[0]
if chunk_charset is None:
chunk_charset = 'us-ascii'
first_text = chunk_text.decode(chunk_charset)
first_text = re.sub(prefix_pattern, '', first_text).lstrip()
rematch = re.match(RE_PATTERN, first_text, re.I)
if rematch:
first_text = 'Re: ' + first_text[rematch.end():]
chunks[0] = (first_text, chunk_charset)
chunks.insert(0, (prefix, list_charset))
return make_header(chunks, continuation_ws=ws)
def process_stream_message(to: str, message: message.Message,
debug_info: Dict[str, Any]) -> None:
subject_header = str(make_header(decode_header(message.get("Subject", ""))))
subject = strip_from_subject(subject_header) or "(no topic)"
stream, show_sender = extract_and_validate(to)
# Don't remove quotations if message is forwarded:
remove_quotations = not is_forwarded(subject_header)
body = construct_zulip_body(message, stream.realm, show_sender, remove_quotations)
debug_info["stream"] = stream
send_zulip(settings.EMAIL_GATEWAY_BOT, stream, subject, body)
logger.info("Successfully processed email to %s (%s)" % (
stream.name, stream.realm.string_id))
def send_email_simple(sender, receiver, subject, text, html, attachments=()):
"""
send_email simple function
"""
charset = "UTF-8"
msg = MIMEMultipart('related')
msg['From'] = str(make_header([(sender, charset)]))
msg['To'] = str(make_header([(receiver, charset)]))
msg['Subject'] = str(make_header([(subject, charset)]))
msg['Date'] = formatdate()
body = MIMEMultipart('alternative')
plain_part = MIMEText(text, 'plain', charset)
plain_part.add_header('Content-Disposition', 'inline')
html_part = MIMEText(html, 'html', charset)
html_part.add_header('Content-Disposition', 'inline')
body.attach(plain_part)
body.attach(html_part)
msg.attach(body)
for filename, cid in attachments:
fp = open(filename, 'rb')
img = MIMEImage(fp.read())
fp.close()
if cid:
img.add_header('Content-ID', '<%s>' % cid)
img.add_header('Content-Disposition', 'inline')
else:
img.add_header('Content-Disposition', 'attachment', filename=os.path.basename(filename))
msg.attach(img)
# Send the message via local SMTP server.
s = smtplib.SMTP('localhost')
s.sendmail(sender, receiver, msg.as_string())
s.quit()
def __init__(self, msg, msgdata, results):
self.command_lines = []
self.ignored_lines = []
self.processed_lines = []
# Depending on where the message was destined to, add some implicit
# commands. For example, if this was sent to the -join or -leave
# addresses, it's the same as if 'join' or 'leave' commands were sent
# to the -request address.
subaddress = msgdata.get('subaddress')
if subaddress == 'join':
self.command_lines.append('join')
elif subaddress == 'leave':
self.command_lines.append('leave')
elif subaddress == 'confirm':
mo = re.match(config.mta.verp_confirm_regexp, msg.get('to', ''))
if mo:
self.command_lines.append('confirm ' + mo.group('cookie'))
# Extract the subject header and do RFC 2047 decoding.
raw_subject = msg.get('subject', '')
try:
subject = unicode(make_header(decode_header(raw_subject)))
# Mail commands must be ASCII.
self.command_lines.append(subject.encode('us-ascii'))
except (HeaderParseError, UnicodeError, LookupError):
# The Subject header was unparseable or not ASCII. If the raw
# subject is a unicode object, convert it to ASCII ignoring all
# bogus characters. Otherwise, there's nothing in the subject
# that we can use.
if isinstance(raw_subject, unicode):
safe_subject = raw_subject.encode('us-ascii', 'ignore')
self.command_lines.append(safe_subject)
# Find the first text/plain part of the message.
part = None
for part in typed_subpart_iterator(msg, 'text', 'plain'):
break
if part is None or part is not msg:
# Either there was no text/plain part or we ignored some
# non-text/plain parts.
print(_('Ignoring non-text/plain MIME parts'), file=results)
if part is None:
# There was no text/plain part to be found.
return
body = part.get_payload(decode=True)
# text/plain parts better have string payloads.
assert isinstance(body, basestring), 'Non-string decoded payload'
lines = body.splitlines()
# Use no more lines than specified
max_lines = int(config.mailman.email_commands_max_lines)
self.command_lines.extend(lines[:max_lines])
self.ignored_lines.extend(lines[max_lines:])
def mixed_charsets(mlist, msgdata, subject, prefix, prefix_pattern, ws):
list_charset = mlist.preferred_language.charset
chunks = decode_header(subject.encode())
if len(chunks) == 0:
with _.push(mlist.preferred_language.code):
subject_text = _('(no subject)')
chunks = [(prefix, list_charset),
(subject_text, list_charset),
]
return make_header(chunks, continuation_ws=ws)
# Only search the first chunk for Re and existing prefix.
chunk_text, chunk_charset = chunks[0]
if chunk_charset is None:
chunk_charset = 'us-ascii'
first_text = chunk_text.decode(chunk_charset)
first_text = re.sub(prefix_pattern, '', first_text).lstrip()
rematch = re.match(RE_PATTERN, first_text, re.I)
if rematch:
first_text = 'Re: ' + first_text[rematch.end():]
chunks[0] = (first_text, chunk_charset)
# The subject text stripped of the prefix, for use in the NNTP gateway.
msgdata['stripped_subject'] = str(make_header(chunks, continuation_ws=ws))
chunks.insert(0, (prefix, list_charset))
return make_header(chunks, continuation_ws=ws)
def buildEmailMessage(from_url, to_url, msg=None,
subject=None, attachment_list=None,
extra_headers=None,
additional_headers=None):
"""
Builds a mail message which is ready to be
sent by Zope MailHost.
* attachment_list is a list of dictionaries with those keys:
- name : name of the attachment,
- content: data of the attachment
- mime_type: mime-type corresponding to the attachment
* extra_headers is a dictionary of custom headers to add to the email.
"X-" prefix is automatically added to those headers.
* additional_headers is similar to extra_headers, but no prefix is added.
"""
if attachment_list == None:
# Create non multi-part MIME message.
message = MIMEText(msg, _charset='utf-8')
attachment_list = []
else:
# Create multi-part MIME message.
message = MIMEMultipart()
message.preamble = "If you can read this, your mailreader\n" \
"can not handle multi-part messages!\n"
message.attach(MIMEText(msg, _charset='utf-8'))
if extra_headers:
for key, value in extra_headers.items():
message.add_header('X-%s' % key, value)
if additional_headers:
for key, value in additional_headers.items():
message.add_header(key, value)
if subject:
message.add_header('Subject',
make_header([(subject, 'utf-8')]).encode())
if from_url:
message.add_header('From', from_url)
if to_url:
message.add_header('To', to_url)
for attachment in attachment_list:
attachment_name = attachment.get('name', '')
# try to guess the mime type
if not attachment.has_key('mime_type'):
mime_type, _ = guess_type( attachment_name )
if mime_type is not None:
attachment['mime_type'] = mime_type
else:
attachment['mime_type'] = 'application/octet-stream'
# attach it
if attachment['mime_type'] == 'text/plain':
part = MIMEText(attachment['content'], _charset='utf-8')
else:
major, minor = attachment['mime_type'].split('/', 1)
if major == 'text':
part = MIMEText(attachment['content'], _subtype=minor)
elif major == 'image':
part = MIMEImage(attachment['content'], _subtype=minor)
elif major == 'audio':
part = MIMEAudio(attachment['content'], _subtype=minor)
else:
# encode non-plaintext attachment in base64
part = MIMEBase(major, minor)
part.set_payload(attachment['content'])
encoders.encode_base64(part)
part.add_header('Content-Disposition', 'attachment',
filename=attachment_name)
part.add_header('Content-ID', '<%s>' % \
''.join(['%s' % ord(i) for i in attachment_name]))
message.attach(part)
return message
def handle_header_content(content: str) -> str:
"""
Deals with converting encoded headers to readable python string.
"""
return str(make_header(decode_header(content)))
def share(self, msgfile):
msg = email.message_from_file(msgfile)
args = {}
files = []
check = getattr(settings, 'EMAIL2POST_CHECK', {})
for lhs in check:
v = six.text_type(make_header(decode_header(msg.get(lhs, ''))))
if not check[lhs] in v:
return 77 # EX_NOPERM
if msg.is_multipart():
for part in msg.walk():
attach = False
t = part.get_content_type()
if t == 'text/plain':
if part.get_filename(None):
attach = True
else:
args['content'] = part.get_payload(decode=True)
if attach or \
t.startswith ('image/') or \
t.startswith ('audio/') or \
t.startswith ('video/') or \
t.startswith('application/'):
payload = part.get_payload(decode=True)
os.umask(0)
tmp = TemporaryUploadedFile(
name=part.get_filename('attachment'),
content_type=t,
size=len(payload),
charset=None)
tmp.write(payload)
tmp.seek(0)
os.chmod(tmp.file.name, 0o644)
files.append(tmp)
else:
args['content'] = msg.get_payload(decode=True)
subject = msg.get('Subject', None)
if subject:
hdr = make_header(decode_header(subject))
args['title'] = six.text_type(hdr)
# Mail subject may contain @foo, a selfposts' class name for which
# this message is post to.
m = re.search(r'(\A|\s)@(\w[\w\-]+)', args['title'])
if m:
cls = m.groups()[1]
args['title'] = re.sub(r'(\A|\s)@(\w[\w\-]+)', '', args['title'])
s = Service.objects.filter(cls=cls, api='selfposts').values('id')
if len(s):
args['id'] = s[0]['id']
# Mail subject may contain "!draft" literal.
if '!draft' in args['title']:
args['title'] = args['title'].replace('!draft', '').strip()
args['draft'] = True
# Mail subject may contain "!friends-only" literal.
if '!friends-only' in args['title']:
args['title'] = args['title'].replace('!friends-only', '').strip()
args['friends_only'] = True
if len(files):
args['files'] = MultiValueDict()
args['files'].setlist('docs', files)
selfposts.API(None).share(args)
return 0 # EX_OK
def header_as_unicode(key):
decoded = header.decode_header(parsed_mail[key])
return str(header.make_header(decoded))
def safe_decode_header(header):
"""Not every email is RFC-conform."""
try:
return make_header(decode_header(header))
except Exception:
return header
def get_subject_from_mail(mail):
text = make_header(decode_header(mail.get("Subject")))
return str(text)
def poll(self, host, mailbox, user, passwd, room, ssl=True):
"""Poll an IMAP mailbox"""
log.info("Polling {0}@{1}".format(user, host))
if 'seen' not in self.shelf.keys():
seen = []
else:
seen = self.shelf['seen']
if ssl:
M = imaplib.IMAP4_SSL(host)
else:
M = imaplib.IMAP4(host)
log.debug("IMAP LOGIN")
code, message = M.login(user, passwd)
log.debug("{0}: {1}".format(code, message))
log.debug("IMAP SELECT: {0}".format(mailbox))
M.select(mailbox)
log.debug("{0}: {1}".format(code, message))
log.debug("IMAP SEARCH")
if self._highest_uid is None:
search = '(SENTSINCE {})'.format(
(datetime.datetime.now() +
datetime.timedelta(weeks=-1)).strftime('%d-%b-%Y'))
else:
search = '(UID {}:*)'.format(self._highest_uid)
typ, data = M.search(None, search)
log.debug("{0}: {1}".format(typ, data))
for num in data[0].split():
typ, data = M.fetch(num, '(RFC822.HEADER)')
raw_mail = data[0][1]
# raw_message is a bytestring which must be decoded to make it usable
mail = email.message_from_string(raw_mail.decode(
"utf-8", "ignore"))
if mail.get('Message-ID') not in seen:
log.debug("New message: {0}".format(mail.get('Message-ID')))
seen.append(mail.get('Message-ID'))
message = 'New email arrived'
for hdrname in ['From', 'To', 'Cc', 'Subject']:
value = mail.get(hdrname) or None
if value:
if PY2:
pairs = decode_header(value)
hdrvalue = ' '.join([
unicode(t[0], t[1] or 'ASCII') for t in pairs
])
message += "\n\t{}: {}".format(hdrname, hdrvalue)
else:
hi = make_header(decode_header(value))
message += "\n\t{}: {}".format(hdrname, str(hi))
self.send(room, message, message_type='groupchat')
else:
log.debug("Seen message: {0}".format(mail.get('Message-ID')))
self._highest_uid = num.decode('ascii')
M.close()
M.logout()
self.shelf['seen'] = seen
self.shelf.sync()
def make_address(email, name=None):
if name:
return str(make_header(decode_header(formataddr((name, email)))))
return email
def process_message(self, peer, mailfrom, rcpttos, data):
message = email.message_from_string(data)
from_str = message.get('From', "").strip()
to_str = message.get('To', "").strip()
cc_str = message.get('Cc', "").strip()
subject = message.get('Subject', "").strip()
try:
subject = str(make_header(decode_header(subject)))
except:
pass
command = subject.strip().lower()
logging.info("Processing: from=%s|to=%s|cc=%s|subject=%s|", from_str,
to_str, cc_str, subject)
_from = normalize([from_str])
to = normalize(to_str.split(','))
cc = normalize(cc_str.split(','))
# Quick hack
mailfrom = list(_from)[0]
# Check cross-post
mls = [_ for _ in (to | cc) if _.endswith(self.at_domain)]
if len(mls) == 0:
logging.error("No ML specified")
return const.SMTP_STATUS_NO_ML_SPECIFIED
elif len(mls) > 1:
logging.error("Can't cross-post a message")
return const.SMTP_STATUS_CANT_CROSS_POST
# Aquire the ML name
ml_address = mls[0]
ml_name = ml_address.replace(self.at_domain, "")
params = dict(ml_name=ml_name,
ml_address=ml_address,
mailfrom=mailfrom)
# Remove the ML name from to'd and cc'd address lists
if ml_address in to:
to.remove(ml_address)
if ml_address in cc:
cc.remove(ml_address)
# Is an error mail?
if ml_name.endswith(ERROR_SUFFIX):
ml_name = ml_name.replace(ERROR_SUFFIX, "")
error_str = REMOVE_RFC822.sub(
"", message.get('Original-Recipient', ""))
error = normalize(error_str.split(','))
if len(error) > 0 and len(ml_name) > 0:
logging.error("not delivered to %s for %s", error, ml_name)
return
# Aquire current tenants
tenants = db.find_tenants({"status": const.TENANT_STATUS_ENABLED})
# Want a new ML?
for config in tenants:
if ml_name == config['new_ml_account']:
tenant_name = config['tenant_name']
ml_name = config['ml_name_format'] % \
db.increase_counter(config['tenant_name'])
members = (to | cc | _from) - config['admins']
db.create_ml(tenant_name, ml_name, subject, members, mailfrom)
ml_address = ml_name + self.at_domain
params = dict(ml_name=ml_name,
ml_address=ml_address,
mailfrom=mailfrom,
members=members)
message = ensure_multipart(message, config['charset'])
self.send_message(config, ml_name, message, mailfrom, params,
config['welcome_msg'], 'Welcome.txt')
return
# Post a message to an existing ML
# Check ML exists
ml = db.get_ml(ml_name)
if ml is None:
logging.error("No such ML: %s", ml_name)
return const.SMTP_STATUS_NO_SUCH_ML
# Set config variable
for config in tenants:
if ml['tenant_name'] == config['new_ml_account']:
break
else:
if config is None:
logging.error("No such tenant: %s", ml['tenant_name'])
return const.SMTP_STATUS_NO_SUCH_TENANT
message = ensure_multipart(message, config['charset'])
# Checking whether the sender is one of the ML members
members = db.get_members(ml_name)
if mailfrom not in (members | config['admins']):
logging.error("Non-member post")
return const.SMTP_STATUS_NOT_MEMBER
# Update parameters
new_ml_address = config['new_ml_account'] + self.at_domain
params = dict(ml_name=ml_name,
ml_address=ml_address,
mailfrom=mailfrom,
new_ml_address=new_ml_address,
members=members)
# Check ML status
ml_status = ml['status']
if ml_status == const.STATUS_CLOSED:
if command == "reopen":
self.send_message(config, ml_name, message, mailfrom, params,
config['reopen_msg'], 'Reopen.txt')
db.change_ml_status(ml_name, const.STATUS_OPEN, mailfrom)
logging.info("reopened %s by %s", ml_name, mailfrom)
return
logging.error("ML is closed: %s", ml_name)
return const.SMTP_STATUS_CLOSED_ML
elif command == "close":
self.send_message(config, ml_name, message, mailfrom, params,
config['goodbye_msg'], 'Goodbye.txt')
db.change_ml_status(ml_name, const.STATUS_CLOSED, mailfrom)
logging.info("closed %s by %s", ml_name, mailfrom)
return
if ml_status != const.STATUS_OPEN:
db.change_ml_status(ml_name, const.STATUS_OPEN, mailfrom)
# Remove admin members from cc
cc -= config['admins']
params["cc"] = cc
# Remove cc'd members from the ML members if the subject is empty
if command == "":
if len(cc) > 0:
params['members'] = members - cc
self.send_message(config, ml_name, message, mailfrom, params,
config['remove_msg'], 'RemoveMembers.txt')
db.del_members(ml_name, cc, mailfrom)
logging.info("removed %s from %s", cc, ml_name)
return
# Checking Cc:
if len(cc) > 0:
db.add_members(ml_name, cc, mailfrom)
logging.info("added %s into %s", cc, ml_name)
members = db.get_members(ml_name)
params['members'] = members
self.send_message(config, ml_name, message, mailfrom, params,
config['add_msg'], 'AddMembers.txt')
return
# Attach readme and send the post
self.send_message(config, ml_name, message, mailfrom, params,
config['readme_msg'], 'Readme.txt')
def _decode_header(self, header_title: str) -> typing.Optional[str]:
# FIXME : Handle exception
if header_title in self._message:
return str(make_header(decode_header(self._message[header_title])))
else:
return None
def _get_file_name(self, input_str):
try:
return str(make_header(decode_header(input_str)))
except Exception:
return self._decode_uni_string(input_str, input_str)
def sanitise_header(header_contents, header_name=None):
"""Clean and individual mail header.
Given a header with header_contents, optionally labelled
header_name, decode it with decode_header, sanitise it to make
sure it decodes correctly and contains no invalid characters,
then encode the result with make_header()
"""
# We have some Py2/Py3 issues here.
#
# Firstly, the email parser (before we get here)
# Python 3: headers with weird chars are email.header.Header
# class, others as str
# Python 2: every header is an str
#
# Secondly, the behaviour of decode_header:
# Python 3: weird headers are labelled as unknown-8bit
# Python 2: weird headers are not labelled differently
#
# Lastly, aking matters worse, in Python2, unknown-8bit doesn't
# seem to be supported as an input to make_header, so not only do
# we have to detect dodgy headers, we have to fix them ourselves.
#
# We solve this by catching any Unicode errors, and then manually
# handling any interesting headers.
value = decode_header(header_contents)
try:
header = make_header(value,
header_name=header_name,
continuation_ws='\t')
except UnicodeDecodeError:
# At least one of the parts cannot be encoded as ascii.
# Find out which one and fix it somehow.
#
# We get here under Py2 when there's non-7-bit chars in header,
# or under Py2 or Py3 where decoding with the coding hint fails.
new_value = []
for (part, coding) in value:
# We have random bytes that aren't properly coded.
# If we had a coding hint, it failed to help.
if six.PY3:
# python3 - force coding to unknown-8bit
new_value += [(part, 'unknown-8bit')]
else:
# python2 - no support in make_header for unknown-8bit
# We should do unknown-8bit coding ourselves.
# For now, we're just going to replace any dubious
# chars with ?.
#
# TODO: replace it with a proper QP unknown-8bit codec.
new_value += [
(part.decode('ascii',
errors='replace').encode('ascii',
errors='replace'),
None)
]
header = make_header(new_value,
header_name=header_name,
continuation_ws='\t')
return header
# Walk the email and pull out all relevant information depending on the subject
if 'User Submission:' in email_subject:
for part in raw.walk():
if part.get_content_type() == 'message/rfc822':
part_string = str(part)
#print(part_string) # Easy way to see the full message for this part if you need to
try:
# Pulling the full email header of the attachment and then decoding it so I can truly get the to, from, and subject without formatting issues
# I had to pull a lot larger section because the email header format/order varied on me
attachment_header = re.search(
'(Content-Disposition:\sattachment;(.*?(\n))+.*?)MIME-Version:',
part_string)
coded_header = attachment_header.group(1)
decoded_header = make_header(decode_header(coded_header))
#print(decoded_header)
original_from = re.search('\sFrom:\s(.+?)\n?(\S+:)',
str(decoded_header))
original_to = re.search('\sTo:\s(.+?)\n?(\S+:)',
str(decoded_header))
original_subject = re.search(
'Subject:\s(.+?)\n?(\w+\-\w+:|From:\s|To:\s|Type:\s|Date:\s|References:\s)',
str(decoded_header))
from_address = original_from.group(1)
to_address = original_to.group(1)
email_subject = original_subject.group(1)
# I need to clean this up. Right now if one fails, they all fail
def _convert(self, text):
decoded = decode_header(text)
if not decoded[0][1] or 'unknown' in decoded[0][1]:
decoded = [(decoded[0][0], 'latin-1')]
return str(make_header(decoded))
def mbox_df(infile):
try:
row = []
for message in mbox(infile):
content = get_content(message)
line = [str(parsedate_to_datetime(message['date'])), message['Message-ID'], message['In-Reply-To'], message['References'], message['from'], message['to'], message['Cc'], str(make_header(decode_header(message['subject']))), content]
row.append(line)
except Exception as e:
print("Something wrong with your mbox file: ", infile, ". Check the file(s) if it is formed correctly. Maybe 'From ' in the mail body has not been correctly escaped as '>From '. Refer to the following resource:\nhttps://pymotw.com/3/mailbox/\nThe specific error message from Python: ", e, sep="")
finally:
df = DataFrame(row, columns=['date', 'message_ID', 'in_reply_to', 'references', 'from', 'to', 'cc', 'subject', 'content'])
return df
def get_formatted_email(user, mail=None):
"""get Email Address of user formatted as: `John Doe <*****@*****.**>`"""
fullname = get_fullname(user)
if not mail:
mail = get_email_address(user)
return cstr(make_header(decode_header(formataddr((fullname, mail)))))
def downloaAttachmentsInEmail(m, emailid, outputdir):
resp, data = m.fetch(emailid, '(RFC822)')
email_body = data[0][1]
mail = email.message_from_bytes(email_body)
# Extract sender name and sender email address from the email
sender = str(make_header(decode_header(mail['From'])))
try:
name, email_addr = sender.split('<')
email_addr = email_addr.replace(">", "")
except:
print("Sender ohne Name")
email_addr = sender
name = " "
if mail.get_content_maintype() != 'multipart':
return -1
# Each email address gets its own folder
new_folder_name = "".join(email_addr.split("@")[0].split("."))
outputdir = outputdir + new_folder_name + "/"
if not os.path.exists(outputdir):
os.makedirs(outputdir)
for part in mail.walk():
if part.get_content_maintype() != 'multipart' and part.get(
'Content-Disposition') is not None:
old_filename = part.get_filename()
# Check if attachement contains "whatsapp chat"
#if "whatsapp chat" not in old_filename.lower():
# return -1
# Write file/attachment to disk
if part.get_filename() is not None:
sv_path = os.path.join(outputdir, old_filename)
if not os.path.isfile(sv_path):
fp = open(sv_path, 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
# iOS: File is zipped; Android: File is a unzipped
if ".txt" in old_filename:
# File is from Android client
# New Filename: DD_MM_YY_OLDFILENAME.txt
new_filename = str((time.strftime("%d_%m_%Y"))) + "_" + \
old_filename
os.rename(outputdir + old_filename, outputdir + new_filename)
else:
# File is from iOS client
# New Filename: DD_MM_YY_OLDFILENAME.txt
try:
new_filename = old_filename[:-3] + "txt"
unzip(outputdir + old_filename, outputdir)
os.rename(outputdir + '_chat.txt',
outputdir + new_filename)
os.remove(outputdir + old_filename) # remove zip file
except:
return -1
return (new_folder_name, name, email_addr,
outputdir + new_filename)
def subject(self):
"""Message subject"""
value = self.get("subject")
return str(make_header(decode_header(value))) if value else None
msg_list = [] # 取得したMIMEメッセージを格納するリスト
for num in datas[len(datas) - fetch_num::]:
typ, data = imapclient.fetch(num, '(RFC822)')
msg = email.message_from_bytes(data[0][1])
msg_list.append(msg)
imapclient.close()
imapclient.logout()
os.system('cls')
c = 0
for msg in msg_list:
print(msg)
for msg in msg_list:
# 各ヘッダ情報はディクショナリのようにアクセスできる
from_addr = str(make_header(decode_header(msg["From"])))
subject = str(make_header(decode_header(msg["Subject"])))
# 本文(payload)を取得する
if msg.is_multipart() is False:
# シングルパートのとき
payload = msg.get_payload(decode=True) # 備考の※1
charset = msg.get_content_charset() # 備考の※2
if charset is not None:
payload = payload.decode(charset, "ignore")
print(payload)
print()
else:
# マルチパートのとき
for part in msg.walk():
payload = part.get_payload(decode=True)
def sanitise_header(header_contents, header_name=None):
"""Clean and individual mail header.
Given a header with header_contents, optionally labelled
header_name, decode it with decode_header, sanitise it to make
sure it decodes correctly and contains no invalid characters,
then encode the result with make_header()
"""
try:
value = decode_header(header_contents)
except HeaderParseError:
# something has gone really wrong with header parsing.
# (e.g. base64 decoding) We probably can't recover, so:
return None
# We have some issues here.
#
# Firstly, in the email parser (before we get here) headers with weird
# chars are email.header.Header class, others as str
#
# Secondly, the behaviour of decode_header: weird headers are labelled
# as unknown-8bit
#
# We solve this by catching any Unicode errors, and then manually
# handling any interesting headers.
try:
header = make_header(value,
header_name=header_name,
continuation_ws='\t')
except (UnicodeDecodeError, LookupError, ValueError):
# - a part cannot be encoded as ascii. (UnicodeDecodeError), or
# - we don't have a codec matching the hint (LookupError)
# - the codec has a null byte (ValueError)
# Find out which part and fix it somehow.
#
# We get here under where decoding with the coding hint fails.
new_value = []
for (part, _) in value:
# We have random bytes that aren't properly coded.
# If we had a coding hint, it failed to help.
# python3 - force coding to unknown-8bit
new_value += [(part, 'unknown-8bit')]
header = make_header(new_value,
header_name=header_name,
continuation_ws='\t')
try:
header.encode()
except (HeaderParseError, IndexError):
# despite our best efforts, the header is stuffed
# HeaderParseError: some very weird multi-line headers
# IndexError: bug, thrown by make_header(decode_header(' ')).encode()
return None
return header
def properly_decode_header(subject):
contents = make_header(decode_header(subject))
return str(contents)
def handle(self, *args, **options):
##########
# 1. Save the email-attachment file to the django model (database)
##########
sticker_email_host = env('STICKER_EMAIL_HOST', '')
sticker_email_port = env('STICKER_EMAIL_PORT', '')
sticker_email_username = env('STICKER_EMAIL_USERNAME', '')
sticker_email_password = env('STICKER_EMAIL_PASSWORD', '')
if not sticker_email_host or not sticker_email_port or not sticker_email_username or not sticker_email_password:
raise Exception(
'Configure email settings at .env to process sticker responses'
)
context = ssl.create_default_context()
imapclient = imaplib.IMAP4_SSL(sticker_email_host,
sticker_email_port,
ssl_context=context)
# login
imapclient.login(sticker_email_username, sticker_email_password)
"""
Retrieve messages
"""
imapclient.select('INBOX') # Select mail box
typ, data = imapclient.search(None, "ALL") # data = [b"1 2 3 4 ..."]
datas = data[0].split()
fetch_num = 5000 # The number of messages to fetch
if (len(datas) - fetch_num) < 0:
fetch_num = len(datas)
# msg_list = []
for num in datas[len(datas) - fetch_num::]:
try:
##########
# 2. Save the attached files into the database
##########
typ, data = imapclient.fetch(num, '(RFC822)')
# typ, data = imapclient.fetch(num, '(BODY[HEADER.FIELDS (MESSAGE-ID)])')
raw_email = data[0][1]
# converts byte literal to string removing b''
raw_email_string = raw_email.decode('utf-8')
email_message = email.message_from_string(raw_email_string)
email_message_id = str(
make_header(decode_header(email_message["Message-ID"])))
if StickerPrintingResponseEmail.objects.filter(
email_message_id=email_message_id):
# This email has been saved in the database already. Skip this email
continue
email_subject = str(
make_header(decode_header(email_message["Subject"])))
email_from = email_message['From']
body = get_text(email_message)
email_body = body.decode()
email_date = email_message['Date']
sticker_printing_response_email = StickerPrintingResponseEmail.objects.create(
email_subject=email_subject,
email_from=email_from,
email_body=email_body,
email_date=email_date,
email_message_id=email_message_id,
)
# downloading attachments
for part in email_message.walk():
try:
# this part comes from the snipped I don't understand yet...
if part.get_content_maintype() == 'multipart':
continue
if part.get('Content-Disposition') is None:
continue
fileName = part.get_filename()
if bool(fileName) and fileName.lower().endswith(
'.xlsx'):
now = timezone.localtime(timezone.now())
# Create sticker_printing_response object. File is not saved yet
sticker_printing_response = StickerPrintingResponse.objects.create(
sticker_printing_response_email=
sticker_printing_response_email,
name=fileName,
)
# Load attachment file
my_bytes = part.get_payload(decode=True)
content_file = ContentFile(my_bytes)
# Save file
sticker_printing_response._file.save(
fileName, content_file)
except Exception as e:
logger.exception(
'Exception has been raised when importing .xlsx file'
)
continue
# imapclient.store(num, "+FLAGS", "\\Deleted")
imapclient.copy(num, "Archive")
imapclient.store(num, "+FLAGS", "\\Deleted")
except:
logger.exception(
'Exception has been raised when processing emails')
continue
imapclient.close()
imapclient.logout()
##########
# 3. Process xlsx file saved in django model
##########
updates, errors = process_sticker_printing_response()
cmd_name = __name__.split('.')[-1].replace('_', ' ').upper()
# error_count = len(errors) + len(error_filenames)
error_count = len(errors)
err_str = '<strong style="color: red;">Errors: {}</strong>'.format(
error_count
) if error_count else '<strong style="color: green;">Errors: 0</strong>'
msg = '<p>{} completed. {}. IDs updated: {}.</p>'.format(
cmd_name, err_str, updates)
logger.info(msg)
print(
msg) # will redirect to cron_tasks.log file, by the parent script
def sanitise_header(header_contents, header_name=None):
"""Clean and individual mail header.
Given a header with header_contents, optionally labelled
header_name, decode it with decode_header, sanitise it to make
sure it decodes correctly and contains no invalid characters,
then encode the result with make_header()
"""
try:
value = decode_header(header_contents)
except HeaderParseError:
# something has gone really wrong with header parsing.
# (e.g. base64 decoding) We probably can't recover, so:
return None
# We have some Py2/Py3 issues here.
#
# Firstly, the email parser (before we get here)
# Python 3: headers with weird chars are email.header.Header
# class, others as str
# Python 2: every header is an str
#
# Secondly, the behaviour of decode_header:
# Python 3: weird headers are labelled as unknown-8bit
# Python 2: weird headers are not labelled differently
#
# Lastly, aking matters worse, in Python2, unknown-8bit doesn't
# seem to be supported as an input to make_header, so not only do
# we have to detect dodgy headers, we have to fix them ourselves.
#
# We solve this by catching any Unicode errors, and then manually
# handling any interesting headers.
try:
header = make_header(value,
header_name=header_name,
continuation_ws='\t')
except (UnicodeDecodeError, LookupError, ValueError, TypeError):
# - a part cannot be encoded as ascii. (UnicodeDecodeError), or
# - we don't have a codec matching the hint (LookupError)
# - the codec has a null byte (Py3 ValueError/Py2 TypeError)
# Find out which part and fix it somehow.
#
# We get here under Py2 when there's non-7-bit chars in header,
# or under Py2 or Py3 where decoding with the coding hint fails.
new_value = []
for (part, coding) in value:
# We have random bytes that aren't properly coded.
# If we had a coding hint, it failed to help.
if six.PY3:
# python3 - force coding to unknown-8bit
new_value += [(part, 'unknown-8bit')]
else:
# python2 - no support in make_header for unknown-8bit
# We should do unknown-8bit coding ourselves.
# For now, we're just going to replace any dubious
# chars with ?.
#
# TODO: replace it with a proper QP unknown-8bit codec.
new_value += [
(part.decode('ascii',
errors='replace').encode('ascii',
errors='replace'),
None)
]
header = make_header(new_value,
header_name=header_name,
continuation_ws='\t')
try:
header.encode()
except (HeaderParseError, IndexError):
# despite our best efforts, the header is stuffed
# HeaderParseError: some very weird multi-line headers
# IndexError: bug, thrown by make_header(decode_header(' ')).encode()
return None
return header
def _handle_part(self, part, part_index, tmp_dir, extract_attach,
parsed_mail):
bodies = parsed_mail[PROC_EMAIL_JSON_BODIES]
files = parsed_mail[PROC_EMAIL_JSON_FILES]
# get the file_name
file_name = part.get_filename()
content_disp = part.get('Content-Disposition')
content_type = part.get('Content-Type')
content_id = part.get('Content-ID')
if file_name is None:
# init name and extension to default values
name = "part_{0}".format(part_index)
extension = ".{0}".format(part_index)
# Try to create an extension from the content type if possible
if content_type is not None:
extension = mimetypes.guess_extension(
re.sub(';.*', '', content_type))
# Try to create a name from the content id if possible
if content_id is not None:
name = content_id
file_name = "{0}{1}".format(name, extension)
else:
try:
file_name = str(make_header(decode_header(file_name)))
except Exception:
file_name = self._decode_uni_string(file_name, file_name)
# Remove any chars that we don't want in the name
file_path = "{0}/{1}_{2}".format(
tmp_dir, part_index,
file_name.translate(str.maketrans("", "", ''.join(['<', '>',
' ']))))
self._base_connector.debug_print("file_path: {0}".format(file_path))
# is the part representing the body of the email
status, process_further = self._handle_if_body(content_disp,
content_type, part,
bodies, file_path,
parsed_mail)
if not process_further:
return phantom.APP_SUCCESS
# is this another email as an attachment
if (content_type is not None) and (
content_type.find(PROC_EMAIL_CONTENT_TYPE_MESSAGE) != -1):
return phantom.APP_SUCCESS
# This is an attachment, first check if it is another email or not
if extract_attach:
_, file_extension = os.path.splitext(file_name)
part_payload = part.get_payload(decode=True)
if not part_payload:
return phantom.APP_SUCCESS
try:
with open(file_path, 'wb') as f: # noqa
f.write(part_payload)
files.append({'file_name': file_name, 'file_path': file_path})
except IOError as e:
error_msg = self._get_error_message_from_exception(e)
if "File name too long" in error_msg:
self.write_with_new_filename(tmp_dir,
part_payload,
file_extension,
files,
file_name,
as_byte=False)
else:
self._base_connector.debug_print(
'Failed to write file: {}'.format(e))
return phantom.APP_SUCCESS
def __init__(self, smtp_path, whitelist):
# Initiate logging.
self.logger = logging.getLogger()
# Save the whitelist.
self.whitelist = whitelist
# Items we parse out of the email.
self.ace_url = ''
self.attachments = []
self.body = ''
self.cc_addresses = []
self.envelope_from = ''
self.envelope_to = ''
self.from_address = ''
self.headers = ''
self.html = ''
self.indicators = []
self.message_id = ''
self.original_recipient = ''
self.path = smtp_path
self.received = ''
self.received_time = ''
self.remediated = False
self.reply_to = ''
self.return_path = ''
self.screenshots = []
self.subject = ''
self.subject_decoded = ''
self.to_addresses = []
self.urls = []
self.x_auth_id = ''
self.x_mailer = ''
self.x_original_sender = ''
self.x_originating_ip = ''
self.x_sender = ''
self.x_sender_id = ''
self.x_sender_ip = ''
# Build the URL to the ACE alert.
ace_uuid_pattern = re.compile(
r'([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})')
match = ace_uuid_pattern.search(self.path)
if match:
self.ace_url = '{}{}'.format(config['ace']['ace_alert_url'],
match.group(1))
with open(self.path, encoding='utf-8', errors='ignore') as s:
smtp_stream = s.read().splitlines()
# Locate any screenshots for this email.
email_dir = os.path.dirname(self.path)
files = os.listdir(email_dir)
for f in files:
if 'text_html' in f and f.endswith(
'.png') and not f.startswith('email_screenshot'):
self.logger.debug('Found email screenshot: {}'.format(
os.path.join(email_dir, f)))
self.screenshots.append(os.path.join(email_dir, f))
# Find the envelope from/to addresses. This will only work if given an
# "smtp.stream" file, since otherwise the SMTP commands will not exist.
envelope_address_pattern = re.compile(r'.*<(.*)>.*')
for line in smtp_stream:
if line.startswith('MAIL FROM:'):
try:
self.envelope_from = envelope_address_pattern.match(
line).group(1)
except:
self.logger.exception('Unable to parse envelope from.')
if line.startswith('RCPT TO:'):
try:
self.envelope_to = envelope_address_pattern.match(
line).group(1)
except:
self.logger.exception('Unable to parse envelope to.')
# Just in case we are dealing with an "smtp.stream" file that still has
# the SMTP commands above the actual e-mail, we need to strip those out.
# This will remove all lines prior to the Received: headers so that the
# email.parser can properly parse out the e-mail. If we were given an
# "smtp.email" type of file with the SMTP commands already removed, this
# should not affect anything. This is legacy code at this point.
try:
while not smtp_stream[0].startswith('Received:'):
smtp_stream.pop(0)
except IndexError:
smtp_stream = []
# Join the header lines into a single string.
self.email_text = '\n'.join(smtp_stream)
# Create the e-mail object.
email_obj = email.message_from_string(self.email_text)
# We want to try and parse an embedded/attached e-mail if there is one.
# Walk the full e-mail's parts.
for part in email_obj.walk():
# Continue if the part looks like a valid e-mail.
if part.get_content_type() == 'message/rfc822':
# Split the part lines into a list.
part_text = str(part).splitlines()
if any('Received:' in line for line in part_text):
# Make sure our part starts with the Received: headers.
try:
while not part_text[0].startswith('Received:'):
part_text.pop(0)
except IndexError:
pass
part_text = '\n'.join(part_text)
# Make the new e-mail object.
email_obj = email.message_from_string(part_text)
# Parse the e-mail object for its content.
parsed_email = self._parse_content(email_obj)
# Now that we have the e-mail object, parse out some of the interesting parts.
self.headers = self._get_all_headers_string(email_obj)
self.received = self.get_header(email_obj, 'received')
# Get the e-mail's plaintext body, HTML body, and the visible text from the HTML.
self.body = parsed_email['body']
self.html = parsed_email['html']
# Get any e-mail attachments.
self.attachments = parsed_email['attachments']
# From address
try:
self.from_address = self._get_address_list(email_obj, 'from')[0][1]
self.indicators.append(
Indicator('Email - Address',
self.from_address,
tags=['from_address']))
except:
pass
# From domain
try:
self.indicators.append(
Indicator('URI - Domain Name',
self.from_address.split('@')[1],
tags=['from_domain']))
except:
pass
# Reply-To address
try:
self.reply_to = self._get_address_list(email_obj, 'reply-to')[0][1]
self.indicators.append(
Indicator('Email - Address', self.reply_to, tags=['reply_to']))
except:
pass
# X-Sender address
try:
self.x_sender = self._get_address_list(email_obj, 'X-Sender')[0][1]
self.indicators.append(
Indicator('Email - Address', self.x_sender, tags=['x_sender']))
except:
pass
# X-Sender-Id address
try:
self.x_sender_id = self._get_address_list(email_obj,
'X-Sender-Id')[0][1]
self.indicators.append(
Indicator('Email - Address',
self.x_sender_id,
tags=['x_sender_id']))
except:
pass
# X-Auth-Id address
try:
self.x_auth_id = self._get_address_list(email_obj,
'X-Auth-ID')[0][1]
self.indicators.append(
Indicator('Email - Address',
self.x_auth_id,
tags=['x_auth_id']))
except:
pass
# Return-Path address
try:
self.return_path = self._get_address_list(email_obj,
'return_path')[0][1]
self.indicators.append(
Indicator('Email - Address',
self.return_path,
tags=['return_path']))
except:
pass
# X-MS-Exchange-Organization-OriginalEnvelopeRecipients address
try:
self.original_recipient = self._get_address_list(
email_obj,
'X-MS-Exchange-Organization-OriginalEnvelopeRecipients'
)[0][1].lower()
self.indicators.append(
Indicator('Email - Address',
self.original_recipient,
status='Informational',
tags=['original_recipient']))
except:
pass
# If the original_recipient was not found, check if this is a POTENTIAL PHISH e-mail and use the from address.
if not self.original_recipient and 'Subject: [POTENTIAL PHISH]' in self.email_text:
try:
temp_email_obj = email.message_from_string(self.email_text)
self.original_recipient = self._get_address_list(
temp_email_obj, 'from')[0][1]
self.indicators.append(
Indicator('Email - Address',
self.original_recipient,
status='Informational',
tags=['original_recipient']))
except:
self.logger.exception(
'Error parsing original recipient from POTENTIAL PHISH e-mail.'
)
# Subject
try:
self.subject = ''.join(
self.get_header(email_obj, 'subject')[0].splitlines())
if not self.subject.startswith('[POTENTIAL PHISH]'):
self.indicators.append(
Indicator('Email - Subject', self.subject))
except:
pass
# Decoded subject
try:
self.subject_decoded = ''.join(
str(
make_header(
decode_header(
self.get_header(email_obj,
'subject')[0]))).splitlines())
if not self.subject_decoded.startswith('[POTENTIAL PHISH]'):
self.indicators.append(
Indicator('Email - Subject', self.subject_decoded))
except:
pass
# To addresses
self.to_addresses = [
x[1].lower() for x in self._get_address_list(email_obj, 'to')
]
# CC addresses
self.cc_addresses = [
x[1].lower() for x in self._get_address_list(email_obj, 'cc')
]
# Message-Id
try:
self.message_id = self.get_header(email_obj, 'message-id')[0]
self.indicators.append(
Indicator('Email Message ID',
self.message_id,
status='Informational'))
except:
pass
# X-Mailer
try:
self.x_mailer = self.get_header(email_obj, 'x-mailer')[0]
self.indicators.append(
Indicator('Email - Xmailer',
self.x_mailer,
status='Informational'))
except:
pass
# X-Original-Sender address
try:
self.x_original_sender = self.get_header(email_obj,
'x-original-sender')[0]
self.indicators.append(
Indicator('Email - Address',
self.x_original_sender,
tags=['x_original_sender']))
except:
pass
# X-Originating-Ip
try:
x_originating_ip = self.get_header(email_obj,
'x-originating-ip')[0]
# Sometimes this field is in the form: [1.1.1.1]
# Make sure we remove any non-IP characters.
ip = RegexHelpers.find_ip_addresses(x_originating_ip)
if ip:
self.x_originating_ip = ip[0]
self.indicators.append(
Indicator('Address - ipv4-addr',
self.x_originating_ip,
tags=['x_originating_ip']))
except:
pass
# X-Sender-Ip
try:
x_sender_ip = self.get_header(email_obj, 'x-sender-ip')[0]
# Make sure like the X-Originating-IP that we only
# get the IP address and no other characters.
ip = RegexHelpers.find_ip_addresses(x_sender_ip)
if ip:
self.x_sender_ip = ip[0]
self.indicators.append(
Indicator('Address - ipv4-addr',
self.x_sender_ip,
tags=['x_sender_ip']))
except:
pass
self.received_time = self._get_received_time(email_obj)
if not self.received_time:
self.received_time = self._get_date_time()
# Find any URLs in the plaintext body.
text_urls = find_urls(self.body)
# Find any URLs in the HTML body.
html_urls = find_urls(self.html)
# Get any strings URLs.
strings_urls = []
"""
for file in self.attachments:
try:
strings_urls += file['strings_urls']
except:
pass
"""
# Try and remove any URLs that look like partial versions of other URLs.
all_urls = set.union(text_urls, html_urls)
unique_urls = set()
for u in all_urls:
if not any(
other_url.startswith(u) and other_url != u
for other_url in all_urls):
unique_urls.add(u)
# Get rid of any invalid URLs.
self.urls = [u for u in unique_urls if RegexHelpers.is_url(u)]
# Make indicators for the URLs.
self.indicators += make_url_indicators(self.urls,
from_email_content=True)
# Get rid of any invalid indicators.
self.indicators = [i for i in self.indicators if i.value]
# Add any extra tags to each indicator.
for i in self.indicators:
i.tags.append('phish')
def _read_source(imap_host, imap_port, imap_user, imap_pass, imap_folder,
email_inreply):
source = {'alreadyLoaded': False}
try: ## Time to search for the original email
try:
if "gmail" in imap_host: # gmail server requires an ssl connection
print("gmail server")
imap = IMAP4_SSL(imap_host, imap_port)
else:
imap = IMAP4_SSL(imap_host, imap_port)
## login to server
#print(imap_user, imap_pass)
imap.login(imap_user, imap_pass)
#imap.starttls()
except:
print("Failed to login")
return False
# while 'INBOX' is standard, many email servers use different methods to label the Sent folder
# We have to iterate through the most common methods to find the right one
if "gmail" in imap_host:
imap.select('"[Gmail]/Sent Mail"') # connect to sent mail.
# Search for the original email ID
messages = imap.search(None, 'HEADER', 'MESSAGE-ID',
email_inreply)
#print("Opening gmail 'Sent'")
if 'gmail' not in imap_host:
sentbox = False
try:
imap.select('Sent') # connect to sent mail.
# Search for the original email ID
messages = imap.search(None, 'HEADER', 'MESSAGE-ID',
email_inreply)
sentbox = True
except:
print('Sent folder not found')
if sentbox == False:
try:
imap.select('INBOX.Sent') # connect to sent mail.
# Search for the original email ID
messages = imap.search(None, 'HEADER', 'MESSAGE-ID',
email_inreply)
except:
print(
'Inbox.Sent folder not found, no folders left to try'
)
# Process the result to get the message id’s
messages = messages[1][0].split()
# Use the first id to view the headers for a message
result, source_data = imap.fetch(messages[0], '(RFC822)')
#print("Opening 'Sent'", messages[0])
raw_source = source_data[0][
1] # here's the body, which is raw headers and html and body of the whole email
s = email.message_from_bytes(
raw_source) # convert to message object
source_subject = s['subject']
try:
sub = make_header(decode_header(source_subject))
except:
sub = email_subject
source['subject'] = str(sub)
print(source_subject)
frm = s['From']
add1 = re.findall(
"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)", str(frm))
source['email'] = add1
if s['Date'] != None:
source['_date'] = s['Date']
if s['Delivery-date'] != None:
source['_date'] = s['Delivery-date']
source['bcc'] = ''
if s['bcc'] != None:
print("bcc is: ", s['bcc'])
source['bcc'] = s['bcc'].split(',')
source['msg_id'] = s['Message-ID']
#print("BCC from source: ", source_bcc)
source_body = s.get_payload(decode=True)
if s.is_multipart(): # search for text in the body
for part in s.walk():
ctype = part.get_content_type()
cdispo = str(part.get('Content-Disposition'))
if ctype == ('text/plain' or
'text/html') and 'attachment' not in cdispo:
source_body = part.get_payload(decode=True)
#print(email_body)
break
src_from = BeautifulSoup(frm, 'html.parser')
try: # extra check for encryption (in case user has encypted email)
src_body = BeautifulSoup(source_body, 'html.parser')
text = src_body.get_text()
urls = re.findall(
'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
text)
# break into lines and remove leading and trailing space on each
lines = (line.strip() for line in text.splitlines())
# drop blank lines
text = '\n'.join(line for line in lines if line)
# double space new lines
text = text.replace('\n', '\n\n')
source['nomarkup_body'] = text
for i in urls:
text = text.replace(
i,
f"[ref=<{i}>][color=6666FF][u]{i}[/u][/color][/ref]")
source['body'] = text #.encode('utf-8')
except: # if email is encrypted it will throw an exception
source['body'] = encription_warning
source['from'] = src_from.get_text().split('@')[0]
ids_list.append(source['msg_id'])
return source
except:
print("no origin found")
return False
def _add_body_in_email_headers(self, parsed_mail, file_path, charset,
content_type):
# Add email_bodies to email_headers
email_headers = parsed_mail[PROC_EMAIL_JSON_EMAIL_HEADERS]
try:
with open(file_path, 'r') as f:
body_content = f.read()
except Exception:
with open(file_path, 'rb') as f:
body_content = f.read()
self._base_connector.debug_print(
"Reading file data using binary mode")
# Add body to the last added Email artifact
body_content = UnicodeDammit(body_content).unicode_markup.encode(
'utf-8').decode('utf-8')
if 'text/plain' in content_type:
try:
email_headers[-1]['cef']['bodyText'] = self._get_string(
body_content, charset)
except Exception as e:
try:
email_headers[-1]['cef']['bodyText'] = str(
make_header(decode_header(body_content)))
except Exception:
email_headers[-1]['cef'][
'bodyText'] = self._decode_uni_string(
body_content, body_content)
error_code, error_msg = self._get_error_message_from_exception(
e)
err = "Error occurred while parsing text/plain body content for creating artifacts"
self._base_connector.debug_print("{}. {}. {}".format(
err, error_code, error_msg))
elif 'text/html' in content_type:
try:
email_headers[-1]['cef']['bodyHtml'] = self._get_string(
body_content, charset)
except Exception as e:
try:
email_headers[-1]['cef']['bodyHtml'] = str(
make_header(decode_header(body_content)))
except Exception:
email_headers[-1]['cef'][
'bodyHtml'] = self._decode_uni_string(
body_content, body_content)
error_code, error_msg = self._get_error_message_from_exception(
e)
err = "Error occurred while parsing text/html body content for creating artifacts"
self._base_connector.debug_print("{}. {}. {}".format(
err, error_code, error_msg))
else:
if not email_headers[-1]['cef'].get('bodyOther'):
email_headers[-1]['cef']['bodyOther'] = {}
try:
email_headers[-1]['cef']['bodyOther'][
content_type] = self._get_string(body_content, charset)
except Exception as e:
try:
email_headers[-1]['cef']['bodyOther'][content_type] = str(
make_header(decode_header(body_content)))
except Exception:
email_headers[-1]['cef']['bodyOther'][
content_type] = self._decode_uni_string(
body_content, body_content)
error_code, error_msg = self._get_error_message_from_exception(
e)
err = "Error occurred while parsing bodyOther content for creating artifacts"
self._base_connector.debug_print("{}. {}. {}".format(
err, error_code, error_msg))
def _read_mail(imap_host, imap_port, imap_user, imap_pass, imap_folder,
eNum): # reads the most recent email and parses the text
### Reading emails from the server. The bulk of the logic is here
### We prosses an email, clean up the text, check if it is a reply
### If the message is a reply, search for the original email in the sent box
### If the original email exists, run a search on the inbox for all emails replying to the original
### And finally, check for and load images
global ids_list
if eNum == -1:
ids_list = []
email_recieved = {'alreadyLoaded': False}
try:
imap = IMAP4_SSL(imap_host, imap_port)
## login to server
print(imap_user, imap_pass)
imap.login(imap_user, imap_pass)
#imap.ehlo()
#imap.starttls()
except:
print("Failed to login")
return False
#print(imap.list()) # for identifying mailboxes on the server
imap.select()
result, data = imap.uid('search', None,
"ALL") # search and return uids instead
#print('data', data)
if -len(data[0].split()) > eNum:
return False
current_email_uid = data[0].split()[eNum]
print('current', current_email_uid)
result, data = imap.uid(
'fetch', current_email_uid, '(RFC822)'
) # fetch the email headers and body (RFC822) for the given ID
raw_email = data[0][
1] # here's the body, which is raw headers and html and body of the whole email
b = email.message_from_bytes(raw_email, policy=email.policy.default)
email_recieved['msg_id'] = b['Message-ID']
for i in ids_list:
if i == email_recieved['msg_id']:
print("mail already loaded")
email_recieved['alreadyLoaded'] = True
return email_recieved
ids_list.append(email_recieved['msg_id'])
print(email_recieved['msg_id'])
email_recieved['date'] = b['Date']
if email_recieved['date'] == None:
email_recieved['date'] = b['Delivery-date']
email_recieved['inreply'] = b['in-reply-to']
email_recieved['refs'] = b['references']
# decode and clean the name out of the From header
email_from = b['from']
# get just the text
frm = BeautifulSoup(email_from, 'html.parser')
frm = frm.get_text()
# remove any latent escape chrs
frm = make_header(decode_header(frm))
# remove unneccisary quotes
frm = str(frm).replace('"', '')
# add result to list
email_recieved['from'] = str(frm)
# find all email addresses in the From header
add1 = re.findall("([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)",
str(email_from))
email_recieved['email'] = add1 # is a list
# clean the subject text
email_subject = b['Subject']
try:
sub = make_header(decode_header(email_subject))
except:
sub = email_subject
email_recieved['subject'] = str(sub)
# find and clean up the body text
email_body = b.get_payload(decode=True)
if b.is_multipart(): # search for text in the body
for part in b.walk():
ctype = part.get_content_type()
cdispo = str(part.get('Content-Disposition'))
if ctype == ('text/plain'
or 'text/html') and 'attachment' not in cdispo:
email_body = part.get_payload(decode=True)
#print(email_body)
break
# Use beautifulsoup to get readable text
try: # Try parsing the body text
body = BeautifulSoup(email_body, 'html.parser')
except: # if email is encrypted it will throw an exception
body = encription_warning
if body != encription_warning:
for script in body(["script", "style", "div", "div style"]):
script.extract() # rip it out
text = body.get_text()
urls = re.findall(
'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
text)
# replace and remove leftover html escape chrs, bad wiziwig
text = text.replace('=0A', '\n')
text = text.replace('=20', '\n')
text = text.replace('=\r\n', '')
text = text.replace('=\n', '')
text = text.replace('=C2=A0', '')
text = text.replace('=C2=A9', '')
text = text.replace('=09', '')
text = text.replace('=0D', '')
text = text.replace('<td>', '')
text = text.replace('</td>', '')
text = text.replace('<tr>', '')
text = text.replace('</a>', '')
# break into lines and remove leading and trailing space on each
lines = (line.strip() for line in text.splitlines())
# drop blank lines
text = '\n'.join(line for line in lines if line)
# double space new lines
text = text.replace('\n', '\n\n')
email_recieved['nomarkup_body'] = text
for i in urls:
text = text.replace(
i, f"[ref=<{i}>][color=6666FF][u]{i}[/u][/color][/ref]")
email_recieved['body'] = text
imap.close()
return email_recieved
items = get_messages()
if len(items) != 0:
for item in items:
resp, data = m.fetch(item, '(RFC822)')
mail_full = email.message_from_bytes(data[0][1])
if mail_full.is_multipart():
for part in mail_full.get_payload():
mail = mail_full.get_payload()[0].get_payload(
decode=True).decode()
else:
mail = mail_full.get_payload(decode=True).decode()
parse = parse_html(mail)
mail_from = parseaddr(mail_full['From'])[1]
mail_from = mail_full['X-Envelope-From']
mail_subj = str(make_header(decode_header(mail_full['Subject'])))
if mail_from == sender:
if parse[0]:
download_res, name = download_file(parse[1])
if download_res:
extract_res, txt_name = extract(name)
if extract_res:
admin_notify(
"✅ Файл-выписка успешно загружена и распакована! \n`{} \n{}`"
.format(str(name), str(txt_name)))
else:
admin_notify(
"‼ *Ошибка!* `Не удалось распаковать файл.`")
else:
admin_notify(
"‼ *Ошибка!* `Ссылка недоступна для скачивания.`")
def getMailDetail(self, mailId):
# == 未送信メールを処理
result = {}
## メッセージ取得(ヘッダ・本文)
typ, data = self.ImapInstance.uid('fetch', mailId, '(RFC822)')
if typ != 'OK':
raise IMAPCommandError(
'search', 'I cannot get Mail data of number' + mailId + "." +
'\n responce: ' + typ)
## ログ出力ぅ
"""
print('mail['+str(mailId)+']')
for i in range(len(data)):
for j in range(len(data[i])):
if type(data[i][j])==bytes:
print("data["+str(i)+"]"+"["+str(j)+"]"+str(data[i][j].decode('iso2022_jp')))
else:
print("data["+str(i)+"]"+"["+str(j)+"]"+str(data[i][j]))
"""
## emailライブラリに渡す
email_message = email.message_from_bytes(data[0][1])
## データ抽出
### From
From_Address = str(make_header(decode_header(email_message['From'])))
From_Name = ''
if "<" in From_Address:
tmp_f = From_Address.split("<")
From_Name = tmp_f[0]
From_Address = tmp_f[1].split(">")[0]
### To
To_Address = str(make_header(decode_header(email_message['To'])))
To_Name = ''
if "<" in To_Address:
tmp_t = To_Address.split("<")
To_Name = tmp_t[0]
To_Address = tmp_t[1].split(">")[0]
### Subject
Subject = str(make_header(decode_header(email_message['Subject'])))
### ログ出力
print(' > ' + str(mailId.hex()) +
':' + Subject + ' ' + From_Address + '<' + From_Name + '> ' +
To_Address + '<' + To_Name + '>')
### 本文抽出
#### 本文取得
body = ''
SucceedFlag = False
try:
if email_message.is_multipart() == False: # シングルパート
print(
' - getting body as single part'
)
byt = bytearray(email_message.get_payload(), 'iso2022_jp')
body = byt.decode(encoding='iso2022_jp')
else: # マルチパート
print(
' - getting body as multi part'
)
prt = email_message.get_payload()[0]
byt = prt.get_payload(decode=True)
body = byt.decode(encoding='iso2022_jp')
print(
' - body decode succeed!')
SucceedFlag = True
except Exception as err_inst:
body = 'メールを受信しましたが変換できませんでした\n\nエラーログ:\n' + str(
err_inst.__class__.__name__) + ":" + str(err_inst)
print(str(err_inst.__class__.__name__) + str(err_inst))
#TODO ここ埋めれるように頑張る
result['Succeed'] = SucceedFlag
result['UID'] = mailId.hex()
result['Subject'] = Subject
result['Body'] = body
result['From_Address'] = From_Address
result['From_Name'] = From_Name
result['To_Address'] = To_Address
result['To_Name'] = To_Name
return result
