def parse_json():
parser = EmailReplyParser(language='en')
with open('english.json', 'rb') as fl:
messages = json.load(fl)
parsed = []
for text in messages:
soup = BeautifulSoup(text, 'lxml')
text = soup.getText('\n')
text = parser.parse_reply(text)
parsed.append(text)
import code
code.interact(local=locals())
def verify():
parser = EmailReplyParser(language='fi')
texts = json.load(open('test/emails/emails.json'))
texts = list(filter(lambda d: type(d) == str, texts))
parsed = []
for text in texts:
print('-'*100)
soup = BeautifulSoup(text, 'lxml')
text = soup.getText('\n')
text = parser.parse_reply(text)
parsed.append(text)
print(text)
def parse_df():
parser = EmailReplyParser(language='en')
path = 'test/emails/zipwrotetest.csv'
df = pd.read_csv(path)
parsed = []
for text in df.sentence.values:
soup = BeautifulSoup(text, 'lxml')
text = soup.getText('\n')
text = parser.parse_reply(text)
parsed.append(text)
df = df.assign(clean=parsed)
df.to_csv(path)
import code
code.interact(local=locals())
def profile():
df = pd.read_csv('test.csv')
ground = time.time()
content = df.content.values[np.argmax([len(d) for d in df.content.values])]
start = time.time()
parser = EmailReplyParser(language='fr')
print(str(time.time() - start) + 'init parser')
start = time.time()
res = parser.parse_reply(content)
print(str(time.time() - start) + 'parse')
start = time.time()
soup = BeautifulSoup(res, 'lxml')
text = soup.getText(' ')
print(str(time.time() - start) + 'soup')
print(f'Total time: {time.time() - ground}')
def extract_reply(msg):
"""Extracts the portion of an email that should contain commands.
"""
for part in msg.walk():
if part.get_content_type() == 'text/plain':
content = part.get_payload(decode=True)
return EmailReplyParser.parse_reply(content)
def collect_project_status():
for data in frappe.get_all("Project Update", {'date': today(), 'sent': 0}):
replies = frappe.get_all('Communication',
fields=['content', 'text_content', 'sender'],
filters=dict(
reference_doctype="Project Update",
reference_name=data.name,
communication_type='Communication',
sent_or_received='Received'),
order_by='creation asc')
for d in replies:
doc = frappe.get_doc("Project Update", data.name)
user_data = frappe.db.get_values(
"User", {"email": d.sender},
["full_name", "user_image", "name"],
as_dict=True)[0]
doc.append(
"users", {
'user':
user_data.name,
'full_name':
user_data.full_name,
'image':
user_data.user_image,
'project_status':
frappe.utils.md_to_html(
EmailReplyParser.parse_reply(d.text_content)
or d.content)
})
doc.save(ignore_permissions=True)
def _parse_mail(response, last_request):
msg = email.message_from_bytes(response[1])
date = msg['Date']
subject = msg['Subject']
epoch = mktime_tz(parsedate_tz(date))
if epoch < last_request:
return False, "", ""
if msg.is_multipart():
# iterate over email parts
bodies = []
for part in msg.walk():
body = _get_body(part)
if body:
bodies.append(body)
break
body = "\n".join(bodies)
else:
body = _get_body(msg)
if body:
# Remove replied to emails
body = EmailReplyParser.parse_reply(body)
# Remove hyperlinks
body = re.sub(r'http\S+', '<REDACTED URL>', body, flags=re.MULTILINE)
# Limit body max amount of characters
body = body[:min(len(body), 1800)]
# Add divider
body = body + "\n" + "=" * 40
return True, subject, body
return False, "", ""
def incoming_letter_email():
body = EmailReplyParser.parse_reply(unicode(request.form.get('text')).encode('ascii','xmlcharrefreplace'))
body = '\n'.join(body.split('\n')).replace("\n","<br />")
regexp = re.findall(r'[\w\.-]+@[\w\.-]+',request.form.get('from'))
try:
attachments = int(request.form.get('attachments'))
except Exception:
attachments = 0
if len(regexp) > 0 and len(regexp[-1]) > 0:
username = regexp[-1].lower()
else:
return_bad_params(username)
return Response(response=jfail("missing parameters"), status=200)
to_name = request.form.get('to')
to_address = unicode(request.form.get('subject')).encode('ascii','xmlcharrefreplace').lower().replace("fw:","").replace("re:","").strip()
if None in [body,username,to_name,to_address]:
return_bad_params(username)
return Response(response=jfail("missing parameters"), status=200)
user = User(username)
if user.is_valid():
send_letter(user,to_name,to_address,body,attachments)
else:
return_unknown_sender(username)
return Response(response=jfail("unknown sender"), status=200)
return Response(response=jsuccess(), status=200)
def get_message_details(self):
'''Return args for template'''
settings = frappe.get_doc('Daily Work Summary Settings')
replies = frappe.get_all('Communication', fields=['content', 'text_content', 'sender'],
filters=dict(reference_doctype=self.doctype, reference_name=self.name,
communication_type='Communication', sent_or_received='Received'),
order_by='creation asc')
did_not_reply = self.email_sent_to.split()
for d in replies:
emp = frappe.db.get_values("Employee", {"user_id": d.sender},
["employee_name", "image"], as_dict=True)
d.sender_name = emp[0].employee_name if emp else d.sender
d.image = emp[0].image if emp and emp[0].image else None
if d.sender in did_not_reply:
did_not_reply.remove(d.sender)
if d.text_content:
d.content = markdown(EmailReplyParser.parse_reply(d.text_content))
did_not_reply = [(frappe.db.get_value("Employee", {"user_id": email}, "employee_name") or email)
for email in did_not_reply]
return dict(replies=replies,
original_message=settings.message,
title=_('Daily Work Summary for {0}'.format(global_date_format(self.creation))),
did_not_reply= ', '.join(did_not_reply) or '',
did_not_reply_title = _('No replies from'))
def test_parse_out_just_top_for_outlook_with_reply_directly_above_line(
self):
with open("test/emails/email_2_2.txt") as f:
self.assertEqual(
"Outlook with a reply directly above line",
EmailReplyParser.parse_reply(f.read()),
)
def set_content_and_type(self):
self.content, self.content_type = '[Blank Email]', 'text/plain'
if self.html_content:
self.content, self.content_type = self.html_content, 'text/html'
else:
self.content, self.content_type = EmailReplyParser.parse_reply(
self.text_content), 'text/plain'
def get_body(
self,
use_html_parsing=True,
use_txt_parsing=True,
) -> typing.Optional[str]:
body_part = self._get_mime_body_message()
body = None
if body_part:
charset = body_part.get_content_charset('iso-8859-1')
content_type = body_part.get_content_type()
if content_type == CONTENT_TYPE_TEXT_PLAIN:
txt_body = body_part.get_payload(decode=True).decode(charset)
if use_txt_parsing:
txt_body = EmailReplyParser.parse_reply(txt_body)
html_body = markdown.markdown(txt_body)
body = HtmlSanitizer.sanitize(html_body)
elif content_type == CONTENT_TYPE_TEXT_HTML:
html_body = body_part.get_payload(decode=True).decode(charset)
if use_html_parsing:
html_body = str(ParsedHTMLMail(html_body))
body = HtmlSanitizer.sanitize(html_body)
if not body:
raise EmptyEmailBody()
return body
def collect_project_status():
for data in frappe.get_all("Project Update",
{'date': today(), 'sent': 0}):
replies = frappe.get_all('Communication',
fields=['content', 'text_content', 'sender'],
filters=dict(reference_doctype="Project Update",
reference_name=data.name,
communication_type='Communication',
sent_or_received='Received'),
order_by='creation asc')
for d in replies:
doc = frappe.get_doc("Project Update", data.name)
user_data = frappe.db.get_values("User", {"email": d.sender},
["full_name", "user_image", "name"], as_dict=True)[0]
doc.append("users", {
'user': user_data.name,
'full_name': user_data.full_name,
'image': user_data.user_image,
'project_status': frappe.utils.md_to_html(
EmailReplyParser.parse_reply(d.text_content) or d.content
)
})
doc.save(ignore_permissions=True)
def post(self, request):
token = request.POST["token"]
signature = request.POST["signature"]
timestamp = request.POST["timestamp"]
key = options.get("mail.mailgun-api-key")
if not key:
logger.error("mailgun.api-key-missing")
return HttpResponse(status=500)
if not self.verify(key, token, timestamp, signature):
logger.info(
"mailgun.invalid-signature",
extra={"token": token, "timestamp": timestamp, "signature": signature},
)
return HttpResponse(status=200)
to_email = request.POST["recipient"]
from_email = request.POST["sender"]
try:
group_id = email_to_group_id(to_email)
except Exception:
logger.info("mailgun.invalid-email", extra={"email": to_email})
return HttpResponse(status=200)
payload = EmailReplyParser.parse_reply(request.POST["body-plain"]).strip()
if not payload:
# If there's no body, we don't need to go any further
return HttpResponse(status=200)
process_inbound_email.delay(from_email, group_id, payload)
return HttpResponse(status=201)
def post(self, request):
token = request.POST['token']
signature = request.POST['signature']
timestamp = request.POST['timestamp']
key = options.get('mail.mailgun-api-key')
if not key:
logging.error('mail.mailgun-api-key is not set')
return HttpResponse(status=500)
if not self.verify(key, token, timestamp, signature):
logging.info('Unable to verify signature for mailgun request')
return HttpResponse(status=403)
to_email = parseaddr(request.POST['To'])[1]
from_email = parseaddr(request.POST['From'])[1]
try:
group_id = email_to_group_id(to_email)
except Exception:
logging.info('%r is not a valid email address', to_email)
return HttpResponse(status=500)
payload = EmailReplyParser.parse_reply(request.POST['body-plain']).strip()
if not payload:
# If there's no body, we don't need to go any further
return HttpResponse(status=200)
process_inbound_email.delay(from_email, group_id, payload)
return HttpResponse(status=201)
def __init__(self, email_text):
"""Decode base64 email and turn it into a Django email object."""
try:
email_text = base64.standard_b64decode(
urllib2.unquote(email_text.rstrip()))
except TypeError:
# Corrupt or invalid base 64.
self.decode_error = True
log.info('Decoding error for CommEmailParser')
return
self.email = message_from_string(email_text)
payload = self.email.get_payload()
if isinstance(payload, list):
# If multipart, get the plain text part.
for part in payload:
# Nested multipart. Go deeper.
if part.get_content_type() == 'multipart/alternative':
payload = part.get_payload()
for part in payload:
if part.get_content_type() == 'text/plain':
# Found the plain text part.
payload = part.get_payload()
break
if part.get_content_type() == 'text/plain':
# Found the plain text part.
payload = part.get_payload()
break
# Decode quoted-printable data and remove non-breaking spaces.
payload = (quopri.decodestring(payload).replace('\xc2\xa0', ' '))
payload = self.extra_email_reply_parse(payload)
self.reply_text = EmailReplyParser.read(payload).reply
def get_summary_message(self):
'''Return summary of replies as HTML'''
settings = frappe.get_doc('Daily Work Summary Settings')
replies = frappe.get_all('Communication', fields=['content', 'text_content', 'sender'],
filters=dict(reference_doctype=self.doctype, reference_name=self.name,
communication_type='Communication', sent_or_received='Received'),
order_by='creation asc')
did_not_reply = self.email_sent_to.split()
for d in replies:
d.sender_name = frappe.db.get_value("Employee", {"user_id": d.sender},
"employee_name") or d.sender
if d.sender in did_not_reply:
did_not_reply.remove(d.sender)
if d.text_content:
d.content = markdown(EmailReplyParser.parse_reply(d.text_content))
did_not_reply = [(frappe.db.get_value("Employee", {"user_id": email}, "employee_name") or email)
for email in did_not_reply]
return frappe.render_template(self.get_summary_template(),
dict(replies=replies,
original_message=settings.message,
title=_('Daily Work Summary for {0}'.format(formatdate(self.creation))),
did_not_reply= ', '.join(did_not_reply) or '',
did_not_reply_title = _('No replies from')))
def get_message_details(self):
'''Return args for template'''
dws_group = dataent.get_doc('Daily Work Summary Group',
self.daily_work_summary_group)
replies = dataent.get_all('Communication',
fields=['content', 'text_content', 'sender'],
filters=dict(
reference_doctype=self.doctype,
reference_name=self.name,
communication_type='Communication',
sent_or_received='Received'),
order_by='creation asc')
did_not_reply = self.email_sent_to.split()
for d in replies:
user = dataent.db.get_values("User", {"email": d.sender},
["full_name", "user_image"],
as_dict=True)
d.sender_name = user[0].full_name if user else d.sender
d.image = user[0].image if user and user[0].image else None
original_image = d.image
# make thumbnail image
try:
if original_image:
file_name = dataent.get_list('File',
{'file_url': original_image})
if file_name:
file_name = file_name[0].name
file_doc = dataent.get_doc('File', file_name)
thumbnail_image = file_doc.make_thumbnail(
set_as_thumbnail=False,
width=100,
height=100,
crop=True)
d.image = thumbnail_image
except:
d.image = original_image
if d.sender in did_not_reply:
did_not_reply.remove(d.sender)
if d.text_content:
d.content = dataent.utils.md_to_html(
EmailReplyParser.parse_reply(d.text_content))
did_not_reply = [
(dataent.db.get_value("User", {"email": email}, "full_name")
or email) for email in did_not_reply
]
return dict(replies=replies,
original_message=dws_group.message,
title=_('Work Summary for {0}'.format(
global_date_format(self.creation))),
did_not_reply=', '.join(did_not_reply) or '',
did_not_reply_title=_('No replies from'))
def post(self, request):
token = request.POST['token']
signature = request.POST['signature']
timestamp = request.POST['timestamp']
if not settings.MAILGUN_API_KEY:
logging.error('MAILGUN_API_KEY is not set')
return HttpResponse(status=500)
if not self.verify(settings.MAILGUN_API_KEY, token, timestamp,
signature):
logging.info('Unable to verify signature for mailgun request')
return HttpResponse(status=403)
to_email = parseaddr(request.POST['To'])[1]
from_email = parseaddr(request.POST['From'])[1]
try:
group_id = email_to_group_id(to_email)
except Exception:
logging.info('%r is not a valid email address', to_email)
return HttpResponse(status=500)
payload = EmailReplyParser.parse_reply(
request.POST['body-plain']).strip()
if not payload:
# If there's no body, we don't need to go any further
return HttpResponse(status=200)
process_inbound_email.delay(from_email, group_id, payload)
return HttpResponse(status=201)
def sentiment_sliding(messages, window=1000, shift=20):
allwords = []
data = {}
for m in messages:
if "\\Sent" not in m.get("folders", tuple()):
continue
if not m.get("body") or not m["body"].get("content"):
continue
allwords.append(EmailReplyParser.parse_reply(m["body"]["content"]))
allwords = " ".join(allwords)
allwords = allwords.encode("ascii", "ignore")
allwords = allwords.split()
current_window = 0
next_window = window
print "number of words", len(allwords)
while True:
if len(allwords) < next_window:
print "sliding-sentiment reached end at lengths:%s" % len(allwords)
break
print "sliding-sentiment start:%s end:%s" % (current_window, next_window)
data[current_window] = " ".join(allwords[current_window:next_window])
data[current_window] = indicoio.sentiment(data[current_window])
print data[current_window]
current_window += shift
next_window += shift
return data
def set_content_and_type(self):
self.content, self.content_type = '[Blank Email]', 'text/plain'
if self.html_content:
self.content, self.content_type = self.html_content, 'text/html'
else:
self.content, self.content_type = EmailReplyParser.read(
self.text_content).text.replace("\n", "\n\n"), 'text/plain'
def extract_reply(msg):
"""Extracts the portion of an email that should contain commands.
"""
for part in msg.walk():
if part.get_content_type() == 'text/plain':
content = part.get_payload(decode=True)
return EmailReplyParser.parse_reply(content)
def test_reply_from_gmail_2_ptBr(self):
with open("test/emails/email_gmail2_ptBr.txt") as f:
self.assertIn(
"entendi, muito obrigado pela informação, vou verificar aqui se tenho outras opções.",
EmailReplyParser.parse_reply(f.read()),
)
with open("test/emails/email_gmail2_ptBr.txt") as f:
self.assertIn(
"Em sex., 18 de dez. de 2020 às 14:12",
EmailReplyParser.read(f.read()).fragments[1].content,
)
with open("test/emails/email_gmail2_ptBr.txt") as f:
self.assertIn(
"Já viu o link desse anuncio?",
EmailReplyParser.read(f.read()).fragments[0].content,
)
def clean_comment(comment):
comment = EmailReplyParser.parse_reply(comment)
block = re.compile(r'`{3,}.*?`{3,}|<pre>.*?</pre>|^(?: {4,}|\t+).*?$', re.MULTILINE | re.DOTALL)
comment = block.sub('', comment)
inline = re.compile(r'`.*?`', re.MULTILINE | re.DOTALL)
comment = inline.sub('', comment)
link = re.compile(r'!?\[(.*?)\]\(.*?\)', re.MULTILINE | re.DOTALL)
comment = link.sub(r'\1', comment)
url = re.compile(r'\(?https?://\S+\)?', re.MULTILINE | re.DOTALL)
comment = url.sub('', comment)
code = re.compile(r'(?:[A-Z][a-z]*){2,}(?:::(?:[A-Z][a-z]*)+)*(?:\.|#\S+)*', re.MULTILINE | re.DOTALL)
comment = code.sub('', comment)
ruby = re.compile(r'(?:\w+/)*\w*\.rb', re.MULTILINE | re.DOTALL)
comment = ruby.sub('', comment)
emoji = re.compile(r':\S+:', re.MULTILINE | re.DOTALL)
comment = emoji.sub('', comment)
return comment
def preprocessing_french(x):
x = BeautifulSoup(x)
x = EmailReplyParser.parse_reply(x.get_text())
x = re.sub(r'<.*?>', '', x)
x = x.replace("\n", " ").strip()
x = re.sub(pattern=r'[\!"#$%&\*+,-./:;<=>?@^_`()|~=]', repl='', string=x)
x = x.replace("\n", " ").strip()
x = x.strip()
x = re.sub(r"(^|\W)\d+", "", x)
x = x.lower()
stopwords = {
'merci', 'de', 'nous', 'aider', 'au', 'plus', 'vite', 'bonjour', 'la',
'le', 'en', 'message', 'cordialement', 'logitech', 'cher', 'mon',
'date', 'je', 'récemment', 'salut', 'produit', 'en série', 'nombre',
'achat', 'soutien', 'http', 'com', 'vous', 'logitech', 'www', 'https',
'logi', 'service à la clientèle', 'contact', 'termes', 'passerelle',
'newark', 'usa', 'logo', 'care', 'ca', 'footer', 'use', 'customer',
'owned', 'us', 'survey', 'americas', 'copyright', 'headquarters',
'owners', 'number', 'respective', 'the', 'rights', 'trademarks',
'reserved', 'property', 'dear', 'regards', 'thanks', 'mail', 'email',
'date', 'like', 'get', 'one', 'set', 'thank', 'also', 'two', 'see',
'able', 'could', 'since', 'last', 'know', 'still', 'got', 'pm', 'p',
'puisque', 'operating', 'system', 'platform', 'ce', 'mr', 'de', 'lfcm',
'sy', 'm', 'kh', 'w', 'ks', 'hs', 'afternoon', 'morning', 'regards',
'thx'
'thanks', 'fri', 'mon', 'tue', 'wed', 'thu', 'sat', 'sun', 'jan',
'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'sep', 'oct', 'nov', 'dec'
}
x = x.split()
x = [word for word in x if word.lower() not in stopwords]
x = ' '.join(x)
return x
def test_reply_from_gmail_ptBr(self):
with open("test/emails/email_gmail_ptBr.txt") as f:
self.assertEqual(
"Esta é uma resposta para mensagens github.",
EmailReplyParser.parse_reply(f.read()),
)
with open("test/emails/email_gmail_ptBr.txt") as f:
self.assertIn(
"Em qua., 18 de mai. de 2016 às 11:10 Someone",
EmailReplyParser.read(f.read()).fragments[1].content,
)
with open("test/emails/email_gmail_ptBr.txt") as f:
self.assertIn(
"Esta é uma resposta para mensagens github.",
EmailReplyParser.read(f.read()).fragments[0].content,
)
def instanciate_answer(self, lines):
answer = self.answer_class()
msgtxt = ''.join(lines)
msg = email.message_from_string(msgtxt)
temporary, permanent = all_failures(msg)
if permanent:
answer.is_bounced = True
answer.email_from = scan_message(msg).pop()
elif temporary:
raise TemporaryFailure
else:
answer.email_from = msg["From"]
the_recipient = msg["To"]
answer.subject = msg["Subject"]
answer.when = msg["Date"]
answer.message_id = msg["Message-ID"]
the_recipient = re.sub(r"\n", "", the_recipient)
regex = re.compile(r".*[\+\-](.*)@.*")
the_match = regex.match(the_recipient)
if the_match is None:
raise CouldNotFindIdentifier
answer.email_to = the_recipient
answer.outbound_message_identifier = the_match.groups()[0]
logging.info("Reading the parts")
for part in msg.walk():
logging.info("Part of type " + part.get_content_type())
content_type_attr = self.content_types_attrs.get(
part.get_content_type())
if content_type_attr:
charset = part.get_content_charset() or "ISO-8859-1"
data = part.get_payload(decode=True).decode(charset)
setattr(
answer,
content_type_attr,
EmailReplyParser.parse_reply(data).strip(),
)
else:
self.handle_not_processed_part(part)
attachment = self.parse_attachment(part)
if attachment:
answer.add_attachment(attachment)
log = 'New incoming email from %(from)s sent on %(date)s with subject %(subject)s and content %(content)s'
log = log % {
'from': answer.email_from,
'date': answer.when,
'subject': answer.subject,
'content': answer.content_text,
}
logging.info(log)
return answer
def text_reply(self):
"""
>>> email_msg = MessageDecorator.from_file('test/test_single.eml')
>>> email_msg.text_reply()
'plain word1, word2, word2, word3, word3, word3'
"""
text = self.text()
return EmailReplyParser.parse_reply(text)
def get_reply_text(data):
"""
Strip signatures and replies from emails
http://stackoverflow.com/a/2193937
Drop all text after and including:
Lines that equal '-- \n' (standard email sig delimiter)
Lines that equal '--\n' (people often forget the space in sig delimiter; and this is not that common outside sigs)
Lines that begin with '-----Original Message-----' (MS Outlook default)
Lines that begin with '________________________________' (32 underscores, Outlook agian)
Lines that begin with 'On ' and end with ' wrote:\n' (OS X Mail.app default)
Lines that begin with 'From: ' (failsafe four Outlook and some other reply formats)
Lines that begin with 'Sent from my iPhone'
Lines that begin with 'Sent from my BlackBerry'
"""
msg = email.message_from_string(data)
if msg.get_content_maintype() == 'text':
message = msg.get_payload(decode=True)
elif msg.get_content_maintype(
) == 'multipart': #If message is multi part we only want the text version of the body, this walks the message and gets the body.
for part in msg.walk():
if part.get_content_type() == "text/plain":
message = part.get_payload(decode=True)
break
else:
continue
else:
return False
msg = message.split('-- \n', 1)[0]
msg = msg.split('--\n', 1)[0]
lines = msg.split("\n")
message_lines = []
for line in lines:
if line.startswith('-----Original Message-----'):
break
elif line.startswith('________________________________'):
break
elif line.startswith('On ') and line.endswith(' wrote:\n'):
break
elif line.startswith('From: '):
break
elif line.startswith('Sent from '):
break
# Trường hợp dạng: 2013/1/16 Pham Tuan Anh <*****@*****.**> (Gmail)
elif re.match('[0-9]{4}/[0-9]?[0-2]/[0-9]?[0-9] .*? <\w+@\w+\.\w+>',
line):
break
else:
message_lines.append(line)
msg = '\n'.join(message_lines)
msg = reply_parser_1.parse_reply(msg)
return msg.strip()
def process_mailbox(M, options):
"""
Do something with emails messages in the folder.
For the sake of this example, print some headers.
"""
rv, data = M.search(None, "ALL")
if rv != 'OK':
print("No messages found!")
return
with open(options.output_file, 'w') as outFile:
message_count = 0
for num in data[0].split():
rv, data = M.fetch(num, '(RFC822)')
if rv != 'OK':
print("ERROR getting message", num)
return
if message_count % 100 == 0 and message_count > 0:
runTime = round((time.time() - start_time), 1)
print("Read %d messages in %s seconds") % (message_count,
runTime)
message_count += 1
msg = email.message_from_string(data[0][1])
decode = email.header.decode_header(msg['Subject'])[0]
body = ""
if msg.is_multipart():
for part in msg.walk():
ctype = part.get_content_type()
cdispo = str(part.get('Content-Disposition'))
# skip any text/plain (txt) attachments
if ctype == 'text/plain' and 'attachment' not in cdispo:
body = part.get_payload(decode=True) # decode
break
# not multipart - i.e. plain text, no attachments, keeping fingers crossed
else:
body = msg.get_payload(decode=True)
# Now convert to local date-time
date_tuple = email.utils.parsedate_tz(msg['Date'])
if date_tuple:
local_date = datetime.datetime.fromtimestamp(
email.utils.mktime_tz(date_tuple))
send_date = local_date.strftime("%a, %d %b %Y %H:%M:%S")
reply = EmailReplyParser.parse_reply(body)
# reply_string = [str(r) for r in reply]
reply_string = reply.replace('\n', ' ').replace('\r', '')
# reply_string = email.utils.decode_rfc2231(reply_string)
outFile.write("%s\t%s\t%s\t%s\n" %
(msg['from'], msg['to'], send_date, reply_string))
def instanciate_answer(self, lines):
answer = self.answer_class()
msgtxt = ''.join(lines)
msg = email.message_from_string(msgtxt)
temporary, permanent = all_failures(msg)
if permanent:
answer.is_bounced = True
answer.email_from = scan_message(msg).pop()
elif temporary:
raise TemporaryFailure
else:
answer.email_from = msg["From"]
the_recipient = msg["To"]
answer.subject = msg["Subject"]
answer.when = msg["Date"]
answer.message_id = msg["Message-ID"]
the_recipient = re.sub(r"\n", "", the_recipient)
regex = re.compile(r".*[\+\-](.*)@.*")
the_match = regex.match(the_recipient)
if the_match is None:
raise CouldNotFindIdentifier
answer.email_to = the_recipient
answer.outbound_message_identifier = the_match.groups()[0]
logging.info("Reading the parts")
for part in msg.walk():
logging.info("Part of type " + part.get_content_type())
content_type_attr = self.content_types_attrs.get(part.get_content_type())
if content_type_attr:
charset = part.get_content_charset() or "ISO-8859-1"
data = part.get_payload(decode=True).decode(charset)
setattr(
answer,
content_type_attr,
EmailReplyParser.parse_reply(data).strip(),
)
else:
self.handle_not_processed_part(part)
attachment = self.parse_attachment(part)
if attachment:
answer.add_attachment(attachment)
log = 'New incoming email from %(from)s sent on %(date)s with subject %(subject)s and content %(content)s'
log = log % {
'from': answer.email_from,
'date': answer.when,
'subject': answer.subject,
'content': answer.content_text,
}
logging.info(log)
return answer
def get_message_details(self):
"""Return args for template"""
dws_group = frappe.get_doc("Daily Work Summary Group", self.daily_work_summary_group)
replies = frappe.get_all(
"Communication",
fields=["content", "text_content", "sender"],
filters=dict(
reference_doctype=self.doctype,
reference_name=self.name,
communication_type="Communication",
sent_or_received="Received",
),
order_by="creation asc",
)
did_not_reply = self.email_sent_to.split()
for d in replies:
user = frappe.db.get_values(
"User", {"email": d.sender}, ["full_name", "user_image"], as_dict=True
)
d.sender_name = user[0].full_name if user else d.sender
d.image = user[0].image if user and user[0].image else None
original_image = d.image
# make thumbnail image
try:
if original_image:
file_name = frappe.get_list("File", {"file_url": original_image})
if file_name:
file_name = file_name[0].name
file_doc = frappe.get_doc("File", file_name)
thumbnail_image = file_doc.make_thumbnail(
set_as_thumbnail=False, width=100, height=100, crop=True
)
d.image = thumbnail_image
except Exception:
d.image = original_image
if d.sender in did_not_reply:
did_not_reply.remove(d.sender)
if d.text_content:
d.content = frappe.utils.md_to_html(EmailReplyParser.parse_reply(d.text_content))
did_not_reply = [
(frappe.db.get_value("User", {"email": email}, "full_name") or email) for email in did_not_reply
]
return dict(
replies=replies,
original_message=dws_group.message,
title=_("Work Summary for {0}").format(global_date_format(self.creation)),
did_not_reply=", ".join(did_not_reply) or "",
did_not_reply_title=_("No replies from"),
)
def __init__(self, message):
if (not isinstance(message, dict) or 'TextBody' not in message):
log.exception('ActivityEmailParser didn\'t get a valid message.')
raise ActivityEmailEncodingError(
'Invalid or malformed json message object.')
self.email = message
reply = self._extra_email_reply_parse(self.email['TextBody'])
self.reply = EmailReplyParser.read(reply).reply
def __init__(self, message):
if (not isinstance(message, dict) or 'TextBody' not in message):
log.exception('ActivityEmailParser didn\'t get a valid message.')
raise ActivityEmailEncodingError(
'Invalid or malformed json message object.')
self.email = message
reply = self._extra_email_reply_parse(self.email['TextBody'])
self.reply = EmailReplyParser.read(reply).reply
def get_reply_message(message):
from email_reply_parser import EmailReplyParser
for payload in message.get_payload():
if payload.get_content_type() == 'text/plain':
content = payload.get_payload()
break
return EmailReplyParser.parse_reply(content)
def set_content_and_type(self):
self.content, self.content_type = "[Blank Email]", "text/plain"
if self.html_content:
self.content, self.content_type = self.html_content, "text/html"
else:
self.content, self.content_type = (
EmailReplyParser.read(self.text_content).text.replace(
"\n", "\n\n"),
"text/plain",
)
def extract_body(self, message):
body = None
for part in message.walk():
if part.get_content_type() == 'text/plain':
text = part.get_payload()
try:
body = erp.parse_reply(text)
except:
body = text
return body
def extract_body(self, message):
body = None
for part in message.walk():
if part.get_content_type() == 'text/plain':
text = part.get_payload()
try:
body = erp.parse_reply(text)
except:
body = text
return body
def relay_from_email(to_email, text_message):
email_id = to_email.split('@')[0]
text = EmailReplyParser.parse_reply(text_message)
try:
participant = Participant.select().where(Participant.email_id == email_id).get()
line_mid = participant.line_mid
line_bot_api.push_message(line_mid, TextSendMessage(text=text))
except:
pass
def test_parse_out_just_top_for_outlook_with_reply_directly_above_line_ptBr(self):
with open("test/emails/email_2_2_ptBr.txt") as f:
self.assertEqual(
"um novo dia testando !! navegador!",
EmailReplyParser.parse_reply(f.read()),
)
with open("test/emails/email_2_2_ptBr.txt") as f:
self.assertIn(
"um novo dia testando",
EmailReplyParser.read(f.read()).fragments[0].content,
)
with open("test/emails/email_2_2_ptBr.txt") as f:
self.assertIn(
"Outlook", EmailReplyParser.read(f.read()).fragments[1].content
)
with open("test/emails/email_2_2_ptBr.txt") as f:
self.assertIn(
"De: Store <*****@*****.**>",
EmailReplyParser.read(f.read()).fragments[3].content,
)
def __init__(self, email_text):
"""Decode base64 email and turn it into a Django email object."""
try:
email_text = base64.standard_b64decode(urllib2.unquote(email_text.rstrip()))
except TypeError:
# Corrupt or invalid base 64.
self.decode_error = True
return
self.email = message_from_string(email_text)
self.reply_text = EmailReplyParser.read(self.email.get_payload()).reply
def __init__(self, email_text):
"""Decode base64 email and turn it into a Django email object."""
try:
email_text = base64.standard_b64decode(
urllib2.unquote(email_text.rstrip()))
except TypeError:
# Corrupt or invalid base 64.
self.decode_error = True
return
self.email = message_from_string(email_text)
self.reply_text = EmailReplyParser.read(self.email.get_payload()).reply
def __init__(self, message):
invalid_email = not isinstance(message, dict) or not message.get(
'TextBody', None)
if invalid_email:
log.exception("ActivityEmailParser didn't get a valid message.")
raise ActivityEmailEncodingError(
'Invalid or malformed json message object.')
self.email = message
reply = self._extra_email_reply_parse(self.email['TextBody'])
self.reply = EmailReplyParser.read(reply).reply
def get_data(start=0):
#frappe.only_for('Employee', 'System Manager')
data = frappe.get_all('Communication',
fields=('content', 'text_content', 'sender', 'creation'),
filters=dict(reference_doctype='Daily Work Summary'),
order_by='creation desc', limit=40, start=start)
for d in data:
d.sender_name = frappe.db.get_value("Employee", {"user_id": d.sender},
"employee_name") or d.sender
if d.text_content:
d.content = markdown(EmailReplyParser.parse_reply(d.text_content))
return data
def handle(self, lines):
answer = self.answer_class()
msgtxt = ''.join(lines)
msg = email.message_from_string(msgtxt)
temporary, permanent = all_failures(msg)
if temporary or permanent:
answer.is_bounced = True
answer.email_from = scan_message(msg).pop()
else:
answer.email_from = msg["From"]
the_recipient = msg["To"]
answer.subject = msg["Subject"]
answer.when = msg["Date"]
the_recipient = re.sub(r"\n", "", the_recipient)
regex = re.compile(r".*[\+\-](.*)@.*")
answer.outbound_message_identifier = regex.match(the_recipient).groups()[0]
charset = msg.get_charset()
logging.info("Reading the parts")
for part in msg.walk():
logging.info("Part of type "+part.get_content_type())
if part.get_content_type() == 'text/plain':
charset = part.get_content_charset()
if not charset:
charset = "ISO-8859-1"
data = part.get_payload(decode=True).decode(charset)
text = EmailReplyParser.parse_reply(data)
text.strip()
answer.content_text = text
#logging stuff
log = 'New incoming email from %(from)s sent on %(date)s with subject %(subject)s and content %(content)s'
log = log % {
'from':answer.email_from,
'date':answer.when,
'subject':answer.subject,
'content':answer.content_text
}
logging.info(log)
return answer
def parse(shitmail):
message_id = shitmail['Message-ID']
date = datetime.fromtimestamp(mktime(email.utils.parsedate(shitmail['Date'])))
from_ = shitmail['From']
subject = shitmail['Subject']
body = extract_body(shitmail)
cleanup_body = EmailReplyParser.parse_reply(body).strip()
in_reply_to = shitmail['In-Reply-To']
c = model.session.query(model.Post).filter(model.Post.message_id == message_id).count()
if c > 0:
logging.error("double message-id: " + message_id)
return
post = model.Post(message_id, date, from_, subject, cleanup_body, in_reply_to)
model.session.add(post)
model.session.commit()
def extract_alert(msg):
"""Extract the original alert from an email thread.
Walk through all replies comprising the message, locate the
original alert email, strip off all pseudo-headers, remove quote
markers, and return the result.
"""
for part in msg.walk():
if part.get_content_type() == 'text/plain':
content = EmailReplyParser.read(
part.get_payload(decode=True))
for fragment in content.fragments:
content = fragment._content
if content != extract_reply(msg):
return sanitize_email_fragment(content)
return ''
def createEmailToCentralServer(self, subj, body, html, email, attachments=[]):
'''
Make an email with the project key and issue id and subj and body
return it.
'''
exists, match, subj = self.findProjectIssueInformation(subj)
description = ""
try:
description = body[0].get_payload()
except:
description = body
description_html = html
issue = None
thisUser = None
try:
thisUser = User.objects.get(email=email)
except:
thisUser = User(username=email, first_name='Anonymous', last_name='User', email=email)
thisUser.set_unusable_password()
thisUser.save()
try:
if exists:
print "Issue exists, attempting to create a comment"
# This strips the HTML twice. Once, to remove the HTML before stripping the reply
# email and once more incase there is HTML in the reply email...
if description is None or description == "":
description = strip_html(description_html)
description = EmailReplyParser.parse_reply(description).strip()
comment = self.createComment(match, strip_html(description).strip(), thisUser, attachments)
issue = comment.issue
else:
print "Issue doesn't exist, attempting to create an issue"
proj = self.findClosestProject(subj+description)
print proj.key, subj, description, thisUser
issue = self.createIssue(proj, subj, description, description_html, thisUser, attachments)
except Exception, e:
print e
def post(self, request):
token = request.POST['token']
signature = request.POST['signature']
timestamp = request.POST['timestamp']
key = options.get('mail.mailgun-api-key')
if not key:
logger.error('mailgun.api-key-missing')
return HttpResponse(status=500)
if not self.verify(key, token, timestamp, signature):
logger.info(
'mailgun.invalid-signature',
extra={
'token': token,
'timestamp': timestamp,
'signature': signature,
}
)
return HttpResponse(status=200)
to_email = request.POST['recipient']
from_email = request.POST['sender']
try:
group_id = email_to_group_id(to_email)
except Exception:
logger.info(
'mailgun.invalid-email', extra={
'email': to_email,
}
)
return HttpResponse(status=200)
payload = EmailReplyParser.parse_reply(request.POST['body-plain']).strip()
if not payload:
# If there's no body, we don't need to go any further
return HttpResponse(status=200)
process_inbound_email.delay(from_email, group_id, payload)
return HttpResponse(status=201)
def parse_file(self, f):
mail = email.message_from_file(f)
_ , to_addr = iemail.utils.parseaddr(mail.get('To'))
local_part = re.match(r'([^@]*)@' + settings.EMAIL_FROM_DOMAIN,to_addr)
if local_part:
local_part = local_part.group(1)
message = []
for part in mail.walk():
if part.get_content_type() == 'text/plain':
message.append(part.get_payload())
body = EmailReplyParser.parse_reply('\n'.join(message))
match = re.match(r'issue-([0-9]*)', local_part)
_ , from_addr = email.utils.parse(mail.get('From'))
if local_part in settings.NEW_ISSUE_ADDRESSES:
subject = mail.get('Subject')
issue = models.Issue(name=subject, desc=body, email=from_addr, opened=timezone.now())
issue.save()
elif match:
key = match.group(1)
issue = models.Issue.objects.get(pk=key)
issue.add_message(from_addr, body)
def __init__(self, email_text):
"""Decode base64 email and turn it into a Django email object."""
try:
email_text = base64.standard_b64decode(
urllib2.unquote(email_text.rstrip()))
except TypeError:
# Corrupt or invalid base 64.
self.decode_error = True
return
self.email = message_from_string(email_text)
payload = self.email.get_payload() # If not multipart, it's a string.
if isinstance(payload, list):
# If multipart, get the plaintext part.
for part in payload:
if part.get_content_type() == 'text/plain':
payload = part.get_payload()
break
self.reply_text = EmailReplyParser.read(payload).reply
def __init__(self, email_text):
"""Decode base64 email and turn it into a Django email object."""
try:
log.info('CommEmailParser received email: ' + email_text)
email_text = base64.standard_b64decode(
urllib2.unquote(email_text.rstrip()))
except TypeError:
# Corrupt or invalid base 64.
self.decode_error = True
log.info('Decoding error for CommEmailParser')
return
self.email = message_from_string(email_text)
payload = self.email.get_payload()
if isinstance(payload, list):
# If multipart, get the plain text part.
for part in payload:
# Nested multipart. Go deeper.
if part.get_content_type() == 'multipart/alternative':
payload = part.get_payload()
for part in payload:
if part.get_content_type() == 'text/plain':
# Found the plain text part.
payload = part.get_payload()
break
if part.get_content_type() == 'text/plain':
# Found the plain text part.
payload = part.get_payload()
break
# Decode quoted-printable data and remove non-breaking spaces.
payload = (quopri.decodestring(payload)
.replace('\xc2\xa0', ' '))
payload = self.extra_email_reply_parse(payload)
self.reply_text = EmailReplyParser.read(payload).reply
def parse_mbox(path):
mbox = mailbox.mbox(path)
messages = []
for message in mbox:
if message['From'] in ("Michael Perrone <*****@*****.**>", "Mike Perrone <*****@*****.**>"):
while message.is_multipart():
message = message.get_payload(0)
text = message.get_payload()
# use Zapier's port of Github's open source reply parser to parse just the reply and not the quoted text.
reply = EmailReplyParser.parse_reply(text)
# I'm very consistent with my signature, so trim off anything after
# it, in case EmailReplyParser couldn't figure that out. Also,
# EmailReplyParser can't figure out gmail's forwarded message thing
signature_index = first_non_neg([
reply.find('-Mike Perrone'),
reply.find('-Michael Perrone'),
reply.find("---------- Forwarded message ----------")
])
if signature_index >= 0:
reply = reply[:signature_index]
messages.append(reply)
return messages
def process_message(self, peer, mailfrom, rcpttos, raw_message):
logger.info('Incoming message received from %s', mailfrom)
if not len(rcpttos):
logger.info('Incoming email had no recipients. Ignoring.')
return STATUS[550]
if len(raw_message) > self.max_message_length:
logger.info('Inbound email message was too long: %d', len(raw_message))
return STATUS[552]
try:
group_id = email_to_group_id(rcpttos[0])
except Exception:
logger.info('%r is not a valid email address', rcpttos)
return STATUS[550]
message = email.message_from_string(raw_message)
payload = None
if message.is_multipart():
for msg in message.walk():
if msg.get_content_type() == 'text/plain':
payload = msg.get_payload()
break
if payload is None:
# No text/plain part, bailing
return STATUS[200]
else:
payload = message.get_payload()
payload = EmailReplyParser.parse_reply(payload).strip()
if not payload:
# If there's no body, we don't need to go any further
return STATUS[200]
process_inbound_email.delay(mailfrom, group_id, payload)
return STATUS[200]
def set_content_and_type(self): self.content, self.content_type = '[Blank Email]', 'text/plain' if self.html_content: self.content, self.content_type = self.html_content, 'text/html' else: self.content, self.content_type = EmailReplyParser.parse_reply(self.text_content), 'text/plain'
def ticket_from_message(message, queue, quiet):
# 'message' must be an RFC822 formatted message.
msg = message
message = email.message_from_string(msg.decode('utf-8'))
subject = message.get('subject', _('Created from e-mail'))
subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject))
subject = subject.replace("Re: ", "").replace("Fw: ", "").replace("RE: ", "").replace("FW: ", "").replace("Automatic reply: ", "").strip()
sender = message.get('from', _('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
sender_email = parseaddr(sender)[1]
body_plain, body_html = '', ''
for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)):
if ignore.test(sender_email):
if ignore.keep_in_mailbox:
# By returning 'False' the message will be kept in the mailbox,
# and the 'True' will cause the message to be deleted.
return False
return True
matchobj = re.match(r'.*\['+re.escape(queue.slug)+r'-(?P<id>\d+)\]', subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
else:
ticket = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name is None:
if part.get_content_subtype() == 'plain':
body_plain = EmailReplyParser.parse_reply(decodeUnknown(part.get_content_charset(), part.get_payload(decode=True)))
else:
body_html = decodeUnknown(part.get_content_charset(), part.get_payload(decode=True))
# make plain text more legible when viewing the ticket
body_html, n = re.subn(r'[\r\n]+', r'', body_html)
body_html, n = re.subn(r'\>\s+\<', r'><', body_html)
body_html = body_html.replace("</h1>", "</h1>\n")
body_html = body_html.replace("</h2>", "</h2>\n")
body_html = body_html.replace("</h3>", "</h3>\n")
body_html = body_html.replace("<p>", "\n<p>")
body_html = body_html.replace("</p>", "</p>\n")
body_html = body_html.replace("</div>", "</div>\n")
body_html = body_html.replace("</tr>", "</tr>\n")
body_html = body_html.replace("</td>", "</td> ")
body_html = body_html.replace("<table>", "\n<table>")
body_html = body_html.replace("</table>", "</table>\n")
body_html = body_html.replace("<br />", "<br />\n")
try:
# strip html tags
body_plain = striptags(body_html)
except DjangoUnicodeDecodeError:
charset = chardet.detect(body_html)['encoding']
body_plain = striptags(str(body_html, charset))
body_plain = unescape(body_plain)
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
files.append({
'filename': name,
'content': part.get_payload(decode=True),
'type': part.get_content_type()},
)
counter += 1
if body_plain:
body = body_plain
if body_html:
body += '\n\n'
body += _('***Note that HTML tags are stripped out. Please see attachment email_html_body.html for the full html content.')
else:
body = _('No plain-text email body available. Please see attachment email_html_body.html.')
if body_html:
files.append({
'filename': _("email_html_body.html"),
'content': body_html,
'type': 'text/html',
})
now = timezone.now()
if ticket:
try:
t = Ticket.objects.get(id=ticket)
new = False
except Ticket.DoesNotExist:
ticket = None
priority = 3
smtp_priority = message.get('priority', '')
smtp_importance = message.get('importance', '')
high_priority_types = ('high', 'important', '1', 'urgent')
if smtp_priority in high_priority_types or smtp_importance in high_priority_types:
priority = 2
if ticket is None:
t = Ticket(
title=subject,
queue=queue,
submitter_email=sender_email,
created=now,
description=body,
priority=priority,
)
t.save()
new = True
#update = ''
elif t.status == Ticket.CLOSED_STATUS:
t.status = Ticket.REOPENED_STATUS
t.save()
f = FollowUp(
ticket = t,
title = _('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}),
date = timezone.now(),
public = True,
comment = body,
)
if t.status == Ticket.REOPENED_STATUS:
f.new_status = Ticket.REOPENED_STATUS
f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email})
f.save()
if not quiet:
print((" [%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace'))
for file in files:
if file['content']:
filename = file['filename'].replace(' ', '_')
filename = re.sub(r'[^a-zA-Z0-9._-]+', '', filename)
a = Attachment(
followup=f,
filename=filename,
mime_type=file['type'],
size=len(file['content']),
)
a.file.save(filename, ContentFile(file['content']), save=False)
a.save()
if not quiet:
print(" - %s" % filename)
context = safe_template_context(t)
if new:
if sender_email and not is_no_reply_address(sender_email):
send_templated_mail(
'newticket_submitter',
context,
recipients=sender_email,
sender=queue.from_address,
fail_silently=True,
)
if queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.new_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
else:
context.update(comment=f.comment)
#if t.status == Ticket.REOPENED_STATUS:
# update = _(' (Reopened)')
#else:
# update = _(' (Updated)')
if t.assigned_to:
send_templated_mail(
'updated_owner',
context,
recipients=t.assigned_to.email,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc:
send_templated_mail(
'updated_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
return t
def __init__(self, email_text):
self.email = message_from_string(email_text)
self.reply_text = EmailReplyParser.read(self.email.get_payload()).reply
def ticket_from_message(message, queue, logger):
# 'message' must be an RFC822 formatted message.
message = email.message_from_string(message) if six.PY3 else email.message_from_string(message.encode('utf-8'))
subject = message.get('subject', _('Comment from e-mail'))
subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject))
for affix in STRIPPED_SUBJECT_STRINGS:
subject = subject.replace(affix, "")
subject = subject.strip()
sender = message.get('from', _('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
sender_email = email.utils.parseaddr(sender)[1]
cc = message.get_all('cc', None)
if cc:
# first, fixup the encoding if necessary
cc = [decode_mail_headers(decodeUnknown(message.get_charset(), x)) for x in cc]
# get_all checks if multiple CC headers, but individual emails may be comma separated too
tempcc = []
for hdr in cc:
tempcc.extend(hdr.split(','))
# use a set to ensure no duplicates
cc = set([x.strip() for x in tempcc])
for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)):
if ignore.test(sender_email):
if ignore.keep_in_mailbox:
# By returning 'False' the message will be kept in the mailbox,
# and the 'True' will cause the message to be deleted.
return False
return True
matchobj = re.match(r".*\[" + queue.slug + r"-(?P<id>\d+)\]", subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket))
else:
logger.info("No tracking ID matched.")
ticket = None
body = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = email.utils.collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name is None:
if part.get_content_subtype() == 'plain':
body = EmailReplyParser.parse_reply(
decodeUnknown(part.get_content_charset(), part.get_payload(decode=True))
)
# workaround to get unicode text out rather than escaped text
try:
body = body.encode('ascii').decode('unicode_escape')
except UnicodeEncodeError:
body.encode('utf-8')
logger.debug("Discovered plain text MIME part")
else:
files.append(
SimpleUploadedFile(_("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html')
)
logger.debug("Discovered HTML MIME part")
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
payload = part.get_payload()
if isinstance(payload, list):
payload = payload.pop().as_string()
payloadToWrite = payload
# check version of python to ensure use of only the correct error type
if six.PY2:
non_b64_err = binascii.Error
else:
non_b64_err = TypeError
try:
logger.debug("Try to base64 decode the attachment payload")
if six.PY2:
payloadToWrite = base64.decodestring(payload)
else:
payloadToWrite = base64.decodebytes(payload)
except non_b64_err:
logger.debug("Payload was not base64 encoded, using raw bytes")
payloadToWrite = payload
files.append(SimpleUploadedFile(name, part.get_payload(decode=True), mimetypes.guess_type(name)[0]))
logger.debug("Found MIME attachment %s" % name)
counter += 1
if not body:
mail = BeautifulSoup(part.get_payload(), "lxml")
if ">" in mail.text:
body = mail.find('body')
body = body.text
body = body.encode('ascii', errors='ignore')
else:
body = mail.text
if ticket:
try:
t = Ticket.objects.get(id=ticket)
except Ticket.DoesNotExist:
logger.info("Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket))
ticket = None
else:
logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id))
if t.status == Ticket.CLOSED_STATUS:
t.status = Ticket.REOPENED_STATUS
t.save()
new = False
smtp_priority = message.get('priority', '')
smtp_importance = message.get('importance', '')
high_priority_types = {'high', 'important', '1', 'urgent'}
priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3
if ticket is None:
if settings.QUEUE_EMAIL_BOX_UPDATE_ONLY:
return None
new = True
t = Ticket.objects.create(
title=subject,
queue=queue,
submitter_email=sender_email,
created=timezone.now(),
description=body,
priority=priority,
)
logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id))
if cc:
# get list of currently CC'd emails
current_cc = TicketCC.objects.filter(ticket=ticket)
current_cc_emails = [x.email for x in current_cc if x.email]
# get emails of any Users CC'd to email, if defined
# (some Users may not have an associated email, e.g, when using LDAP)
current_cc_users = [x.user.email for x in current_cc if x.user and x.user.email]
# ensure submitter, assigned user, queue email not added
other_emails = [queue.email_address]
if t.submitter_email:
other_emails.append(t.submitter_email)
if t.assigned_to:
other_emails.append(t.assigned_to.email)
current_cc = set(current_cc_emails + current_cc_users + other_emails)
# first, add any User not previously CC'd (as identified by User's email)
all_users = User.objects.all()
all_user_emails = set([x.email for x in all_users])
users_not_currently_ccd = all_user_emails.difference(set(current_cc))
users_to_cc = cc.intersection(users_not_currently_ccd)
for user in users_to_cc:
tcc = TicketCC.objects.create(
ticket=t,
user=User.objects.get(email=user),
can_view=True,
can_update=False
)
tcc.save()
# then add remaining emails alphabetically, makes testing easy
new_cc = cc.difference(current_cc).difference(all_user_emails)
new_cc = sorted(list(new_cc))
for ccemail in new_cc:
tcc = TicketCC.objects.create(
ticket=t,
email=ccemail.replace('\n', ' ').replace('\r', ' '),
can_view=True,
can_update=False
)
tcc.save()
f = FollowUp(
ticket=t,
title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}),
date=timezone.now(),
public=True,
comment=body,
)
if t.status == Ticket.REOPENED_STATUS:
f.new_status = Ticket.REOPENED_STATUS
f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email})
f.save()
logger.debug("Created new FollowUp for Ticket")
if six.PY2:
logger.info(("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace'))
elif six.PY3:
logger.info("[%s-%s] %s" % (t.queue.slug, t.id, t.title,))
attached = process_attachments(f, files)
for att_file in attached:
logger.info("Attachment '%s' (with size %s) successfully added to ticket from email." % (att_file[0], att_file[1].size))
context = safe_template_context(t)
if new:
if sender_email:
send_templated_mail(
'newticket_submitter',
context,
recipients=sender_email,
sender=queue.from_address,
fail_silently=True,
)
if queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.new_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
else:
context.update(comment=f.comment)
if t.assigned_to:
send_templated_mail(
'updated_owner',
context,
recipients=t.assigned_to.email,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc:
send_templated_mail(
'updated_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
return t
def test_email_one_is_not_on(self):
with open('test/emails/email_one_is_not_on.txt') as email:
self.assertTrue(
"On Oct 1, 2012, at 11:55 PM, Dave Tapley wrote:" not in EmailReplyParser.parse_reply(email.read()))
def get_email(self, name):
""" Return EmailMessage instance
"""
with open('test/emails/%s.txt' % name) as f:
text = f.read()
return EmailReplyParser.read(text)
