def clean_comment(comment):
comment = EmailReplyParser.parse_reply(comment)
block = re.compile(r'`{3,}.*?`{3,}|<pre>.*?</pre>|^(?: {4,}|\t+).*?$', re.MULTILINE | re.DOTALL)
comment = block.sub('', comment)
inline = re.compile(r'`.*?`', re.MULTILINE | re.DOTALL)
comment = inline.sub('', comment)
link = re.compile(r'!?\[(.*?)\]\(.*?\)', re.MULTILINE | re.DOTALL)
comment = link.sub(r'\1', comment)
url = re.compile(r'\(?https?://\S+\)?', re.MULTILINE | re.DOTALL)
comment = url.sub('', comment)
code = re.compile(r'(?:[A-Z][a-z]*){2,}(?:::(?:[A-Z][a-z]*)+)*(?:\.|#\S+)*', re.MULTILINE | re.DOTALL)
comment = code.sub('', comment)
ruby = re.compile(r'(?:\w+/)*\w*\.rb', re.MULTILINE | re.DOTALL)
comment = ruby.sub('', comment)
emoji = re.compile(r':\S+:', re.MULTILINE | re.DOTALL)
comment = emoji.sub('', comment)
return comment
def get_summary_message(self):
'''Return summary of replies as HTML'''
settings = frappe.get_doc('Daily Work Summary Settings')
replies = frappe.get_all('Communication', fields=['content', 'text_content', 'sender'],
filters=dict(reference_doctype=self.doctype, reference_name=self.name,
communication_type='Communication', sent_or_received='Received'),
order_by='creation asc')
did_not_reply = self.email_sent_to.split()
for d in replies:
d.sender_name = frappe.db.get_value("Employee", {"user_id": d.sender},
"employee_name") or d.sender
if d.sender in did_not_reply:
did_not_reply.remove(d.sender)
if d.text_content:
d.content = markdown(EmailReplyParser.parse_reply(d.text_content))
did_not_reply = [(frappe.db.get_value("Employee", {"user_id": email}, "employee_name") or email)
for email in did_not_reply]
return frappe.render_template(self.get_summary_template(),
dict(replies=replies,
original_message=settings.message,
title=_('Daily Work Summary for {0}'.format(formatdate(self.creation))),
did_not_reply= ', '.join(did_not_reply) or '',
did_not_reply_title = _('No replies from')))
def extract_reply(msg):
"""Extracts the portion of an email that should contain commands.
"""
for part in msg.walk():
if part.get_content_type() == 'text/plain':
content = part.get_payload(decode=True)
return EmailReplyParser.parse_reply(content)
def incoming_letter_email():
body = EmailReplyParser.parse_reply(unicode(request.form.get('text')).encode('ascii','xmlcharrefreplace'))
body = '\n'.join(body.split('\n')).replace("\n","<br />")
regexp = re.findall(r'[\w\.-]+@[\w\.-]+',request.form.get('from'))
try:
attachments = int(request.form.get('attachments'))
except Exception:
attachments = 0
if len(regexp) > 0 and len(regexp[-1]) > 0:
username = regexp[-1].lower()
else:
return_bad_params(username)
return Response(response=jfail("missing parameters"), status=200)
to_name = request.form.get('to')
to_address = unicode(request.form.get('subject')).encode('ascii','xmlcharrefreplace').lower().replace("fw:","").replace("re:","").strip()
if None in [body,username,to_name,to_address]:
return_bad_params(username)
return Response(response=jfail("missing parameters"), status=200)
user = User(username)
if user.is_valid():
send_letter(user,to_name,to_address,body,attachments)
else:
return_unknown_sender(username)
return Response(response=jfail("unknown sender"), status=200)
return Response(response=jsuccess(), status=200)
def sentiment_sliding(messages, window=1000, shift=20):
allwords = []
data = {}
for m in messages:
if "\\Sent" not in m.get("folders", tuple()):
continue
if not m.get("body") or not m["body"].get("content"):
continue
allwords.append(EmailReplyParser.parse_reply(m["body"]["content"]))
allwords = " ".join(allwords)
allwords = allwords.encode("ascii", "ignore")
allwords = allwords.split()
current_window = 0
next_window = window
print "number of words", len(allwords)
while True:
if len(allwords) < next_window:
print "sliding-sentiment reached end at lengths:%s" % len(allwords)
break
print "sliding-sentiment start:%s end:%s" % (current_window, next_window)
data[current_window] = " ".join(allwords[current_window:next_window])
data[current_window] = indicoio.sentiment(data[current_window])
print data[current_window]
current_window += shift
next_window += shift
return data
def collect_project_status():
for data in frappe.get_all("Project Update",
{'date': today(), 'sent': 0}):
replies = frappe.get_all('Communication',
fields=['content', 'text_content', 'sender'],
filters=dict(reference_doctype="Project Update",
reference_name=data.name,
communication_type='Communication',
sent_or_received='Received'),
order_by='creation asc')
for d in replies:
doc = frappe.get_doc("Project Update", data.name)
user_data = frappe.db.get_values("User", {"email": d.sender},
["full_name", "user_image", "name"], as_dict=True)[0]
doc.append("users", {
'user': user_data.name,
'full_name': user_data.full_name,
'image': user_data.user_image,
'project_status': frappe.utils.md_to_html(
EmailReplyParser.parse_reply(d.text_content) or d.content
)
})
doc.save(ignore_permissions=True)
def post(self, request):
token = request.POST['token']
signature = request.POST['signature']
timestamp = request.POST['timestamp']
key = options.get('mail.mailgun-api-key')
if not key:
logging.error('mail.mailgun-api-key is not set')
return HttpResponse(status=500)
if not self.verify(key, token, timestamp, signature):
logging.info('Unable to verify signature for mailgun request')
return HttpResponse(status=403)
to_email = parseaddr(request.POST['To'])[1]
from_email = parseaddr(request.POST['From'])[1]
try:
group_id = email_to_group_id(to_email)
except Exception:
logging.info('%r is not a valid email address', to_email)
return HttpResponse(status=500)
payload = EmailReplyParser.parse_reply(request.POST['body-plain']).strip()
if not payload:
# If there's no body, we don't need to go any further
return HttpResponse(status=200)
process_inbound_email.delay(from_email, group_id, payload)
return HttpResponse(status=201)
def text_reply(self):
"""
>>> email_msg = MessageDecorator.from_file('test/test_single.eml')
>>> email_msg.text_reply()
'plain word1, word2, word2, word3, word3, word3'
"""
text = self.text()
return EmailReplyParser.parse_reply(text)
def instanciate_answer(self, lines):
answer = self.answer_class()
msgtxt = ''.join(lines)
msg = email.message_from_string(msgtxt)
temporary, permanent = all_failures(msg)
if permanent:
answer.is_bounced = True
answer.email_from = scan_message(msg).pop()
elif temporary:
raise TemporaryFailure
else:
answer.email_from = msg["From"]
the_recipient = msg["To"]
answer.subject = msg["Subject"]
answer.when = msg["Date"]
answer.message_id = msg["Message-ID"]
the_recipient = re.sub(r"\n", "", the_recipient)
regex = re.compile(r".*[\+\-](.*)@.*")
the_match = regex.match(the_recipient)
if the_match is None:
raise CouldNotFindIdentifier
answer.email_to = the_recipient
answer.outbound_message_identifier = the_match.groups()[0]
logging.info("Reading the parts")
for part in msg.walk():
logging.info("Part of type " + part.get_content_type())
content_type_attr = self.content_types_attrs.get(part.get_content_type())
if content_type_attr:
charset = part.get_content_charset() or "ISO-8859-1"
data = part.get_payload(decode=True).decode(charset)
setattr(
answer,
content_type_attr,
EmailReplyParser.parse_reply(data).strip(),
)
else:
self.handle_not_processed_part(part)
attachment = self.parse_attachment(part)
if attachment:
answer.add_attachment(attachment)
log = 'New incoming email from %(from)s sent on %(date)s with subject %(subject)s and content %(content)s'
log = log % {
'from': answer.email_from,
'date': answer.when,
'subject': answer.subject,
'content': answer.content_text,
}
logging.info(log)
return answer
def get_reply_message(message):
from email_reply_parser import EmailReplyParser
for payload in message.get_payload():
if payload.get_content_type() == 'text/plain':
content = payload.get_payload()
break
return EmailReplyParser.parse_reply(content)
def extract_body(self, message):
body = None
for part in message.walk():
if part.get_content_type() == 'text/plain':
text = part.get_payload()
try:
body = erp.parse_reply(text)
except:
body = text
return body
def get_data(start=0):
#frappe.only_for('Employee', 'System Manager')
data = frappe.get_all('Communication',
fields=('content', 'text_content', 'sender', 'creation'),
filters=dict(reference_doctype='Daily Work Summary'),
order_by='creation desc', limit=40, start=start)
for d in data:
d.sender_name = frappe.db.get_value("Employee", {"user_id": d.sender},
"employee_name") or d.sender
if d.text_content:
d.content = markdown(EmailReplyParser.parse_reply(d.text_content))
return data
def handle(self, lines):
answer = self.answer_class()
msgtxt = ''.join(lines)
msg = email.message_from_string(msgtxt)
temporary, permanent = all_failures(msg)
if temporary or permanent:
answer.is_bounced = True
answer.email_from = scan_message(msg).pop()
else:
answer.email_from = msg["From"]
the_recipient = msg["To"]
answer.subject = msg["Subject"]
answer.when = msg["Date"]
the_recipient = re.sub(r"\n", "", the_recipient)
regex = re.compile(r".*[\+\-](.*)@.*")
answer.outbound_message_identifier = regex.match(the_recipient).groups()[0]
charset = msg.get_charset()
logging.info("Reading the parts")
for part in msg.walk():
logging.info("Part of type "+part.get_content_type())
if part.get_content_type() == 'text/plain':
charset = part.get_content_charset()
if not charset:
charset = "ISO-8859-1"
data = part.get_payload(decode=True).decode(charset)
text = EmailReplyParser.parse_reply(data)
text.strip()
answer.content_text = text
#logging stuff
log = 'New incoming email from %(from)s sent on %(date)s with subject %(subject)s and content %(content)s'
log = log % {
'from':answer.email_from,
'date':answer.when,
'subject':answer.subject,
'content':answer.content_text
}
logging.info(log)
return answer
def parse(shitmail):
message_id = shitmail['Message-ID']
date = datetime.fromtimestamp(mktime(email.utils.parsedate(shitmail['Date'])))
from_ = shitmail['From']
subject = shitmail['Subject']
body = extract_body(shitmail)
cleanup_body = EmailReplyParser.parse_reply(body).strip()
in_reply_to = shitmail['In-Reply-To']
c = model.session.query(model.Post).filter(model.Post.message_id == message_id).count()
if c > 0:
logging.error("double message-id: " + message_id)
return
post = model.Post(message_id, date, from_, subject, cleanup_body, in_reply_to)
model.session.add(post)
model.session.commit()
def createEmailToCentralServer(self, subj, body, html, email, attachments=[]):
'''
Make an email with the project key and issue id and subj and body
return it.
'''
exists, match, subj = self.findProjectIssueInformation(subj)
description = ""
try:
description = body[0].get_payload()
except:
description = body
description_html = html
issue = None
thisUser = None
try:
thisUser = User.objects.get(email=email)
except:
thisUser = User(username=email, first_name='Anonymous', last_name='User', email=email)
thisUser.set_unusable_password()
thisUser.save()
try:
if exists:
print "Issue exists, attempting to create a comment"
# This strips the HTML twice. Once, to remove the HTML before stripping the reply
# email and once more incase there is HTML in the reply email...
if description is None or description == "":
description = strip_html(description_html)
description = EmailReplyParser.parse_reply(description).strip()
comment = self.createComment(match, strip_html(description).strip(), thisUser, attachments)
issue = comment.issue
else:
print "Issue doesn't exist, attempting to create an issue"
proj = self.findClosestProject(subj+description)
print proj.key, subj, description, thisUser
issue = self.createIssue(proj, subj, description, description_html, thisUser, attachments)
except Exception, e:
print e
def post(self, request):
token = request.POST['token']
signature = request.POST['signature']
timestamp = request.POST['timestamp']
key = options.get('mail.mailgun-api-key')
if not key:
logger.error('mailgun.api-key-missing')
return HttpResponse(status=500)
if not self.verify(key, token, timestamp, signature):
logger.info(
'mailgun.invalid-signature',
extra={
'token': token,
'timestamp': timestamp,
'signature': signature,
}
)
return HttpResponse(status=200)
to_email = request.POST['recipient']
from_email = request.POST['sender']
try:
group_id = email_to_group_id(to_email)
except Exception:
logger.info(
'mailgun.invalid-email', extra={
'email': to_email,
}
)
return HttpResponse(status=200)
payload = EmailReplyParser.parse_reply(request.POST['body-plain']).strip()
if not payload:
# If there's no body, we don't need to go any further
return HttpResponse(status=200)
process_inbound_email.delay(from_email, group_id, payload)
return HttpResponse(status=201)
def parse_file(self, f):
mail = email.message_from_file(f)
_ , to_addr = iemail.utils.parseaddr(mail.get('To'))
local_part = re.match(r'([^@]*)@' + settings.EMAIL_FROM_DOMAIN,to_addr)
if local_part:
local_part = local_part.group(1)
message = []
for part in mail.walk():
if part.get_content_type() == 'text/plain':
message.append(part.get_payload())
body = EmailReplyParser.parse_reply('\n'.join(message))
match = re.match(r'issue-([0-9]*)', local_part)
_ , from_addr = email.utils.parse(mail.get('From'))
if local_part in settings.NEW_ISSUE_ADDRESSES:
subject = mail.get('Subject')
issue = models.Issue(name=subject, desc=body, email=from_addr, opened=timezone.now())
issue.save()
elif match:
key = match.group(1)
issue = models.Issue.objects.get(pk=key)
issue.add_message(from_addr, body)
def get_body(self,
use_html_parsing=True,
use_txt_parsing=True) -> typing.Optional[str]:
body_part = self._get_mime_body_message()
body = None
if body_part:
charset = body_part.get_content_charset("iso-8859-1")
content_type = body_part.get_content_type()
if content_type == CONTENT_TYPE_TEXT_PLAIN:
txt_body = body_part.get_payload(decode=True).decode(charset)
if use_txt_parsing:
txt_body = EmailReplyParser.parse_reply(txt_body)
html_body = markdown.markdown(txt_body)
body = HtmlSanitizer.sanitize(html_body)
elif content_type == CONTENT_TYPE_TEXT_HTML:
html_body = body_part.get_payload(decode=True).decode(charset)
if use_html_parsing:
html_body = str(ParsedHTMLMail(html_body))
body = HtmlSanitizer.sanitize(html_body)
if not body:
raise EmptyEmailBody()
return body
def preprocessing_english(x):
x = BeautifulSoup(x)
x = EmailReplyParser.parse_reply(x.get_text())
x = re.sub(r'<.*?>', '', x)
x = x.replace("\n", " ").strip()
x = re.sub(pattern=r'[\!"#$%&\*+,-./:;<=>?@^_`()|~=]', repl='', string=x)
x = x.replace("\n", " ").strip()
x = x.strip()
x = re.sub(r"(^|\W)\d+", "", x)
x = x.lower()
x = re.sub(r'[^a-zA-Z]', ' ', x)
x = re.sub("\s\s+", " ", x)
stopwords = {
'forwarded', 'message', 'lz', 'logitech', 'dear', 'my', 'date', 'i',
'recently', 'hi', 'hello', 'product', 'serial', 'number', '1', '2',
'3', '4', '5', '6', '7', '8', '9', '0', 'purchased', 'purchase',
'support', 'http', 'com', 'logitech', 'www', 'https', 'logi',
'customercare', 'contact', 'terms', 'blvd', 'gateway', 'newark', 'usa',
'logo', 'care', 'ca', 'footer', 'use', 'customer', 'owned', 'us',
'survey', 'americas', 'copyright', 'headquarters', 'owners',
'respective', 'the', 'rights', 'trademarks', 'reserved', 'property',
'dear', 'regards', 'thanks', 'mail', 'email', 'lz', 'g', 'x', 'k',
'date', 'like', 'get', 'one', 'set', 'thank', 'also', 'two', 'see',
'able', 'n', 'could', 'since', 'last', 'know', 'still', 'got', 'pm',
'p', 'n', 's'
'operating', 'system', 'platform', 'ce', 's', 'hs', 'y', 'mr', 'de',
'lfcm', 'sy', 'm', 'kh', 'w', 'ks', 'hs', 'afternoon', 'morning',
'regards', 'thx'
'thanks', 'fri', 'mon', 'tue', 'wed', 'thu', 'sat', 'sun', 'jan',
'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'sep', 'oct', 'nov', 'dec'
}
x = x.split()
x = [word for word in x if word.lower() not in stopwords]
x = ' '.join(x)
return x
def process_message(self, peer, mailfrom, rcpttos, raw_message):
logger.info('Incoming message received from %s', mailfrom)
if not len(rcpttos):
logger.info('Incoming email had no recipients. Ignoring.')
return STATUS[550]
if len(raw_message) > self.max_message_length:
logger.info('Inbound email message was too long: %d',
len(raw_message))
return STATUS[552]
try:
group_id = email_to_group_id(rcpttos[0])
except Exception:
logger.info('%r is not a valid email address', rcpttos)
return STATUS[550]
message = email.message_from_string(raw_message)
payload = None
if message.is_multipart():
for msg in message.walk():
if msg.get_content_type() == 'text/plain':
payload = msg.get_payload()
break
if payload is None:
# No text/plain part, bailing
return STATUS[200]
else:
payload = message.get_payload()
payload = EmailReplyParser.parse_reply(payload).strip()
if not payload:
# If there's no body, we don't need to go any further
return STATUS[200]
process_inbound_email.delay(mailfrom, group_id, payload)
return STATUS[200]
def get_summary_message(self):
'''Return summary of replies as HTML'''
settings = frappe.get_doc('Daily Work Summary Settings')
replies = frappe.get_all('Communication',
fields=['content', 'text_content', 'sender'],
filters=dict(
reference_doctype=self.doctype,
reference_name=self.name,
communication_type='Communication',
sent_or_received='Received'),
order_by='creation asc')
did_not_reply = self.email_sent_to.split()
for d in replies:
d.sender_name = frappe.db.get_value(
"Employee", {"user_id": d.sender}, "employee_name") or d.sender
if d.sender in did_not_reply:
did_not_reply.remove(d.sender)
if d.text_content:
d.content = markdown(
EmailReplyParser.parse_reply(d.text_content))
did_not_reply = [(frappe.db.get_value("Employee", {"user_id": email},
"employee_name") or email)
for email in did_not_reply]
return frappe.render_template(
self.get_summary_template(),
dict(replies=replies,
original_message=settings.message,
title=_('Daily Work Summary for {0}'.format(
formatdate(self.creation))),
did_not_reply=', '.join(did_not_reply) or '',
did_not_reply_title=_('No replies from')))
def parse_mbox(path):
mbox = mailbox.mbox(path)
messages = []
for message in mbox:
if message['From'] in ("Michael Perrone <*****@*****.**>", "Mike Perrone <*****@*****.**>"):
while message.is_multipart():
message = message.get_payload(0)
text = message.get_payload()
# use Zapier's port of Github's open source reply parser to parse just the reply and not the quoted text.
reply = EmailReplyParser.parse_reply(text)
# I'm very consistent with my signature, so trim off anything after
# it, in case EmailReplyParser couldn't figure that out. Also,
# EmailReplyParser can't figure out gmail's forwarded message thing
signature_index = first_non_neg([
reply.find('-Mike Perrone'),
reply.find('-Michael Perrone'),
reply.find("---------- Forwarded message ----------")
])
if signature_index >= 0:
reply = reply[:signature_index]
messages.append(reply)
return messages
def collect_project_status():
for data in dataent.get_all("Project Update", {
'date': today(),
'sent': 0
}):
replies = dataent.get_all('Communication',
fields=['content', 'text_content', 'sender'],
filters=dict(
reference_doctype="Project Update",
reference_name=data.name,
communication_type='Communication',
sent_or_received='Received'),
order_by='creation asc')
for d in replies:
doc = dataent.get_doc("Project Update", data.name)
user_data = dataent.db.get_values(
"User", {"email": d.sender},
["full_name", "user_image", "name"],
as_dict=True)[0]
doc.append(
"users", {
'user':
user_data.name,
'full_name':
user_data.full_name,
'image':
user_data.user_image,
'project_status':
dataent.utils.md_to_html(
EmailReplyParser.parse_reply(d.text_content)
or d.content)
})
doc.save(ignore_permissions=True)
def process_message(self, peer, mailfrom, rcpttos, raw_message):
logger.info('Incoming message received from %s', mailfrom)
if not len(rcpttos):
logger.info('Incoming email had no recipients. Ignoring.')
return STATUS[550]
if len(raw_message) > self.max_message_length:
logger.info('Inbound email message was too long: %d', len(raw_message))
return STATUS[552]
try:
group_id = email_to_group_id(rcpttos[0])
except Exception:
logger.info('%r is not a valid email address', rcpttos)
return STATUS[550]
message = email.message_from_string(raw_message)
payload = None
if message.is_multipart():
for msg in message.walk():
if msg.get_content_type() == 'text/plain':
payload = msg.get_payload()
break
if payload is None:
# No text/plain part, bailing
return STATUS[200]
else:
payload = message.get_payload()
payload = EmailReplyParser.parse_reply(payload).strip()
if not payload:
# If there's no body, we don't need to go any further
return STATUS[200]
process_inbound_email.delay(mailfrom, group_id, payload)
return STATUS[200]
def handle_DATA(self, server, session, envelope):
mail_from = envelope.mail_from
message = BytesParser(policy=policy.default).parsebytes(
envelope.content)
body = message.get_body(preferencelist=('plain', ))
if body:
content = body.get_content()
reply = EmailReplyParser.parse_reply(content)
author, _ = User.objects.get_or_create(email=mail_from)
ticket, message_id = self.get_ticket(message)
if ticket:
if not ticket.inbox.enable_reply_by_email:
return '450 Reply by email is disabled for the inbox'
Comment.objects.create(
ticket=ticket,
author=author,
is_reply=ticket.reply_message_id == message_id,
content=reply)
UserInbox.objects.get_or_create(user=author,
inbox=ticket.inbox)
else:
inbox = Inbox.objects.get(email__in=envelope.rcpt_tos, )
if not inbox.enable_create_new_ticket_by_email:
return '450 Creation of ticket by email is disabled for the inbox'
Ticket.objects.create(author=author,
inbox=inbox,
title=message["Subject"],
content=reply)
UserInbox.objects.get_or_create(user=author, inbox=inbox)
return '250 OK'
def ticket_from_message(message, queue, logger):
# 'message' must be an RFC822 formatted message.
message = email.message_from_string(message) if six.PY3 else email.message_from_string(message.encode('utf-8'))
subject = message.get('subject', _('Comment from e-mail'))
subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject))
for affix in STRIPPED_SUBJECT_STRINGS:
subject = subject.replace(affix, "")
subject = subject.strip()
sender = message.get('from', _('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
sender_email = email.utils.parseaddr(sender)[1]
cc = message.get_all('cc', None)
if cc:
# first, fixup the encoding if necessary
cc = [decode_mail_headers(decodeUnknown(message.get_charset(), x)) for x in cc]
# get_all checks if multiple CC headers, but individual emails may be comma separated too
tempcc = []
for hdr in cc:
tempcc.extend(hdr.split(','))
# use a set to ensure no duplicates
cc = set([x.strip() for x in tempcc])
for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)):
if ignore.test(sender_email):
if ignore.keep_in_mailbox:
# By returning 'False' the message will be kept in the mailbox,
# and the 'True' will cause the message to be deleted.
return False
return True
matchobj = re.match(r".*\[" + queue.slug + r"-(?P<id>\d+)\]", subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket))
else:
logger.info("No tracking ID matched.")
ticket = None
body = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = email.utils.collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name is None:
if part.get_content_subtype() == 'plain':
body = EmailReplyParser.parse_reply(
decodeUnknown(part.get_content_charset(), part.get_payload(decode=True))
)
# workaround to get unicode text out rather than escaped text
try:
body = body.encode('ascii').decode('unicode_escape')
except UnicodeEncodeError:
body.encode('utf-8')
logger.debug("Discovered plain text MIME part")
else:
files.append(
SimpleUploadedFile(_("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html')
)
logger.debug("Discovered HTML MIME part")
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
payload = part.get_payload()
if isinstance(payload, list):
payload = payload.pop().as_string()
payloadToWrite = payload
# check version of python to ensure use of only the correct error type
if six.PY2:
non_b64_err = binascii.Error
else:
non_b64_err = TypeError
try:
logger.debug("Try to base64 decode the attachment payload")
if six.PY2:
payloadToWrite = base64.decodestring(payload)
else:
payloadToWrite = base64.decodebytes(payload)
except non_b64_err:
logger.debug("Payload was not base64 encoded, using raw bytes")
payloadToWrite = payload
files.append(SimpleUploadedFile(name, part.get_payload(decode=True), mimetypes.guess_type(name)[0]))
logger.debug("Found MIME attachment %s" % name)
counter += 1
if not body:
mail = BeautifulSoup(part.get_payload(), "lxml")
if ">" in mail.text:
body = mail.find('body')
body = body.text
body = body.encode('ascii', errors='ignore')
else:
body = mail.text
if ticket:
try:
t = Ticket.objects.get(id=ticket)
except Ticket.DoesNotExist:
logger.info("Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket))
ticket = None
else:
logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id))
if t.status == Ticket.CLOSED_STATUS:
t.status = Ticket.REOPENED_STATUS
t.save()
new = False
smtp_priority = message.get('priority', '')
smtp_importance = message.get('importance', '')
high_priority_types = {'high', 'important', '1', 'urgent'}
priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3
if ticket is None:
if settings.QUEUE_EMAIL_BOX_UPDATE_ONLY:
return None
new = True
t = Ticket.objects.create(
title=subject,
queue=queue,
submitter_email=sender_email,
created=timezone.now(),
description=body,
priority=priority,
)
logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id))
if cc:
# get list of currently CC'd emails
current_cc = TicketCC.objects.filter(ticket=ticket)
current_cc_emails = [x.email for x in current_cc if x.email]
# get emails of any Users CC'd to email, if defined
# (some Users may not have an associated email, e.g, when using LDAP)
current_cc_users = [x.user.email for x in current_cc if x.user and x.user.email]
# ensure submitter, assigned user, queue email not added
other_emails = [queue.email_address]
if t.submitter_email:
other_emails.append(t.submitter_email)
if t.assigned_to:
other_emails.append(t.assigned_to.email)
current_cc = set(current_cc_emails + current_cc_users + other_emails)
# first, add any User not previously CC'd (as identified by User's email)
all_users = User.objects.all()
all_user_emails = set([x.email for x in all_users])
users_not_currently_ccd = all_user_emails.difference(set(current_cc))
users_to_cc = cc.intersection(users_not_currently_ccd)
for user in users_to_cc:
tcc = TicketCC.objects.create(
ticket=t,
user=User.objects.get(email=user),
can_view=True,
can_update=False
)
tcc.save()
# then add remaining emails alphabetically, makes testing easy
new_cc = cc.difference(current_cc).difference(all_user_emails)
new_cc = sorted(list(new_cc))
for ccemail in new_cc:
tcc = TicketCC.objects.create(
ticket=t,
email=ccemail.replace('\n', ' ').replace('\r', ' '),
can_view=True,
can_update=False
)
tcc.save()
f = FollowUp(
ticket=t,
title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}),
date=timezone.now(),
public=True,
comment=body,
)
if t.status == Ticket.REOPENED_STATUS:
f.new_status = Ticket.REOPENED_STATUS
f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email})
f.save()
logger.debug("Created new FollowUp for Ticket")
if six.PY2:
logger.info(("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace'))
elif six.PY3:
logger.info("[%s-%s] %s" % (t.queue.slug, t.id, t.title,))
attached = process_attachments(f, files)
for att_file in attached:
logger.info("Attachment '%s' (with size %s) successfully added to ticket from email." % (att_file[0], att_file[1].size))
context = safe_template_context(t)
if new:
if sender_email:
send_templated_mail(
'newticket_submitter',
context,
recipients=sender_email,
sender=queue.from_address,
fail_silently=True,
)
if queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.new_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
else:
context.update(comment=f.comment)
if t.assigned_to:
send_templated_mail(
'updated_owner',
context,
recipients=t.assigned_to.email,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc:
send_templated_mail(
'updated_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
return t
def object_from_message(message, queue, logger):
# 'message' must be an RFC822 formatted message.
message = email.message_from_string(message)
subject = message.get('subject', _('Comment from e-mail'))
subject = decode_mail_headers(decodeUnknown(message.get_charset(),
subject))
for affix in STRIPPED_SUBJECT_STRINGS:
subject = subject.replace(affix, "")
subject = subject.strip()
sender = message.get('from', _('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
# to address bug #832, we wrap all the text in front of the email address in
# double quotes by using replace() on the email string. Then,
# take first item of list, second item of tuple is the actual email address.
# Note that the replace won't work on just an email with no real name,
# but the getaddresses() function seems to be able to handle just unclosed quotes
# correctly. Not ideal, but this seems to work for now.
sender_email = email.utils.getaddresses(
['\"' + sender.replace('<', '\" <')])[0][1]
body_plain, body_html = '', ''
cc = message.get_all('cc', None)
if cc:
# first, fixup the encoding if necessary
cc = [
decode_mail_headers(decodeUnknown(message.get_charset(), x))
for x in cc
]
# get_all checks if multiple CC headers, but individual emails may be comma separated too
tempcc = []
for hdr in cc:
tempcc.extend(hdr.split(','))
# use a set to ensure no duplicates
cc = set([x.strip() for x in tempcc])
for ignore in IgnoreEmail.objects.filter(
Q(queues=queue) | Q(queues__isnull=True)):
if ignore.test(sender_email):
if ignore.keep_in_mailbox:
# By returning 'False' the message will be kept in the mailbox,
# and the 'True' will cause the message to be deleted.
return False
return True
matchobj = re.match(r".*\[" + queue.slug + r"-(?P<id>\d+)\]", subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket))
else:
logger.info("No tracking ID matched.")
ticket = None
body = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = email.utils.collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name is None:
if part.get_content_subtype() == 'plain':
body = part.get_payload(decode=True)
# https://github.com/django-helpdesk/django-helpdesk/issues/732
if part['Content-Transfer-Encoding'] == '8bit' and part.get_content_charset(
) == 'utf-8':
body = body.decode('unicode_escape')
body = decodeUnknown(part.get_content_charset(), body)
body = EmailReplyParser.parse_reply(body)
# workaround to get unicode text out rather than escaped text
try:
body = body.encode('ascii').decode('unicode_escape')
except UnicodeEncodeError:
body.encode('utf-8')
logger.debug("Discovered plain text MIME part")
else:
try:
email_body = encoding.smart_text(
part.get_payload(decode=True))
except UnicodeDecodeError:
email_body = encoding.smart_text(
part.get_payload(decode=False))
payload = """
<html>
<head>
<meta charset="utf-8"/>
</head>
%s
</html>""" % email_body
files.append(
SimpleUploadedFile(_("email_html_body.html"),
payload.encode("utf-8"), 'text/html'))
logger.debug("Discovered HTML MIME part")
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
payload = part.get_payload()
if isinstance(payload, list):
payload = payload.pop().as_string()
payloadToWrite = payload
# check version of python to ensure use of only the correct error type
non_b64_err = TypeError
try:
logger.debug("Try to base64 decode the attachment payload")
payloadToWrite = base64.decodebytes(payload)
except non_b64_err:
logger.debug("Payload was not base64 encoded, using raw bytes")
payloadToWrite = payload
files.append(
SimpleUploadedFile(name, part.get_payload(decode=True),
mimetypes.guess_type(name)[0]))
logger.debug("Found MIME attachment %s" % name)
counter += 1
if not body:
mail = BeautifulSoup(str(message), "html.parser")
beautiful_body = mail.find('body')
if beautiful_body:
try:
body = beautiful_body.text
except AttributeError:
pass
if not body:
body = ""
smtp_priority = message.get('priority', '')
smtp_importance = message.get('importance', '')
high_priority_types = {'high', 'important', '1', 'urgent'}
priority = 2 if high_priority_types & {smtp_priority, smtp_importance
} else 3
payload = {
'body': body,
'subject': subject,
'queue': queue,
'sender_email': sender_email,
'priority': priority,
'files': files,
}
return create_object_from_email_message(message,
ticket,
payload,
files,
logger=logger)
def test_email_one_is_not_on(self):
with open('test/emails/email_one_is_not_on.txt') as email:
self.assertTrue(
"On Oct 1, 2012, at 11:55 PM, Dave Tapley wrote:" not in EmailReplyParser.parse_reply(email.read()))
def test_parse_out_just_top_for_outlook_with_reply_directly_above_line(self):
with open('test/emails/email_2_2.txt') as f:
self.assertEqual("Outlook with a reply directly above line", EmailReplyParser.parse_reply(f.read()))
def test_reply_from_gmail(self):
with open('test/emails/email_gmail.txt') as f:
self.assertEqual('This is a test for inbox replying to a github message.',
EmailReplyParser.parse_reply(f.read()))
def set_content_and_type(self): self.content, self.content_type = '[Blank Email]', 'text/plain' if self.html_content: self.content, self.content_type = self.html_content, 'text/html' else: self.content, self.content_type = EmailReplyParser.parse_reply(self.text_content), 'text/plain'
def inbox():
# Parse E-Mail
parsed_email = message_from_string(request.form.to_dict()['email'])
parsed_email_from = parseaddr(parsed_email['From'])[1]
parsed_email_to = parseaddr(parsed_email['To'])[1]
parsed_email_to_domain = parsed_email_to.split('@')[1]
parsed_email_session = md5(
bytes(parsed_email.get('Subject', '').replace('Re: ', '') +
parsed_email_from,
encoding='utf8')).hexdigest()
parsed_email_body = ''
for part in parsed_email.walk():
if part.get_content_type() == 'text/plain':
parsed_email_body += part.get_payload()
parsed_email_body = EmailReplyParser.parse_reply(parsed_email_body)
parsed_email_lang = fallback_lang
try:
parsed_email_lang = detect(parsed_email_body)
except:
pass
# Log E-Mail
app.logger.info('Received new E-Mail')
app.logger.info('From: ' + parsed_email_from)
app.logger.info('To: ' + parsed_email_to)
app.logger.info('Text: ' + parsed_email_body)
app.logger.info('Message ID: ' + parsed_email['Message-ID'])
app.logger.info('Session ID: ' + parsed_email_session)
# Build Request
agent_id = parsed_email_to.split('@')[0]
req = {
'session': parsed_email_session,
'queryInput': {
'text': {
'text': parsed_email_body,
'languageCode': parsed_email_lang
}
},
'queryParams': {
'payload': {
'email': {
'from': parsed_email_from,
'to': parsed_email_to,
'subject': parsed_email['Subject'],
'body': parsed_email_body
}
}
}
}
# Make the request
agent = requests.get(endpoint.replace('*', agent_id))
r = requests.post(endpoint.replace('*', agent_id), json=req)
if r.status_code == 200:
# Make new E-Mail for the response
message = MIMEMultipart()
message['Message-ID'] = make_msgid()
message['In-Reply-To'] = parsed_email['Message-ID']
message['References'] = parsed_email['Message-ID']
message['From'] = agent.json(
)['displayName'] + ' <' + parsed_email_to + '>' if agent.json().get(
'displayName') else parsed_email['To']
message['To'] = parsed_email['From']
message['Subject'] = parsed_email['Subject']
# Attach the components
result = r.json()['queryResult']
if 'fulfillmentMessages' in result:
for component in result['fulfillmentMessages']:
if 'text' in component:
message.attach(
MIMEText(component['text']['text'][0], 'plain'))
elif 'simpleResponses' in component:
message.attach(
MIMEText(
component['simpleResponses']['simpleResponses'][0]
['textToSpeech'], 'plain'))
if 'webhookPayload' in result:
if 'google' in result['webhookPayload']:
for component in result['webhookPayload']['google'][
'richResponse']['items']:
if 'simpleResponse' in component:
message.attach(
MIMEText(
component['simpleResponse']['textToSpeech'],
'plain'))
# Send the E-Mail
session = smtplib.SMTP(host, 587)
session.ehlo()
session.starttls()
session.ehlo()
session.login(user, password)
session.sendmail(message['From'], message['To'], message.as_string())
# Log response status
app.logger.info('E-Mail response sent to ' + parsed_email_from)
elif r.status_code == 404 and catchall:
# Make new E-Mail for the response
message = MIMEMultipart()
message['Message-ID'] = make_msgid()
message['In-Reply-To'] = parsed_email['Message-ID']
message['Reply-To'] = parsed_email['From']
message['References'] = parsed_email['Message-ID']
message['From'] = 'no-reply@' + parsed_email_to_domain
message['To'] = catchall
message['Subject'] = parsed_email['Subject']
message.attach(MIMEText(parsed_email_body, 'plain'))
# Send the E-Mail
session = smtplib.SMTP(host, 587)
session.ehlo()
session.starttls()
session.ehlo()
session.login(user, password)
session.sendmail(message['From'], message['To'], message.as_string())
# Log response status
app.logger.info('E-Mail response sent to ' + parsed_email_from)
else:
# Log request error
app.logger.error('Request failed')
app.logger.error('Status: ' + str(r.status_code))
app.logger.error(str(r.json()))
return "OK", 200
def set_content_and_type(self):
self.content, self.content_type = "[Blank Email]", "text/plain"
if self.html_content:
self.content, self.content_type = self.html_content, "text/html"
else:
self.content, self.content_type = EmailReplyParser.parse_reply(self.text_content), "text/plain"
def test_parse_android_gmail_inline(self):
with open('test/emails/email_android_gmail_inline.txt') as f:
self.assertEqual("This is a test", EmailReplyParser.parse_reply(f.read()))
def get_reply_text(self):
reply_text = EmailReplyParser.parse_reply(self.plain_content)
return reply_text
def parse_text():
parser = EmailReplyParser(language='en')
with open('test/emails/caution.txt', 'r') as fl:
message = fl.read()
text = parser.parse_reply(message)
print(text)
def receive(from_addr, to_plus_cc_addrs, current_email, thread_id, fulist, bu):
try:
# the following is simply to fix the state among all free users. If a new
# free guy is added by the busy guy, please take the most recent state
# among the free guys and add him to that. <-- this adding a free guy in the
# middle has not been included in the below snippet within the if statement
if from_addr in fulist:
audience = [addr for addr in to_plus_cc_addrs if addr != bu]
if audience and audience != fulist:
adding_others_reply(fulist, current_email, thread_id)
to_plus_cc_addrs = fulist
person_list = []
for item in to_plus_cc_addrs:
person_list.append({
'email': item,
'first_name': address_dict[item][0]
})
from_entry = {
'email': from_addr,
'first_name': address_dict[from_addr][0]
}
input_obj = {
'email': {
'from': from_entry,
'to': person_list,
'body':
EmailReplyParser.parse_reply(mail.get_body(current_email)),
},
'availability': {
'dt': get_free_slots(bu), # list of tuples of dt objects
'loc': 'Location',
}
}
sara_debug('INPUTTT' + input_obj.__str__())
# dsobj = ds(thread_id) # if tid is None ds will pass a brand new object
dsobj = gds(thread_id)
output_obj = dsobj.take_turn(input_obj)
sara_debug('OUTPUTTT ' + output_obj.__str__())
if output_obj is not None:
for each_email in output_obj['emails']:
to_addrs = list(each_email['to'])
sara_debug("INPUTTTTTTTT" + ','.join(to_addrs))
to_addrs = [address_dict[item][1] for item in to_addrs]
body = each_email['body']
send(to_addrs, body, thread_id)
if output_obj['meeting'] is not None:
send_invite(SARA_F, list(output_obj['meeting']['to']),
output_obj['meeting']['loc'],
output_obj['meeting']['dt']['start'],
output_obj['meeting']['dt']['end'])
sara_debug("Finished receiving...Returning")
return 'success'
except Exception as e:
print e
print sys.exc_info()[0]
sara_debug("Failed receiving...Returning")
return 'Failure'
def ticket_from_message(message, queue, quiet):
# 'message' must be an RFC822 formatted message.
msg = message
message = email.message_from_string(msg.decode('utf-8'))
subject = message.get('subject', _('Created from e-mail'))
subject = decode_mail_headers(decodeUnknown(message.get_charset(),
subject))
subject = subject.replace("Re: ", "").replace("Fw: ", "").replace(
"RE: ", "").replace("FW: ", "").replace("Automatic reply: ",
"").strip()
sender = message.get('from', _('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
sender_email = parseaddr(sender)[1]
body_plain, body_html = '', ''
for ignore in IgnoreEmail.objects.filter(
Q(queues=queue) | Q(queues__isnull=True)):
if ignore.test(sender_email):
if ignore.keep_in_mailbox:
# By returning 'False' the message will be kept in the mailbox,
# and the 'True' will cause the message to be deleted.
return False
return True
matchobj = re.match(r'.*\[' + re.escape(queue.slug) + r'-(?P<id>\d+)\]',
subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
else:
ticket = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name is None:
if part.get_content_subtype() == 'plain':
body_plain = EmailReplyParser.parse_reply(
decodeUnknown(part.get_content_charset(),
part.get_payload(decode=True)))
else:
body_html = decodeUnknown(part.get_content_charset(),
part.get_payload(decode=True))
# make plain text more legible when viewing the ticket
body_html, n = re.subn(r'[\r\n]+', r'', body_html)
body_html, n = re.subn(r'\>\s+\<', r'><', body_html)
body_html = body_html.replace("</h1>", "</h1>\n")
body_html = body_html.replace("</h2>", "</h2>\n")
body_html = body_html.replace("</h3>", "</h3>\n")
body_html = body_html.replace("<p>", "\n<p>")
body_html = body_html.replace("</p>", "</p>\n")
body_html = body_html.replace("</div>", "</div>\n")
body_html = body_html.replace("</tr>", "</tr>\n")
body_html = body_html.replace("</td>", "</td> ")
body_html = body_html.replace("<table>", "\n<table>")
body_html = body_html.replace("</table>", "</table>\n")
body_html = body_html.replace("<br />", "<br />\n")
try:
# strip html tags
body_plain = striptags(body_html)
except DjangoUnicodeDecodeError:
charset = chardet.detect(body_html)['encoding']
body_plain = striptags(str(body_html, charset))
body_plain = unescape(body_plain)
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
files.append(
{
'filename': name,
'content': part.get_payload(decode=True),
'type': part.get_content_type()
}, )
counter += 1
if body_plain:
body = body_plain
if body_html:
body += '\n\n'
body += _(
'***Note that HTML tags are stripped out. Please see attachment email_html_body.html for the full html content.'
)
else:
body = _(
'No plain-text email body available. Please see attachment email_html_body.html.'
)
if body_html:
files.append({
'filename': _("email_html_body.html"),
'content': body_html,
'type': 'text/html',
})
now = timezone.now()
if ticket:
try:
t = Ticket.objects.get(id=ticket)
new = False
except Ticket.DoesNotExist:
ticket = None
priority = 3
smtp_priority = message.get('priority', '')
smtp_importance = message.get('importance', '')
high_priority_types = ('high', 'important', '1', 'urgent')
if smtp_priority in high_priority_types or smtp_importance in high_priority_types:
priority = 2
if ticket is None:
t = Ticket(
title=subject,
queue=queue,
submitter_email=sender_email,
created=now,
description=body,
priority=priority,
)
t.save()
new = True
#update = ''
elif t.status == Ticket.CLOSED_STATUS:
t.status = Ticket.REOPENED_STATUS
t.save()
f = FollowUp(
ticket=t,
title=_('E-Mail Received from %(sender_email)s' %
{'sender_email': sender_email}),
date=timezone.now(),
public=True,
comment=body,
)
if t.status == Ticket.REOPENED_STATUS:
f.new_status = Ticket.REOPENED_STATUS
f.title = _(
'Ticket Re-Opened by E-Mail Received from %(sender_email)s' %
{'sender_email': sender_email})
f.save()
if not quiet:
print((" [%s-%s] %s" % (
t.queue.slug,
t.id,
t.title,
)).encode('ascii', 'replace'))
for file in files:
if file['content']:
filename = file['filename'].replace(' ', '_')
filename = re.sub(r'[^a-zA-Z0-9._-]+', '', filename)
a = Attachment(
followup=f,
filename=filename,
mime_type=file['type'],
size=len(file['content']),
)
a.file.save(filename, ContentFile(file['content']), save=False)
a.save()
if not quiet:
print(" - %s" % filename)
context = safe_template_context(t)
if new:
if sender_email and not is_no_reply_address(sender_email):
send_templated_mail(
'newticket_submitter',
context,
recipients=sender_email,
sender=queue.from_address,
fail_silently=True,
)
if queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.new_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
else:
context.update(comment=f.comment)
#if t.status == Ticket.REOPENED_STATUS:
# update = _(' (Reopened)')
#else:
# update = _(' (Updated)')
if t.assigned_to:
send_templated_mail(
'updated_owner',
context,
recipients=t.assigned_to.email,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc:
send_templated_mail(
'updated_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
return t
def parse_reply_from_email(message):
# not working right now
return EmailReplyParser.parse_reply(message)
def set_content_and_type(self): self.content, self.content_type = '[Blank Email]', 'text/plain' if self.html_content: self.content, self.content_type = self.html_content, 'text/html' else: self.content, self.content_type = EmailReplyParser.parse_reply(self.text_content), 'text/plain'
def object_from_message(message, queue, logger):
# 'message' must be an RFC822 formatted message.
message = email.message_from_string(message)
subject = message.get('subject', _('Comment from e-mail'))
subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject))
for affix in STRIPPED_SUBJECT_STRINGS:
subject = subject.replace(affix, "")
subject = subject.strip()
sender = message.get('from', _('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
# to address bug #832, we wrap all the text in front of the email address in
# double quotes by using replace() on the email string. Then,
# take first item of list, second item of tuple is the actual email address.
# Note that the replace won't work on just an email with no real name,
# but the getaddresses() function seems to be able to handle just unclosed quotes
# correctly. Not ideal, but this seems to work for now.
sender_email = email.utils.getaddresses(['\"' + sender.replace('<', '\" <')])[0][1]
cc = message.get_all('cc', None)
if cc:
# first, fixup the encoding if necessary
cc = [decode_mail_headers(decodeUnknown(message.get_charset(), x)) for x in cc]
# get_all checks if multiple CC headers, but individual emails may be comma separated too
tempcc = []
for hdr in cc:
tempcc.extend(hdr.split(','))
# use a set to ensure no duplicates
cc = set([x.strip() for x in tempcc])
for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)):
if ignore.test(sender_email):
if ignore.keep_in_mailbox:
# By returning 'False' the message will be kept in the mailbox,
# and the 'True' will cause the message to be deleted.
return False
return True
matchobj = re.match(r".*\[" + queue.slug + r"-(?P<id>\d+)\]", subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket))
else:
logger.info("No tracking ID matched.")
ticket = None
body = None
full_body = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = email.utils.collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name is None:
if part.get_content_subtype() == 'plain':
body = part.get_payload(decode=True)
# https://github.com/django-helpdesk/django-helpdesk/issues/732
if part['Content-Transfer-Encoding'] == '8bit' and part.get_content_charset() == 'utf-8':
body = body.decode('unicode_escape')
body = decodeUnknown(part.get_content_charset(), body)
# have to use django_settings here so overwritting it works in tests
# the default value is False anyway
if ticket is None and getattr(django_settings, 'HELPDESK_FULL_FIRST_MESSAGE_FROM_EMAIL', False):
# first message in thread, we save full body to avoid losing forwards and things like that
body_parts = []
for f in EmailReplyParser.read(body).fragments:
body_parts.append(f.content)
full_body = '\n\n'.join(body_parts)
body = EmailReplyParser.parse_reply(body)
else:
# second and other reply, save only first part of the message
body = EmailReplyParser.parse_reply(body)
full_body = body
# workaround to get unicode text out rather than escaped text
try:
body = body.encode('ascii').decode('unicode_escape')
except UnicodeEncodeError:
body.encode('utf-8')
logger.debug("Discovered plain text MIME part")
else:
try:
email_body = encoding.smart_text(part.get_payload(decode=True))
except UnicodeDecodeError:
email_body = encoding.smart_text(part.get_payload(decode=False))
if not body and not full_body:
# no text has been parsed so far - try such deep parsing for some messages
altered_body = email_body.replace("</p>", "</p>\n").replace("<br", "\n<br")
mail = BeautifulSoup(str(altered_body), "html.parser")
full_body = mail.get_text()
if "<body" not in email_body:
email_body = f"<body>{email_body}</body>"
payload = (
'<html>'
'<head>'
'<meta charset="utf-8" />'
'</head>'
'%s'
'</html>'
) % email_body
files.append(
SimpleUploadedFile(_("email_html_body.html"), payload.encode("utf-8"), 'text/html')
)
logger.debug("Discovered HTML MIME part")
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
else:
name = ("part-%i_" % counter) + name
# # FIXME: this code gets the paylods, then does something with it and then completely ignores it
# # writing the part.get_payload(decode=True) instead; and then the payload variable is
# # replaced by some dict later.
# # the `payloadToWrite` has been also ignored so was commented
# payload = part.get_payload()
# if isinstance(payload, list):
# payload = payload.pop().as_string()
# # payloadToWrite = payload
# # check version of python to ensure use of only the correct error type
# non_b64_err = TypeError
# try:
# logger.debug("Try to base64 decode the attachment payload")
# # payloadToWrite = base64.decodebytes(payload)
# except non_b64_err:
# logger.debug("Payload was not base64 encoded, using raw bytes")
# # payloadToWrite = payload
files.append(SimpleUploadedFile(name, part.get_payload(decode=True), mimetypes.guess_type(name)[0]))
logger.debug("Found MIME attachment %s" % name)
counter += 1
if not body:
mail = BeautifulSoup(str(message), "html.parser")
beautiful_body = mail.find('body')
if beautiful_body:
try:
body = beautiful_body.text
full_body = body
except AttributeError:
pass
if not body:
body = ""
if getattr(django_settings, 'HELPDESK_ALWAYS_SAVE_INCOMING_EMAIL_MESSAGE', False):
# save message as attachment in case of some complex markup renders wrong
files.append(
SimpleUploadedFile(
_("original_message.eml").replace(
".eml",
timezone.localtime().strftime("_%d-%m-%Y_%H:%M") + ".eml"
),
str(message).encode("utf-8"),
'text/plain'
)
)
smtp_priority = message.get('priority', '')
smtp_importance = message.get('importance', '')
high_priority_types = {'high', 'important', '1', 'urgent'}
priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3
payload = {
'body': body,
'full_body': full_body or body,
'subject': subject,
'queue': queue,
'sender_email': sender_email,
'priority': priority,
'files': files,
}
return create_object_from_email_message(message, ticket, payload, files, logger=logger)
import sys from email_reply_parser import EmailReplyParser print EmailReplyParser.parse_reply(sys.stdin.read())
def ticket_from_message(message, quiet):
"""
Create a ticket or a followup (if ticket id in subject)
"""
msg = message
message = email.message_from_string(msg)
subject = message.get("subject", "Created from e-mail")
subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject))
sender = message.get("from", ("Unknown Sender"))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
sender_email = parseaddr(sender)[1]
body_plain, body_html = "", ""
matchobj = re.match(r".*\[" + "-(?P<id>\d+)\]", subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group("id")
else:
ticket = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == "multipart":
continue
name = part.get_param("name")
if name:
name = collapse_rfc2231_value(name)
if part.get_content_maintype() == "text" and name == None:
if part.get_content_subtype() == "plain":
body_plain = EmailReplyParser.parse_reply(
decodeUnknown(part.get_content_charset(), part.get_payload(decode=True))
)
else:
body_html = part.get_payload(decode=True)
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
files.append({"filename": name, "content": part.get_payload(decode=True), "type": part.get_content_type()})
counter += 1
if body_plain:
body = body_plain
else:
body = "No plain-text email body available. Please see attachment email_html_body.html."
if body_html:
files.append({"filename": "email_html_body.html", "content": body_html, "type": "text/html"})
now = timezone.now()
if ticket:
try:
t = Ticket.objects.get(id=ticket)
new = False
except Ticket.DoesNotExist:
ticket = None
if ticket == None:
# set owner depending on sender_email
# list of all email addresses from the user model
users = User.objects.all()
email_addresses = []
for user in users:
email_addresses.append(user.email)
############################################################
# if ticket id in subject => new followup instead of new ticket
tickets = Ticket.objects.all()
ticket_ids = []
for ticket in tickets:
ticket_ids.append(ticket.id)
# extract id from subject
subject_id = re.search(r"\[#(\d*)\]\s.*", subject)
try:
subject_id = subject_id.group(1)
except:
subject_id = "0000" # no valid id
# if there was an ID in the subject, create followup
if int(subject_id) in ticket_ids:
if sender_email in email_addresses:
f = FollowUp(
title=subject,
created=now,
text=body,
ticket=Ticket.objects.get(id=subject_id),
user=User.objects.get(email=sender_email),
)
else:
f = FollowUp(title=subject, created=now, text=body, ticket=Ticket.objects.get(id=subject_id))
f.save()
# if no ID in the subject, create ticket
else:
# if known sender, set also the field owner
if sender_email in email_addresses:
t = Ticket(
title=subject,
status="TODO",
created=now,
description=body,
owner=User.objects.get(email=sender_email),
)
# if unknown sender, skip the field owner
else:
t = Ticket(title=subject, status="TODO", created=now, description=body)
t.save()
from django.core.mail import send_mail
notification_subject = "[#" + str(t.id) + "] New ticket created"
notification_body = "Hi,\n\na new ticket was created: http://localhost:8000/ticket/" + str(t.id) + "/"
send_mail(
notification_subject,
notification_body,
os.environ["DJANGO_TICKET_EMAIL_NOTIFICATIONS_FROM"],
[os.environ["DJANGO_TICKET_EMAIL_NOTIFICATIONS_TO"]],
fail_silently=False,
)
############################################################
new = True
update = ""
elif t.status == Ticket.CLOSED_STATUS:
t.status = Ticket.REOPENED_STATUS
t.save()
# files of followups should be assigned to the corresponding ticket
for file in files:
if file["content"]:
filename = file["filename"].encode("ascii", "replace").replace(" ", "_")
filename = re.sub("[^a-zA-Z0-9._-]+", "", filename)
# if followup
if int(subject_id) in ticket_ids:
a = Attachment(
ticket=Ticket.objects.get(id=subject_id),
filename=filename,
# mime_type=file['type'],
# size=len(file['content']),
)
# if new ticket
else:
a = Attachment(
ticket=t,
filename=filename,
# mime_type=file['type'],
# size=len(file['content']),
)
a.file.save(filename, ContentFile(file["content"]), save=False)
a.save()
if not quiet:
print " - %s" % filename
if int(subject_id) in ticket_ids:
return f
else:
return t
def ticket_from_message(message, queue, logger):
# 'message' must be an RFC822 formatted message.
message = email.message_from_string(
message) if six.PY3 else email.message_from_string(
message.encode('utf-8'))
subject = message.get('subject', _('Created from e-mail'))
subject = decode_mail_headers(decodeUnknown(message.get_charset(),
subject))
for affix in STRIPPED_SUBJECT_STRINGS:
subject = subject.replace(affix, "")
subject = subject.strip()
sender = message.get('from', _('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
sender_email = email.utils.parseaddr(sender)[1]
for ignore in IgnoreEmail.objects.filter(
Q(queues=queue) | Q(queues__isnull=True)):
if ignore.test(sender_email):
if ignore.keep_in_mailbox:
# By returning 'False' the message will be kept in the mailbox,
# and the 'True' will cause the message to be deleted.
return False
return True
matchobj = re.match(r".*\[" + queue.slug + "-(?P<id>\d+)\]", subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket))
else:
logger.info("No tracking ID matched.")
ticket = None
body = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = email.utils.collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name is None:
if part.get_content_subtype() == 'plain':
body = EmailReplyParser.parse_reply(
decodeUnknown(part.get_content_charset(),
part.get_payload(decode=True)))
# workaround to get unicode text out rather than escaped text
body = body.encode('ascii').decode(
'unicode_escape') if six.PY3 else body.encode('utf-8')
logger.debug("Discovered plain text MIME part")
else:
files.append(
SimpleUploadedFile(
_("email_html_body.html"),
encoding.smart_bytes(part.get_payload()), 'text/html'))
logger.debug("Discovered HTML MIME part")
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
files.append(
SimpleUploadedFile(name,
encoding.smart_bytes(part.get_payload()),
part.get_content_type()))
logger.debug("Found MIME attachment %s" % name)
counter += 1
if not body:
body = _(
'No plain-text email body available. Please see attachment "email_html_body.html".'
)
if ticket:
try:
t = Ticket.objects.get(id=ticket)
except Ticket.DoesNotExist:
logger.info(
"Tracking ID %s-%s not associated with existing ticket. Creating new ticket."
% (queue.slug, ticket))
ticket = None
else:
logger.info("Found existing ticket with Tracking ID %s-%s" %
(t.queue.slug, t.id))
if t.status == Ticket.CLOSED_STATUS:
t.status = Ticket.REOPENED_STATUS
t.save()
new = False
smtp_priority = message.get('priority', '')
smtp_importance = message.get('importance', '')
high_priority_types = {'high', 'important', '1', 'urgent'}
priority = 2 if high_priority_types & {smtp_priority, smtp_importance
} else 3
if ticket is None:
new = True
t = Ticket.objects.create(
title=subject,
queue=queue,
submitter_email=sender_email,
created=timezone.now(),
description=body,
priority=priority,
)
logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id))
f = FollowUp(
ticket=t,
title=_('E-Mail Received from %(sender_email)s' %
{'sender_email': sender_email}),
date=timezone.now(),
public=True,
comment=body,
)
if t.status == Ticket.REOPENED_STATUS:
f.new_status = Ticket.REOPENED_STATUS
f.title = _(
'Ticket Re-Opened by E-Mail Received from %(sender_email)s' %
{'sender_email': sender_email})
f.save()
logger.debug("Created new FollowUp for Ticket")
if six.PY2:
logger.info(("[%s-%s] %s" % (
t.queue.slug,
t.id,
t.title,
)).encode('ascii', 'replace'))
elif six.PY3:
logger.info("[%s-%s] %s" % (
t.queue.slug,
t.id,
t.title,
))
attached = process_attachments(f, files)
for att_file in attached:
logger.info(
"Attachment '%s' successfully added to ticket from email." %
att_file[0])
context = safe_template_context(t)
if new:
if sender_email:
send_templated_mail(
'newticket_submitter',
context,
recipients=sender_email,
sender=queue.from_address,
fail_silently=True,
)
if queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.new_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
else:
context.update(comment=f.comment)
if t.assigned_to:
send_templated_mail(
'updated_owner',
context,
recipients=t.assigned_to.email,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc:
send_templated_mail(
'updated_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
return t
shortcode = ShortCode.objects.get(alias=email_alias)
if isinstance(shortcode, Introduction):
intro = shortcode
else:
raise Introduction.DoesNotExist("%s does not appear to represent an existing introduction. Message: %s" % (delivered_to, message.pk))
# If the sender is the user that owns the intro, add it as a comment
person = None
try:
person = PersonEmail.objects.get(email_address=get_from_addr(msg))
except Exception, e:
pass
if person.person == intro.person:
new_comment = IntroductionComment(
person = person.person,
introduction = intro,
comment = EmailReplyParser.parse_reply(message.text)
)
new_comment.save()
return intro
def process_message_as_new_intro(message):
msg = message.get_email_object()
from_addr = get_from_addr(msg)
delivered_to = get_delivered_to_addr(msg)
to_addrs = ','.join(get_to_addrs(msg))
cc_addrs = ','.join(get_to_addrs(msg))
try:
person = PersonEmail.objects.get(email_address=from_addr).person
except PersonEmail.DoesNotExist:
raise Introduction.DoesNotExist("from address: %s doesn't represent a person in our system. Message: %s" % (from_addr, message.pk))
if not person:
def test_parse_out_just_top_for_outlook_reply(self):
with open('test/emails/email_2_1.txt') as f:
self.assertEqual("Outlook with a reply", EmailReplyParser.parse_reply(f.read()))
def test_parse_out_just_top_for_outlook_with_reply_directly_above_line(
self):
with open('test/emails/email_2_2.txt') as f:
self.assertEqual("Outlook with a reply directly above line",
EmailReplyParser.parse_reply(f.read()))
def test_sent_from_iphone(self):
with open('test/emails/email_iPhone.txt') as email:
self.assertTrue("Sent from my iPhone" not in EmailReplyParser.parse_reply(email.read()))
def test_reply_from_gmail(self):
with open("test/emails/email_gmail.txt") as f:
self.assertEqual(
"This is a test for inbox replying to a github message.",
EmailReplyParser.parse_reply(f.read()),
)
def ticket_from_message(message, quiet):
"""
Create a ticket or a followup (if ticket id in subject)
"""
msg = message
message = email.message_from_string(msg)
subject = message.get('subject', 'Created from e-mail')
subject = decode_mail_headers(decodeUnknown(message.get_charset(),
subject))
sender = message.get('from', ('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
sender_email = parseaddr(sender)[1]
body_plain, body_html = '', ''
matchobj = re.match(r".*\[" + "-(?P<id>\d+)\]", subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
else:
ticket = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name == None:
if part.get_content_subtype() == 'plain':
body_plain = EmailReplyParser.parse_reply(
decodeUnknown(part.get_content_charset(),
part.get_payload(decode=True)))
else:
body_html = part.get_payload(decode=True)
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
files.append(
{
'filename': name,
'content': part.get_payload(decode=True),
'type': part.get_content_type()
}, )
counter += 1
if body_plain:
body = body_plain
else:
body = 'No plain-text email body available. Please see attachment email_html_body.html.'
if body_html:
files.append({
'filename': 'email_html_body.html',
'content': body_html,
'type': 'text/html',
})
now = timezone.now()
if ticket:
try:
t = Ticket.objects.get(id=ticket)
new = False
except Ticket.DoesNotExist:
ticket = None
if ticket == None:
# set owner depending on sender_email
# list of all email addresses from the user model
users = User.objects.all()
email_addresses = []
for user in users:
email_addresses.append(user.email)
############################################################
# if ticket id in subject => new followup instead of new ticket
tickets = Ticket.objects.all()
ticket_ids = []
for ticket in tickets:
ticket_ids.append(ticket.id)
# extract id from subject
subject_id = re.search(r'\[#(\d*)\]\s.*', subject)
try:
subject_id = subject_id.group(1)
except:
subject_id = "0000" # no valid id
# if there was an ID in the subject, create followup
if int(subject_id) in ticket_ids:
if sender_email in email_addresses:
f = FollowUp(
title=subject,
created=now,
text=body,
ticket=Ticket.objects.get(id=subject_id),
user=User.objects.get(email=sender_email),
)
else:
f = FollowUp(
title=subject,
created=now,
text=body,
ticket=Ticket.objects.get(id=subject_id),
)
f.save()
# if no ID in the subject, create ticket
else:
# if known sender, set also the field owner
if sender_email in email_addresses:
t = Ticket(
title=subject,
status="TODO",
created=now,
description=body,
owner=User.objects.get(email=sender_email),
)
# if unknown sender, skip the field owner
else:
t = Ticket(
title=subject,
status="TODO",
created=now,
description=body,
)
t.save()
from django.core.mail import send_mail
notification_subject = "[#" + str(t.id) + "] New ticket created"
notification_body = "Hi,\n\na new ticket was created: http://localhost:8000/ticket/" \
+ str(t.id) + "/"
send_mail(notification_subject,
notification_body,
os.environ["DJANGO_TICKET_EMAIL_NOTIFICATIONS_FROM"],
[os.environ["DJANGO_TICKET_EMAIL_NOTIFICATIONS_TO"]],
fail_silently=False)
############################################################
new = True
update = ''
elif t.status == Ticket.CLOSED_STATUS:
t.status = Ticket.REOPENED_STATUS
t.save()
# files of followups should be assigned to the corresponding ticket
for file in files:
if file['content']:
filename = file['filename'].encode('ascii',
'replace').replace(' ', '_')
filename = re.sub('[^a-zA-Z0-9._-]+', '', filename)
# if followup
if int(subject_id) in ticket_ids:
a = Attachment(
ticket=Ticket.objects.get(id=subject_id),
filename=filename,
#mime_type=file['type'],
#size=len(file['content']),
)
# if new ticket
else:
a = Attachment(
ticket=t,
filename=filename,
#mime_type=file['type'],
#size=len(file['content']),
)
a.file.save(filename, ContentFile(file['content']), save=False)
a.save()
if not quiet:
print " - %s" % filename
if int(subject_id) in ticket_ids:
return f
else:
return t
def test_parse_out_just_top_for_outlook_reply(self):
with open("test/emails/email_2_1.txt") as f:
self.assertEqual("Outlook with a reply",
EmailReplyParser.parse_reply(f.read()))
def get_message_details(self):
'''Return args for template'''
dws_group = frappe.get_doc('Daily Work Summary Group',
self.daily_work_summary_group)
replies = frappe.get_all('Communication',
fields=['content', 'text_content', 'sender'],
filters=dict(reference_doctype=self.doctype,
reference_name=self.name,
communication_type='Communication',
sent_or_received='Received'),
order_by='creation asc')
did_not_reply = self.email_sent_to.split()
for d in replies:
user = frappe.db.get_values("User",
{"email": d.sender},
["full_name", "user_image"],
as_dict=True)
d.sender_name = user[0].full_name if user else d.sender
d.image = user[0].image if user and user[0].image else None
original_image = d.image
# make thumbnail image
try:
if original_image:
file_name = frappe.get_list('File',
{'file_url': original_image})
if file_name:
file_name = file_name[0].name
file_doc = frappe.get_doc('File', file_name)
thumbnail_image = file_doc.make_thumbnail(
set_as_thumbnail=False,
width=100,
height=100,
crop=True
)
d.image = thumbnail_image
except Exception:
d.image = original_image
if d.sender in did_not_reply:
did_not_reply.remove(d.sender)
if d.text_content:
d.content = frappe.utils.md_to_html(
EmailReplyParser.parse_reply(d.text_content)
)
did_not_reply = [(frappe.db.get_value("User", {"email": email}, "full_name") or email)
for email in did_not_reply]
return dict(replies=replies,
original_message=dws_group.message,
title=_('Work Summary for {0}').format(
global_date_format(self.creation)
),
did_not_reply=', '.join(did_not_reply) or '',
did_not_reply_title=_('No replies from'))
def test_parse_out_just_top_for_outlook_with_unusual_headers_format(self):
with open("test/emails/email_2_3.txt") as f:
self.assertEqual(
"Outlook with a reply above headers using unusual format",
EmailReplyParser.parse_reply(f.read()),
)
# Parse E-Mail
parsed_email = message_from_string(data[0][1].decode('utf-8'))
parsed_email_from = parseaddr(parsed_email['From'])[1]
parsed_email_to = parseaddr(parsed_email['To'])[1]
parsed_email_to_domain = parsed_email_to.split('@')[1]
parsed_email_session = md5(
bytes(parsed_email.get('Subject', '').replace('Re: ', '') +
parsed_email_from,
encoding='utf8')).hexdigest()
parsed_email_body = ''
for part in parsed_email.walk():
if part.get_content_type() == 'text/plain':
parsed_email_body += part.get_payload()
parsed_email_body = EmailReplyParser.parse_reply(parsed_email_body)
parsed_email_lang = fallback_lang
try:
parsed_email_lang = detect(parsed_email_body)
except:
pass
# Log E-Mail
logging.info('Received new E-Mail')
logging.info('From: ' + parsed_email_from)
logging.info('To: ' + parsed_email_to)
logging.info('Text: ' + parsed_email_body)
logging.info('Message ID: ' + parsed_email['Message-ID'])
logging.info('Session ID: ' + parsed_email_session)
# Build Request
def test_sent_from_iphone(self):
with open("test/emails/email_iPhone.txt") as email:
self.assertTrue("Sent from my iPhone" not in
EmailReplyParser.parse_reply(email.read()))
def ticket_from_message(message, queue, quiet):
# 'message' must be an RFC822 formatted message.
msg = message
message = email.message_from_string(msg.decode('utf-8'))
subject = message.get('subject', _('Created from e-mail'))
subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject))
subject = subject.replace("Re: ", "").replace("Fw: ", "").replace("RE: ", "").replace("FW: ", "").replace("Automatic reply: ", "").strip()
sender = message.get('from', _('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
sender_email = parseaddr(sender)[1]
body_plain, body_html = '', ''
for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)):
if ignore.test(sender_email):
if ignore.keep_in_mailbox:
# By returning 'False' the message will be kept in the mailbox,
# and the 'True' will cause the message to be deleted.
return False
return True
matchobj = re.match(r'.*\['+re.escape(queue.slug)+r'-(?P<id>\d+)\]', subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
else:
ticket = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name is None:
if part.get_content_subtype() == 'plain':
body_plain = EmailReplyParser.parse_reply(decodeUnknown(part.get_content_charset(), part.get_payload(decode=True)))
else:
body_html = decodeUnknown(part.get_content_charset(), part.get_payload(decode=True))
# make plain text more legible when viewing the ticket
body_html, n = re.subn(r'[\r\n]+', r'', body_html)
body_html, n = re.subn(r'\>\s+\<', r'><', body_html)
body_html = body_html.replace("</h1>", "</h1>\n")
body_html = body_html.replace("</h2>", "</h2>\n")
body_html = body_html.replace("</h3>", "</h3>\n")
body_html = body_html.replace("<p>", "\n<p>")
body_html = body_html.replace("</p>", "</p>\n")
body_html = body_html.replace("</div>", "</div>\n")
body_html = body_html.replace("</tr>", "</tr>\n")
body_html = body_html.replace("</td>", "</td> ")
body_html = body_html.replace("<table>", "\n<table>")
body_html = body_html.replace("</table>", "</table>\n")
body_html = body_html.replace("<br />", "<br />\n")
try:
# strip html tags
body_plain = striptags(body_html)
except DjangoUnicodeDecodeError:
charset = chardet.detect(body_html)['encoding']
body_plain = striptags(str(body_html, charset))
body_plain = unescape(body_plain)
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
files.append({
'filename': name,
'content': part.get_payload(decode=True),
'type': part.get_content_type()},
)
counter += 1
if body_plain:
body = body_plain
if body_html:
body += '\n\n'
body += _('***Note that HTML tags are stripped out. Please see attachment email_html_body.html for the full html content.')
else:
body = _('No plain-text email body available. Please see attachment email_html_body.html.')
if body_html:
files.append({
'filename': _("email_html_body.html"),
'content': body_html,
'type': 'text/html',
})
now = timezone.now()
if ticket:
try:
t = Ticket.objects.get(id=ticket)
new = False
except Ticket.DoesNotExist:
ticket = None
priority = 3
smtp_priority = message.get('priority', '')
smtp_importance = message.get('importance', '')
high_priority_types = ('high', 'important', '1', 'urgent')
if smtp_priority in high_priority_types or smtp_importance in high_priority_types:
priority = 2
if ticket is None:
t = Ticket(
title=subject,
queue=queue,
submitter_email=sender_email,
created=now,
description=body,
priority=priority,
)
t.save()
new = True
#update = ''
elif t.status == Ticket.CLOSED_STATUS:
t.status = Ticket.REOPENED_STATUS
t.save()
f = FollowUp(
ticket = t,
title = _('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}),
date = timezone.now(),
public = True,
comment = body,
)
if t.status == Ticket.REOPENED_STATUS:
f.new_status = Ticket.REOPENED_STATUS
f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email})
f.save()
if not quiet:
print((" [%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace'))
for file in files:
if file['content']:
filename = file['filename'].replace(' ', '_')
filename = re.sub(r'[^a-zA-Z0-9._-]+', '', filename)
a = Attachment(
followup=f,
filename=filename,
mime_type=file['type'],
size=len(file['content']),
)
a.file.save(filename, ContentFile(file['content']), save=False)
a.save()
if not quiet:
print(" - %s" % filename)
context = safe_template_context(t)
if new:
if sender_email and not is_no_reply_address(sender_email):
send_templated_mail(
'newticket_submitter',
context,
recipients=sender_email,
sender=queue.from_address,
fail_silently=True,
)
if queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.new_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
else:
context.update(comment=f.comment)
#if t.status == Ticket.REOPENED_STATUS:
# update = _(' (Reopened)')
#else:
# update = _(' (Updated)')
if t.assigned_to:
send_templated_mail(
'updated_owner',
context,
recipients=t.assigned_to.email,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc:
send_templated_mail(
'updated_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
return t
def test_email_one_is_not_on(self):
with open("test/emails/email_one_is_not_on.txt") as email:
self.assertTrue("On Oct 1, 2012, at 11:55 PM, Dave Tapley wrote:"
not in EmailReplyParser.parse_reply(email.read()))
def ticket_from_message(message, queue, quiet):
# 'message' must be an RFC822 formatted message.
msg = message
message = email.message_from_string(msg)
subject = message.get('subject', _('Created from e-mail'))
subject = decode_mail_headers(decodeUnknown(message.get_charset(),
subject))
subject = subject.replace("Re: ", "").replace("Fw: ", "").replace(
"RE: ", "").replace("FW: ", "").replace("Automatic reply: ",
"").strip()
sender = message.get('from', _('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
sender_email = parseaddr(sender)[1]
body_plain, body_html = '', ''
for ignore in IgnoreEmail.objects.filter(
Q(queues=queue) | Q(queues__isnull=True)):
if ignore.test(sender_email):
if ignore.keep_in_mailbox:
# By returning 'False' the message will be kept in the mailbox,
# and the 'True' will cause the message to be deleted.
return False
return True
matchobj = re.match(r".*\[" + queue.slug + "-(?P<id>\d+)\]", subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
else:
ticket = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name == None:
if part.get_content_subtype() == 'plain':
body_plain = EmailReplyParser.parse_reply(
decodeUnknown(part.get_content_charset(),
part.get_payload(decode=True)))
else:
body_html = part.get_payload(decode=True)
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
files.append(
{
'filename': name,
'content': part.get_payload(decode=True),
'type': part.get_content_type()
}, )
counter += 1
if body_plain:
body = body_plain
else:
body = _(
'No plain-text email body available. Please see attachment email_html_body.html.'
)
if body_html:
files.append({
'filename': _("email_html_body.html"),
'content': body_html,
'type': 'text/html',
})
now = timezone.now()
if ticket:
try:
t = Ticket.objects.get(id=ticket)
new = False
except Ticket.DoesNotExist:
ticket = None
priority = 3
smtp_priority = message.get('priority', '')
smtp_importance = message.get('importance', '')
high_priority_types = ('high', 'important', '1', 'urgent')
if smtp_priority in high_priority_types or smtp_importance in high_priority_types:
priority = 2
if ticket == None:
t = Ticket(
title=subject,
queue=queue,
submitter_email=sender_email,
created=now,
description=body,
priority=priority,
)
t.save()
new = True
update = ''
elif t.status == Ticket.CLOSED_STATUS:
t.status = Ticket.REOPENED_STATUS
t.save()
f = FollowUp(
ticket=t,
title=_('E-Mail Received from %(sender_email)s' %
{'sender_email': sender_email}),
date=timezone.now(),
public=True,
comment=body,
)
if t.status == Ticket.REOPENED_STATUS:
f.new_status = Ticket.REOPENED_STATUS
f.title = _(
'Ticket Re-Opened by E-Mail Received from %(sender_email)s' %
{'sender_email': sender_email})
f.save()
if not quiet:
print(" [%s-%s] %s" % (
t.queue.slug,
t.id,
t.title,
)).encode('ascii', 'replace')
print files
for file in files:
print file
if file['content']:
filename = file['filename'].encode('ascii',
'replace').replace(' ', '_')
print filename
filename = re.sub('[^a-zA-Z0-9._-]+', '', filename)
print filename
a = Attachment(
followup=f,
filename=filename,
mime_type=file['type'],
size=len(file['content']),
)
a.file.save(filename, ContentFile(file['content']), save=False)
a.save()
if not quiet:
print " - %s" % filename
context = safe_template_context(t)
if new:
if sender_email:
send_templated_mail(
'newticket_submitter',
context,
recipients=sender_email,
sender=queue.from_address,
fail_silently=True,
)
if queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.new_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
else:
context.update(comment=f.comment)
if t.status == Ticket.REOPENED_STATUS:
update = _(' (Reopened)')
else:
update = _(' (Updated)')
if t.assigned_to:
send_templated_mail(
'updated_owner',
context,
recipients=t.assigned_to.email,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc:
send_templated_mail(
'updated_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
return t
def get_reply_contents(email):
return EmailReplyParser.parse_reply(email.text)
