def test_unicode_filename_to_filesystem(self, mock_att_save, mock_queue_save, mock_ticket_save, mock_follow_up_save):
""" don't mock saving to filesystem to test file renames caused by storage layer """
filename, fileobj = lib.process_attachments(self.follow_up, [self.test_file])[0]
# Attachment object was zeroth positional arg (i.e. self) of att.save call
attachment_obj = mock_att_save.call_args[0][0]
mock_att_save.assert_called_once_with(attachment_obj)
self.assertIsInstance(attachment_obj, models.FollowUpAttachment)
self.assertEqual(attachment_obj.filename, self.file_attrs['filename'])
def test_unicode_filename_to_filesystem(self, mock_att_save, mock_queue_save, mock_ticket_save, mock_follow_up_save):
""" don't mock saving to filesystem to test file renames caused by storage layer """
filename, fileobj = lib.process_attachments(self.follow_up, [self.test_file])[0]
# Attachment object was zeroth positional arg (i.e. self) of att.save call
attachment_obj = mock_att_save.call_args[0][0]
mock_att_save.assert_called_once_with(attachment_obj)
self.assertIsInstance(attachment_obj, models.Attachment)
self.assertEqual(attachment_obj.filename, self.file_attrs['filename'])
def test_unicode_attachment_filename(self, mock_att_save, mock_queue_save, mock_ticket_save, mock_follow_up_save):
""" check utf-8 data is parsed correctly """
filename, fileobj = lib.process_attachments(self.follow_up, [self.test_file])[0]
mock_att_save.assert_called_with(
file=self.test_file,
filename=self.file_attrs['filename'],
mime_type=self.file_attrs['content-type'],
size=len(self.file_attrs['content']),
followup=self.follow_up
)
self.assertEqual(filename, self.file_attrs['filename'])
def test_unicode_attachment_filename(self, mock_att_save, mock_queue_save, mock_ticket_save, mock_follow_up_save):
""" check utf-8 data is parsed correcltly """
filename, fileobj = lib.process_attachments(self.follow_up, [self.test_file])[0]
mock_att_save.assert_called_with(
file=self.test_file,
filename=self.file_attrs['filename'],
mime_type=self.file_attrs['content-type'],
size=len(self.file_attrs['content']),
followup=self.follow_up
)
self.assertEqual(filename, self.file_attrs['filename'])
def _attach_files_to_follow_up(self, followup):
files = self.cleaned_data['attachment']
if files:
files = process_attachments(followup, [files])
return files
def ticket_from_message(message, queue, logger):
# 'message' must be an RFC822 formatted message.
message = email.message_from_string(message) if six.PY3 else email.message_from_string(message.encode('utf-8'))
subject = message.get('subject', _('Comment from e-mail'))
subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject))
for affix in STRIPPED_SUBJECT_STRINGS:
subject = subject.replace(affix, "")
subject = subject.strip()
sender = message.get('from', _('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
sender_email = email.utils.parseaddr(sender)[1]
cc = message.get_all('cc', None)
if cc:
# first, fixup the encoding if necessary
cc = [decode_mail_headers(decodeUnknown(message.get_charset(), x)) for x in cc]
# get_all checks if multiple CC headers, but individual emails may be comma separated too
tempcc = []
for hdr in cc:
tempcc.extend(hdr.split(','))
# use a set to ensure no duplicates
cc = set([x.strip() for x in tempcc])
for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)):
if ignore.test(sender_email):
if ignore.keep_in_mailbox:
# By returning 'False' the message will be kept in the mailbox,
# and the 'True' will cause the message to be deleted.
return False
return True
matchobj = re.match(r".*\[" + queue.slug + r"-(?P<id>\d+)\]", subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket))
else:
logger.info("No tracking ID matched.")
ticket = None
body = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = email.utils.collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name is None:
if part.get_content_subtype() == 'plain':
body = EmailReplyParser.parse_reply(
decodeUnknown(part.get_content_charset(), part.get_payload(decode=True))
)
# workaround to get unicode text out rather than escaped text
try:
body = body.encode('ascii').decode('unicode_escape')
except UnicodeEncodeError:
body.encode('utf-8')
logger.debug("Discovered plain text MIME part")
else:
files.append(
SimpleUploadedFile(_("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html')
)
logger.debug("Discovered HTML MIME part")
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
payload = part.get_payload()
if isinstance(payload, list):
payload = payload.pop().as_string()
payloadToWrite = payload
# check version of python to ensure use of only the correct error type
if six.PY2:
non_b64_err = binascii.Error
else:
non_b64_err = TypeError
try:
logger.debug("Try to base64 decode the attachment payload")
if six.PY2:
payloadToWrite = base64.decodestring(payload)
else:
payloadToWrite = base64.decodebytes(payload)
except non_b64_err:
logger.debug("Payload was not base64 encoded, using raw bytes")
payloadToWrite = payload
files.append(SimpleUploadedFile(name, part.get_payload(decode=True), mimetypes.guess_type(name)[0]))
logger.debug("Found MIME attachment %s" % name)
counter += 1
if not body:
mail = BeautifulSoup(part.get_payload(), "lxml")
if ">" in mail.text:
body = mail.find('body')
body = body.text
body = body.encode('ascii', errors='ignore')
else:
body = mail.text
if ticket:
try:
t = Ticket.objects.get(id=ticket)
except Ticket.DoesNotExist:
logger.info("Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket))
ticket = None
else:
logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id))
if t.status == Ticket.CLOSED_STATUS:
t.status = Ticket.REOPENED_STATUS
t.save()
new = False
smtp_priority = message.get('priority', '')
smtp_importance = message.get('importance', '')
high_priority_types = {'high', 'important', '1', 'urgent'}
priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3
if ticket is None:
if settings.QUEUE_EMAIL_BOX_UPDATE_ONLY:
return None
new = True
t = Ticket.objects.create(
title=subject,
queue=queue,
submitter_email=sender_email,
created=timezone.now(),
description=body,
priority=priority,
)
logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id))
if cc:
# get list of currently CC'd emails
current_cc = TicketCC.objects.filter(ticket=ticket)
current_cc_emails = [x.email for x in current_cc if x.email]
# get emails of any Users CC'd to email, if defined
# (some Users may not have an associated email, e.g, when using LDAP)
current_cc_users = [x.user.email for x in current_cc if x.user and x.user.email]
# ensure submitter, assigned user, queue email not added
other_emails = [queue.email_address]
if t.submitter_email:
other_emails.append(t.submitter_email)
if t.assigned_to:
other_emails.append(t.assigned_to.email)
current_cc = set(current_cc_emails + current_cc_users + other_emails)
# first, add any User not previously CC'd (as identified by User's email)
all_users = User.objects.all()
all_user_emails = set([x.email for x in all_users])
users_not_currently_ccd = all_user_emails.difference(set(current_cc))
users_to_cc = cc.intersection(users_not_currently_ccd)
for user in users_to_cc:
tcc = TicketCC.objects.create(
ticket=t,
user=User.objects.get(email=user),
can_view=True,
can_update=False
)
tcc.save()
# then add remaining emails alphabetically, makes testing easy
new_cc = cc.difference(current_cc).difference(all_user_emails)
new_cc = sorted(list(new_cc))
for ccemail in new_cc:
tcc = TicketCC.objects.create(
ticket=t,
email=ccemail.replace('\n', ' ').replace('\r', ' '),
can_view=True,
can_update=False
)
tcc.save()
f = FollowUp(
ticket=t,
title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}),
date=timezone.now(),
public=True,
comment=body,
)
if t.status == Ticket.REOPENED_STATUS:
f.new_status = Ticket.REOPENED_STATUS
f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email})
f.save()
logger.debug("Created new FollowUp for Ticket")
if six.PY2:
logger.info(("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace'))
elif six.PY3:
logger.info("[%s-%s] %s" % (t.queue.slug, t.id, t.title,))
attached = process_attachments(f, files)
for att_file in attached:
logger.info("Attachment '%s' (with size %s) successfully added to ticket from email." % (att_file[0], att_file[1].size))
context = safe_template_context(t)
if new:
if sender_email:
send_templated_mail(
'newticket_submitter',
context,
recipients=sender_email,
sender=queue.from_address,
fail_silently=True,
)
if queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.new_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
else:
context.update(comment=f.comment)
if t.assigned_to:
send_templated_mail(
'updated_owner',
context,
recipients=t.assigned_to.email,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc:
send_templated_mail(
'updated_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
return t
def create_object_from_email_message(message, ticket_id, payload, files,
logger):
ticket, previous_followup, new = None, None, False
now = timezone.now()
queue = payload['queue']
sender_email = payload['sender_email']
to_list = getaddresses(message.get_all('To', []))
cc_list = getaddresses(message.get_all('Cc', []))
message_id = message.get('Message-Id')
in_reply_to = message.get('In-Reply-To')
if in_reply_to is not None:
try:
queryset = FollowUp.objects.filter(
message_id=in_reply_to).order_by('-date')
if queryset.count() > 0:
previous_followup = queryset.first()
ticket = previous_followup.ticket
except FollowUp.DoesNotExist:
pass # play along. The header may be wrong
if previous_followup is None and ticket_id is not None:
try:
ticket = Ticket.objects.get(id=ticket_id)
except Ticket.DoesNotExist:
ticket = None
else:
new = False
# Check if the ticket has been merged to another ticket
if ticket.merged_to:
logger.info("Ticket has been merged to %s" %
ticket.merged_to.ticket)
# Use the ticket in which it was merged to for next operations
ticket = ticket.merged_to
# New issue, create a new <Ticket> instance
if ticket is None:
if not settings.QUEUE_EMAIL_BOX_UPDATE_ONLY:
ticket = Ticket.objects.create(
title=payload['subject'],
queue=queue,
submitter_email=sender_email,
created=now,
description=payload['body'],
priority=payload['priority'],
)
ticket.save()
logger.debug("Created new ticket %s-%s" %
(ticket.queue.slug, ticket.id))
new = True
update = ''
# Old issue being re-opened
elif ticket.status == Ticket.CLOSED_STATUS:
ticket.status = Ticket.REOPENED_STATUS
ticket.save()
f = FollowUp(ticket=ticket,
title=_('E-Mail Received from %(sender_email)s' %
{'sender_email': sender_email}),
date=now,
public=True,
comment=payload['body'],
message_id=message_id)
if ticket.status == Ticket.REOPENED_STATUS:
f.new_status = Ticket.REOPENED_STATUS
f.title = _(
'Ticket Re-Opened by E-Mail Received from %(sender_email)s' %
{'sender_email': sender_email})
f.save()
logger.debug("Created new FollowUp for Ticket")
logger.info("[%s-%s] %s" % (
ticket.queue.slug,
ticket.id,
ticket.title,
))
attached = process_attachments(f, files)
for att_file in attached:
logger.info(
"Attachment '%s' (with size %s) successfully added to ticket from email."
% (att_file[0], att_file[1].size))
context = safe_template_context(ticket)
new_ticket_ccs = []
new_ticket_ccs.append(create_ticket_cc(ticket, to_list + cc_list))
notifications_to_be_sent = [sender_email]
if queue.enable_notifications_on_email_events and len(
notifications_to_be_sent):
ticket_cc_list = TicketCC.objects.filter(
ticket=ticket).all().values_list('email', flat=True)
for email in ticket_cc_list:
notifications_to_be_sent.append(email)
# send mail to appropriate people now depending on what objects
# were created and who was CC'd
if new:
ticket.send(
{
'submitter': ('newticket_submitter', context),
'new_ticket_cc': ('newticket_cc', context),
'ticket_cc': ('newticket_cc', context)
},
fail_silently=True,
extra_headers={'In-Reply-To': message_id},
)
else:
context.update(comment=f.comment)
ticket.send(
{
'submitter': ('newticket_submitter', context),
'assigned_to': ('updated_owner', context)
},
fail_silently=True,
extra_headers={'In-Reply-To': message_id},
)
if queue.enable_notifications_on_email_events:
ticket.send(
{'ticket_cc': ('updated_cc', context)},
fail_silently=True,
extra_headers={'In-Reply-To': message_id},
)
return ticket
def ticket_from_message(message, queue, logger):
# 'message' must be an RFC822 formatted message.
message = email.message_from_string(message)
subject = message.get('subject', _('Created from e-mail'))
subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject))
for affix in STRIPPED_SUBJECT_STRINGS:
subject = subject.replace(affix, "")
subject = subject.strip()
sender = message.get('from', _('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
sender_email = email.utils.parseaddr(sender)[1]
for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)):
if ignore.test(sender_email):
if ignore.keep_in_mailbox:
# By returning 'False' the message will be kept in the mailbox,
# and the 'True' will cause the message to be deleted.
return False
return True
matchobj = re.match(r".*\[" + queue.slug + "-(?P<id>\d+)\]", subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket))
else:
logger.info("No tracking ID matched.")
ticket = None
body = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = email.utils.collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name is None:
if part.get_content_subtype() == 'plain':
body = EmailReplyParser.parse_reply(
decodeUnknown(part.get_content_charset(), part.get_payload(decode=True))
)
logger.debug("Discovered plain text MIME part")
else:
files.append(
SimpleUploadedFile(_("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html')
)
logger.debug("Discovered HTML MIME part")
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
files.append(SimpleUploadedFile(name, encoding.smart_bytes(part.get_payload()), part.get_content_type()))
logger.debug("Found MIME attachment %s" % name)
counter += 1
if not body:
body = _('No plain-text email body available. Please see attachment "email_html_body.html".')
if ticket:
try:
t = Ticket.objects.get(id=ticket)
except Ticket.DoesNotExist:
logger.info("Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket))
ticket = None
else:
logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id))
if t.status == Ticket.CLOSED_STATUS:
t.status = Ticket.REOPENED_STATUS
t.save()
new = False
smtp_priority = message.get('priority', '')
smtp_importance = message.get('importance', '')
high_priority_types = {'high', 'important', '1', 'urgent'}
priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3
if ticket is None:
new = True
t = Ticket.objects.create(
title=subject,
queue=queue,
submitter_email=sender_email,
created=timezone.now(),
description=body,
priority=priority,
)
logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id))
f = FollowUp(
ticket=t,
title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}),
date=timezone.now(),
public=True,
comment=body,
)
if t.status == Ticket.REOPENED_STATUS:
f.new_status = Ticket.REOPENED_STATUS
f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email})
f.save()
logger.debug("Created new FollowUp for Ticket")
if six.PY2:
logger.info(("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace'))
elif six.PY3:
logger.info("[%s-%s] %s" % (t.queue.slug, t.id, t.title,))
attached = process_attachments(f, files)
for att_file in attached:
logger.info("Attachment '%s' successfully added to ticket from email." % att_file[0])
context = safe_template_context(t)
if new:
if sender_email:
send_templated_mail(
'newticket_submitter',
context,
recipients=sender_email,
sender=queue.from_address,
fail_silently=True,
)
if queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.new_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
else:
context.update(comment=f.comment)
if t.assigned_to:
send_templated_mail(
'updated_owner',
context,
recipients=t.assigned_to.email,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc:
send_templated_mail(
'updated_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
return t
def ticket_from_message(message, queue, logger):
# 'message' must be an RFC822 formatted message.
message = email.message_from_string(
message) if six.PY3 else email.message_from_string(
message.encode('utf-8'))
subject = message.get('subject', _('Created from e-mail'))
subject = decode_mail_headers(decodeUnknown(message.get_charset(),
subject))
for affix in STRIPPED_SUBJECT_STRINGS:
subject = subject.replace(affix, "")
subject = subject.strip()
sender = message.get('from', _('Unknown Sender'))
sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
sender_email = email.utils.parseaddr(sender)[1]
for ignore in IgnoreEmail.objects.filter(
Q(queues=queue) | Q(queues__isnull=True)):
if ignore.test(sender_email):
if ignore.keep_in_mailbox:
# By returning 'False' the message will be kept in the mailbox,
# and the 'True' will cause the message to be deleted.
return False
return True
matchobj = re.match(r".*\[" + queue.slug + "-(?P<id>\d+)\]", subject)
if matchobj:
# This is a reply or forward.
ticket = matchobj.group('id')
logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket))
else:
logger.info("No tracking ID matched.")
ticket = None
body = None
counter = 0
files = []
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = email.utils.collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name is None:
if part.get_content_subtype() == 'plain':
body = EmailReplyParser.parse_reply(
decodeUnknown(part.get_content_charset(),
part.get_payload(decode=True)))
# workaround to get unicode text out rather than escaped text
body = body.encode('ascii').decode(
'unicode_escape') if six.PY3 else body.encode('utf-8')
logger.debug("Discovered plain text MIME part")
else:
files.append(
SimpleUploadedFile(
_("email_html_body.html"),
encoding.smart_bytes(part.get_payload()), 'text/html'))
logger.debug("Discovered HTML MIME part")
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
files.append(
SimpleUploadedFile(name,
encoding.smart_bytes(part.get_payload()),
part.get_content_type()))
logger.debug("Found MIME attachment %s" % name)
counter += 1
if not body:
body = _(
'No plain-text email body available. Please see attachment "email_html_body.html".'
)
if ticket:
try:
t = Ticket.objects.get(id=ticket)
except Ticket.DoesNotExist:
logger.info(
"Tracking ID %s-%s not associated with existing ticket. Creating new ticket."
% (queue.slug, ticket))
ticket = None
else:
logger.info("Found existing ticket with Tracking ID %s-%s" %
(t.queue.slug, t.id))
if t.status == Ticket.CLOSED_STATUS:
t.status = Ticket.REOPENED_STATUS
t.save()
new = False
smtp_priority = message.get('priority', '')
smtp_importance = message.get('importance', '')
high_priority_types = {'high', 'important', '1', 'urgent'}
priority = 2 if high_priority_types & {smtp_priority, smtp_importance
} else 3
if ticket is None:
new = True
t = Ticket.objects.create(
title=subject,
queue=queue,
submitter_email=sender_email,
created=timezone.now(),
description=body,
priority=priority,
)
logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id))
f = FollowUp(
ticket=t,
title=_('E-Mail Received from %(sender_email)s' %
{'sender_email': sender_email}),
date=timezone.now(),
public=True,
comment=body,
)
if t.status == Ticket.REOPENED_STATUS:
f.new_status = Ticket.REOPENED_STATUS
f.title = _(
'Ticket Re-Opened by E-Mail Received from %(sender_email)s' %
{'sender_email': sender_email})
f.save()
logger.debug("Created new FollowUp for Ticket")
if six.PY2:
logger.info(("[%s-%s] %s" % (
t.queue.slug,
t.id,
t.title,
)).encode('ascii', 'replace'))
elif six.PY3:
logger.info("[%s-%s] %s" % (
t.queue.slug,
t.id,
t.title,
))
attached = process_attachments(f, files)
for att_file in attached:
logger.info(
"Attachment '%s' successfully added to ticket from email." %
att_file[0])
context = safe_template_context(t)
if new:
if sender_email:
send_templated_mail(
'newticket_submitter',
context,
recipients=sender_email,
sender=queue.from_address,
fail_silently=True,
)
if queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.new_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc:
send_templated_mail(
'newticket_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
else:
context.update(comment=f.comment)
if t.assigned_to:
send_templated_mail(
'updated_owner',
context,
recipients=t.assigned_to.email,
sender=queue.from_address,
fail_silently=True,
)
if queue.updated_ticket_cc:
send_templated_mail(
'updated_cc',
context,
recipients=queue.updated_ticket_cc,
sender=queue.from_address,
fail_silently=True,
)
return t
def _attach_files_to_follow_up(self, followup):
files = self.cleaned_data['attachment']
if files:
files = process_attachments(followup, [files])
return files