def test_unicode_filename_to_filesystem(self, mock_att_save, mock_queue_save, mock_ticket_save, mock_follow_up_save): """ don't mock saving to filesystem to test file renames caused by storage layer """ filename, fileobj = lib.process_attachments(self.follow_up, [self.test_file])[0] # Attachment object was zeroth positional arg (i.e. self) of att.save call attachment_obj = mock_att_save.call_args[0][0] mock_att_save.assert_called_once_with(attachment_obj) self.assertIsInstance(attachment_obj, models.FollowUpAttachment) self.assertEqual(attachment_obj.filename, self.file_attrs['filename'])
def test_unicode_filename_to_filesystem(self, mock_att_save, mock_queue_save, mock_ticket_save, mock_follow_up_save): """ don't mock saving to filesystem to test file renames caused by storage layer """ filename, fileobj = lib.process_attachments(self.follow_up, [self.test_file])[0] # Attachment object was zeroth positional arg (i.e. self) of att.save call attachment_obj = mock_att_save.call_args[0][0] mock_att_save.assert_called_once_with(attachment_obj) self.assertIsInstance(attachment_obj, models.Attachment) self.assertEqual(attachment_obj.filename, self.file_attrs['filename'])
def test_unicode_attachment_filename(self, mock_att_save, mock_queue_save, mock_ticket_save, mock_follow_up_save): """ check utf-8 data is parsed correctly """ filename, fileobj = lib.process_attachments(self.follow_up, [self.test_file])[0] mock_att_save.assert_called_with( file=self.test_file, filename=self.file_attrs['filename'], mime_type=self.file_attrs['content-type'], size=len(self.file_attrs['content']), followup=self.follow_up ) self.assertEqual(filename, self.file_attrs['filename'])
def test_unicode_attachment_filename(self, mock_att_save, mock_queue_save, mock_ticket_save, mock_follow_up_save): """ check utf-8 data is parsed correcltly """ filename, fileobj = lib.process_attachments(self.follow_up, [self.test_file])[0] mock_att_save.assert_called_with( file=self.test_file, filename=self.file_attrs['filename'], mime_type=self.file_attrs['content-type'], size=len(self.file_attrs['content']), followup=self.follow_up ) self.assertEqual(filename, self.file_attrs['filename'])
def _attach_files_to_follow_up(self, followup): files = self.cleaned_data['attachment'] if files: files = process_attachments(followup, [files]) return files
def ticket_from_message(message, queue, logger): # 'message' must be an RFC822 formatted message. message = email.message_from_string(message) if six.PY3 else email.message_from_string(message.encode('utf-8')) subject = message.get('subject', _('Comment from e-mail')) subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject)) for affix in STRIPPED_SUBJECT_STRINGS: subject = subject.replace(affix, "") subject = subject.strip() sender = message.get('from', _('Unknown Sender')) sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender)) sender_email = email.utils.parseaddr(sender)[1] cc = message.get_all('cc', None) if cc: # first, fixup the encoding if necessary cc = [decode_mail_headers(decodeUnknown(message.get_charset(), x)) for x in cc] # get_all checks if multiple CC headers, but individual emails may be comma separated too tempcc = [] for hdr in cc: tempcc.extend(hdr.split(',')) # use a set to ensure no duplicates cc = set([x.strip() for x in tempcc]) for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)): if ignore.test(sender_email): if ignore.keep_in_mailbox: # By returning 'False' the message will be kept in the mailbox, # and the 'True' will cause the message to be deleted. return False return True matchobj = re.match(r".*\[" + queue.slug + r"-(?P<id>\d+)\]", subject) if matchobj: # This is a reply or forward. ticket = matchobj.group('id') logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket)) else: logger.info("No tracking ID matched.") ticket = None body = None counter = 0 files = [] for part in message.walk(): if part.get_content_maintype() == 'multipart': continue name = part.get_param("name") if name: name = email.utils.collapse_rfc2231_value(name) if part.get_content_maintype() == 'text' and name is None: if part.get_content_subtype() == 'plain': body = EmailReplyParser.parse_reply( decodeUnknown(part.get_content_charset(), part.get_payload(decode=True)) ) # workaround to get unicode text out rather than escaped text try: body = body.encode('ascii').decode('unicode_escape') except UnicodeEncodeError: body.encode('utf-8') logger.debug("Discovered plain text MIME part") else: files.append( SimpleUploadedFile(_("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html') ) logger.debug("Discovered HTML MIME part") else: if not name: ext = mimetypes.guess_extension(part.get_content_type()) name = "part-%i%s" % (counter, ext) payload = part.get_payload() if isinstance(payload, list): payload = payload.pop().as_string() payloadToWrite = payload # check version of python to ensure use of only the correct error type if six.PY2: non_b64_err = binascii.Error else: non_b64_err = TypeError try: logger.debug("Try to base64 decode the attachment payload") if six.PY2: payloadToWrite = base64.decodestring(payload) else: payloadToWrite = base64.decodebytes(payload) except non_b64_err: logger.debug("Payload was not base64 encoded, using raw bytes") payloadToWrite = payload files.append(SimpleUploadedFile(name, part.get_payload(decode=True), mimetypes.guess_type(name)[0])) logger.debug("Found MIME attachment %s" % name) counter += 1 if not body: mail = BeautifulSoup(part.get_payload(), "lxml") if ">" in mail.text: body = mail.find('body') body = body.text body = body.encode('ascii', errors='ignore') else: body = mail.text if ticket: try: t = Ticket.objects.get(id=ticket) except Ticket.DoesNotExist: logger.info("Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket)) ticket = None else: logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id)) if t.status == Ticket.CLOSED_STATUS: t.status = Ticket.REOPENED_STATUS t.save() new = False smtp_priority = message.get('priority', '') smtp_importance = message.get('importance', '') high_priority_types = {'high', 'important', '1', 'urgent'} priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3 if ticket is None: if settings.QUEUE_EMAIL_BOX_UPDATE_ONLY: return None new = True t = Ticket.objects.create( title=subject, queue=queue, submitter_email=sender_email, created=timezone.now(), description=body, priority=priority, ) logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id)) if cc: # get list of currently CC'd emails current_cc = TicketCC.objects.filter(ticket=ticket) current_cc_emails = [x.email for x in current_cc if x.email] # get emails of any Users CC'd to email, if defined # (some Users may not have an associated email, e.g, when using LDAP) current_cc_users = [x.user.email for x in current_cc if x.user and x.user.email] # ensure submitter, assigned user, queue email not added other_emails = [queue.email_address] if t.submitter_email: other_emails.append(t.submitter_email) if t.assigned_to: other_emails.append(t.assigned_to.email) current_cc = set(current_cc_emails + current_cc_users + other_emails) # first, add any User not previously CC'd (as identified by User's email) all_users = User.objects.all() all_user_emails = set([x.email for x in all_users]) users_not_currently_ccd = all_user_emails.difference(set(current_cc)) users_to_cc = cc.intersection(users_not_currently_ccd) for user in users_to_cc: tcc = TicketCC.objects.create( ticket=t, user=User.objects.get(email=user), can_view=True, can_update=False ) tcc.save() # then add remaining emails alphabetically, makes testing easy new_cc = cc.difference(current_cc).difference(all_user_emails) new_cc = sorted(list(new_cc)) for ccemail in new_cc: tcc = TicketCC.objects.create( ticket=t, email=ccemail.replace('\n', ' ').replace('\r', ' '), can_view=True, can_update=False ) tcc.save() f = FollowUp( ticket=t, title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}), date=timezone.now(), public=True, comment=body, ) if t.status == Ticket.REOPENED_STATUS: f.new_status = Ticket.REOPENED_STATUS f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}) f.save() logger.debug("Created new FollowUp for Ticket") if six.PY2: logger.info(("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace')) elif six.PY3: logger.info("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)) attached = process_attachments(f, files) for att_file in attached: logger.info("Attachment '%s' (with size %s) successfully added to ticket from email." % (att_file[0], att_file[1].size)) context = safe_template_context(t) if new: if sender_email: send_templated_mail( 'newticket_submitter', context, recipients=sender_email, sender=queue.from_address, fail_silently=True, ) if queue.new_ticket_cc: send_templated_mail( 'newticket_cc', context, recipients=queue.new_ticket_cc, sender=queue.from_address, fail_silently=True, ) if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc: send_templated_mail( 'newticket_cc', context, recipients=queue.updated_ticket_cc, sender=queue.from_address, fail_silently=True, ) else: context.update(comment=f.comment) if t.assigned_to: send_templated_mail( 'updated_owner', context, recipients=t.assigned_to.email, sender=queue.from_address, fail_silently=True, ) if queue.updated_ticket_cc: send_templated_mail( 'updated_cc', context, recipients=queue.updated_ticket_cc, sender=queue.from_address, fail_silently=True, ) return t
def create_object_from_email_message(message, ticket_id, payload, files, logger): ticket, previous_followup, new = None, None, False now = timezone.now() queue = payload['queue'] sender_email = payload['sender_email'] to_list = getaddresses(message.get_all('To', [])) cc_list = getaddresses(message.get_all('Cc', [])) message_id = message.get('Message-Id') in_reply_to = message.get('In-Reply-To') if in_reply_to is not None: try: queryset = FollowUp.objects.filter( message_id=in_reply_to).order_by('-date') if queryset.count() > 0: previous_followup = queryset.first() ticket = previous_followup.ticket except FollowUp.DoesNotExist: pass # play along. The header may be wrong if previous_followup is None and ticket_id is not None: try: ticket = Ticket.objects.get(id=ticket_id) except Ticket.DoesNotExist: ticket = None else: new = False # Check if the ticket has been merged to another ticket if ticket.merged_to: logger.info("Ticket has been merged to %s" % ticket.merged_to.ticket) # Use the ticket in which it was merged to for next operations ticket = ticket.merged_to # New issue, create a new <Ticket> instance if ticket is None: if not settings.QUEUE_EMAIL_BOX_UPDATE_ONLY: ticket = Ticket.objects.create( title=payload['subject'], queue=queue, submitter_email=sender_email, created=now, description=payload['body'], priority=payload['priority'], ) ticket.save() logger.debug("Created new ticket %s-%s" % (ticket.queue.slug, ticket.id)) new = True update = '' # Old issue being re-opened elif ticket.status == Ticket.CLOSED_STATUS: ticket.status = Ticket.REOPENED_STATUS ticket.save() f = FollowUp(ticket=ticket, title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}), date=now, public=True, comment=payload['body'], message_id=message_id) if ticket.status == Ticket.REOPENED_STATUS: f.new_status = Ticket.REOPENED_STATUS f.title = _( 'Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}) f.save() logger.debug("Created new FollowUp for Ticket") logger.info("[%s-%s] %s" % ( ticket.queue.slug, ticket.id, ticket.title, )) attached = process_attachments(f, files) for att_file in attached: logger.info( "Attachment '%s' (with size %s) successfully added to ticket from email." % (att_file[0], att_file[1].size)) context = safe_template_context(ticket) new_ticket_ccs = [] new_ticket_ccs.append(create_ticket_cc(ticket, to_list + cc_list)) notifications_to_be_sent = [sender_email] if queue.enable_notifications_on_email_events and len( notifications_to_be_sent): ticket_cc_list = TicketCC.objects.filter( ticket=ticket).all().values_list('email', flat=True) for email in ticket_cc_list: notifications_to_be_sent.append(email) # send mail to appropriate people now depending on what objects # were created and who was CC'd if new: ticket.send( { 'submitter': ('newticket_submitter', context), 'new_ticket_cc': ('newticket_cc', context), 'ticket_cc': ('newticket_cc', context) }, fail_silently=True, extra_headers={'In-Reply-To': message_id}, ) else: context.update(comment=f.comment) ticket.send( { 'submitter': ('newticket_submitter', context), 'assigned_to': ('updated_owner', context) }, fail_silently=True, extra_headers={'In-Reply-To': message_id}, ) if queue.enable_notifications_on_email_events: ticket.send( {'ticket_cc': ('updated_cc', context)}, fail_silently=True, extra_headers={'In-Reply-To': message_id}, ) return ticket
def ticket_from_message(message, queue, logger): # 'message' must be an RFC822 formatted message. message = email.message_from_string(message) subject = message.get('subject', _('Created from e-mail')) subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject)) for affix in STRIPPED_SUBJECT_STRINGS: subject = subject.replace(affix, "") subject = subject.strip() sender = message.get('from', _('Unknown Sender')) sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender)) sender_email = email.utils.parseaddr(sender)[1] for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)): if ignore.test(sender_email): if ignore.keep_in_mailbox: # By returning 'False' the message will be kept in the mailbox, # and the 'True' will cause the message to be deleted. return False return True matchobj = re.match(r".*\[" + queue.slug + "-(?P<id>\d+)\]", subject) if matchobj: # This is a reply or forward. ticket = matchobj.group('id') logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket)) else: logger.info("No tracking ID matched.") ticket = None body = None counter = 0 files = [] for part in message.walk(): if part.get_content_maintype() == 'multipart': continue name = part.get_param("name") if name: name = email.utils.collapse_rfc2231_value(name) if part.get_content_maintype() == 'text' and name is None: if part.get_content_subtype() == 'plain': body = EmailReplyParser.parse_reply( decodeUnknown(part.get_content_charset(), part.get_payload(decode=True)) ) logger.debug("Discovered plain text MIME part") else: files.append( SimpleUploadedFile(_("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html') ) logger.debug("Discovered HTML MIME part") else: if not name: ext = mimetypes.guess_extension(part.get_content_type()) name = "part-%i%s" % (counter, ext) files.append(SimpleUploadedFile(name, encoding.smart_bytes(part.get_payload()), part.get_content_type())) logger.debug("Found MIME attachment %s" % name) counter += 1 if not body: body = _('No plain-text email body available. Please see attachment "email_html_body.html".') if ticket: try: t = Ticket.objects.get(id=ticket) except Ticket.DoesNotExist: logger.info("Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket)) ticket = None else: logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id)) if t.status == Ticket.CLOSED_STATUS: t.status = Ticket.REOPENED_STATUS t.save() new = False smtp_priority = message.get('priority', '') smtp_importance = message.get('importance', '') high_priority_types = {'high', 'important', '1', 'urgent'} priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3 if ticket is None: new = True t = Ticket.objects.create( title=subject, queue=queue, submitter_email=sender_email, created=timezone.now(), description=body, priority=priority, ) logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id)) f = FollowUp( ticket=t, title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}), date=timezone.now(), public=True, comment=body, ) if t.status == Ticket.REOPENED_STATUS: f.new_status = Ticket.REOPENED_STATUS f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}) f.save() logger.debug("Created new FollowUp for Ticket") if six.PY2: logger.info(("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace')) elif six.PY3: logger.info("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)) attached = process_attachments(f, files) for att_file in attached: logger.info("Attachment '%s' successfully added to ticket from email." % att_file[0]) context = safe_template_context(t) if new: if sender_email: send_templated_mail( 'newticket_submitter', context, recipients=sender_email, sender=queue.from_address, fail_silently=True, ) if queue.new_ticket_cc: send_templated_mail( 'newticket_cc', context, recipients=queue.new_ticket_cc, sender=queue.from_address, fail_silently=True, ) if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc: send_templated_mail( 'newticket_cc', context, recipients=queue.updated_ticket_cc, sender=queue.from_address, fail_silently=True, ) else: context.update(comment=f.comment) if t.assigned_to: send_templated_mail( 'updated_owner', context, recipients=t.assigned_to.email, sender=queue.from_address, fail_silently=True, ) if queue.updated_ticket_cc: send_templated_mail( 'updated_cc', context, recipients=queue.updated_ticket_cc, sender=queue.from_address, fail_silently=True, ) return t
def ticket_from_message(message, queue, logger): # 'message' must be an RFC822 formatted message. message = email.message_from_string( message) if six.PY3 else email.message_from_string( message.encode('utf-8')) subject = message.get('subject', _('Created from e-mail')) subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject)) for affix in STRIPPED_SUBJECT_STRINGS: subject = subject.replace(affix, "") subject = subject.strip() sender = message.get('from', _('Unknown Sender')) sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender)) sender_email = email.utils.parseaddr(sender)[1] for ignore in IgnoreEmail.objects.filter( Q(queues=queue) | Q(queues__isnull=True)): if ignore.test(sender_email): if ignore.keep_in_mailbox: # By returning 'False' the message will be kept in the mailbox, # and the 'True' will cause the message to be deleted. return False return True matchobj = re.match(r".*\[" + queue.slug + "-(?P<id>\d+)\]", subject) if matchobj: # This is a reply or forward. ticket = matchobj.group('id') logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket)) else: logger.info("No tracking ID matched.") ticket = None body = None counter = 0 files = [] for part in message.walk(): if part.get_content_maintype() == 'multipart': continue name = part.get_param("name") if name: name = email.utils.collapse_rfc2231_value(name) if part.get_content_maintype() == 'text' and name is None: if part.get_content_subtype() == 'plain': body = EmailReplyParser.parse_reply( decodeUnknown(part.get_content_charset(), part.get_payload(decode=True))) # workaround to get unicode text out rather than escaped text body = body.encode('ascii').decode( 'unicode_escape') if six.PY3 else body.encode('utf-8') logger.debug("Discovered plain text MIME part") else: files.append( SimpleUploadedFile( _("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html')) logger.debug("Discovered HTML MIME part") else: if not name: ext = mimetypes.guess_extension(part.get_content_type()) name = "part-%i%s" % (counter, ext) files.append( SimpleUploadedFile(name, encoding.smart_bytes(part.get_payload()), part.get_content_type())) logger.debug("Found MIME attachment %s" % name) counter += 1 if not body: body = _( 'No plain-text email body available. Please see attachment "email_html_body.html".' ) if ticket: try: t = Ticket.objects.get(id=ticket) except Ticket.DoesNotExist: logger.info( "Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket)) ticket = None else: logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id)) if t.status == Ticket.CLOSED_STATUS: t.status = Ticket.REOPENED_STATUS t.save() new = False smtp_priority = message.get('priority', '') smtp_importance = message.get('importance', '') high_priority_types = {'high', 'important', '1', 'urgent'} priority = 2 if high_priority_types & {smtp_priority, smtp_importance } else 3 if ticket is None: new = True t = Ticket.objects.create( title=subject, queue=queue, submitter_email=sender_email, created=timezone.now(), description=body, priority=priority, ) logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id)) f = FollowUp( ticket=t, title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}), date=timezone.now(), public=True, comment=body, ) if t.status == Ticket.REOPENED_STATUS: f.new_status = Ticket.REOPENED_STATUS f.title = _( 'Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}) f.save() logger.debug("Created new FollowUp for Ticket") if six.PY2: logger.info(("[%s-%s] %s" % ( t.queue.slug, t.id, t.title, )).encode('ascii', 'replace')) elif six.PY3: logger.info("[%s-%s] %s" % ( t.queue.slug, t.id, t.title, )) attached = process_attachments(f, files) for att_file in attached: logger.info( "Attachment '%s' successfully added to ticket from email." % att_file[0]) context = safe_template_context(t) if new: if sender_email: send_templated_mail( 'newticket_submitter', context, recipients=sender_email, sender=queue.from_address, fail_silently=True, ) if queue.new_ticket_cc: send_templated_mail( 'newticket_cc', context, recipients=queue.new_ticket_cc, sender=queue.from_address, fail_silently=True, ) if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc: send_templated_mail( 'newticket_cc', context, recipients=queue.updated_ticket_cc, sender=queue.from_address, fail_silently=True, ) else: context.update(comment=f.comment) if t.assigned_to: send_templated_mail( 'updated_owner', context, recipients=t.assigned_to.email, sender=queue.from_address, fail_silently=True, ) if queue.updated_ticket_cc: send_templated_mail( 'updated_cc', context, recipients=queue.updated_ticket_cc, sender=queue.from_address, fail_silently=True, ) return t