def test_angle_brackets_dont_contribute_to_hash(self): # According to RFC 5322, the [matching] angle brackets do not # contribute to the hash. msg = mfs("""\ Message-ID: aardvark """) add_message_hash(msg) self.assertEqual(msg['message-id-hash'], '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT')
def test_angle_brackets_dont_contribute_to_hash(self): # According to RFC 5322, the [matching] angle brackets do not # contribute to the hash. msg = mfs("""\ Message-ID: aardvark """) add_message_hash(msg) self.assertEqual(msg['x-message-id-hash'], '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT')
def test_adding_the_message_hash(self): # When the message has a Message-ID header, this will add the # X-Mailman-Hash-ID header. msg = mfs("""\ Message-ID: <aardvark> """) add_message_hash(msg) self.assertEqual(msg['x-message-id-hash'], '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT')
def test_adding_the_message_hash(self): # When the message has a Message-ID header, this will add the # X-Mailman-Hash-ID header. msg = mfs("""\ Message-ID: <aardvark> """) add_message_hash(msg) self.assertEqual(msg['message-id-hash'], '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT')
def test_hash_header_left_alone_if_no_message_id(self): # If the original message has no Message-ID header, then any existing # X-Message-ID-Hash headers are left intact. msg = mfs("""\ X-Message-ID-Hash: abc """) add_message_hash(msg) headers = msg.get_all('x-message-id-hash') self.assertEqual(len(headers), 1) self.assertEqual(headers[0], 'abc')
def test_hash_header_left_alone_if_no_message_id(self): # If the original message has no Message-ID header, then any existing # Message-ID-Hash headers are left intact. msg = mfs("""\ Message-ID-Hash: abc """) add_message_hash(msg) headers = msg.get_all('message-id-hash') self.assertEqual(len(headers), 1) self.assertEqual(headers[0], 'abc')
def test_remove_hash_headers_first(self): # Any existing X-Mailman-Hash-ID header is removed first. msg = mfs("""\ Message-ID: <aardvark> X-Message-ID-Hash: abc """) add_message_hash(msg) headers = msg.get_all('x-message-id-hash') self.assertEqual(len(headers), 1) self.assertEqual(headers[0], '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT')
def test_remove_hash_headers_first(self): # Any existing X-Mailman-Hash-ID header is removed first. msg = mfs("""\ Message-ID: <aardvark> Message-ID-Hash: abc """) add_message_hash(msg) headers = msg.get_all('message-id-hash') self.assertEqual(len(headers), 1) self.assertEqual(headers[0], '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT')
def test_return_value(self): msg = mfs("""\ Message-ID: aardvark> """) hash32 = add_message_hash(msg) self.assertEqual(hash32, '5KH3RA7ZM4VM6XOZXA7AST2XN2X4S3WY')
def test_mismatched_angle_brackets_do_contribute_to_hash(self): # According to RFC 5322, the [matching] angle brackets do not # contribute to the hash. msg = mfs("""\ Message-ID: <aardvark """) add_message_hash(msg) self.assertEqual(msg['x-message-id-hash'], 'AOJ545GHRYD2Y3RUFG2EWMPHUABTG4SM') msg = mfs("""\ Message-ID: aardvark> """) add_message_hash(msg) self.assertEqual(msg['x-message-id-hash'], '5KH3RA7ZM4VM6XOZXA7AST2XN2X4S3WY')
def test_mismatched_angle_brackets_do_contribute_to_hash(self): # According to RFC 5322, the [matching] angle brackets do not # contribute to the hash. msg = mfs("""\ Message-ID: <aardvark """) add_message_hash(msg) self.assertEqual(msg['message-id-hash'], 'AOJ545GHRYD2Y3RUFG2EWMPHUABTG4SM') msg = mfs("""\ Message-ID: aardvark> """) add_message_hash(msg) self.assertEqual(msg['message-id-hash'], '5KH3RA7ZM4VM6XOZXA7AST2XN2X4S3WY')
def test_get_message_by_hash(self): # Messages have an X-Message-ID-Hash header, the value of which can be # used to look the message up in the message store. msg = mfs("""\ Subject: An important message Message-ID: <ant> This message is very important. """) add_message_hash(msg) self._store.add(msg) self.assertEqual(msg['x-message-id-hash'], 'MS6QLWERIJLGCRF44J7USBFDELMNT2BW') found = self._store.get_message_by_hash( 'MS6QLWERIJLGCRF44J7USBFDELMNT2BW') self.assertEqual(found['message-id'], '<ant>') self.assertEqual(found['x-message-id-hash'], 'MS6QLWERIJLGCRF44J7USBFDELMNT2BW')
def test_get_message_by_hash(self): # Messages have an X-Message-ID-Hash header, the value of which can be # used to look the message up in the message store. message = mfs("""\ Subject: An important message Message-ID: <ant> This message is very important. """) add_message_hash(message) self._store.add(message) self.assertEqual(message['x-message-id-hash'], 'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG') found = self._store.get_message_by_hash( 'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG') self.assertEqual(found['message-id'], '<ant>') self.assertEqual(found['x-message-id-hash'], 'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG')
def test_get_message_by_hash(self): # Messages have an X-Message-ID-Hash header, the value of which can be # used to look the message up in the message store. msg = mfs("""\ Subject: An important message Message-ID: <ant> This message is very important. """) add_message_hash(msg) self._store.add(msg) self.assertEqual(msg['x-message-id-hash'], 'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG') found = self._store.get_message_by_hash( 'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG') self.assertEqual(found['message-id'], '<ant>') self.assertEqual(found['x-message-id-hash'], 'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG')
def test_archive_maildir_existence_does_not_raise(self): # Archiving a second message does not cause an EEXIST to be raised # when a second message is archived. new_dir = None Prototype.archive_message(self._mlist, self._msg) for directory in ("cur", "new", "tmp"): path = os.path.join(config.ARCHIVE_DIR, "prototype", self._mlist.fqdn_listname, directory) if directory == "new": new_dir = path self.assertTrue(os.path.isdir(path)) # There should be one message in the 'new' directory. self.assertEqual(len(os.listdir(new_dir)), 1) # Archive a second message. If an exception occurs, let it fail the # test. Afterward, two messages should be in the 'new' directory. del self._msg["message-id"] del self._msg["x-message-id-hash"] self._msg["Message-ID"] = "<bee>" add_message_hash(self._msg) Prototype.archive_message(self._mlist, self._msg) self.assertEqual(len(os.listdir(new_dir)), 2)
def test_archive_maildir_existence_does_not_raise(self): # Archiving a second message does not cause an EEXIST to be raised # when a second message is archived. new_dir = None Prototype.archive_message(self._mlist, self._msg) for directory in ('cur', 'new', 'tmp'): path = os.path.join(config.ARCHIVE_DIR, 'prototype', self._mlist.fqdn_listname, directory) if directory == 'new': new_dir = path self.assertTrue(os.path.isdir(path)) # There should be one message in the 'new' directory. self.assertEqual(len(os.listdir(new_dir)), 1) # Archive a second message. If an exception occurs, let it fail the # test. Afterward, two messages should be in the 'new' directory. del self._msg['message-id'] del self._msg['message-id-hash'] self._msg['Message-ID'] = '<bee>' add_message_hash(self._msg) Prototype.archive_message(self._mlist, self._msg) self.assertEqual(len(os.listdir(new_dir)), 2)
def inject_message(mlist, msg, recipients=None, switchboard=None, **kws): """Inject a message into a queue. If the message does not have a Message-ID header, one is added. An Message-ID-Hash header is also always added. :param mlist: The mailing list this message is destined for. :type mlist: IMailingList :param msg: The Message object to inject. :type msg: a Message object :param recipients: Optional set of recipients to put into the message's metadata. :type recipients: sequence of strings :param switchboard: Optional name of switchboard to inject this message into. If not given, the 'in' switchboard is used. :type switchboard: string :param kws: Additional values for the message metadata. :type kws: dictionary :return: filebase of enqueued message :rtype: string """ if switchboard is None: switchboard = 'in' # Since we're crafting the message from whole cloth, let's make sure this # message has a Message-ID. if 'message-id' not in msg: msg['Message-ID'] = make_msgid() add_message_hash(msg) # Ditto for Date: as required by RFC 2822. if 'date' not in msg: msg['Date'] = formatdate(localtime=True) msg.original_size = len(msg.as_string()) msgdata = dict( listid=mlist.list_id, original_size=msg.original_size, ) msgdata.update(kws) if recipients is not None: msgdata['recipients'] = recipients return config.switchboards[switchboard].enqueue(msg, **msgdata)
def test_archive_message_replay(self): # If there are messages in the spool directory, they must be processed # before any other message. # Create a previously failed message in the spool queue. msg_1 = self._get_msg() msg_1["Message-ID"] = "<dummy-1>" del msg_1["Message-ID-Hash"] add_message_hash(msg_1) self.archiver._switchboard.enqueue(msg_1, mlist=self.mlist) # Now send another message msg_2 = self._get_msg() msg_2["Message-ID"] = "<dummy-2>" del msg_2["Message-ID-Hash"] add_message_hash(msg_2) self.fake_response = FakeResponse(200, {"url": "dummy"}) with patch("mailman_hyperkitty.logger") as logger: self.archiver.archive_message(self.mlist, msg_2) # Two messages must have been archived self.assertEqual(logger.info.call_count, 2) self.assertEqual(self.requests.post.call_args_list, [ (("http://localhost/api/mailman/archive", ), dict( params={'key': 'DummyKey'}, data={'mlist': '*****@*****.**'}, files={'message': ('message.txt', msg_1.as_string())}, )), (("http://localhost/api/mailman/archive", ), dict( params={'key': 'DummyKey'}, data={'mlist': '*****@*****.**'}, files={'message': ('message.txt', msg_2.as_string())}, )), ]) # Make sure the spool directory is empty now. self.assertEqual( len(os.listdir(self.archiver._switchboard.queue_directory)), 0) self.assertEqual(len(self.archiver._switchboard.files), 0)
def add(self, store, message): # Ensure that the message has the requisite headers. message_ids = message.get_all('message-id', []) if len(message_ids) != 1: raise ValueError('Exactly one Message-ID header required') # Calculate and insert the Message-ID-Hash. message_id = message_ids[0] if isinstance(message_id, bytes): message_id = message_id.decode('ascii') # Complain if the Message-ID already exists in the storage. existing = store.query(Message).filter( Message.message_id == message_id).first() if existing is not None: raise ValueError( 'Message ID already exists in message store: {0}'.format( message_id)) hash32 = add_message_hash(message) # Calculate the path on disk where we're going to store this message # object, in pickled format. parts = [] split = list(hash32) while split and len(parts) < MAX_SPLITS: parts.append(split.pop(0) + split.pop(0)) parts.append(hash32) relpath = os.path.join(*parts) # Store the message in the database. This relies on the database # providing a unique serial number, but to get this information, we # have to use a straight insert instead of relying on Elixir to create # the object. Message(message_id=message_id, message_id_hash=hash32, path=relpath) # Now calculate the full file system path. path = os.path.join(config.MESSAGES_DIR, relpath) # Write the file to the path, but catch the appropriate exception in # case the parent directories don't yet exist. In that case, create # them and try again. while True: try: with open(path, 'wb') as fp: # -1 says to use the highest protocol available. pickle.dump(message, fp, -1) break except IOError as error: if error.errno != errno.ENOENT: raise makedirs(os.path.dirname(path)) return hash32
def process_message(self, peer, mailfrom, rcpttos, data): try: # Refresh the list of list names every time we process a message # since the set of mailing lists could have changed. listnames = set(getUtility(IListManager).names) # Parse the message data. If there are any defects in the # message, reject it right away; it's probably spam. msg = email.message_from_string(data, Message) except Exception: elog.exception('LMTP message parsing') config.db.abort() return CRLF.join(ERR_451 for to in rcpttos) # Do basic post-processing of the message, checking it for defects or # other missing information. message_id = msg.get('message-id') if message_id is None: return ERR_550_MID if msg.defects: return ERR_501 msg.original_size = len(data) add_message_hash(msg) msg['X-MailFrom'] = mailfrom # RFC 2033 requires us to return a status code for every recipient. status = [] # Now for each address in the recipients, parse the address to first # see if it's destined for a valid mailing list. If so, then queue # the message to the appropriate place and record a 250 status for # that recipient. If not, record a failure status for that recipient. received_time = now() for to in rcpttos: try: to = parseaddr(to)[1].lower() listname, subaddress, domain = split_recipient(to) slog.debug('%s to: %s, list: %s, sub: %s, dom: %s', message_id, to, listname, subaddress, domain) listname += '@' + domain if listname not in listnames: status.append(ERR_550) continue # The recipient is a valid mailing list. Find the subaddress # if there is one, and set things up to enqueue to the proper # queue. queue = None msgdata = dict(listname=listname, original_size=msg.original_size, received_time=received_time) canonical_subaddress = SUBADDRESS_NAMES.get(subaddress) queue = SUBADDRESS_QUEUES.get(canonical_subaddress) if subaddress is None: # The message is destined for the mailing list. msgdata['to_list'] = True queue = 'in' elif canonical_subaddress is None: # The subaddress was bogus. slog.error('%s unknown sub-address: %s', message_id, subaddress) status.append(ERR_550) continue else: # A valid subaddress. msgdata['subaddress'] = canonical_subaddress if canonical_subaddress == 'owner': msgdata.update(dict( to_owner=True, envsender=config.mailman.site_owner, )) queue = 'in' # If we found a valid destination, enqueue the message and add # a success status for this recipient. if queue is not None: config.switchboards[queue].enqueue(msg, msgdata) slog.debug('%s subaddress: %s, queue: %s', message_id, canonical_subaddress, queue) status.append(b'250 Ok') except Exception: slog.exception('Queue detection: %s', msg['message-id']) config.db.abort() status.append(ERR_550) # All done; returning this big status string should give the expected # response to the LMTP client. return CRLF.join(status)
def handle_DATA(self, server, session, envelope): try: # Refresh the list of list names every time we process a message # since the set of mailing lists could have changed. listnames = set(getUtility(IListManager).names) # Parse the message data. If there are any defects in the # message, reject it right away; it's probably spam. msg = email.message_from_bytes(envelope.content, Message) except Exception: elog.exception('LMTP message parsing') config.db.abort() return CRLF.join(ERR_451 for to in envelope.rcpt_tos) # Do basic post-processing of the message, checking it for defects or # other missing information. message_id = msg.get('message-id') if message_id is None: return ERR_550_MID if msg.defects: return ERR_501 msg.original_size = len(envelope.content) add_message_hash(msg) msg['X-MailFrom'] = envelope.mail_from # RFC 2033 requires us to return a status code for every recipient. status = [] # Now for each address in the recipients, parse the address to first # see if it's destined for a valid mailing list. If so, then queue # the message to the appropriate place and record a 250 status for # that recipient. If not, record a failure status for that recipient. received_time = now() for to in envelope.rcpt_tos: try: to = parseaddr(to)[1].lower() local, subaddress, domain = split_recipient(to) if subaddress is not None: # Check that local-subaddress is not an actual list name. listname = '{}-{}@{}'.format(local, subaddress, domain) if listname in listnames: local = '{}-{}'.format(local, subaddress) subaddress = None slog.debug('%s to: %s, list: %s, sub: %s, dom: %s', message_id, to, local, subaddress, domain) listname = '{}@{}'.format(local, domain) if listname not in listnames: status.append(ERR_550) continue mlist = getUtility(IListManager).get_by_fqdn(listname) # The recipient is a valid mailing list. Find the subaddress # if there is one, and set things up to enqueue to the proper # queue. queue = None msgdata = dict(listid=mlist.list_id, original_size=msg.original_size, received_time=received_time) canonical_subaddress = SUBADDRESS_NAMES.get(subaddress) queue = SUBADDRESS_QUEUES.get(canonical_subaddress) if subaddress is None: # The message is destined for the mailing list. msgdata['to_list'] = True queue = 'in' elif canonical_subaddress is None: # The subaddress was bogus. slog.error('%s unknown sub-address: %s', message_id, subaddress) status.append(ERR_550) continue else: # A valid subaddress. msgdata['subaddress'] = canonical_subaddress if canonical_subaddress == 'owner': msgdata.update(dict( to_owner=True, envsender=config.mailman.site_owner, )) queue = 'in' # If we found a valid destination, enqueue the message and add # a success status for this recipient. if queue is not None: config.switchboards[queue].enqueue(msg, msgdata) slog.debug('%s subaddress: %s, queue: %s', message_id, canonical_subaddress, queue) status.append('250 Ok') except Exception: slog.exception('Queue detection: %s', msg['message-id']) config.db.abort() status.append(ERR_550) # All done; returning this big status string should give the expected # response to the LMTP client. return CRLF.join(status)