Example #1
0
    def test_angle_brackets_dont_contribute_to_hash(self):
        # According to RFC 5322, the [matching] angle brackets do not
        # contribute to the hash.
        msg = mfs("""\
Message-ID: aardvark

""")
        add_message_hash(msg)
        self.assertEqual(msg['message-id-hash'],
                         '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT')
Example #2
0
    def test_angle_brackets_dont_contribute_to_hash(self):
        # According to RFC 5322, the [matching] angle brackets do not
        # contribute to the hash.
        msg = mfs("""\
Message-ID: aardvark

""")
        add_message_hash(msg)
        self.assertEqual(msg['x-message-id-hash'],
                         '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT')
Example #3
0
    def test_adding_the_message_hash(self):
        # When the message has a Message-ID header, this will add the
        # X-Mailman-Hash-ID header.
        msg = mfs("""\
Message-ID: <aardvark>

""")
        add_message_hash(msg)
        self.assertEqual(msg['x-message-id-hash'],
                         '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT')
Example #4
0
    def test_adding_the_message_hash(self):
        # When the message has a Message-ID header, this will add the
        # X-Mailman-Hash-ID header.
        msg = mfs("""\
Message-ID: <aardvark>

""")
        add_message_hash(msg)
        self.assertEqual(msg['message-id-hash'],
                         '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT')
Example #5
0
    def test_hash_header_left_alone_if_no_message_id(self):
        # If the original message has no Message-ID header, then any existing
        # X-Message-ID-Hash headers are left intact.
        msg = mfs("""\
X-Message-ID-Hash: abc

""")
        add_message_hash(msg)
        headers = msg.get_all('x-message-id-hash')
        self.assertEqual(len(headers), 1)
        self.assertEqual(headers[0], 'abc')
Example #6
0
    def test_hash_header_left_alone_if_no_message_id(self):
        # If the original message has no Message-ID header, then any existing
        # Message-ID-Hash headers are left intact.
        msg = mfs("""\
Message-ID-Hash: abc

""")
        add_message_hash(msg)
        headers = msg.get_all('message-id-hash')
        self.assertEqual(len(headers), 1)
        self.assertEqual(headers[0], 'abc')
Example #7
0
    def test_remove_hash_headers_first(self):
        # Any existing X-Mailman-Hash-ID header is removed first.
        msg = mfs("""\
Message-ID: <aardvark>
X-Message-ID-Hash: abc

""")
        add_message_hash(msg)
        headers = msg.get_all('x-message-id-hash')
        self.assertEqual(len(headers), 1)
        self.assertEqual(headers[0], '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT')
Example #8
0
    def test_remove_hash_headers_first(self):
        # Any existing X-Mailman-Hash-ID header is removed first.
        msg = mfs("""\
Message-ID: <aardvark>
Message-ID-Hash: abc

""")
        add_message_hash(msg)
        headers = msg.get_all('message-id-hash')
        self.assertEqual(len(headers), 1)
        self.assertEqual(headers[0], '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT')
Example #9
0
    def test_return_value(self):
        msg = mfs("""\
Message-ID: aardvark>

""")
        hash32 = add_message_hash(msg)
        self.assertEqual(hash32, '5KH3RA7ZM4VM6XOZXA7AST2XN2X4S3WY')
Example #10
0
    def test_return_value(self):
        msg = mfs("""\
Message-ID: aardvark>

""")
        hash32 = add_message_hash(msg)
        self.assertEqual(hash32, '5KH3RA7ZM4VM6XOZXA7AST2XN2X4S3WY')
Example #11
0
    def test_mismatched_angle_brackets_do_contribute_to_hash(self):
        # According to RFC 5322, the [matching] angle brackets do not
        # contribute to the hash.
        msg = mfs("""\
Message-ID: <aardvark

""")
        add_message_hash(msg)
        self.assertEqual(msg['x-message-id-hash'],
                         'AOJ545GHRYD2Y3RUFG2EWMPHUABTG4SM')
        msg = mfs("""\
Message-ID: aardvark>

""")
        add_message_hash(msg)
        self.assertEqual(msg['x-message-id-hash'],
                         '5KH3RA7ZM4VM6XOZXA7AST2XN2X4S3WY')
Example #12
0
    def test_mismatched_angle_brackets_do_contribute_to_hash(self):
        # According to RFC 5322, the [matching] angle brackets do not
        # contribute to the hash.
        msg = mfs("""\
Message-ID: <aardvark

""")
        add_message_hash(msg)
        self.assertEqual(msg['message-id-hash'],
                         'AOJ545GHRYD2Y3RUFG2EWMPHUABTG4SM')
        msg = mfs("""\
Message-ID: aardvark>

""")
        add_message_hash(msg)
        self.assertEqual(msg['message-id-hash'],
                         '5KH3RA7ZM4VM6XOZXA7AST2XN2X4S3WY')
Example #13
0
    def test_get_message_by_hash(self):
        # Messages have an X-Message-ID-Hash header, the value of which can be
        # used to look the message up in the message store.
        msg = mfs("""\
Subject: An important message
Message-ID: <ant>

This message is very important.
""")
        add_message_hash(msg)
        self._store.add(msg)
        self.assertEqual(msg['x-message-id-hash'],
                         'MS6QLWERIJLGCRF44J7USBFDELMNT2BW')
        found = self._store.get_message_by_hash(
            'MS6QLWERIJLGCRF44J7USBFDELMNT2BW')
        self.assertEqual(found['message-id'], '<ant>')
        self.assertEqual(found['x-message-id-hash'],
                         'MS6QLWERIJLGCRF44J7USBFDELMNT2BW')
Example #14
0
    def test_get_message_by_hash(self):
        # Messages have an X-Message-ID-Hash header, the value of which can be
        # used to look the message up in the message store.
        message = mfs("""\
Subject: An important message
Message-ID: <ant>

This message is very important.
""")
        add_message_hash(message)
        self._store.add(message)
        self.assertEqual(message['x-message-id-hash'],
                         'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG')
        found = self._store.get_message_by_hash(
            'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG')
        self.assertEqual(found['message-id'], '<ant>')
        self.assertEqual(found['x-message-id-hash'],
                         'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG')
Example #15
0
    def test_get_message_by_hash(self):
        # Messages have an X-Message-ID-Hash header, the value of which can be
        # used to look the message up in the message store.
        msg = mfs("""\
Subject: An important message
Message-ID: <ant>

This message is very important.
""")
        add_message_hash(msg)
        self._store.add(msg)
        self.assertEqual(msg['x-message-id-hash'],
                         'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG')
        found = self._store.get_message_by_hash(
            'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG')
        self.assertEqual(found['message-id'], '<ant>')
        self.assertEqual(found['x-message-id-hash'],
                         'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG')
Example #16
0
    def test_get_message_by_hash(self):
        # Messages have an X-Message-ID-Hash header, the value of which can be
        # used to look the message up in the message store.
        msg = mfs("""\
Subject: An important message
Message-ID: <ant>

This message is very important.
""")
        add_message_hash(msg)
        self._store.add(msg)
        self.assertEqual(msg['x-message-id-hash'],
                         'MS6QLWERIJLGCRF44J7USBFDELMNT2BW')
        found = self._store.get_message_by_hash(
            'MS6QLWERIJLGCRF44J7USBFDELMNT2BW')
        self.assertEqual(found['message-id'], '<ant>')
        self.assertEqual(found['x-message-id-hash'],
                         'MS6QLWERIJLGCRF44J7USBFDELMNT2BW')
Example #17
0
 def test_archive_maildir_existence_does_not_raise(self):
     # Archiving a second message does not cause an EEXIST to be raised
     # when a second message is archived.
     new_dir = None
     Prototype.archive_message(self._mlist, self._msg)
     for directory in ("cur", "new", "tmp"):
         path = os.path.join(config.ARCHIVE_DIR, "prototype", self._mlist.fqdn_listname, directory)
         if directory == "new":
             new_dir = path
         self.assertTrue(os.path.isdir(path))
     # There should be one message in the 'new' directory.
     self.assertEqual(len(os.listdir(new_dir)), 1)
     # Archive a second message.  If an exception occurs, let it fail the
     # test.  Afterward, two messages should be in the 'new' directory.
     del self._msg["message-id"]
     del self._msg["x-message-id-hash"]
     self._msg["Message-ID"] = "<bee>"
     add_message_hash(self._msg)
     Prototype.archive_message(self._mlist, self._msg)
     self.assertEqual(len(os.listdir(new_dir)), 2)
Example #18
0
 def test_archive_maildir_existence_does_not_raise(self):
     # Archiving a second message does not cause an EEXIST to be raised
     # when a second message is archived.
     new_dir = None
     Prototype.archive_message(self._mlist, self._msg)
     for directory in ('cur', 'new', 'tmp'):
         path = os.path.join(config.ARCHIVE_DIR, 'prototype',
                             self._mlist.fqdn_listname, directory)
         if directory == 'new':
             new_dir = path
         self.assertTrue(os.path.isdir(path))
     # There should be one message in the 'new' directory.
     self.assertEqual(len(os.listdir(new_dir)), 1)
     # Archive a second message.  If an exception occurs, let it fail the
     # test.  Afterward, two messages should be in the 'new' directory.
     del self._msg['message-id']
     del self._msg['message-id-hash']
     self._msg['Message-ID'] = '<bee>'
     add_message_hash(self._msg)
     Prototype.archive_message(self._mlist, self._msg)
     self.assertEqual(len(os.listdir(new_dir)), 2)
Example #19
0
def inject_message(mlist, msg, recipients=None, switchboard=None, **kws):
    """Inject a message into a queue.

    If the message does not have a Message-ID header, one is added.  An
    Message-ID-Hash header is also always added.

    :param mlist: The mailing list this message is destined for.
    :type mlist: IMailingList
    :param msg: The Message object to inject.
    :type msg: a Message object
    :param recipients: Optional set of recipients to put into the message's
        metadata.
    :type recipients: sequence of strings
    :param switchboard: Optional name of switchboard to inject this message
        into.  If not given, the 'in' switchboard is used.
    :type switchboard: string
    :param kws: Additional values for the message metadata.
    :type kws: dictionary
    :return: filebase of enqueued message
    :rtype: string
    """
    if switchboard is None:
        switchboard = 'in'
    # Since we're crafting the message from whole cloth, let's make sure this
    # message has a Message-ID.
    if 'message-id' not in msg:
        msg['Message-ID'] = make_msgid()
    add_message_hash(msg)
    # Ditto for Date: as required by RFC 2822.
    if 'date' not in msg:
        msg['Date'] = formatdate(localtime=True)
    msg.original_size = len(msg.as_string())
    msgdata = dict(
        listid=mlist.list_id,
        original_size=msg.original_size,
        )
    msgdata.update(kws)
    if recipients is not None:
        msgdata['recipients'] = recipients
    return config.switchboards[switchboard].enqueue(msg, **msgdata)
    def test_archive_message_replay(self):
        # If there are messages in the spool directory, they must be processed
        # before any other message.

        # Create a previously failed message in the spool queue.
        msg_1 = self._get_msg()
        msg_1["Message-ID"] = "<dummy-1>"
        del msg_1["Message-ID-Hash"]
        add_message_hash(msg_1)
        self.archiver._switchboard.enqueue(msg_1, mlist=self.mlist)
        # Now send another message
        msg_2 = self._get_msg()
        msg_2["Message-ID"] = "<dummy-2>"
        del msg_2["Message-ID-Hash"]
        add_message_hash(msg_2)

        self.fake_response = FakeResponse(200, {"url": "dummy"})
        with patch("mailman_hyperkitty.logger") as logger:
            self.archiver.archive_message(self.mlist, msg_2)
        # Two messages must have been archived
        self.assertEqual(logger.info.call_count, 2)

        self.assertEqual(self.requests.post.call_args_list, [
            (("http://localhost/api/mailman/archive", ),
             dict(
                 params={'key': 'DummyKey'},
                 data={'mlist': '*****@*****.**'},
                 files={'message': ('message.txt', msg_1.as_string())},
             )),
            (("http://localhost/api/mailman/archive", ),
             dict(
                 params={'key': 'DummyKey'},
                 data={'mlist': '*****@*****.**'},
                 files={'message': ('message.txt', msg_2.as_string())},
             )),
        ])
        # Make sure the spool directory is empty now.
        self.assertEqual(
            len(os.listdir(self.archiver._switchboard.queue_directory)), 0)
        self.assertEqual(len(self.archiver._switchboard.files), 0)
Example #21
0
def inject_message(mlist, msg, recipients=None, switchboard=None, **kws):
    """Inject a message into a queue.

    If the message does not have a Message-ID header, one is added.  An
    Message-ID-Hash header is also always added.

    :param mlist: The mailing list this message is destined for.
    :type mlist: IMailingList
    :param msg: The Message object to inject.
    :type msg: a Message object
    :param recipients: Optional set of recipients to put into the message's
        metadata.
    :type recipients: sequence of strings
    :param switchboard: Optional name of switchboard to inject this message
        into.  If not given, the 'in' switchboard is used.
    :type switchboard: string
    :param kws: Additional values for the message metadata.
    :type kws: dictionary
    :return: filebase of enqueued message
    :rtype: string
    """
    if switchboard is None:
        switchboard = 'in'
    # Since we're crafting the message from whole cloth, let's make sure this
    # message has a Message-ID.
    if 'message-id' not in msg:
        msg['Message-ID'] = make_msgid()
    add_message_hash(msg)
    # Ditto for Date: as required by RFC 2822.
    if 'date' not in msg:
        msg['Date'] = formatdate(localtime=True)
    msg.original_size = len(msg.as_string())
    msgdata = dict(
        listid=mlist.list_id,
        original_size=msg.original_size,
    )
    msgdata.update(kws)
    if recipients is not None:
        msgdata['recipients'] = recipients
    return config.switchboards[switchboard].enqueue(msg, **msgdata)
Example #22
0
 def add(self, store, message):
     # Ensure that the message has the requisite headers.
     message_ids = message.get_all('message-id', [])
     if len(message_ids) != 1:
         raise ValueError('Exactly one Message-ID header required')
     # Calculate and insert the Message-ID-Hash.
     message_id = message_ids[0]
     if isinstance(message_id, bytes):
         message_id = message_id.decode('ascii')
     # Complain if the Message-ID already exists in the storage.
     existing = store.query(Message).filter(
         Message.message_id == message_id).first()
     if existing is not None:
         raise ValueError(
             'Message ID already exists in message store: {0}'.format(
                 message_id))
     hash32 = add_message_hash(message)
     # Calculate the path on disk where we're going to store this message
     # object, in pickled format.
     parts = []
     split = list(hash32)
     while split and len(parts) < MAX_SPLITS:
         parts.append(split.pop(0) + split.pop(0))
     parts.append(hash32)
     relpath = os.path.join(*parts)
     # Store the message in the database.  This relies on the database
     # providing a unique serial number, but to get this information, we
     # have to use a straight insert instead of relying on Elixir to create
     # the object.
     Message(message_id=message_id,
             message_id_hash=hash32,
             path=relpath)
     # Now calculate the full file system path.
     path = os.path.join(config.MESSAGES_DIR, relpath)
     # Write the file to the path, but catch the appropriate exception in
     # case the parent directories don't yet exist.  In that case, create
     # them and try again.
     while True:
         try:
             with open(path, 'wb') as fp:
                 # -1 says to use the highest protocol available.
                 pickle.dump(message, fp, -1)
                 break
         except IOError as error:
             if error.errno != errno.ENOENT:
                 raise
         makedirs(os.path.dirname(path))
     return hash32
Example #23
0
 def add(self, store, message):
     # Ensure that the message has the requisite headers.
     message_ids = message.get_all('message-id', [])
     if len(message_ids) != 1:
         raise ValueError('Exactly one Message-ID header required')
     # Calculate and insert the Message-ID-Hash.
     message_id = message_ids[0]
     if isinstance(message_id, bytes):
         message_id = message_id.decode('ascii')
     # Complain if the Message-ID already exists in the storage.
     existing = store.query(Message).filter(
         Message.message_id == message_id).first()
     if existing is not None:
         raise ValueError(
             'Message ID already exists in message store: {0}'.format(
                 message_id))
     hash32 = add_message_hash(message)
     # Calculate the path on disk where we're going to store this message
     # object, in pickled format.
     parts = []
     split = list(hash32)
     while split and len(parts) < MAX_SPLITS:
         parts.append(split.pop(0) + split.pop(0))
     parts.append(hash32)
     relpath = os.path.join(*parts)
     # Store the message in the database.  This relies on the database
     # providing a unique serial number, but to get this information, we
     # have to use a straight insert instead of relying on Elixir to create
     # the object.
     Message(message_id=message_id, message_id_hash=hash32, path=relpath)
     # Now calculate the full file system path.
     path = os.path.join(config.MESSAGES_DIR, relpath)
     # Write the file to the path, but catch the appropriate exception in
     # case the parent directories don't yet exist.  In that case, create
     # them and try again.
     while True:
         try:
             with open(path, 'wb') as fp:
                 # -1 says to use the highest protocol available.
                 pickle.dump(message, fp, -1)
                 break
         except IOError as error:
             if error.errno != errno.ENOENT:
                 raise
         makedirs(os.path.dirname(path))
     return hash32
Example #24
0
 def process_message(self, peer, mailfrom, rcpttos, data):
     try:
         # Refresh the list of list names every time we process a message
         # since the set of mailing lists could have changed.
         listnames = set(getUtility(IListManager).names)
         # Parse the message data.  If there are any defects in the
         # message, reject it right away; it's probably spam.
         msg = email.message_from_string(data, Message)
     except Exception:
         elog.exception('LMTP message parsing')
         config.db.abort()
         return CRLF.join(ERR_451 for to in rcpttos)
     # Do basic post-processing of the message, checking it for defects or
     # other missing information.
     message_id = msg.get('message-id')
     if message_id is None:
         return ERR_550_MID
     if msg.defects:
         return ERR_501
     msg.original_size = len(data)
     add_message_hash(msg)
     msg['X-MailFrom'] = mailfrom
     # RFC 2033 requires us to return a status code for every recipient.
     status = []
     # Now for each address in the recipients, parse the address to first
     # see if it's destined for a valid mailing list.  If so, then queue
     # the message to the appropriate place and record a 250 status for
     # that recipient.  If not, record a failure status for that recipient.
     received_time = now()
     for to in rcpttos:
         try:
             to = parseaddr(to)[1].lower()
             listname, subaddress, domain = split_recipient(to)
             slog.debug('%s to: %s, list: %s, sub: %s, dom: %s',
                        message_id, to, listname, subaddress, domain)
             listname += '@' + domain
             if listname not in listnames:
                 status.append(ERR_550)
                 continue
             # The recipient is a valid mailing list.  Find the subaddress
             # if there is one, and set things up to enqueue to the proper
             # queue.
             queue = None
             msgdata = dict(listname=listname,
                            original_size=msg.original_size,
                            received_time=received_time)
             canonical_subaddress = SUBADDRESS_NAMES.get(subaddress)
             queue = SUBADDRESS_QUEUES.get(canonical_subaddress)
             if subaddress is None:
                 # The message is destined for the mailing list.
                 msgdata['to_list'] = True
                 queue = 'in'
             elif canonical_subaddress is None:
                 # The subaddress was bogus.
                 slog.error('%s unknown sub-address: %s',
                            message_id, subaddress)
                 status.append(ERR_550)
                 continue
             else:
                 # A valid subaddress.
                 msgdata['subaddress'] = canonical_subaddress
                 if canonical_subaddress == 'owner':
                     msgdata.update(dict(
                         to_owner=True,
                         envsender=config.mailman.site_owner,
                         ))
                     queue = 'in'
             # If we found a valid destination, enqueue the message and add
             # a success status for this recipient.
             if queue is not None:
                 config.switchboards[queue].enqueue(msg, msgdata)
                 slog.debug('%s subaddress: %s, queue: %s',
                            message_id, canonical_subaddress, queue)
                 status.append(b'250 Ok')
         except Exception:
             slog.exception('Queue detection: %s', msg['message-id'])
             config.db.abort()
             status.append(ERR_550)
     # All done; returning this big status string should give the expected
     # response to the LMTP client.
     return CRLF.join(status)
Example #25
0
 def handle_DATA(self, server, session, envelope):
     try:
         # Refresh the list of list names every time we process a message
         # since the set of mailing lists could have changed.
         listnames = set(getUtility(IListManager).names)
         # Parse the message data.  If there are any defects in the
         # message, reject it right away; it's probably spam.
         msg = email.message_from_bytes(envelope.content, Message)
     except Exception:
         elog.exception('LMTP message parsing')
         config.db.abort()
         return CRLF.join(ERR_451 for to in envelope.rcpt_tos)
     # Do basic post-processing of the message, checking it for defects or
     # other missing information.
     message_id = msg.get('message-id')
     if message_id is None:
         return ERR_550_MID
     if msg.defects:
         return ERR_501
     msg.original_size = len(envelope.content)
     add_message_hash(msg)
     msg['X-MailFrom'] = envelope.mail_from
     # RFC 2033 requires us to return a status code for every recipient.
     status = []
     # Now for each address in the recipients, parse the address to first
     # see if it's destined for a valid mailing list.  If so, then queue
     # the message to the appropriate place and record a 250 status for
     # that recipient.  If not, record a failure status for that recipient.
     received_time = now()
     for to in envelope.rcpt_tos:
         try:
             to = parseaddr(to)[1].lower()
             local, subaddress, domain = split_recipient(to)
             if subaddress is not None:
                 # Check that local-subaddress is not an actual list name.
                 listname = '{}-{}@{}'.format(local, subaddress, domain)
                 if listname in listnames:
                     local = '{}-{}'.format(local, subaddress)
                     subaddress = None
             slog.debug('%s to: %s, list: %s, sub: %s, dom: %s',
                        message_id, to, local, subaddress, domain)
             listname = '{}@{}'.format(local, domain)
             if listname not in listnames:
                 status.append(ERR_550)
                 continue
             mlist = getUtility(IListManager).get_by_fqdn(listname)
             # The recipient is a valid mailing list.  Find the subaddress
             # if there is one, and set things up to enqueue to the proper
             # queue.
             queue = None
             msgdata = dict(listid=mlist.list_id,
                            original_size=msg.original_size,
                            received_time=received_time)
             canonical_subaddress = SUBADDRESS_NAMES.get(subaddress)
             queue = SUBADDRESS_QUEUES.get(canonical_subaddress)
             if subaddress is None:
                 # The message is destined for the mailing list.
                 msgdata['to_list'] = True
                 queue = 'in'
             elif canonical_subaddress is None:
                 # The subaddress was bogus.
                 slog.error('%s unknown sub-address: %s',
                            message_id, subaddress)
                 status.append(ERR_550)
                 continue
             else:
                 # A valid subaddress.
                 msgdata['subaddress'] = canonical_subaddress
                 if canonical_subaddress == 'owner':
                     msgdata.update(dict(
                         to_owner=True,
                         envsender=config.mailman.site_owner,
                         ))
                     queue = 'in'
             # If we found a valid destination, enqueue the message and add
             # a success status for this recipient.
             if queue is not None:
                 config.switchboards[queue].enqueue(msg, msgdata)
                 slog.debug('%s subaddress: %s, queue: %s',
                            message_id, canonical_subaddress, queue)
                 status.append('250 Ok')
         except Exception:
             slog.exception('Queue detection: %s', msg['message-id'])
             config.db.abort()
             status.append(ERR_550)
     # All done; returning this big status string should give the expected
     # response to the LMTP client.
     return CRLF.join(status)