def test_load_all(self):
     self.store.load().AndReturn([(3, 'one'), (5, 'two'), (1, 'three')])
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay)
     queue._load_all()
     self.assertEqual([(1, 'three'), (3, 'one'), (5, 'two')], queue.queued)
     self.assertTrue(queue.wake.isSet())
Beispiel #2
0
 def test_load_all(self):
     self.store.load().AndReturn([(3, 'one'), (5, 'two'), (1, 'three')])
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay)
     queue._load_all()
     self.assertEqual([(1, 'three'), (3, 'one'), (5, 'two')], queue.queued)
     self.assertTrue(queue.wake.isSet())
 def test_check_ready_missing(self):
     self.store.get('1234').AndRaise(KeyError)
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, store_pool=Pool(5))
     queue._add_queued((10, '1234'))
     queue._check_ready(20)
     queue.store_pool.join()
 def test_load_all_empty(self):
     self.store.load().AndReturn([])
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay)
     queue._load_all()
     self.assertEqual([], queue.queued)
     self.assertFalse(queue.wake.isSet())
Beispiel #5
0
    def test_enqueue_wait_partial_relay(self):
        env = Envelope(
            '*****@*****.**',
            ['*****@*****.**', '*****@*****.**', '*****@*****.**'])
        self.store.write(env, IsA(float)).AndReturn('1234')
        self.relay._attempt(env, 0).AndReturn({
            '*****@*****.**':
            None,
            '*****@*****.**':
            TransientRelayError('transient', Reply('450', 'transient')),
            '*****@*****.**':
            PermanentRelayError('permanent', Reply('550', 'permanent'))
        })
        self.store.increment_attempts('1234')
        self.store.set_timestamp('1234', IsA(float))
        self.store.set_recipients_delivered('1234', set([0, 2]))
        self.mox.ReplayAll()

        def backoff(envelope, attempts):
            return 0

        def no_bounce(envelope, reply):
            return None

        queue = Queue(self.store,
                      self.relay,
                      backoff=backoff,
                      bounce_factory=no_bounce,
                      relay_pool=5)
        queue.enqueue(env)
        queue.relay_pool.join()
Beispiel #6
0
 def test_enqueue_wait_partial_relay_expired(self):
     env = Envelope(
         '*****@*****.**',
         ['*****@*****.**', '*****@*****.**', '*****@*****.**'])
     bounce_mock = self.mox.CreateMockAnything()
     bounce_mock(IsA(Envelope), IsA(Reply)).AndReturn(None)
     bounce_mock(IsA(Envelope), IsA(Reply)).AndReturn(None)
     self.store.write(env, IsA(float)).AndReturn('1234')
     self.relay._attempt(env, 0).AndReturn({
         '*****@*****.**':
         TransientRelayError('transient', Reply('450', 'transient 1')),
         '*****@*****.**':
         TransientRelayError('transient', Reply('450', 'transient 1')),
         '*****@*****.**':
         TransientRelayError('transient', Reply('450', 'transient 2'))
     })
     self.store.increment_attempts('1234')
     self.store.remove('1234')
     self.mox.ReplayAll()
     queue = Queue(self.store,
                   self.relay,
                   bounce_factory=bounce_mock,
                   relay_pool=5)
     queue.enqueue(env)
     queue.relay_pool.join()
Beispiel #7
0
 def test_load_all_empty(self):
     self.store.load().AndReturn([])
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay)
     queue._load_all()
     self.assertEqual([], queue.queued)
     self.assertFalse(queue.wake.isSet())
Beispiel #8
0
 def test_add_queued(self):
     queue = Queue(self.store, self.relay)
     queue._add_queued((10, 'one'))
     queue._add_queued((5, 'two'))
     queue._add_queued((99, 'one'))
     queue._add_queued((7, 'three'))
     self.assertEqual([(5, 'two'), (7, 'three'), (10, 'one')], queue.queued)
     self.assertTrue(queue.wake.isSet())
 def test_enqueue_wait(self):
     self.store.write(self.env, IsA(float)).AndReturn('1234')
     self.relay._attempt(self.env, 0)
     self.store.remove('1234')
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, relay_pool=5)
     self.assertEqual([(self.env, '1234')], queue.enqueue(self.env))
     queue.relay_pool.join()
Beispiel #10
0
 def test_enqueue_wait(self):
     self.store.write(self.env, IsA(float)).AndReturn('1234')
     self.relay._attempt(self.env, 0)
     self.store.remove('1234')
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, relay_pool=5)
     self.assertEqual([(self.env, '1234')], queue.enqueue(self.env))
     queue.relay_pool.join()
 def test_check_ready(self):
     self.store.get('1234').AndReturn((self.env, 0))
     self.relay._attempt(self.env, 0)
     self.store.remove('1234')
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, store_pool=Pool(5))
     queue._add_queued((10, '1234'))
     queue._check_ready(20)
     queue.store_pool.join()
 def test_enqueue_wait_permanentfail(self):
     self.store.write(self.env, IsA(float)).AndReturn('1234')
     self.relay._attempt(self.env, 0).AndRaise(PermanentRelayError('permanent', Reply('550', 'permanent')))
     self.store.remove('1234')
     self.mox.ReplayAll()
     def no_bounce(envelope, reply):
         return None
     queue = Queue(self.store, self.relay, bounce_factory=no_bounce, relay_pool=5)
     queue.enqueue(self.env)
     queue.relay_pool.join()
 def test_enqueue_wait_permanentfail(self):
     self.store.write(self.env, IsA(float)).AndReturn('1234')
     self.relay._attempt(self.env, 0).AndRaise(PermanentRelayError('permanent', Reply('550', 'permanent')))
     self.store.remove('1234')
     self.mox.ReplayAll()
     def no_bounce(envelope, reply):
         return None
     queue = Queue(self.store, self.relay, bounce_factory=no_bounce, relay_pool=5)
     queue.enqueue(self.env)
     queue.relay_pool.join()
 def test_wait_ready_nonequeued(self):
     queue = Queue(self.store, self.relay)
     def wait_func():
         queue._wait_ready(20)
     thread = gevent.spawn(wait_func)
     gevent.sleep(0)
     self.assertFalse(thread.ready())
     queue._add_queued((10, '1234'))
     gevent.sleep(0)
     self.assertTrue(thread.ready())
 def test_wait_ready_nonequeued(self):
     queue = Queue(self.store, self.relay)
     def wait_func():
         queue._wait_ready(20)
     thread = gevent.spawn(wait_func)
     gevent.sleep(0)
     self.assertFalse(thread.ready())
     queue._add_queued((10, '1234'))
     gevent.sleep(0)
     self.assertTrue(thread.ready())
Beispiel #16
0
 def test_enqueue_wait_splitpolicy(self):
     splitpolicy1 = self.mox.CreateMock(QueuePolicy)
     splitpolicy2 = self.mox.CreateMock(QueuePolicy)
     regpolicy = self.mox.CreateMock(QueuePolicy)
     env1 = Envelope('*****@*****.**', ['*****@*****.**'])
     env2 = Envelope('*****@*****.**', ['*****@*****.**'])
     env3 = Envelope('*****@*****.**', ['*****@*****.**'])
     splitpolicy1.apply(self.env).AndReturn([env1, env2])
     regpolicy.apply(env1)
     splitpolicy2.apply(env1)
     regpolicy.apply(env2)
     splitpolicy2.apply(env2).AndReturn([env2, env3])
     self.store.write(env1, IsA(float)).AndReturn('1234')
     self.store.write(env2, IsA(float)).AndReturn('5678')
     self.store.write(env3, IsA(float)).AndReturn('90AB')
     self.relay._attempt(env1, 0).InAnyOrder('relay')
     self.relay._attempt(env2, 0).InAnyOrder('relay')
     self.relay._attempt(env3, 0).InAnyOrder('relay')
     self.store.remove('1234').InAnyOrder('relay')
     self.store.remove('5678').InAnyOrder('relay')
     self.store.remove('90AB').InAnyOrder('relay')
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, relay_pool=5)
     queue.add_policy(splitpolicy1)
     queue.add_policy(regpolicy)
     queue.add_policy(splitpolicy2)
     self.assertEqual([(env1, '1234'), (env2, '5678'), (env3, '90AB')],
                      queue.enqueue(self.env))
     queue.relay_pool.join()
Beispiel #17
0
 def test_wait_store(self):
     queue = Queue(self.store, self.relay, relay_pool=5)
     queue.wake = self.mox.CreateMockAnything()
     self.store.wait().AndReturn([(1234567890, '1234')])
     queue.wake.set()
     self.store.wait().AndReturn([])
     self.store.wait().AndReturn([(2345678901, '5678')])
     queue.wake.set()
     self.store.wait().AndRaise(NotImplementedError)
     self.mox.ReplayAll()
     queue._wait_store()
 def test_enqueue_wait_unhandledfail(self):
     self.store.write(self.env, IsA(float)).AndReturn('1234')
     self.relay._attempt(self.env, 0).AndRaise(Exception('unhandled error'))
     self.store.increment_attempts('1234')
     self.store.set_timestamp('1234', IsA(float))
     self.mox.ReplayAll()
     def backoff(envelope, attempts):
         return 0
     queue = Queue(self.store, self.relay, backoff=backoff, relay_pool=5)
     queue.enqueue(self.env)
     queue.relay_pool.join()
 def test_policies(self):
     p1 = self.mox.CreateMock(QueuePolicy)
     p2 = self.mox.CreateMock(QueuePolicy)
     p1.apply(self.env)
     p2.apply(self.env)
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay)
     queue.add_policy(p1)
     queue.add_policy(p2)
     self.assertRaises(TypeError, queue.add_policy, None)
     queue._run_policies(self.env)
 def test_enqueue_wait_transientfail(self):
     self.store.write(self.env, IsA(float)).AndReturn('1234')
     self.relay._attempt(self.env, 0).AndRaise(TransientRelayError('transient', Reply('450', 'transient')))
     self.store.increment_attempts('1234')
     self.store.set_timestamp('1234', IsA(float))
     self.mox.ReplayAll()
     def backoff(envelope, attempts):
         return 0
     queue = Queue(self.store, self.relay, backoff=backoff, relay_pool=5)
     queue.enqueue(self.env)
     queue.relay_pool.join()
 def test_enqueue_wait_unhandledfail(self):
     self.store.write(self.env, IsA(float)).AndReturn('1234')
     self.relay._attempt(self.env, 0).AndRaise(Exception('unhandled error'))
     self.store.increment_attempts('1234')
     self.store.set_timestamp('1234', IsA(float))
     self.mox.ReplayAll()
     def backoff(envelope, attempts):
         return 0
     queue = Queue(self.store, self.relay, backoff=backoff, relay_pool=5)
     queue.enqueue(self.env)
     queue.relay_pool.join()
 def test_wait_store(self):
     queue = Queue(self.store, self.relay, relay_pool=5)
     queue.wake = self.mox.CreateMock(AsyncResult)
     self.store.wait().AndReturn((1234567890, '1234'))
     queue.wake.set()
     self.store.wait().AndReturn(None)
     self.store.wait().AndReturn((2345678901, '5678'))
     queue.wake.set()
     self.store.wait().AndRaise(NotImplementedError)
     self.mox.ReplayAll()
     queue._wait_store()
 def test_enqueue_wait_transientfail(self):
     self.store.write(self.env, IsA(float)).AndReturn('1234')
     self.relay._attempt(self.env, 0).AndRaise(TransientRelayError('transient', Reply('450', 'transient')))
     self.store.increment_attempts('1234')
     self.store.set_timestamp('1234', IsA(float))
     self.mox.ReplayAll()
     def backoff(envelope, attempts):
         return 0
     queue = Queue(self.store, self.relay, backoff=backoff, relay_pool=5)
     queue.enqueue(self.env)
     queue.relay_pool.join()
 def test_add_queued(self):
     queue = Queue(self.store, self.relay)
     queue._add_queued((10, 'one'))
     queue._add_queued((5, 'two'))
     queue._add_queued((99, 'one'))
     queue._add_queued((7, 'three'))
     self.assertEqual([(5, 'two'), (7, 'three'), (10, 'one')], queue.queued)
     self.assertTrue(queue.wake.isSet())
 def test_wait_ready_noneready(self):
     queue = Queue(self.store, self.relay)
     queue._add_queued((20, '1234'))
     queue.wake.clear()
     def wait_func():
         queue._wait_ready(10)
     thread = gevent.spawn(wait_func)
     gevent.sleep(0)
     self.assertFalse(thread.ready())
     queue._add_queued((5, '5678'))
     gevent.sleep(0)
     self.assertTrue(thread.ready())
 def test_enqueue_wait_splitpolicy(self):
     splitpolicy1 = self.mox.CreateMock(QueuePolicy)
     splitpolicy2 = self.mox.CreateMock(QueuePolicy)
     regpolicy = self.mox.CreateMock(QueuePolicy)
     env1 = Envelope('*****@*****.**', ['*****@*****.**'])
     env2 = Envelope('*****@*****.**', ['*****@*****.**'])
     env3 = Envelope('*****@*****.**', ['*****@*****.**'])
     splitpolicy1.apply(self.env).AndReturn([env1, env2])
     regpolicy.apply(env1)
     splitpolicy2.apply(env1)
     regpolicy.apply(env2)
     splitpolicy2.apply(env2).AndReturn([env2, env3])
     self.store.write(env1, IsA(float)).AndReturn('1234')
     self.store.write(env2, IsA(float)).AndReturn('5678')
     self.store.write(env3, IsA(float)).AndReturn('90AB')
     self.relay._attempt(env1, 0).InAnyOrder('relay')
     self.relay._attempt(env2, 0).InAnyOrder('relay')
     self.relay._attempt(env3, 0).InAnyOrder('relay')
     self.store.remove('1234').InAnyOrder('relay')
     self.store.remove('5678').InAnyOrder('relay')
     self.store.remove('90AB').InAnyOrder('relay')
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, relay_pool=5)
     queue.add_policy(splitpolicy1)
     queue.add_policy(regpolicy)
     queue.add_policy(splitpolicy2)
     self.assertEqual([(env1, '1234'), (env2, '5678'), (env3, '90AB')],
                      queue.enqueue(self.env))
     queue.relay_pool.join()
Beispiel #27
0
 def test_enqueue_wait_partial_relay_expired(self):
     env = Envelope('*****@*****.**', ['*****@*****.**',
                                           '*****@*****.**',
                                           '*****@*****.**'])
     bounce_mock = self.mox.CreateMockAnything()
     bounce_mock(IsA(Envelope), IsA(Reply)).AndReturn(None)
     bounce_mock(IsA(Envelope), IsA(Reply)).AndReturn(None)
     self.store.write(env, IsA(float)).AndReturn('1234')
     self.relay._attempt(env, 0).AndReturn([TransientRelayError('transient', Reply('450', 'transient 1')),
                                            TransientRelayError('transient', Reply('450', 'transient 1')),
                                            TransientRelayError('transient', Reply('450', 'transient 2'))])
     self.store.increment_attempts('1234')
     self.store.remove('1234')
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, bounce_factory=bounce_mock, relay_pool=5)
     queue.enqueue(env)
     queue.relay_pool.join()
Beispiel #28
0
class MDA(MTA):
    def __init__(self,
                 msa,
                 mail_list_url,
                 mda_domain,
                 list_subject_prefix=None):

        self.msa = msa

        # Relay:
        relay = DovecotLdaRelay(config['LDA']['dovecot_path'], timeout=10.0)

        # Queue:
        #env_db = shelve.open('envelope')
        #meta_db = shelve.open('meta')
        #storage = DictStorage(env_db, meta_db) # !!! replace with DiskStorage!  (installed via pip install python-slimta-diskstorage)
        storage = DiskStorage(config['MDA']['ds_env'],
                              config['MDA']['ds_meta'])

        self.queue = Queue(
            storage,
            relay)  # no backoff - just fail local delivery immediately
        self.queue.start()

        # Headers:
        self.queue.add_policy(AddDateHeader())
        self.queue.add_policy(AddMessageIdHeader())
        self.queue.add_policy(AddReceivedHeader())
        # Mailing List:
        self.queue.add_policy(
            MailingListDistribution(self.msa, mail_list_url, mda_domain,
                                    list_subject_prefix))
        # SpamAssassin:
        #self.queue.add_policy(SpamAssassin())

        # Edge:
        #tls_args = {'keyfile': '/home/jmcaine/dev/temp/slimta/tls/key.pem', 'certfile': '/home/jmcaine/dev/temp/slimta/tls/certificate.pem'} -- gone, see https://docs.slimta.org/en/latest/blog/2016-11-14.html
        ssl = SSLContext(PROTOCOL_SSLv23)
        ssl.load_cert_chain(config['SSL']['certificate_path'],
                            config['SSL']['key_path'])
        self.edge = SmtpEdge(('0.0.0.0', 25),
                             self.queue,
                             validator_class=MDA_Validators,
                             hostname=mda_domain,
                             context=ssl)
        self.edge.start()
Beispiel #29
0
 def test_flush(self):
     self.store.get('three').AndReturn((self.env, 1))
     self.store.get('two').AndReturn((self.env, 2))
     self.store.get('one').AndReturn((self.env, 3))
     self.relay._attempt(self.env, 1)
     self.store.remove('three')
     self.relay._attempt(self.env, 2)
     self.store.remove('two')
     self.relay._attempt(self.env, 3)
     self.store.remove('one')
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, store_pool=5, relay_pool=5)
     queue._add_queued((float('inf'), 'one'))
     queue._add_queued((0, 'two'))
     queue._add_queued((float('-inf'), 'three'))
     queue.flush()
     queue.store_pool.join()
     queue.relay_pool.join()
Beispiel #30
0
 def test_enqueue_wait_partial_relay(self):
     env = Envelope('*****@*****.**', ['*****@*****.**',
                                           '*****@*****.**',
                                           '*****@*****.**'])
     self.store.write(env, IsA(float)).AndReturn('1234')
     self.relay._attempt(env, 0).AndReturn([None,
                                            TransientRelayError('transient', Reply('450', 'transient')),
                                            PermanentRelayError('permanent', Reply('550', 'permanent'))])
     self.store.increment_attempts('1234')
     self.store.set_timestamp('1234', IsA(float))
     self.store.set_recipients_delivered('1234', [0, 2])
     self.mox.ReplayAll()
     def backoff(envelope, attempts):
         return 0
     def no_bounce(envelope, reply):
         return None
     queue = Queue(self.store, self.relay, backoff=backoff, bounce_factory=no_bounce, relay_pool=5)
     queue.enqueue(env)
     queue.relay_pool.join()
Beispiel #31
0
class MSA(MTA):
    def __init__(self):

        # Relay:
        ssl = SSLContext(PROTOCOL_SSLv23)
        ssl.load_cert_chain(config['SSL']['certificate_path'],
                            config['SSL']['key_path'])
        self.relay = MxSmtpRelay(context=ssl,
                                 connect_timeout=20,
                                 command_timeout=10,
                                 data_timeout=20,
                                 idle_timeout=30)

        # Queue:
        #env_db = shelve.open('msa_envelope')
        #meta_db = shelve.open('msa_meta')
        #storage = DictStorage(env_db, meta_db) # !!! replace with DiskStorage!  (installed via pip install python-slimta-diskstorage)
        storage = DiskStorage(config['MSA']['ds_env'],
                              config['MSA']['ds_meta'])

        def retry_backoff(envelope, attempts):
            if attempts < 10:
                return 60 * attempts * attempts  # try again at increasingly long intervals; give up after 10 tries (100 minutes)
            return None

        self.queue = Queue(storage, self.relay, backoff=retry_backoff)
        self.queue.start()

        # Headers:
        self.queue.add_policy(AddDateHeader())
        self.queue.add_policy(AddMessageIdHeader())
        self.queue.add_policy(AddReceivedHeader())
        self.queue.add_policy(RecipientDomainSplit())
        # !!! Add Forward policy here, to manage general forwarding (but not list distribution - do that in mda!)

        # Edge:
        self.edge = SmtpEdge(
            ('localhost', 587), self.queue, auth=False
        )  #, auth=True, validator_class=MSA_Validators) # ?!!! context=ssl, tls_immediately=True,
        self.edge.start()
Beispiel #32
0
 def test_check_ready_missing(self):
     self.store.get('1234').AndRaise(KeyError)
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, store_pool=Pool(5))
     queue._add_queued((10, '1234'))
     queue._check_ready(20)
     queue.store_pool.join()
 def test_flush(self):
     self.store.get('three').AndReturn((self.env, 1))
     self.store.get('two').AndReturn((self.env, 2))
     self.store.get('one').AndReturn((self.env, 3))
     self.relay._attempt(self.env, 1)
     self.store.remove('three')
     self.relay._attempt(self.env, 2)
     self.store.remove('two')
     self.relay._attempt(self.env, 3)
     self.store.remove('one')
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, store_pool=5, relay_pool=5)
     queue._add_queued((float('inf'), 'one'))
     queue._add_queued((0, 'two'))
     queue._add_queued((float('-inf'), 'three'))
     queue.flush()
     queue.store_pool.join()
     queue.relay_pool.join()
Beispiel #34
0
 def test_check_ready(self):
     self.store.get('1234').AndReturn((self.env, 0))
     self.relay._attempt(self.env, 0)
     self.store.remove('1234')
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, store_pool=Pool(5))
     queue._add_queued((10, '1234'))
     queue._check_ready(20)
     queue.store_pool.join()
Beispiel #35
0
 def test_policies(self):
     p1 = self.mox.CreateMock(QueuePolicy)
     p2 = self.mox.CreateMock(QueuePolicy)
     p1.apply(self.env)
     p2.apply(self.env)
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay)
     queue.add_policy(p1)
     queue.add_policy(p2)
     self.assertRaises(TypeError, queue.add_policy, None)
     queue._run_policies(self.env)
 def test_wait_ready_noneready(self):
     queue = Queue(self.store, self.relay)
     queue._add_queued((20, '1234'))
     queue.wake.clear()
     def wait_func():
         queue._wait_ready(10)
     thread = gevent.spawn(wait_func)
     gevent.sleep(0)
     self.assertFalse(thread.ready())
     queue._add_queued((5, '5678'))
     gevent.sleep(0)
     self.assertTrue(thread.ready())
Beispiel #37
0
 def test_enqueue_wait_norelay(self):
     self.store.write(self.env, IsA(float)).AndReturn('1234')
     self.mox.ReplayAll()
     queue = Queue(self.store, relay_pool=5)
     assert_equal([(self.env, '1234')], queue.enqueue(self.env))
     queue.relay_pool.join()
Beispiel #38
0
 def _start_queue(self, name, options=None):
     if name in self.queues:
         return self.queues[name]
     if not options:
         options = getattr(self.cfg.queue, name)
     from .helpers import add_queue_policies, build_backoff_function
     new_queue = None
     relay_name = options.relay
     relay = self._start_relay(relay_name) if relay_name else None
     bounce_queue_name = options.get('bounce_queue', name)
     bounce_queue = self._start_queue(bounce_queue_name) \
                    if bounce_queue_name != name else None
     if options.type == 'memory':
         from slimta.queue import Queue
         from slimta.queue.dict import DictStorage
         store = DictStorage()
         backoff = build_backoff_function(options.retry)
         new_queue = Queue(store, relay, backoff=backoff,
                           bounce_queue=bounce_queue)
         new_queue.start()
     elif options.type == 'disk':
         from slimta.queue import Queue
         from slimta.diskstorage import DiskStorage
         env_dir = options.envelope_dir
         meta_dir = options.meta_dir
         tmp_dir = options.tmp_dir
         store = DiskStorage(env_dir, meta_dir, tmp_dir)
         backoff = build_backoff_function(options.retry)
         new_queue = Queue(store, relay, backoff=backoff,
                           bounce_queue=bounce_queue)
         new_queue.start()
     elif options.type == 'redis':
         from slimta.queue import Queue
         from slimta.redisstorage import RedisStorage
         kwargs = {}
         if 'host' in options:
             kwargs['host'] = options.host
         if 'port' in options:
             kwargs['port'] = int(options.port)
         if 'db' in options:
             kwargs['db'] = int(options.db)
         if 'password' in options:
             kwargs['password'] = options.password
         if 'socket_timeout' in options:
             kwargs['socket_timeout'] = float(options.socket_timeout)
         if 'prefix' in options:
             kwargs['prefix'] = options.prefix
         store = RedisStorage(**kwargs)
         backoff = build_backoff_function(options.retry)
         new_queue = Queue(store, relay, backoff=backoff,
                           bounce_queue=bounce_queue)
         new_queue.start()
     elif options.type == 'rackspace':
         from slimta.queue import Queue
         from slimta.cloudstorage import CloudStorage
         from slimta.cloudstorage.rackspace import RackspaceCloudAuth, \
                 RackspaceCloudFiles, RackspaceCloudQueues
         credentials = {'username': options.username}
         if 'password' in options:
             credentials['password'] = options.password
         if 'api_key' in options:
             credentials['api_key'] = options.api_key
         if 'tenant_id' in options:
             credentials['tenant_id'] = options.tenant_id
         auth_kwargs = {'region': options.region,
                        'timeout': 10.0}
         if 'endpoint' in options:
             auth_kwargs['endpoint'] = options.endpoint
         auth = RackspaceCloudAuth(credentials, **auth_kwargs)
         cloud_files = RackspaceCloudFiles(auth,
                 container=options.container_name, timeout=20.0)
         cloud_queues = None
         if 'queue_name' in options:
             cloud_queues = RackspaceCloudQueues(auth,
                     queue_name=options.queue_name, timeout=10.0)
         store = CloudStorage(cloud_files, cloud_queues)
         new_queue = Queue(store, relay, backoff=backoff,
                           bounce_queue=bounce_queue)
         new_queue.start()
     elif options.type == 'aws':
         from slimta.queue import Queue
         from slimta.cloudstorage import CloudStorage
         from slimta.cloudstorage.aws import SimpleStorageService, \
                 SimpleQueueService
         import boto
         if 'access_key_id' in options:
             from boto.s3.connection import S3Connection
             s3_conn = S3Connection(options.access_key_id,
                                    options.secret_access_key)
         else:
             s3_conn = boto.connect_s3()
         s3_bucket = s3_conn.get_bucket(options.bucket_name)
         s3 = SimpleStorageService(s3_bucket, timeout=20.0)
         sqs = None
         if 'queue_name' in options:
             from boto.sqs import connect_to_region
             region = options.get('queue_region', 'us-west-2')
             if 'access_key_id' in options:
                 sqs_conn = connect_to_region(region,
                         aws_access_key_id=options.access_key_id,
                         aws_secret_access_key=options.secret_access_key)
             else:
                 sqs_conn = connect_to_region(region)
             sqs_queue = sqs_conn.create_queue(options.queue_name)
             sqs = SimpleQueueService(sqs_queue, timeout=10.0)
         store = CloudStorage(s3, sqs)
         new_queue = Queue(store, relay, backoff=backoff,
                           bounce_queue=bounce_queue)
         new_queue.start()
     elif options.type == 'proxy':
         from slimta.queue.proxy import ProxyQueue
         new_queue = ProxyQueue(relay)
     elif options.type == 'custom':
         new_queue = custom_factory(options, relay)
     else:
         msg = 'queue type does not exist: '+options.type
         raise ConfigValidationError(msg)
     add_queue_policies(new_queue, options.get('policies', []))
     self.queues[name] = new_queue
     return new_queue
 def test_kill(self):
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay)
     assert_false(queue.ready())
     queue.kill()
     assert_true(queue.ready())
 def test_enqueue_wait_norelay(self):
     self.store.write(self.env, IsA(float)).AndReturn('1234')
     self.mox.ReplayAll()
     queue = Queue(self.store, relay_pool=5)
     assert_equal([(self.env, '1234')], queue.enqueue(self.env))
     queue.relay_pool.join()
Beispiel #41
0
 def test_wait_ready_nowait(self):
     queue = Queue(self.store, self.relay)
     queue._add_queued((10, '1234'))
     with gevent.Timeout(1.0):
         queue._wait_ready(20)
Beispiel #42
0
def _start_inbound_queue(args, relay):
    from slimta.queue.dict import DictStorage
    from slimta.queue import Queue
    from slimta.policy.headers import AddDateHeader, \
        AddMessageIdHeader, AddReceivedHeader
    from slimta.policy.spamassassin import SpamAssassin

    envelope_db = {}
    meta_db = {}

    storage = DictStorage(envelope_db, meta_db)
    queue = Queue(storage, relay)
    queue.start()

    queue.add_policy(AddDateHeader())
    queue.add_policy(AddMessageIdHeader())
    queue.add_policy(AddReceivedHeader())
    if args.spamassassin:
        queue.add_policy(SpamAssassin())

    return queue
Beispiel #43
0
from slimta.edge.smtp import SmtpEdge
from slimta.queue import Queue
from slimta.queue.dict import DictStorage
from slimta.relay import RelayError
from slimta.relay.smtp.static import StaticSmtpRelay

def backoff(envelope, attempts):
    if attempts <= 5:
        return 5.0 * attempts

relay = StaticSmtpRelay('mail.example.com', 25, pool_size=2)

env_db = shelve.open('envelope.db')
meta_db = shelve.open('meta.db')
queue_storage = DictStorage(env_db, meta_db)
queue = Queue(queue_storage, relay, backoff)

edge = SmtpEdge(('127.0.0.1', 1337), queue)
edge.start()
queue.start()
try:
    edge.get()
except KeyboardInterrupt:
    print
finally:
    for key in env_db.keys():
        print 'env', key
    for key in meta_db.keys():
        print 'meta', key
    env_db.close()
    meta_db.close()
Beispiel #44
0
 def test_kill(self):
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay)
     self.assertFalse(queue.ready())
     queue.kill()
     self.assertTrue(queue.ready())
Beispiel #45
0
from slimta.edge.smtp import SmtpEdge
from slimta.queue import Queue
from slimta.queue.dict import DictStorage
from slimta.relay.smtp.mx import MxSmtpRelay
from slimta.policy.headers import *
from slimta.policy.split import RecipientDomainSplit

logging.basicConfig(level=logging.DEBUG)

relay = MxSmtpRelay()

env_db = shelve.open('envelope.db')
meta_db = shelve.open('meta.db')
queue_storage = DictStorage(env_db, meta_db)
queue = Queue(queue_storage, relay)

queue.add_prequeue_policy(AddDateHeader())
queue.add_prequeue_policy(AddMessageIdHeader())
queue.add_prequeue_policy(RecipientDomainSplit())
queue.add_prequeue_policy(AddReceivedHeader())

edge = SmtpEdge(('127.0.0.1', 1337), queue)
edge.start()
queue.start()
try:
    edge.get()
except KeyboardInterrupt:
    print
finally:
    for key in env_db.keys():
from slimta.edge.smtp import SmtpEdge
from slimta.queue import Queue
from slimta.queue.dict import DictStorage
from slimta.relay.maildrop import MaildropRelay
from slimta.policy.headers import *
from slimta.bounce import Bounce

logging.basicConfig(level=logging.DEBUG)

relay = MaildropRelay()

env_db = shelve.open('envelope.db')
meta_db = shelve.open('meta.db')
queue_storage = DictStorage(env_db, meta_db)
queue = Queue(queue_storage, relay)

queue.add_prequeue_policy(AddDateHeader())
queue.add_prequeue_policy(AddMessageIdHeader())
queue.add_prequeue_policy(AddReceivedHeader())

edge = SmtpEdge(('127.0.0.1', 1337), queue)
edge.start()
queue.start()
try:
    edge.get()
except KeyboardInterrupt:
    print
finally:
    for key in env_db.keys():
        print 'env', key
Beispiel #47
0
 def _start_queue(self, name, options=None):
     if name in self.queues:
         return self.queues[name]
     if not options:
         options = getattr(self.cfg.queue, name)
     from .helpers import add_queue_policies, build_backoff_function
     new_queue = None
     relay_name = options.get('relay')
     relay = self._start_relay(relay_name) if relay_name else None
     bounce_queue_name = options.get('bounce_queue', name)
     bounce_queue = self._start_queue(bounce_queue_name) \
                    if bounce_queue_name != name else None
     if options.type == 'memory':
         from slimta.queue import Queue
         from slimta.queue.dict import DictStorage
         store = DictStorage()
         backoff = build_backoff_function(options.get('retry'))
         new_queue = Queue(store, relay, backoff=backoff,
                           bounce_queue=bounce_queue)
         new_queue.start()
     elif options.type == 'disk':
         from slimta.queue import Queue
         from slimta.diskstorage import DiskStorage
         env_dir = options.envelope_dir
         meta_dir = options.meta_dir
         tmp_dir = options.get('tmp_dir')
         store = DiskStorage(env_dir, meta_dir, tmp_dir)
         backoff = build_backoff_function(options.get('retry'))
         new_queue = Queue(store, relay, backoff=backoff,
                           bounce_queue=bounce_queue)
         new_queue.start()
     elif options.type == 'redis':
         from slimta.queue import Queue
         from slimta.redisstorage import RedisStorage
         kwargs = {}
         if 'host' in options:
             kwargs['host'] = options.host
         if 'port' in options:
             kwargs['port'] = int(options.port)
         if 'db' in options:
             kwargs['db'] = int(options.db)
         if 'password' in options:
             kwargs['password'] = options.password
         if 'socket_timeout' in options:
             kwargs['socket_timeout'] = float(options.socket_timeout)
         if 'prefix' in options:
             kwargs['prefix'] = options.prefix
         store = RedisStorage(**kwargs)
         backoff = build_backoff_function(options.get('retry'))
         new_queue = Queue(store, relay, backoff=backoff,
                           bounce_queue=bounce_queue)
         new_queue.start()
     elif options.type == 'rackspace':
         from slimta.queue import Queue
         from slimta.cloudstorage import CloudStorage
         from slimta.cloudstorage.rackspace import RackspaceCloudAuth, \
                 RackspaceCloudFiles, RackspaceCloudQueues
         credentials = {'username': options.username}
         if 'password' in options:
             credentials['password'] = options.password
         if 'api_key' in options:
             credentials['api_key'] = options.api_key
         if 'tenant_id' in options:
             credentials['tenant_id'] = options.tenant_id
         auth_kwargs = {'region': options.get('region'),
                        'timeout': 10.0}
         if 'endpoint' in options:
             auth_kwargs['endpoint'] = options.endpoint
         auth = RackspaceCloudAuth(credentials, **auth_kwargs)
         cloud_files = RackspaceCloudFiles(auth,
                 container=options.container_name, timeout=20.0)
         cloud_queues = None
         if 'queue_name' in options:
             cloud_queues = RackspaceCloudQueues(auth,
                     queue_name=options.queue_name, timeout=10.0)
         store = CloudStorage(cloud_files, cloud_queues)
         new_queue = Queue(store, relay, backoff=backoff,
                           bounce_queue=bounce_queue)
         new_queue.start()
     elif options.type == 'aws':
         from slimta.queue import Queue
         from slimta.cloudstorage import CloudStorage
         from slimta.cloudstorage.aws import SimpleStorageService, \
                 SimpleQueueService
         import boto
         if 'access_key_id' in options:
             from boto.s3.connection import S3Connection
             s3_conn = S3Connection(options.access_key_id,
                                    options.secret_access_key)
         else:
             s3_conn = boto.connect_s3()
         s3_bucket = s3_conn.get_bucket(options.bucket_name)
         s3 = SimpleStorageService(s3_bucket, timeout=20.0)
         sqs = None
         if 'queue_name' in options:
             from boto.sqs import connect_to_region
             region = options.get('queue_region', 'us-west-2')
             if 'access_key_id' in options:
                 sqs_conn = connect_to_region(region,
                         aws_access_key_id=options.access_key_id,
                         aws_secret_access_key=options.secret_access_key)
             else:
                 sqs_conn = connect_to_region(region)
             sqs_queue = sqs_conn.create_queue(options.queue_name)
             sqs = SimpleQueueService(sqs_queue, timeout=10.0)
         store = CloudStorage(s3, sqs)
         new_queue = Queue(store, relay, backoff=backoff,
                           bounce_queue=bounce_queue)
         new_queue.start()
     elif options.type == 'proxy':
         from slimta.queue.proxy import ProxyQueue
         new_queue = ProxyQueue(relay)
     elif options.type == 'custom':
         new_queue = self._load_from_custom(options, relay)
     else:
         msg = 'queue type does not exist: '+options.type
         raise ConfigValidationError()
     add_queue_policies(new_queue, options.get('policies', []))
     self.queues[name] = new_queue
     return new_queue
Beispiel #48
0
 def test_enqueue_randomfail(self):
     self.store.write(self.env, IsA(float)).AndRaise(gevent.GreenletExit)
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, relay_pool=5)
     self.assertRaises(gevent.GreenletExit, queue.enqueue, self.env)
Beispiel #49
0
 def test_check_ready_empty(self):
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, store_pool=5)
     queue._add_queued((20, '1234'))
     queue._check_ready(10)
     queue.store_pool.join()
Beispiel #50
0
 def test_kill(self):
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay)
     assert_false(queue.ready())
     queue.kill()
     assert_true(queue.ready())
 def test_check_ready_empty(self):
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay, store_pool=5)
     queue._add_queued((20, '1234'))
     queue._check_ready(10)
     queue.store_pool.join()
 def test_kill(self):
     self.mox.ReplayAll()
     queue = Queue(self.store, self.relay)
     self.assertFalse(queue.ready())
     queue.kill()
     self.assertTrue(queue.ready())
Beispiel #53
0
def _start_outbound_queue(args, relay, inbound_queue):
    from slimta.queue.dict import DictStorage
    from slimta.queue import Queue
    from slimta.policy.headers import AddDateHeader, \
        AddMessageIdHeader, AddReceivedHeader
    from slimta.policy.split import RecipientDomainSplit

    envelope_db = {}
    meta_db = {}

    storage = DictStorage(envelope_db, meta_db)
    queue = Queue(storage, relay, bounce_queue=inbound_queue)
    queue.start()

    queue.add_policy(AddDateHeader())
    queue.add_policy(AddMessageIdHeader())
    queue.add_policy(AddReceivedHeader())
    queue.add_policy(RecipientDomainSplit())

    return queue
 def test_wait_ready_nowait(self):
     queue = Queue(self.store, self.relay)
     queue._add_queued((10, '1234'))
     with gevent.Timeout(1.0):
         queue._wait_ready(20)
Beispiel #55
0
def _start_outbound_queue(args, relay):
    from slimta.queue.dict import DictStorage
    from slimta.queue import Queue
    from slimta.policy.headers import AddDateHeader, \
            AddMessageIdHeader, AddReceivedHeader
    from slimta.policy.split import RecipientDomainSplit
    import shelve

    envelope_db = shelve.open(args.envelope_db)
    meta_db = shelve.open(args.meta_db)

    storage = DictStorage(envelope_db, meta_db)
    queue = Queue(storage, relay)
    queue.start()

    queue.add_policy(AddDateHeader())
    queue.add_policy(AddMessageIdHeader())
    queue.add_policy(AddReceivedHeader())
    queue.add_policy(RecipientDomainSplit())

    return queue
Beispiel #56
0
def _start_inbound_queue(args, relay):
    from slimta.queue.dict import DictStorage
    from slimta.queue import Queue
    from slimta.policy.headers import AddDateHeader, \
        AddMessageIdHeader, AddReceivedHeader
    from slimta.policy.spamassassin import SpamAssassin

    envelope_db = {}
    meta_db = {}

    storage = DictStorage(envelope_db, meta_db)
    queue = Queue(storage, relay)
    queue.start()

    queue.add_policy(AddDateHeader())
    queue.add_policy(AddMessageIdHeader())
    queue.add_policy(AddReceivedHeader())
    if args.spamassassin:
        queue.add_policy(SpamAssassin())

    return queue