def _start_outbound_queue(args, relay, inbound_queue):
from slimta.queue.dict import DictStorage
from slimta.queue import Queue
from slimta.policy.headers import AddDateHeader, \
AddMessageIdHeader, AddReceivedHeader
from slimta.policy.split import RecipientDomainSplit
envelope_db = {}
meta_db = {}
storage = DictStorage(envelope_db, meta_db)
queue = Queue(storage, relay, bounce_queue=inbound_queue)
queue.start()
queue.add_policy(AddDateHeader())
queue.add_policy(AddMessageIdHeader())
queue.add_policy(AddReceivedHeader())
queue.add_policy(RecipientDomainSplit())
return queue
def _start_inbound_queue(args, relay):
from slimta.queue.dict import DictStorage
from slimta.queue import Queue
from slimta.policy.headers import AddDateHeader, \
AddMessageIdHeader, AddReceivedHeader
from slimta.policy.spamassassin import SpamAssassin
envelope_db = {}
meta_db = {}
storage = DictStorage(envelope_db, meta_db)
queue = Queue(storage, relay)
queue.start()
queue.add_policy(AddDateHeader())
queue.add_policy(AddMessageIdHeader())
queue.add_policy(AddReceivedHeader())
if args.spamassassin:
queue.add_policy(SpamAssassin())
return queue
def setUp(self):
self.env = {}
self.meta = {}
self.dict = DictStorage(self.env, self.meta)
class TestDictStorage(unittest.TestCase):
id_pattern = re.compile(r'[0-9a-fA-F]{32}')
def setUp(self):
self.env = {}
self.meta = {}
self.dict = DictStorage(self.env, self.meta)
def _write_test_envelope(self, rcpts=None):
env = Envelope('*****@*****.**', rcpts or ['*****@*****.**'])
env.timestamp = 9876543210
id = self.dict.write(env, 1234567890)
return id, env
def test_write(self):
id, env = self._write_test_envelope()
assert_true(self.id_pattern.match(id))
assert_equal(env, self.env[id])
assert_equal(1234567890, self.meta[id]['timestamp'])
assert_equal(0, self.meta[id]['attempts'])
assert_equal('*****@*****.**', self.env[id].sender)
assert_equal(['*****@*****.**'], self.env[id].recipients)
assert_equal(9876543210, self.env[id].timestamp)
def test_set_timestamp(self):
id, env = self._write_test_envelope()
self.dict.set_timestamp(id, 1111)
assert_equal(env, self.env[id])
assert_equal(1111, self.meta[id]['timestamp'])
def test_increment_attempts(self):
id, env = self._write_test_envelope()
assert_equal(1, self.dict.increment_attempts(id))
assert_equal(2, self.dict.increment_attempts(id))
assert_equal(env, self.env[id])
assert_equal(2, self.meta[id]['attempts'])
def test_load(self):
queued = [self._write_test_envelope(),
self._write_test_envelope()]
loaded = [info for info in self.dict.load()]
assert_equal(len(queued), len(loaded))
for timestamp, loaded_id in loaded:
for queued_id, env in queued:
if loaded_id == queued_id:
assert_equal(env, self.env[loaded_id])
assert_equal(timestamp, self.meta[queued_id]['timestamp'])
break
else:
raise ValueError('Queued does not match loaded')
def test_get(self):
id, env = self._write_test_envelope()
self.dict.increment_attempts(id)
get_env, get_attempts = self.dict.get(id)
assert_equal(env, get_env)
assert_equal(1, get_attempts)
def test_remove(self):
id, env = self._write_test_envelope()
self.dict.remove(id)
id, env = self._write_test_envelope()
del self.env[id]
self.dict.remove(id)
id, env = self._write_test_envelope()
del self.meta[id]
self.dict.remove(id)
def test_get_info(self):
id1, _ = self._write_test_envelope()
id2, _ = self._write_test_envelope()
id3, _ = self._write_test_envelope()
self.dict.remove(id2)
info = self.dict.get_info()
assert_equal(2, info['size'])
assert_equal(2, info['meta_size'])
class TestDictStorage(unittest.TestCase):
id_pattern = re.compile(r'[0-9a-fA-F]{32}')
def setUp(self):
self.env = {}
self.meta = {}
self.dict = DictStorage(self.env, self.meta)
def _write_test_envelope(self, rcpts=None):
env = Envelope('*****@*****.**', rcpts or ['*****@*****.**'])
env.timestamp = 9876543210
id = self.dict.write(env, 1234567890)
return id, env
def test_write(self):
id, env = self._write_test_envelope()
self.assertTrue(self.id_pattern.match(id))
self.assertEqual(env, self.env[id])
self.assertEqual(1234567890, self.meta[id]['timestamp'])
self.assertEqual(0, self.meta[id]['attempts'])
self.assertEqual('*****@*****.**', self.env[id].sender)
self.assertEqual(['*****@*****.**'], self.env[id].recipients)
self.assertEqual(9876543210, self.env[id].timestamp)
def test_set_timestamp(self):
id, env = self._write_test_envelope()
self.dict.set_timestamp(id, 1111)
self.assertEqual(env, self.env[id])
self.assertEqual(1111, self.meta[id]['timestamp'])
def test_increment_attempts(self):
id, env = self._write_test_envelope()
self.assertEqual(1, self.dict.increment_attempts(id))
self.assertEqual(2, self.dict.increment_attempts(id))
self.assertEqual(env, self.env[id])
self.assertEqual(2, self.meta[id]['attempts'])
def test_load(self):
queued = [self._write_test_envelope(), self._write_test_envelope()]
loaded = [info for info in self.dict.load()]
self.assertEqual(len(queued), len(loaded))
for timestamp, loaded_id in loaded:
for queued_id, env in queued:
if loaded_id == queued_id:
self.assertEqual(env, self.env[loaded_id])
self.assertEqual(timestamp,
self.meta[queued_id]['timestamp'])
break
else:
raise ValueError('Queued does not match loaded')
def test_get(self):
id, env = self._write_test_envelope()
self.dict.increment_attempts(id)
get_env, get_attempts = self.dict.get(id)
self.assertEqual(env, get_env)
self.assertEqual(1, get_attempts)
def test_remove(self):
id, env = self._write_test_envelope()
self.dict.remove(id)
id, env = self._write_test_envelope()
del self.env[id]
self.dict.remove(id)
id, env = self._write_test_envelope()
del self.meta[id]
self.dict.remove(id)
def setUp(self):
self.env = {}
self.meta = {}
self.dict = DictStorage(self.env, self.meta)
def _start_queue(self, name, options=None):
if name in self.queues:
return self.queues[name]
if not options:
options = getattr(self.cfg.queue, name)
from .helpers import add_queue_policies, build_backoff_function
new_queue = None
relay_name = options.relay
relay = self._start_relay(relay_name) if relay_name else None
bounce_queue_name = options.get('bounce_queue', name)
bounce_queue = self._start_queue(bounce_queue_name) \
if bounce_queue_name != name else None
if options.type == 'memory':
from slimta.queue import Queue
from slimta.queue.dict import DictStorage
store = DictStorage()
backoff = build_backoff_function(options.retry)
new_queue = Queue(store, relay, backoff=backoff,
bounce_queue=bounce_queue)
new_queue.start()
elif options.type == 'disk':
from slimta.queue import Queue
from slimta.diskstorage import DiskStorage
env_dir = options.envelope_dir
meta_dir = options.meta_dir
tmp_dir = options.tmp_dir
store = DiskStorage(env_dir, meta_dir, tmp_dir)
backoff = build_backoff_function(options.retry)
new_queue = Queue(store, relay, backoff=backoff,
bounce_queue=bounce_queue)
new_queue.start()
elif options.type == 'redis':
from slimta.queue import Queue
from slimta.redisstorage import RedisStorage
kwargs = {}
if 'host' in options:
kwargs['host'] = options.host
if 'port' in options:
kwargs['port'] = int(options.port)
if 'db' in options:
kwargs['db'] = int(options.db)
if 'password' in options:
kwargs['password'] = options.password
if 'socket_timeout' in options:
kwargs['socket_timeout'] = float(options.socket_timeout)
if 'prefix' in options:
kwargs['prefix'] = options.prefix
store = RedisStorage(**kwargs)
backoff = build_backoff_function(options.retry)
new_queue = Queue(store, relay, backoff=backoff,
bounce_queue=bounce_queue)
new_queue.start()
elif options.type == 'rackspace':
from slimta.queue import Queue
from slimta.cloudstorage import CloudStorage
from slimta.cloudstorage.rackspace import RackspaceCloudAuth, \
RackspaceCloudFiles, RackspaceCloudQueues
credentials = {'username': options.username}
if 'password' in options:
credentials['password'] = options.password
if 'api_key' in options:
credentials['api_key'] = options.api_key
if 'tenant_id' in options:
credentials['tenant_id'] = options.tenant_id
auth_kwargs = {'region': options.region,
'timeout': 10.0}
if 'endpoint' in options:
auth_kwargs['endpoint'] = options.endpoint
auth = RackspaceCloudAuth(credentials, **auth_kwargs)
cloud_files = RackspaceCloudFiles(auth,
container=options.container_name, timeout=20.0)
cloud_queues = None
if 'queue_name' in options:
cloud_queues = RackspaceCloudQueues(auth,
queue_name=options.queue_name, timeout=10.0)
store = CloudStorage(cloud_files, cloud_queues)
new_queue = Queue(store, relay, backoff=backoff,
bounce_queue=bounce_queue)
new_queue.start()
elif options.type == 'aws':
from slimta.queue import Queue
from slimta.cloudstorage import CloudStorage
from slimta.cloudstorage.aws import SimpleStorageService, \
SimpleQueueService
import boto
if 'access_key_id' in options:
from boto.s3.connection import S3Connection
s3_conn = S3Connection(options.access_key_id,
options.secret_access_key)
else:
s3_conn = boto.connect_s3()
s3_bucket = s3_conn.get_bucket(options.bucket_name)
s3 = SimpleStorageService(s3_bucket, timeout=20.0)
sqs = None
if 'queue_name' in options:
from boto.sqs import connect_to_region
region = options.get('queue_region', 'us-west-2')
if 'access_key_id' in options:
sqs_conn = connect_to_region(region,
aws_access_key_id=options.access_key_id,
aws_secret_access_key=options.secret_access_key)
else:
sqs_conn = connect_to_region(region)
sqs_queue = sqs_conn.create_queue(options.queue_name)
sqs = SimpleQueueService(sqs_queue, timeout=10.0)
store = CloudStorage(s3, sqs)
new_queue = Queue(store, relay, backoff=backoff,
bounce_queue=bounce_queue)
new_queue.start()
elif options.type == 'proxy':
from slimta.queue.proxy import ProxyQueue
new_queue = ProxyQueue(relay)
elif options.type == 'custom':
new_queue = custom_factory(options, relay)
else:
msg = 'queue type does not exist: '+options.type
raise ConfigValidationError(msg)
add_queue_policies(new_queue, options.get('policies', []))
self.queues[name] = new_queue
return new_queue
