def test_oversize_protections(self):
# first just make an oversize limited queue
overq = queue.Queue("run/queue", pop_limit=10)
overq.clear()
for i in range(5):
overq.push("HELLO" * 100)
self.assertEqual(overq.count(), 5)
key, msg = overq.pop()
assert not key and not msg, "Should get no messages."
self.assertEqual(overq.count(), 0)
# now make sure that oversize mail is moved to the overq
overq = queue.Queue("run/queue",
pop_limit=10,
oversize_dir="run/big_queue")
moveq = queue.Queue("run/big_queue")
moveq.clear()
for i in range(5):
overq.push("HELLO" * 100)
key, msg = overq.pop()
assert not key and not msg, "Should get no messages."
self.assertEqual(overq.count(), 0)
self.assertEqual(moveq.count(), 5)
moveq.clear()
overq.clear()
def setUp(self):
super().setUp()
# re-create destoryed queues
queue.Queue(
os.path.join(self._cwd,
server_settings.UNDELIVERABLE_QUEUE)).clear()
queue.Queue(os.path.join(self._cwd,
server_settings.QUEUE_PATH)).clear()
def test_we_dont_get_message():
client = smtplib.SMTP(**server_settings.receiver_config)
client.helo()
client.sendmail("*****@*****.**", "*****@*****.**", "hello")
undelivered = queue.Queue(server_settings.UNDELIVERABLE_QUEUE)
assert_equal(len(undelivered.mbox), 1)
inbox = queue.Queue(server_settings.QUEUE_PATH)
assert_equal(len(inbox.mbox), 0)
def test_we_dont_get_message(self):
client = smtplib.SMTP(**server_settings.receiver_config)
client.helo()
client.sendmail("*****@*****.**", "*****@*****.**", "hello")
undelivered = queue.Queue(
os.path.join(self._cwd, server_settings.UNDELIVERABLE_QUEUE))
self.assertEqual(len(undelivered), 1)
inbox = queue.Queue(os.path.join(self._cwd,
server_settings.QUEUE_PATH))
self.assertEqual(len(inbox), 0)
def START(message, to=None, host=None):
"""
@stateless and routes however handlers.log.START routes (everything).
Has @nolocking, but that's alright since it's just writing to a maildir.
"""
q = queue.Queue('run/queue')
q.push(message)
def POSTING(message, post_name=None, host=None):
user, address = parseaddr(message['from'])
user = user or address
post_url = "posts/%s/%s.html" % (address, post_name)
index_q = queue.Queue("run/indexed")
post_keys = sorted(index_q.keys(), reverse=True)
old_keys = post_keys[50:]
del post_keys[50:]
# find the old one and remove it
posts = []
for key in post_keys:
msg = index_q.get(key)
if msg['x-post-url'] == post_url:
# this is the old one, take it out
index_q.remove(key)
else:
posts.append(msg)
# update the index and our posts
message['X-Post-URL'] = post_url
index_q.push(message)
posts.insert(0, message)
# and generate the index with what we got now
index = view.render(locals(), "web/index.html")
f = open("app/data/index.html", "w")
f.write(index.encode("utf-8"))
f.close()
# finally, zap all the old keys
for old in old_keys: index_q.remove(old)
def command(name,
pop=False,
get=False,
keys=False,
remove=False,
count=False,
clear=False):
print("Using queue: %r" % name)
inq = queue.Queue(name)
if pop:
key, msg = inq.pop()
if key:
print("KEY: %s" % key)
print(msg)
elif get:
print(inq.get(get))
elif remove:
inq.remove(remove)
elif count:
print("Queue %s contains %d messages" % (name, inq.count()))
elif clear:
inq.clear()
elif keys:
print("\n".join(inq.keys()))
def POSTING(message, post_name=None, host=None, action=None):
name, address = parseaddr(message['from'])
if not action:
post.post(post_name, address, host, message)
msg = view.respond(locals(),
'mail/page_ready.msg',
From="noreply@%(host)s",
To=message['from'],
Subject="Your page '%(post_name)s' is ready.")
relay.deliver(msg)
# first real message, now we can index it
index_q = queue.Queue("run/posts")
index_q.push(message)
elif action == "delete":
post.delete(post_name, address)
msg = view.respond(locals(),
'mail/deleted.msg',
From="noreply@%(host)s",
To=message['from'],
Subject="Your page '%(post_name)s' was deleted.")
relay.deliver(msg)
else:
logging.debug("Invalid action: %r", action)
return POSTING
def START(message, to=None, host=None):
logging.debug("MESSAGE to %s@%s added to queue.", to, host)
q = queue.Queue('run/queue')
email = "%s@%s" % (to, host)
message = str(message).replace("%", "%%")
new_msg = re.sub(r'(?m)^\To:.*\n?', 'To: %s\n', message, 1) % (email,)
q.push(new_msg)
def start(self, one_shot=False):
"""
Start simply loops indefinitely sleeping and pulling messages
off for processing when they are available.
If you give one_shot=True it will stop once it has exhausted the queue
"""
logging.info("Queue receiver started on queue dir %s", self.queue_dir)
salmonscheduler.schedule()
inq = queue.Queue(self.queue_dir)
while True:
keys = inq.keys()
for key in keys:
msg = inq.get(key)
if msg:
logging.info("Pulled message with key: %r off", key)
salmonmailparser.process_email(key, msg)
logging.info("Removed %r key from queue.\n\n", key)
inq.remove(key)
if one_shot:
return
else:
time.sleep(self.sleep)
def queue(name,
pop=False,
get=False,
keys=False,
remove=False,
count=False,
clear=False):
"""
Lets you do most of the operations available to a queue.
"""
click.echo("Using queue: %r" % name)
inq = queue_module.Queue(name)
if pop:
key, msg = inq.pop()
if key:
click.echo("KEY: %s" % key)
click.echo(msg)
elif get:
click.echo(inq.get(get))
elif remove:
inq.remove(remove)
elif count:
click.echo("Queue %s contains %d messages" % (name, inq.count()))
elif clear:
inq.clear()
elif keys:
click.echo("\n".join(inq.keys()))
def start(self, one_shot=False):
"""
Start simply loops indefinitely sleeping and pulling messages
off for processing when they are available.
If you give one_shot=True it will run once rather than do a big
while loop with a sleep.
"""
logging.info("Queue receiver started on queue dir %s", self.queue_dir)
logging.debug("Sleeping for %d seconds...", self.sleep)
inq = queue.Queue(self.queue_dir)
while True:
keys = inq.keys()
for key in keys:
msg = inq.get(key)
if msg:
logging.debug("Pulled message with key: %r off", key)
self.process_message(msg)
logging.debug("Removed %r key from queue.", key)
inq.remove(key)
if one_shot:
return
else:
time.sleep(self.sleep)
def queue_command(pop=False,
get=False,
keys=False,
remove=False,
count=False,
clear=False,
name="run/queue"):
"""
Let's you do most of the operations available to a queue.
salmon queue (-pop | -get | -remove | -count | -clear | -keys) -name run/queue
"""
print "Using queue: %r" % name
inq = queue.Queue(name)
if pop:
key, msg = inq.pop()
if key:
print "KEY: ", key
print msg
elif get:
print inq.get(get)
elif remove:
inq.remove(remove)
elif count:
print "Queue %s contains %d messages" % (name, inq.count())
elif clear:
inq.clear()
elif keys:
print "\n".join(inq.keys())
else:
print "Give something to do. Try salmon help -for queue to find out what."
sys.exit(1)
return # for unit tests mocking sys.exit
def START(message, to=None, host=None):
"""
@stateless and routes however handlers.log.START routes (everything).
Has @nolocking, but that's alright since it's just writing to a Maildir.
"""
logging.debug("MESSAGE to %s@%s added to queue.", to, host)
q = queue.Queue('run/queue')
q.push(message)
def __init__(self, queue_dir, sleep=10, size_limit=0, oversize_dir=None):
"""
The router should be fully configured and ready to work, the
queue_dir can be a fully qualified path or relative.
"""
self.queue = queue.Queue(queue_dir,
pop_limit=size_limit,
oversize_dir=oversize_dir)
self.queue_dir = queue_dir
self.sleep = sleep
def test_queue_receiver():
receiver = server.QueueReceiver('run/queue')
run_queue = queue.Queue('run/queue')
run_queue.push(str(test_mail_response_plain_text()))
assert run_queue.count() > 0
receiver.start(one_shot=True)
assert_equal(run_queue.count(), 0)
routing.Router.deliver.side_effect = raises_exception
receiver.process_message(mail.MailRequest('localhost', 'test@localhost', 'test@localhost', 'Fake body.'))
def enqueue(list_name, message):
qpath = store_path(list_name, 'queue')
pending = queue.Queue(qpath, safe=True)
white_list_cleanse(message)
key = pending.push(message)
fix_permissions(qpath)
update_json(list_name, key, message)
return key
def test_cleanse_command_with_encoding_error(self, from_message):
runner = CliRunner()
from_message.side_effect = encoding.EncodingError
in_queue = "run/queue"
q = queue.Queue(in_queue)
q.push("hello")
result = runner.invoke(commands.main,
("cleanse", in_queue, "run/cleased"))
self.assertEqual(result.exit_code, 1)
def __init__(self, queue_dir, sleep=2, size_limit=0, oversize_dir=None):
"""
The router should be fully configured and ready to work, the queue_dir
can be a fully qualified path or relative. The option workers dictates
how many threads are started to process messages. Consider adding
``@nolocking`` to your handlers if you are able to.
"""
self.queue = queue.Queue(queue_dir, pop_limit=size_limit,
oversize_dir=oversize_dir)
self.queue_dir = queue_dir
self.sleep = sleep
def __init__(self, pending_queue, storage):
"""
The pending_queue should be a string with the path to the salmon.queue.Queue
that will store pending messages. These messages are the originals the user
sent when they tried to confirm.
Storage should be something that is like ConfirmationStorage so that this
can store things for later verification.
"""
self.pending = queue.Queue(pending_queue)
self.storage = storage
def test_queue_receiver(self, router_mock):
receiver = server.QueueReceiver('run/queue')
run_queue = queue.Queue('run/queue')
run_queue.push(str(generate_mail(factory=mail.MailResponse)))
assert run_queue.count() > 0
receiver.start(one_shot=True)
self.assertEqual(run_queue.count(), 0)
self.assertEqual(run_queue.count(), 0)
self.assertEqual(router_mock.deliver.call_count, 1)
router_mock.deliver.side_effect = RuntimeError("Raised on purpose")
receiver.process_message(mail.MailRequest('localhost', 'test@localhost', 'test@localhost', 'Fake body.'))
def test_push():
q = queue.Queue("run/queue", safe=USE_SAFE)
q.clear()
# the queue doesn't really care if its a request or response, as long
# as the object answers to str(msg)
msg = mail.MailResponse(To="test@localhost", From="test@localhost", Subject="Test", Body="Test")
key = q.push(msg)
assert key, "Didn't get a key for test_get push."
return q
def test_no_connection(self):
q = queue.Queue("run/queue")
msg = mail.MailResponse(To="tests@localhost",
From="tests@localhost",
Subject="Hello",
Body="Test body.")
q.push(msg)
runner = CliRunner()
result = runner.invoke(
commands.main,
("blast", "--host", "127.0.1.2", "--port", "8901", "run/queue"))
self.assertEqual(result.exit_code, 1)
self.assertEqual(result.output,
"Error: [Errno 111] Connection refused\n")
def COMMENTING(message, user_id=None, domain=None, post_name=None, host=None):
address = user_id + '@' + domain
user_dir = post.get_user_dir(address)
if post.user_exists(address):
# stuff it here for now, but we'll just build the file rolling
comments = queue.Queue("%s/comments" % user_dir)
comments.push(message)
contents = markdown(message.body())
comment_file = "%s/%s-comments.html" % (user_dir, post_name)
snippet = view.render(locals(), "web/comments.html")
with open(comment_file, "a") as out:
out.write(snippet)
else:
logging.warning("Attempt to post to user %r but user doesn't exist.", address)
def test_bytes(self):
"""Test that passing a queue raw data works, i.e. as happens in the
undeliverable queue"""
q = queue.Queue("run/queue", safe=self.use_safe)
q.clear()
key = q.push(BYTES_MESSAGE)
assert key, "Didn't get a key"
mail = q.get(key)
assert mail is not None, "Failed to get email from queue"
self.assertEqual(mail['from'], "me@localhost")
self.assertEqual(mail['to'], "you@localhost")
self.assertEqual(mail['subject'], "bob!")
self.assertEqual(mail.body(), "Blobcat")
def test_cleanse_command(self):
q = queue.Queue("run/queue")
msg_count = 3
for i in range(msg_count):
msg = mail.MailResponse(To="tests%s@localhost" % i,
From="tests%s@localhost" % i,
Subject="Hello",
Body="Test body.")
q.push(msg)
self.assertEqual(q.count(), msg_count)
runner = CliRunner()
result = runner.invoke(commands.main,
("cleanse", "run/queue", "run/cleansed"))
self.assertEqual(result.exit_code, 0)
self.assertEqual(q.count(), msg_count)
outbox = mailbox.Maildir("run/cleansed", create=False)
self.assertEqual(len(outbox), msg_count)
def test_ConfirmationEngine_send():
queue.Queue('run/queue').clear()
engine.clear()
list_name = 'testing'
action = 'subscribing to'
host = 'localhost'
message = mail.MailRequest('fakepeer', 'somedude@localhost',
'testing-subscribe@localhost', 'Fake body.')
engine.send(relay(port=8899), 'testing', message, 'confirmation.msg', locals())
confirm = delivered('confirm')
assert delivered('somedude', to_queue=engine.pending)
assert confirm
return confirm
def test_queue_receiver_pool(self, pool_mock):
run_queue = queue.Queue('run/queue')
msg = str(generate_mail(factory=mail.MailResponse))
run_queue.push(msg)
receiver = server.QueueReceiver('run/queue', sleep=10, workers=1)
receiver.start(one_shot=True)
self.assertEqual(receiver.workers.apply_async.call_count, 1)
self.assertEqual(receiver.workers.apply_async.call_args[0], (receiver.process_message,))
args = receiver.workers.apply_async.call_args[1]["args"]
del receiver.workers.apply_async.call_args[1]["args"]
# onlly the "args" kwarg should be present
self.assertEqual(receiver.workers.apply_async.call_args[1], {})
# we can't compare two Mail* objects, so we'll just check the type
self.assertEqual(len(args), 1)
self.assertEqual(type(args[0]), mail.MailRequest)
def test_blast_three_messages(self, client_mock):
q = queue.Queue("run/queue")
msg_count = 3
for i in range(msg_count):
msg = mail.MailResponse(To="tests%s@localhost" % i,
From="tests%s@localhost" % i,
Subject="Hello",
Body="Test body.")
q.push(msg)
runner = CliRunner()
result = runner.invoke(
commands.main,
("blast", "--host", "127.0.0.2", "--port", "8900", "run/queue"))
self.assertEqual(result.exit_code, 0)
self.assertEqual(client_mock.call_count, msg_count)
self.assertEqual(client_mock.call_args_list, [
(("127.0.0.2", 8900), {}),
(("127.0.0.2", 8900), {}),
(("127.0.0.2", 8900), {}),
])
self.assertEqual(client_mock.return_value.sendmail.call_count,
msg_count)
import logging.config
from salmon.routing import Router
from salmon import queue
from . import settings
import os
dir_path = os.path.dirname(os.path.abspath(__file__))
loggingConfig = os.path.join(dir_path, "logging.conf")
logging.config.fileConfig(loggingConfig)
Router.defaults(**settings.router_defaults)
Router.load(settings.handlers)
Router.RELOAD = True
Router.UNDELIVERABLE_QUEUE = queue.Queue("run/undeliverable")
