def test_oversize_protections(self): # first just make an oversize limited queue overq = queue.Queue("run/queue", pop_limit=10) overq.clear() for i in range(5): overq.push("HELLO" * 100) self.assertEqual(overq.count(), 5) key, msg = overq.pop() assert not key and not msg, "Should get no messages." self.assertEqual(overq.count(), 0) # now make sure that oversize mail is moved to the overq overq = queue.Queue("run/queue", pop_limit=10, oversize_dir="run/big_queue") moveq = queue.Queue("run/big_queue") moveq.clear() for i in range(5): overq.push("HELLO" * 100) key, msg = overq.pop() assert not key and not msg, "Should get no messages." self.assertEqual(overq.count(), 0) self.assertEqual(moveq.count(), 5) moveq.clear() overq.clear()
def setUp(self): super().setUp() # re-create destoryed queues queue.Queue( os.path.join(self._cwd, server_settings.UNDELIVERABLE_QUEUE)).clear() queue.Queue(os.path.join(self._cwd, server_settings.QUEUE_PATH)).clear()
def test_we_dont_get_message(): client = smtplib.SMTP(**server_settings.receiver_config) client.helo() client.sendmail("*****@*****.**", "*****@*****.**", "hello") undelivered = queue.Queue(server_settings.UNDELIVERABLE_QUEUE) assert_equal(len(undelivered.mbox), 1) inbox = queue.Queue(server_settings.QUEUE_PATH) assert_equal(len(inbox.mbox), 0)
def test_we_dont_get_message(self): client = smtplib.SMTP(**server_settings.receiver_config) client.helo() client.sendmail("*****@*****.**", "*****@*****.**", "hello") undelivered = queue.Queue( os.path.join(self._cwd, server_settings.UNDELIVERABLE_QUEUE)) self.assertEqual(len(undelivered), 1) inbox = queue.Queue(os.path.join(self._cwd, server_settings.QUEUE_PATH)) self.assertEqual(len(inbox), 0)
def START(message, to=None, host=None): """ @stateless and routes however handlers.log.START routes (everything). Has @nolocking, but that's alright since it's just writing to a maildir. """ q = queue.Queue('run/queue') q.push(message)
def POSTING(message, post_name=None, host=None): user, address = parseaddr(message['from']) user = user or address post_url = "posts/%s/%s.html" % (address, post_name) index_q = queue.Queue("run/indexed") post_keys = sorted(index_q.keys(), reverse=True) old_keys = post_keys[50:] del post_keys[50:] # find the old one and remove it posts = [] for key in post_keys: msg = index_q.get(key) if msg['x-post-url'] == post_url: # this is the old one, take it out index_q.remove(key) else: posts.append(msg) # update the index and our posts message['X-Post-URL'] = post_url index_q.push(message) posts.insert(0, message) # and generate the index with what we got now index = view.render(locals(), "web/index.html") f = open("app/data/index.html", "w") f.write(index.encode("utf-8")) f.close() # finally, zap all the old keys for old in old_keys: index_q.remove(old)
def command(name, pop=False, get=False, keys=False, remove=False, count=False, clear=False): print("Using queue: %r" % name) inq = queue.Queue(name) if pop: key, msg = inq.pop() if key: print("KEY: %s" % key) print(msg) elif get: print(inq.get(get)) elif remove: inq.remove(remove) elif count: print("Queue %s contains %d messages" % (name, inq.count())) elif clear: inq.clear() elif keys: print("\n".join(inq.keys()))
def POSTING(message, post_name=None, host=None, action=None): name, address = parseaddr(message['from']) if not action: post.post(post_name, address, host, message) msg = view.respond(locals(), 'mail/page_ready.msg', From="noreply@%(host)s", To=message['from'], Subject="Your page '%(post_name)s' is ready.") relay.deliver(msg) # first real message, now we can index it index_q = queue.Queue("run/posts") index_q.push(message) elif action == "delete": post.delete(post_name, address) msg = view.respond(locals(), 'mail/deleted.msg', From="noreply@%(host)s", To=message['from'], Subject="Your page '%(post_name)s' was deleted.") relay.deliver(msg) else: logging.debug("Invalid action: %r", action) return POSTING
def START(message, to=None, host=None): logging.debug("MESSAGE to %s@%s added to queue.", to, host) q = queue.Queue('run/queue') email = "%s@%s" % (to, host) message = str(message).replace("%", "%%") new_msg = re.sub(r'(?m)^\To:.*\n?', 'To: %s\n', message, 1) % (email,) q.push(new_msg)
def start(self, one_shot=False): """ Start simply loops indefinitely sleeping and pulling messages off for processing when they are available. If you give one_shot=True it will stop once it has exhausted the queue """ logging.info("Queue receiver started on queue dir %s", self.queue_dir) salmonscheduler.schedule() inq = queue.Queue(self.queue_dir) while True: keys = inq.keys() for key in keys: msg = inq.get(key) if msg: logging.info("Pulled message with key: %r off", key) salmonmailparser.process_email(key, msg) logging.info("Removed %r key from queue.\n\n", key) inq.remove(key) if one_shot: return else: time.sleep(self.sleep)
def queue(name, pop=False, get=False, keys=False, remove=False, count=False, clear=False): """ Lets you do most of the operations available to a queue. """ click.echo("Using queue: %r" % name) inq = queue_module.Queue(name) if pop: key, msg = inq.pop() if key: click.echo("KEY: %s" % key) click.echo(msg) elif get: click.echo(inq.get(get)) elif remove: inq.remove(remove) elif count: click.echo("Queue %s contains %d messages" % (name, inq.count())) elif clear: inq.clear() elif keys: click.echo("\n".join(inq.keys()))
def start(self, one_shot=False): """ Start simply loops indefinitely sleeping and pulling messages off for processing when they are available. If you give one_shot=True it will run once rather than do a big while loop with a sleep. """ logging.info("Queue receiver started on queue dir %s", self.queue_dir) logging.debug("Sleeping for %d seconds...", self.sleep) inq = queue.Queue(self.queue_dir) while True: keys = inq.keys() for key in keys: msg = inq.get(key) if msg: logging.debug("Pulled message with key: %r off", key) self.process_message(msg) logging.debug("Removed %r key from queue.", key) inq.remove(key) if one_shot: return else: time.sleep(self.sleep)
def queue_command(pop=False, get=False, keys=False, remove=False, count=False, clear=False, name="run/queue"): """ Let's you do most of the operations available to a queue. salmon queue (-pop | -get | -remove | -count | -clear | -keys) -name run/queue """ print "Using queue: %r" % name inq = queue.Queue(name) if pop: key, msg = inq.pop() if key: print "KEY: ", key print msg elif get: print inq.get(get) elif remove: inq.remove(remove) elif count: print "Queue %s contains %d messages" % (name, inq.count()) elif clear: inq.clear() elif keys: print "\n".join(inq.keys()) else: print "Give something to do. Try salmon help -for queue to find out what." sys.exit(1) return # for unit tests mocking sys.exit
def START(message, to=None, host=None): """ @stateless and routes however handlers.log.START routes (everything). Has @nolocking, but that's alright since it's just writing to a Maildir. """ logging.debug("MESSAGE to %s@%s added to queue.", to, host) q = queue.Queue('run/queue') q.push(message)
def __init__(self, queue_dir, sleep=10, size_limit=0, oversize_dir=None): """ The router should be fully configured and ready to work, the queue_dir can be a fully qualified path or relative. """ self.queue = queue.Queue(queue_dir, pop_limit=size_limit, oversize_dir=oversize_dir) self.queue_dir = queue_dir self.sleep = sleep
def test_queue_receiver(): receiver = server.QueueReceiver('run/queue') run_queue = queue.Queue('run/queue') run_queue.push(str(test_mail_response_plain_text())) assert run_queue.count() > 0 receiver.start(one_shot=True) assert_equal(run_queue.count(), 0) routing.Router.deliver.side_effect = raises_exception receiver.process_message(mail.MailRequest('localhost', 'test@localhost', 'test@localhost', 'Fake body.'))
def enqueue(list_name, message): qpath = store_path(list_name, 'queue') pending = queue.Queue(qpath, safe=True) white_list_cleanse(message) key = pending.push(message) fix_permissions(qpath) update_json(list_name, key, message) return key
def test_cleanse_command_with_encoding_error(self, from_message): runner = CliRunner() from_message.side_effect = encoding.EncodingError in_queue = "run/queue" q = queue.Queue(in_queue) q.push("hello") result = runner.invoke(commands.main, ("cleanse", in_queue, "run/cleased")) self.assertEqual(result.exit_code, 1)
def __init__(self, queue_dir, sleep=2, size_limit=0, oversize_dir=None): """ The router should be fully configured and ready to work, the queue_dir can be a fully qualified path or relative. The option workers dictates how many threads are started to process messages. Consider adding ``@nolocking`` to your handlers if you are able to. """ self.queue = queue.Queue(queue_dir, pop_limit=size_limit, oversize_dir=oversize_dir) self.queue_dir = queue_dir self.sleep = sleep
def __init__(self, pending_queue, storage): """ The pending_queue should be a string with the path to the salmon.queue.Queue that will store pending messages. These messages are the originals the user sent when they tried to confirm. Storage should be something that is like ConfirmationStorage so that this can store things for later verification. """ self.pending = queue.Queue(pending_queue) self.storage = storage
def test_queue_receiver(self, router_mock): receiver = server.QueueReceiver('run/queue') run_queue = queue.Queue('run/queue') run_queue.push(str(generate_mail(factory=mail.MailResponse))) assert run_queue.count() > 0 receiver.start(one_shot=True) self.assertEqual(run_queue.count(), 0) self.assertEqual(run_queue.count(), 0) self.assertEqual(router_mock.deliver.call_count, 1) router_mock.deliver.side_effect = RuntimeError("Raised on purpose") receiver.process_message(mail.MailRequest('localhost', 'test@localhost', 'test@localhost', 'Fake body.'))
def test_push(): q = queue.Queue("run/queue", safe=USE_SAFE) q.clear() # the queue doesn't really care if its a request or response, as long # as the object answers to str(msg) msg = mail.MailResponse(To="test@localhost", From="test@localhost", Subject="Test", Body="Test") key = q.push(msg) assert key, "Didn't get a key for test_get push." return q
def test_no_connection(self): q = queue.Queue("run/queue") msg = mail.MailResponse(To="tests@localhost", From="tests@localhost", Subject="Hello", Body="Test body.") q.push(msg) runner = CliRunner() result = runner.invoke( commands.main, ("blast", "--host", "127.0.1.2", "--port", "8901", "run/queue")) self.assertEqual(result.exit_code, 1) self.assertEqual(result.output, "Error: [Errno 111] Connection refused\n")
def COMMENTING(message, user_id=None, domain=None, post_name=None, host=None): address = user_id + '@' + domain user_dir = post.get_user_dir(address) if post.user_exists(address): # stuff it here for now, but we'll just build the file rolling comments = queue.Queue("%s/comments" % user_dir) comments.push(message) contents = markdown(message.body()) comment_file = "%s/%s-comments.html" % (user_dir, post_name) snippet = view.render(locals(), "web/comments.html") with open(comment_file, "a") as out: out.write(snippet) else: logging.warning("Attempt to post to user %r but user doesn't exist.", address)
def test_bytes(self): """Test that passing a queue raw data works, i.e. as happens in the undeliverable queue""" q = queue.Queue("run/queue", safe=self.use_safe) q.clear() key = q.push(BYTES_MESSAGE) assert key, "Didn't get a key" mail = q.get(key) assert mail is not None, "Failed to get email from queue" self.assertEqual(mail['from'], "me@localhost") self.assertEqual(mail['to'], "you@localhost") self.assertEqual(mail['subject'], "bob!") self.assertEqual(mail.body(), "Blobcat")
def test_cleanse_command(self): q = queue.Queue("run/queue") msg_count = 3 for i in range(msg_count): msg = mail.MailResponse(To="tests%s@localhost" % i, From="tests%s@localhost" % i, Subject="Hello", Body="Test body.") q.push(msg) self.assertEqual(q.count(), msg_count) runner = CliRunner() result = runner.invoke(commands.main, ("cleanse", "run/queue", "run/cleansed")) self.assertEqual(result.exit_code, 0) self.assertEqual(q.count(), msg_count) outbox = mailbox.Maildir("run/cleansed", create=False) self.assertEqual(len(outbox), msg_count)
def test_ConfirmationEngine_send(): queue.Queue('run/queue').clear() engine.clear() list_name = 'testing' action = 'subscribing to' host = 'localhost' message = mail.MailRequest('fakepeer', 'somedude@localhost', 'testing-subscribe@localhost', 'Fake body.') engine.send(relay(port=8899), 'testing', message, 'confirmation.msg', locals()) confirm = delivered('confirm') assert delivered('somedude', to_queue=engine.pending) assert confirm return confirm
def test_queue_receiver_pool(self, pool_mock): run_queue = queue.Queue('run/queue') msg = str(generate_mail(factory=mail.MailResponse)) run_queue.push(msg) receiver = server.QueueReceiver('run/queue', sleep=10, workers=1) receiver.start(one_shot=True) self.assertEqual(receiver.workers.apply_async.call_count, 1) self.assertEqual(receiver.workers.apply_async.call_args[0], (receiver.process_message,)) args = receiver.workers.apply_async.call_args[1]["args"] del receiver.workers.apply_async.call_args[1]["args"] # onlly the "args" kwarg should be present self.assertEqual(receiver.workers.apply_async.call_args[1], {}) # we can't compare two Mail* objects, so we'll just check the type self.assertEqual(len(args), 1) self.assertEqual(type(args[0]), mail.MailRequest)
def test_blast_three_messages(self, client_mock): q = queue.Queue("run/queue") msg_count = 3 for i in range(msg_count): msg = mail.MailResponse(To="tests%s@localhost" % i, From="tests%s@localhost" % i, Subject="Hello", Body="Test body.") q.push(msg) runner = CliRunner() result = runner.invoke( commands.main, ("blast", "--host", "127.0.0.2", "--port", "8900", "run/queue")) self.assertEqual(result.exit_code, 0) self.assertEqual(client_mock.call_count, msg_count) self.assertEqual(client_mock.call_args_list, [ (("127.0.0.2", 8900), {}), (("127.0.0.2", 8900), {}), (("127.0.0.2", 8900), {}), ]) self.assertEqual(client_mock.return_value.sendmail.call_count, msg_count)
import logging.config from salmon.routing import Router from salmon import queue from . import settings import os dir_path = os.path.dirname(os.path.abspath(__file__)) loggingConfig = os.path.join(dir_path, "logging.conf") logging.config.fileConfig(loggingConfig) Router.defaults(**settings.router_defaults) Router.load(settings.handlers) Router.RELOAD = True Router.UNDELIVERABLE_QUEUE = queue.Queue("run/undeliverable")