def test_rabbitmq_broker_can_enqueue_messages_with_priority(rabbitmq_broker):
max_priority = 10
message_processing_order = []
queue_name = "prioritized"
# Given that I have an actor that store priorities
@remoulade.actor(queue_name=queue_name)
def do_work(message_priority):
message_processing_order.append(message_priority)
remoulade.declare_actors([do_work])
worker = Worker(rabbitmq_broker, worker_threads=1)
worker.queue_prefetch = 1
try:
# When I send that actor messages with increasing priorities
for priority in range(max_priority):
do_work.send_with_options(args=(priority, ), priority=priority)
worker.start()
# And then tell the broker to wait for all messages
rabbitmq_broker.join(queue_name, timeout=5000)
worker.join()
# I expect the stored priorities to be saved in decreasing order
assert message_processing_order == list(reversed(range(max_priority)))
finally:
worker.stop()
def worker(*args, **kwargs):
try:
worker = Worker(*args, **kwargs)
worker.start()
yield worker
finally:
worker.stop()
def test_max_tasks(stub_broker):
worker = Worker(stub_broker, worker_timeout=100, worker_threads=1)
worker.start()
stub_broker.add_middleware(MaxTasks(max_tasks=20))
result = 0
@remoulade.actor
def do_work():
nonlocal result
result += 1
stub_broker.declare_actor(do_work)
for _ in range(30):
do_work.send()
time.sleep(0.1)
# The worker should have run tasks until he reached max_tasks and then stopped
assert result == 20
assert worker.worker_stopped
def worker_process(args, worker_id, logging_fd):
try:
# Re-seed the random number generator from urandom on
# supported platforms. This should make it so that worker
# processes don't all follow the same sequence.
random.seed()
logging_pipe = os.fdopen(logging_fd, "w")
logger = setup_worker_logging(args, worker_id, logging_pipe)
for module in args.modules:
importlib.import_module(module)
broker = get_broker()
broker.emit_after("process_boot")
worker = Worker(broker,
queues=args.queues,
worker_threads=args.threads,
prefetch_multiplier=args.prefetch_multiplier)
worker.start()
except ImportError:
logger.exception("Failed to import module.")
return os._exit(RET_IMPORT)
def termhandler(signum, frame):
nonlocal running # type: ignore
if running:
logger.info("Stopping worker process...")
running = False
else:
logger.warning("Killing worker process...")
return os._exit(RET_KILLED)
logger.info("Worker process is ready for action.")
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, termhandler)
signal.signal(signal.SIGHUP, termhandler)
running = True
ret_code = RET_OK
while running:
if worker.consumer_stopped:
running = False
ret_code = RET_CONNECT
else:
time.sleep(1)
worker.stop()
broker.emit_before("process_stop")
broker.close()
logging_pipe.close()
return ret_code
def start_worker(args, logger):
try:
for module in args.modules:
importlib.import_module(module)
broker = get_broker()
broker.emit_after("process_boot")
worker = Worker(
broker, queues=args.queues, worker_threads=args.threads, prefetch_multiplier=args.prefetch_multiplier
)
worker.start()
except ImportError:
logger.exception("Failed to import module.")
return os._exit(RET_IMPORT)
def termhandler(signum, frame):
nonlocal running # type: ignore
if running:
logger.info("Stopping worker...")
running = False
else:
logger.warning("Killing worker...")
return os._exit(RET_KILLED)
logger.info("Worker is ready for action.")
signal.signal(signal.SIGINT, termhandler)
signal.signal(signal.SIGTERM, termhandler)
signal.signal(signal.SIGHUP, termhandler)
running = True
ret_code = RET_OK
while running:
if worker.consumer_stopped:
running = False
ret_code = RET_CONNECT
if worker.worker_stopped:
running = False
ret_code = RET_OK
logger.info("Worker thread is not running anymore, stopping Worker.")
else:
time.sleep(1)
worker.stop()
broker.emit_before("process_stop")
broker.close()
return ret_code
def test_local_broker_worker_forbidden(local_broker):
with pytest.raises(RemouladeError):
Worker(broker=local_broker)
def rabbitmq_worker(rabbitmq_broker):
worker = Worker(rabbitmq_broker, worker_timeout=100, worker_threads=32)
worker.start()
yield worker
worker.stop()
def stub_worker(stub_broker):
worker = Worker(stub_broker, worker_timeout=100, worker_threads=32)
worker.start()
yield worker
worker.stop()
