def test_rabbitmq_broker_can_enqueue_messages_with_priority(rabbitmq_broker):
max_priority = 10
message_processing_order = []
queue_name = "prioritized"
# Given that I have an actor that store priorities
@dramatiq.actor(queue_name=queue_name)
def do_work(message_priority):
message_processing_order.append(message_priority)
worker = Worker(rabbitmq_broker, worker_threads=1)
worker.queue_prefetch = 1
worker.start()
worker.pause()
try:
# When I send that actor messages with increasing priorities
for priority in range(max_priority):
do_work.send_with_options(args=(priority, ),
broker_priority=priority)
# And then tell the broker to wait for all messages
worker.resume()
rabbitmq_broker.join(queue_name, timeout=5000)
worker.join()
# I expect the stored priorities to be saved in decreasing order
assert message_processing_order == list(reversed(range(max_priority)))
finally:
worker.stop()
def test_abort_polling(
stub_broker: dramatiq.Broker,
stub_worker: dramatiq.Worker,
stub_event_backend: EventBackend,
) -> None:
sentinel = []
abortable = Abortable(backend=stub_event_backend)
stub_broker.add_middleware(abortable)
@dramatiq.actor(abortable=True, max_retries=0)
def abort_with_delay() -> None:
try:
sentinel.append(True)
time.sleep(5)
abortable.abort(message.message_id)
for _ in range(20):
time.sleep(0.1)
sentinel.append(True)
except Abort:
sentinel.append(False)
raise
sentinel.append(True)
stub_broker.emit_after("process_boot")
# If I send it a message
message = abort_with_delay.send()
# Then join on the queue
stub_broker.join(abort_with_delay.queue_name)
stub_worker.join()
# I expect it to shutdown
assert sentinel == [True, False]
def test_actors_can_prioritize_work(stub_broker):
# Given that I have a database of calls
calls = []
# And an actor with high priority
@dramatiq.actor(priority=0)
def hi():
calls.append("hi")
# And an actor with low priority
@dramatiq.actor(priority=10)
def lo():
calls.append("lo")
# If I send both actors a message
lo.send_with_options()
hi.send_with_options()
# Then start a worker and join on their queue
worker = Worker(stub_broker)
worker.start()
stub_broker.join("default")
worker.join()
worker.stop()
# I expect the high priority worker to run first
assert calls == ["hi", "lo"]
def test_rabbitmq_broker_retries_declaring_queues_when_connection_related_errors_occur(rabbitmq_broker):
executed, declare_called = False, False
original_declare = rabbitmq_broker._declare_queue
def flaky_declare_queue(*args, **kwargs):
nonlocal declare_called
if not declare_called:
declare_called = True
raise pika.exceptions.AMQPConnectionError
return original_declare(*args, **kwargs)
# Given that I have a flaky connection to a rabbitmq server
with patch.object(rabbitmq_broker, "_declare_queue", flaky_declare_queue):
# When I declare an actor
@dramatiq.actor(queue_name="flaky_queue")
def do_work():
nonlocal executed
executed = True
# And I send that actor a message
do_work.send()
# And wait for the worker to process the message
worker = Worker(rabbitmq_broker, worker_threads=1)
worker.start()
try:
rabbitmq_broker.join(do_work.queue_name, timeout=5000)
worker.join()
# Then the queue should eventually be declared and the message executed
assert declare_called
assert executed
finally:
worker.stop()
def test_not_abortable(
stub_broker: dramatiq.Broker,
stub_worker: dramatiq.Worker,
stub_event_backend: EventBackend,
) -> None:
aborts, successes = [], []
abortable = Abortable(backend=stub_event_backend)
stub_broker.add_middleware(abortable)
@dramatiq.actor(abortable=False)
def not_abortable() -> None:
try:
for _ in range(10):
time.sleep(0.1)
except Abort:
aborts.append(1)
raise
successes.append(1)
stub_broker.emit_after("process_boot")
# If I send it a message
message = not_abortable.send()
# Then wait and signal the task to terminate
time.sleep(0.1)
abort(message.message_id)
# Then join on the queue
stub_broker.join(not_abortable.queue_name)
stub_worker.join()
# I expect it to shutdown
assert sum(aborts) == 0
assert sum(successes) == 1
def test_abort_notifications_are_received(
stub_broker: dramatiq.Broker,
stub_worker: dramatiq.Worker,
event_backend: EventBackend,
) -> None:
# Given that I have a database
aborts, successes = [], []
abortable = Abortable(backend=event_backend)
stub_broker.add_middleware(abortable)
# And an actor that handles shutdown interrupts
@dramatiq.actor(abortable=True, max_retries=0)
def do_work() -> None:
try:
for _ in range(10):
time.sleep(0.1)
except Abort:
aborts.append(1)
raise
successes.append(1)
stub_broker.emit_after("process_boot")
# If I send it a message
message = do_work.send()
# Then wait and signal the task to terminate
time.sleep(0.1)
abort(message.message_id)
# Then join on the queue
stub_broker.join(do_work.queue_name)
stub_worker.join()
def test_dramatiq_error(
broker: Broker, worker: Worker, backend: WriterBackend, frozen_time: Any
) -> None:
@dramatiq.actor(queue_name="test")
def simple_task_error() -> None:
raise ValueError("Expected")
message = simple_task_error.send_with_options(time_limit=10000)
assert backend.enqueued() == [
EnqueuedLog(
type=LogType.ENQUEUED,
timestamp=datetime.now(),
job_id=message.message_id,
task_id="simple_task_error",
job=JobDetails(
queue="test",
task_path="tests.test_dramatiq.test_dramatiq_error.<locals>"
".simple_task_error",
execute_at=None,
args=[],
kwargs={},
options={"time_limit": 10000},
),
)
]
assert backend.dequeued() == []
assert backend.completed() == []
worker.start()
broker.join(simple_task_error.queue_name)
worker.join()
assert backend.dequeued() == [
DequeuedLog(
job_id=message.message_id,
task_id="simple_task_error",
timestamp=datetime.now(),
type=LogType.DEQUEUED,
)
]
exceptions = backend.exception()
for exception in exceptions:
assert "Expected" in exception.exception
exception.exception = ""
assert exceptions == [
ExceptionLog(
job_id=message.message_id,
task_id="simple_task_error",
timestamp=datetime.now(),
type=LogType.EXCEPTION,
exception="",
)
]
def test_dramatiq_completion(
broker: StubBroker, worker: Worker, backend: WriterBackend, frozen_time: Any
) -> None:
@dramatiq.actor(queue_name="test")
def simple_task(a: str, b: str) -> str:
return "hello"
message = simple_task.send("a", b="b")
assert backend.enqueued() == [
EnqueuedLog(
type=LogType.ENQUEUED,
timestamp=datetime.now(),
job_id=message.message_id,
task_id="simple_task",
job=JobDetails(
queue="test",
task_path=(
"tests.test_dramatiq.test_dramatiq_completion.<locals>"
".simple_task"
),
execute_at=None,
args=["a"],
kwargs={"b": "b"},
options={},
),
)
]
assert backend.dequeued() == []
assert backend.completed() == []
worker.start()
broker.join(simple_task.queue_name)
worker.join()
assert backend.dequeued() == [
DequeuedLog(
job_id=message.message_id,
task_id="simple_task",
timestamp=datetime.now(),
type=LogType.DEQUEUED,
)
]
assert backend.completed() == [
CompletedLog(
job_id=message.message_id,
task_id="simple_task",
timestamp=datetime.now(),
result="hello",
type=LogType.COMPLETED,
)
]
def test_create_range_for_stats_async(
self,
transactional_db,
broker: stub.StubBroker,
worker: dramatiq.Worker,
hosting_provider_with_sample_user: ac_models.Hostingprovider,
green_ip: gc_models.GreencheckIp,
client,
):
"""
Create a collection of daily stats, for a range of dates provided.
"""
broker.declare_queue("default")
generated_dates = self._set_up_dates_for_last_week()
for date in generated_dates:
gc = gc_factories.GreencheckFactory.create(date=date +
relativedelta(hours=2))
# logger.info(f"gc {date}: {gc.__dict__}")
logger.info(f"just this date: { generated_dates[0] }")
gc_models.DailyStat.create_jobs_for_date_range_async(
generated_dates, "total_count")
# Wait for all the tasks to be processed
broker.join("default")
worker.join()
green_stats = gc_models.DailyStat.objects.filter(
green=gc_choices.BoolChoice.YES)
grey_stats = gc_models.DailyStat.objects.filter(
green=gc_choices.BoolChoice.NO)
mixed_stats = gc_models.DailyStat.objects.exclude(
green__in=[gc_choices.BoolChoice.YES, gc_choices.BoolChoice.NO])
# have we generated the expected stats per day?
assert green_stats.count() == 7
assert grey_stats.count() == 7
assert mixed_stats.count() == 7
# we should one count showing zero green checks for each day
assert [stat.count for stat in green_stats] == [0, 0, 0, 0, 0, 0, 0]
# mixed and grey should be the same
assert [stat.count for stat in grey_stats] == [1, 1, 1, 1, 1, 1, 1]
assert [stat.count for stat in grey_stats] == [1, 1, 1, 1, 1, 1, 1]
def test_disabled_log(
broker: Broker,
worker: Worker,
backend: WriterBackend,
actor_log: Optional[bool],
task_log: Optional[bool],
log_expected: Optional[bool],
) -> None:
@dramatiq.actor(queue_name="test", log=actor_log)
def simple_task_with_log_option() -> None:
pass
simple_task_with_log_option.send_with_options(log=task_log)
worker.start()
broker.join(simple_task_with_log_option.queue_name)
worker.join()
expected = 1 if log_expected else 0
assert len(backend.dequeued()) == expected
assert len(backend.completed()) == expected
def test_cancel_notifications_are_received(
stub_broker: dramatiq.Broker,
stub_worker: dramatiq.Worker,
event_backend: EventBackend,
) -> None:
# Given that I have a database
aborts, successes = [], []
abortable = Abortable(backend=event_backend)
stub_broker.add_middleware(abortable)
test_event = Event()
# And an actor that handles shutdown interrupts
@dramatiq.actor(abortable=True, max_retries=0)
def do_work() -> None:
try:
test_event.set()
for _ in range(10):
time.sleep(0.1)
except Abort:
aborts.append(1)
raise
successes.append(1)
stub_broker.emit_after("process_boot")
# If I send it a message
message = do_work.send()
# Then wait
test_event.wait()
abort(message.message_id, mode=AbortMode.CANCEL)
# Then join on the queue
stub_broker.join(do_work.queue_name)
stub_worker.join()
# Task will finished, the cancel won't take any effect.
assert successes
assert not aborts
def test_actors_can_be_assigned_message_age_limits(stub_broker):
# Given that I have a database
runs = []
# And an actor whose messages have an age limit
@dramatiq.actor(max_age=100)
def do_work():
runs.append(1)
# If I send it a message
do_work.send()
# And join on its queue after the age limit has passed
time.sleep(0.1)
worker = Worker(stub_broker, worker_timeout=100)
worker.start()
stub_broker.join(do_work.queue_name)
worker.join()
worker.stop()
# I expect the message to have been skipped
assert sum(runs) == 0
def test_dramatiq_failed(
broker: Broker, worker: Worker, backend: WriterBackend, frozen_time: Any
) -> None:
class FailMessage(Middleware):
def after_process_message(
self,
broker: Broker,
message: Message,
*,
result: Any = None,
exception: Optional[BaseException] = None,
) -> None:
message.fail()
@dramatiq.actor(queue_name="test")
def simple_task_failed() -> None:
return
broker.add_middleware(FailMessage())
message = simple_task_failed.send()
simple_task_failed.send_with_options(log=False)
assert backend.enqueued() == [
EnqueuedLog(
type=LogType.ENQUEUED,
timestamp=datetime.now(),
job_id=message.message_id,
task_id="simple_task_failed",
job=JobDetails(
queue="test",
task_path="tests.test_dramatiq.test_dramatiq_failed.<locals>"
".simple_task_failed",
execute_at=None,
args=[],
kwargs={},
options={},
),
)
]
assert backend.dequeued() == []
assert backend.completed() == []
worker.start()
broker.join(simple_task_failed.queue_name)
worker.join()
assert backend.dequeued() == [
DequeuedLog(
job_id=message.message_id,
task_id="simple_task_failed",
timestamp=datetime.now(),
type=LogType.DEQUEUED,
)
]
exceptions = backend.exception()
for exception in exceptions:
assert "Failed" in exception.exception
exception.exception = ""
assert exceptions == [
ExceptionLog(
job_id=message.message_id,
task_id="simple_task_failed",
timestamp=datetime.now(),
type=LogType.EXCEPTION,
exception="",
)
]
def test_create_stat_async(
self,
transactional_db,
broker: stub.StubBroker,
worker: dramatiq.Worker,
hosting_provider_with_sample_user: ac_models.Hostingprovider,
green_ip: gc_models.GreencheckIp,
client,
):
"""
Create a collection of daily stats, for a range of dates provided,
but have a worker create the stats asynchronously.
"""
broker.declare_queue("default")
assert gc_models.DailyStat.objects.count() == 0
# set up our date range
generated_dates = self._set_up_dates_for_last_week()
for date in generated_dates:
gc_factories.GreencheckFactory.create(date=date +
relativedelta(hours=2))
chosen_date = str(generated_dates[0].date())
# we use the 'send' with the 'transactional_db' fixture here instead of db
# because if we use the regular db fixture, the workers can not see what is
# happening 'inside' this test. TODO: check that this really is the
# explanation for this strange test behaviour
gc_tasks.create_stat_async.send(date_string=chosen_date,
query_name="total_count")
# import ipdb
# ipdb.set_trace()
# Wait for all the tasks to be processed
broker.join(gc_tasks.create_stat_async.queue_name)
worker.join()
# import ipdb
# ipdb.set_trace()
# hae we generate the daily stats?
assert gc_models.DailyStat.objects.count() == 3
# do that they have the right date?
for stat in gc_models.DailyStat.objects.all():
assert str(stat.stat_date) == chosen_date
# do the stats count up to what we expect?
green_daily_stat = gc_models.DailyStat.objects.filter(
green=gc_choices.BoolChoice.YES).first()
grey_daily_stat = gc_models.DailyStat.objects.filter(
green=gc_choices.BoolChoice.NO).first()
mixed_daily_stat = gc_models.DailyStat.objects.exclude(
green__in=[gc_choices.BoolChoice.YES, gc_choices.BoolChoice.NO
]).first()
assert green_daily_stat.count == 0
assert grey_daily_stat.count == 1
assert mixed_daily_stat.count == 1
