def test_on_failure_runs_only_once(stub_broker, stub_worker):
on_failure_count = 0
@remoulade.actor
def on_failure(actor_name, exception_name, args, kwargs):
nonlocal on_failure_count
on_failure_count += 1
@remoulade.actor
def fail_actor():
raise Exception()
remoulade.declare_actors([on_failure, fail_actor])
fail_actor.send_with_options(on_failure=on_failure,
max_retries=3,
min_backoff=1,
backoff_strategy="constant")
stub_broker.join(fail_actor.queue_name)
stub_broker.join(on_failure.queue_name)
stub_worker.join()
# The on_failure should have run only once
assert on_failure_count == 1
def test_rabbitmq_broker_can_enqueue_messages_with_priority(rabbitmq_broker):
max_priority = 10
message_processing_order = []
queue_name = "prioritized"
# Given that I have an actor that store priorities
@remoulade.actor(queue_name=queue_name)
def do_work(message_priority):
message_processing_order.append(message_priority)
remoulade.declare_actors([do_work])
worker = Worker(rabbitmq_broker, worker_threads=1)
worker.queue_prefetch = 1
try:
# When I send that actor messages with increasing priorities
for priority in range(max_priority):
do_work.send_with_options(args=(priority, ), priority=priority)
worker.start()
# And then tell the broker to wait for all messages
rabbitmq_broker.join(queue_name, timeout=5000)
worker.join()
# I expect the stored priorities to be saved in decreasing order
assert message_processing_order == list(reversed(range(max_priority)))
finally:
worker.stop()
def test_compositions_are_canceled_on_actor_failure(stub_broker, stub_worker, cancel_backend):
# Given a cancel backend
# And a broker with the cancel middleware
stub_broker.add_middleware(Cancel(backend=cancel_backend))
# And an actor who doesn't fail
@remoulade.actor
def do_work(arg=None):
return 1
# And an actor who fails
@remoulade.actor
def do_fail(arg):
raise Exception
# And those actors are declared
remoulade.declare_actors([do_work, do_fail])
g = group([do_work.message(), do_fail.message() | do_work.message()], cancel_on_error=True)
# When I group a few jobs together and run it
g.run()
stub_broker.join(do_fail.queue_name)
stub_worker.join()
# All actors should have been canceled
assert cancel_backend.is_canceled("", g.group_id)
def test_failure_state_message(self, stub_broker, stub_worker,
state_middleware):
@remoulade.actor
def error():
raise Exception()
remoulade.declare_actors([error])
msg = error.send()
stub_broker.join(error.queue_name)
stub_worker.join()
assert state_middleware.backend.get_state(
msg.message_id).name == StateNamesEnum.Failure
def test_failure_state_message(self, stub_broker, state_middleware, stub_worker, frozen_datetime):
@remoulade.actor
def error():
raise Exception()
remoulade.declare_actors([error])
msg = error.send()
stub_broker.join(error.queue_name)
stub_worker.join()
state = state_middleware.backend.get_state(msg.message_id)
assert state.status == StateStatusesEnum.Failure
assert state.end_datetime.isoformat() == "2020-02-03T00:00:00+00:00"
def test_declare_actors(stub_broker):
# Given that some actors
@remoulade.actor
def add(x, y):
return x + y
@remoulade.actor
def modulo(x, y):
return x % y
actors = [add, modulo]
remoulade.declare_actors(actors)
assert set(stub_broker.actors.values()) == set(actors)
def test_on_failure(stub_broker, stub_worker):
on_failure_count = 0
@remoulade.actor
def on_failure(actor_name, exception_name, args, kwargs):
nonlocal on_failure_count
on_failure_count += 1
@remoulade.actor
def fail_actor():
raise Exception()
remoulade.declare_actors([on_failure, fail_actor])
fail_actor.send_with_options(on_failure=on_failure)
stub_broker.join(fail_actor.queue_name)
stub_broker.join(on_failure.queue_name)
stub_worker.join()
# The on_failure should have run
assert on_failure_count == 1
def test_clean_runs_on_timeout(stub_broker, stub_worker):
on_failure_count = 0
@remoulade.actor
def on_failure(actor_name, exception_name, args, kwargs):
nonlocal on_failure_count
on_failure_count += 1
@remoulade.actor
def do_work():
time.sleep(1)
remoulade.declare_actors([on_failure, do_work])
do_work.send_with_options(on_failure=on_failure, time_limit=1)
stub_broker.join(do_work.queue_name)
stub_broker.join(on_failure.queue_name)
stub_worker.join()
# The on_failure should have run as messages should not be passed in on_failure as actor options
assert on_failure_count == 1
def test_declare_actors_no_broker():
remoulade.broker.global_broker = None
with pytest.raises(ValueError):
remoulade.declare_actors([])
def latency():
p = random.randint(1, 100)
if p == 1:
durations = [3, 3, 3, 1]
elif p <= 10:
durations = [2, 3]
elif p <= 40:
durations = [1, 2]
else:
durations = [1]
for duration in durations:
time.sleep(duration)
remoulade.declare_actors([throughput, fib, latency])
@pytest.mark.skipif(os.getenv("CI") == "1", reason="test skipped on CI")
@pytest.mark.benchmark(group="rabbitmq-100k-throughput")
def test_rabbitmq_process_100k_messages_with_cli(benchmark, info_logging, start_cli):
# Given that I've loaded 100k messages into RabbitMQ
def setup():
# The connection error tests have the side-effect of
# disconnecting this broker so we need to force it to
# reconnect.
broker.channel_pool.clear()
del broker.connection
for _ in range(100000):
throughput.send()
def long_running():
logger = logging.getLogger("long_running")
while True:
logger.info("Sleeping...")
time.sleep(1)
@remoulade.actor(time_limit=5000)
def long_running_with_catch():
logger = logging.getLogger("long_running_with_catch")
try:
while True:
logger.info("Sleeping...")
time.sleep(1)
except remoulade.middleware.time_limit.TimeLimitExceeded:
logger.warning("Time limit exceeded. Aborting...")
remoulade.declare_actors([long_running, long_running_with_catch])
def main():
long_running.send()
long_running_with_catch.send()
if __name__ == "__main__":
sys.exit(main())
broker.add_middleware(Results(backend=backend))
remoulade.set_broker(broker)
remoulade.set_encoder(encoder)
@remoulade.actor(store_results=True)
def request(uri):
return requests.get(uri)
@remoulade.actor(store_results=True)
def count_words(response):
return len(response.text.split(" "))
remoulade.declare_actors([request, count_words])
def main():
parser = argparse.ArgumentParser()
parser.add_argument("uri", nargs="+", help="A website URI.")
arguments = parser.parse_args()
jobs = group([
request.message(uri) | count_words.message() for uri in arguments.uri
]).run()
for uri, count in zip(arguments.uri, jobs.results.get(block=True)):
print(" * {uri} has {count} words".format(uri=uri, count=count))
return 0