def test_on_failure_runs_only_once(stub_broker, stub_worker): on_failure_count = 0 @remoulade.actor def on_failure(actor_name, exception_name, args, kwargs): nonlocal on_failure_count on_failure_count += 1 @remoulade.actor def fail_actor(): raise Exception() remoulade.declare_actors([on_failure, fail_actor]) fail_actor.send_with_options(on_failure=on_failure, max_retries=3, min_backoff=1, backoff_strategy="constant") stub_broker.join(fail_actor.queue_name) stub_broker.join(on_failure.queue_name) stub_worker.join() # The on_failure should have run only once assert on_failure_count == 1
def test_rabbitmq_broker_can_enqueue_messages_with_priority(rabbitmq_broker): max_priority = 10 message_processing_order = [] queue_name = "prioritized" # Given that I have an actor that store priorities @remoulade.actor(queue_name=queue_name) def do_work(message_priority): message_processing_order.append(message_priority) remoulade.declare_actors([do_work]) worker = Worker(rabbitmq_broker, worker_threads=1) worker.queue_prefetch = 1 try: # When I send that actor messages with increasing priorities for priority in range(max_priority): do_work.send_with_options(args=(priority, ), priority=priority) worker.start() # And then tell the broker to wait for all messages rabbitmq_broker.join(queue_name, timeout=5000) worker.join() # I expect the stored priorities to be saved in decreasing order assert message_processing_order == list(reversed(range(max_priority))) finally: worker.stop()
def test_compositions_are_canceled_on_actor_failure(stub_broker, stub_worker, cancel_backend): # Given a cancel backend # And a broker with the cancel middleware stub_broker.add_middleware(Cancel(backend=cancel_backend)) # And an actor who doesn't fail @remoulade.actor def do_work(arg=None): return 1 # And an actor who fails @remoulade.actor def do_fail(arg): raise Exception # And those actors are declared remoulade.declare_actors([do_work, do_fail]) g = group([do_work.message(), do_fail.message() | do_work.message()], cancel_on_error=True) # When I group a few jobs together and run it g.run() stub_broker.join(do_fail.queue_name) stub_worker.join() # All actors should have been canceled assert cancel_backend.is_canceled("", g.group_id)
def test_failure_state_message(self, stub_broker, stub_worker, state_middleware): @remoulade.actor def error(): raise Exception() remoulade.declare_actors([error]) msg = error.send() stub_broker.join(error.queue_name) stub_worker.join() assert state_middleware.backend.get_state( msg.message_id).name == StateNamesEnum.Failure
def test_failure_state_message(self, stub_broker, state_middleware, stub_worker, frozen_datetime): @remoulade.actor def error(): raise Exception() remoulade.declare_actors([error]) msg = error.send() stub_broker.join(error.queue_name) stub_worker.join() state = state_middleware.backend.get_state(msg.message_id) assert state.status == StateStatusesEnum.Failure assert state.end_datetime.isoformat() == "2020-02-03T00:00:00+00:00"
def test_declare_actors(stub_broker): # Given that some actors @remoulade.actor def add(x, y): return x + y @remoulade.actor def modulo(x, y): return x % y actors = [add, modulo] remoulade.declare_actors(actors) assert set(stub_broker.actors.values()) == set(actors)
def test_on_failure(stub_broker, stub_worker): on_failure_count = 0 @remoulade.actor def on_failure(actor_name, exception_name, args, kwargs): nonlocal on_failure_count on_failure_count += 1 @remoulade.actor def fail_actor(): raise Exception() remoulade.declare_actors([on_failure, fail_actor]) fail_actor.send_with_options(on_failure=on_failure) stub_broker.join(fail_actor.queue_name) stub_broker.join(on_failure.queue_name) stub_worker.join() # The on_failure should have run assert on_failure_count == 1
def test_clean_runs_on_timeout(stub_broker, stub_worker): on_failure_count = 0 @remoulade.actor def on_failure(actor_name, exception_name, args, kwargs): nonlocal on_failure_count on_failure_count += 1 @remoulade.actor def do_work(): time.sleep(1) remoulade.declare_actors([on_failure, do_work]) do_work.send_with_options(on_failure=on_failure, time_limit=1) stub_broker.join(do_work.queue_name) stub_broker.join(on_failure.queue_name) stub_worker.join() # The on_failure should have run as messages should not be passed in on_failure as actor options assert on_failure_count == 1
def test_declare_actors_no_broker(): remoulade.broker.global_broker = None with pytest.raises(ValueError): remoulade.declare_actors([])
def latency(): p = random.randint(1, 100) if p == 1: durations = [3, 3, 3, 1] elif p <= 10: durations = [2, 3] elif p <= 40: durations = [1, 2] else: durations = [1] for duration in durations: time.sleep(duration) remoulade.declare_actors([throughput, fib, latency]) @pytest.mark.skipif(os.getenv("CI") == "1", reason="test skipped on CI") @pytest.mark.benchmark(group="rabbitmq-100k-throughput") def test_rabbitmq_process_100k_messages_with_cli(benchmark, info_logging, start_cli): # Given that I've loaded 100k messages into RabbitMQ def setup(): # The connection error tests have the side-effect of # disconnecting this broker so we need to force it to # reconnect. broker.channel_pool.clear() del broker.connection for _ in range(100000): throughput.send()
def long_running(): logger = logging.getLogger("long_running") while True: logger.info("Sleeping...") time.sleep(1) @remoulade.actor(time_limit=5000) def long_running_with_catch(): logger = logging.getLogger("long_running_with_catch") try: while True: logger.info("Sleeping...") time.sleep(1) except remoulade.middleware.time_limit.TimeLimitExceeded: logger.warning("Time limit exceeded. Aborting...") remoulade.declare_actors([long_running, long_running_with_catch]) def main(): long_running.send() long_running_with_catch.send() if __name__ == "__main__": sys.exit(main())
broker.add_middleware(Results(backend=backend)) remoulade.set_broker(broker) remoulade.set_encoder(encoder) @remoulade.actor(store_results=True) def request(uri): return requests.get(uri) @remoulade.actor(store_results=True) def count_words(response): return len(response.text.split(" ")) remoulade.declare_actors([request, count_words]) def main(): parser = argparse.ArgumentParser() parser.add_argument("uri", nargs="+", help="A website URI.") arguments = parser.parse_args() jobs = group([ request.message(uri) | count_words.message() for uri in arguments.uri ]).run() for uri, count in zip(arguments.uri, jobs.results.get(block=True)): print(" * {uri} has {count} words".format(uri=uri, count=count)) return 0