def test_pipelines_expose_completion_stats(stub_broker, stub_worker, result_backend): # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # And an actor that waits some amount of time result_backend.store_results, event_result = mock_func( result_backend.store_results) event_count = [threading.Event() for _ in range(4)] @remoulade.actor(store_results=True) def wait(n): event_count[n].wait(3) return n + 1 # And this actor is declared stub_broker.declare_actor(wait) # When I pipe some messages intended for that actor together and run the pipeline pipe = wait.message(0) | wait.message() | wait.message() | wait.message() pipe.run() # Then every time a job in the pipeline completes, the completed_count should increase for count in range(0, len(pipe)): event_count[count].set() event_result.wait(2) event_result.clear() assert pipe.results.completed_count == count + 1 # Finally, completed should be true assert pipe.results.completed
def test_complex_pipelines(stub_broker, stub_worker, result_backend): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # Given an actor that stores results @remoulade.actor(store_results=True) def do_work(): return 1 # Given an actor that stores results @remoulade.actor(store_results=True) def add(a): return 1 + a # Given an actor that stores results @remoulade.actor(store_results=True) def do_sum(results): return sum(results) # And this actor is declared stub_broker.declare_actor(do_work) stub_broker.declare_actor(do_sum) stub_broker.declare_actor(add) pipe = do_work.message_with_options(pipe_ignore=True) | add.message() | add.message() # return 3 [1, 2, 3] ? g = group([pipe, add.message(), add.message(), do_work.message_with_options(pipe_ignore=True)]) # return [3,2,2,1] final_pipe = do_work.message() | g | do_sum.message() | add.message() # return 9 final_pipe.run() result = final_pipe.result.get(block=True) assert 9 == result
def test_multiple_groups_pipelines(stub_broker, stub_worker, result_backend): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # Given an actor that stores results @remoulade.actor(store_results=True) def do_work(): return 1 # Given an actor that stores results @remoulade.actor(store_results=True) def do_sum(results): return sum(results) # And this actor is declared stub_broker.declare_actor(do_work) stub_broker.declare_actor(do_sum) pipe = pipeline( [group([do_work.message(), do_work.message()]), group([do_sum.message(), do_sum.message()]), do_sum.message()] ).run() result = pipe.result.get(block=True) assert 4 == result
def test_actors_can_store_results(stub_broker, stub_worker, result_backend, forget, block): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # And an actor that stores results @remoulade.actor(store_results=True) def do_work(): return 42 # And this actor is declared stub_broker.declare_actor(do_work) # When I send that actor a message message = do_work.send() # And wait for a result if not block: stub_broker.join(do_work.queue_name) stub_worker.join() result = message.result.get(block=block, forget=forget) assert isinstance(message.result, Result) # Then the result should be what the actor returned assert result == 42
def test_pipelines_expose_completion_stats(stub_broker, stub_worker, result_backend): # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # And an actor that waits some amount of time condition = Condition() @remoulade.actor(store_results=True) def wait(n): time.sleep(n) with condition: condition.notify_all() return n # And this actor is declared stub_broker.declare_actor(wait) # When I pipe some messages intended for that actor together and run the pipeline pipe = wait.message(1) | wait.message() pipe.run() # Then every time a job in the pipeline completes, the completed_count should increase for count in range(1, len(pipe) + 1): with condition: condition.wait(2) time.sleep(0.1) # give the worker time to set the result assert pipe.results.completed_count == count # Finally, completed should be true assert pipe.results.completed
def test_redis_backend_keep_ttl_all_time(stub_broker, stub_worker, redis_result_backend, block, forget): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=redis_result_backend)) # And an actor that stores a result with a result_ttl result_ttl = 100 * 1000 @remoulade.actor(store_results=True, result_ttl=result_ttl) def do_work(): return 42 # And this actor is declared stub_broker.declare_actor(do_work) # When I send that actor a message message = do_work.send() # And wait for a result if not block: stub_broker.join(do_work.queue_name) stub_worker.join() # The result should have a TTL in redis assert redis_result_backend.client.ttl( redis_result_backend.build_message_key(message.message_id)) > 0 # Then I should get that result back assert message.result.get(block=block, forget=forget) == 42 # The forgotten result should still have a TTL in redis assert redis_result_backend.client.ttl( redis_result_backend.build_message_key(message.message_id)) > 0
def test_retry_if_saving_result_fail(stub_broker, stub_worker): with patch.object(ResultBackend, "store_results") as mock_store_results: mock_store_results.side_effect = Exception("Cannot save result") middleware = Results(backend=StubBackend()) stub_broker.add_middleware(middleware) attempts = [] # And an actor that stores results @remoulade.actor(store_results=True) def do_work(): attempts.append(1) # And this actor is declared stub_broker.declare_actor(do_work) # When I send that actor a message do_work.send() # And wait for a result stub_broker.join(do_work.queue_name) stub_worker.join() # The actor has been tried 4 times assert len(attempts) == 4
def test_retrieving_a_result_can_time_out(stub_broker, stub_worker, result_backend, forget): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # And an actor that sleeps for a long time before it stores a result @remoulade.actor(store_results=True) def do_work(): time.sleep(0.5) return 42 # And this actor is declared stub_broker.declare_actor(do_work) # When I send that actor a message message = do_work.send() # And wait for a result # Then a ResultTimeout error should be raised with pytest.raises(ResultTimeout): result_backend.get_result(message.message_id, block=True, timeout=200, forget=forget)
def test_messages_can_get_completed(stub_broker, stub_worker, result_backend, error): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # And an actor that stores results @remoulade.actor(store_results=True) def do_work(): if error: raise ValueError() else: return 42 # And this actor is declared stub_broker.declare_actor(do_work) # When I send that actor a message message = do_work.send() # And wait for a result stub_broker.join(do_work.queue_name) stub_worker.join() result = message.result # we can get the completion assert result.completed result.get(forget=True, raise_on_error=False) # even after a forget assert result.completed
def test_result_get_forget_not_store_if_no_result(stub_broker, stub_worker, result_backend): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) event = threading.Event() # And an actor that stores results @remoulade.actor(store_results=True) def do_work(): event.wait(2) return 42 # And this actor is declared stub_broker.declare_actor(do_work) # When I send that actor a message message = do_work.send() result = message.result # If I get the result and forget it with pytest.raises(ResultMissing): result.get(forget=True) event.set() # It should not store a forgotten result if there is no key assert result.get(block=True) == 42
def test_messages_can_get_results_and_forget(stub_broker, stub_worker, result_backend, block): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # And an actor that stores a result @remoulade.actor(store_results=True) def do_work(): return 42 # And this actor is declared stub_broker.declare_actor(do_work) # When I send that actor a message message = do_work.send() # And wait for a result if not block: stub_broker.join(do_work.queue_name) stub_worker.join() # Then I should get that result back assert message.result.get(block=block, forget=True) == 42 # If I ask again for the same result it should have been forgotten assert message.result.get() is None
def test_groups_expose_completion_stats(stub_broker, stub_worker, result_backend): # Given that I have a result backend stub_broker.add_middleware(Results(backend=result_backend)) # And an actor that waits some amount of time condition = Condition() @remoulade.actor(store_results=True) def wait(n): time.sleep(n) with condition: condition.notify_all() return n # And this actor is declared stub_broker.declare_actor(wait) # When I group messages of varying durations together and run the group g = group(wait.message(n) for n in range(1, 4)) g.run() # Then every time a job in the group completes, the completed_count should increase for count in range(1, len(g) + 1): with condition: condition.wait(5) time.sleep(0.1) # give the worker time to set the result assert g.results.completed_count == count # Finally, completed should be true assert g.results.completed
def test_store_errors(stub_broker, result_backend, stub_worker, block): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # And an actor that store a result and fail @remoulade.actor(store_results=True) def do_work(): raise ValueError() # And this actor is declared stub_broker.declare_actor(do_work) # When I send that actor a message message = do_work.send() # And wait for a result if not block: stub_broker.join(do_work.queue_name) stub_worker.join() # Then I should get an ErrorStored error_stored = message.result.get(block=block, raise_on_error=False) assert isinstance(error_stored, ErrorStored) assert str(error_stored) == "ValueError()"
def test_store_errors_after_no_more_retry(stub_broker, result_backend, stub_worker): # Given that I have a database failures = [] # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # Given an actor that stores results @remoulade.actor(max_retries=3, store_results=True, min_backoff=10, max_backoff=100) def do_work(): failures.append(1) raise ValueError() # And this actor is declared stub_broker.declare_actor(do_work) # When I send that actor a message, message = do_work.send() stub_broker.join(do_work.queue_name) stub_worker.join() # I get an error with pytest.raises(Exception) as e: message.result.get(block=True) assert str(e.value) == "ValueError()" # all the retries have been made assert sum(failures) == 4
def test_groups_execute_jobs_in_parallel(stub_broker, stub_worker, result_backend): # Given that I have a result backend stub_broker.add_middleware(Results(backend=result_backend)) # And I have an actor that sleeps for 100ms @remoulade.actor(store_results=True) def wait(): time.sleep(0.1) # And this actor is declared stub_broker.declare_actor(wait) # When I group multiple of these actors together and run them t = time.monotonic() g = group([wait.message() for _ in range(5)]) g.run() # group message_ids are no stored if not needed with pytest.raises(MessageIdsMissing): result_backend.get_group_message_ids(g.group_id) # And wait on the group to complete results = list(g.results.get(block=True)) # Then the total elapsed time should be less than 500ms assert time.monotonic() - t <= 0.5 # And I should get back as many results as there were jobs in the group assert len(results) == len(g) # And the group should be completed assert g.results.completed assert isinstance(g.results, CollectionResults)
def test_raise_on_error(stub_broker, result_backend, stub_worker, block): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # And an actor that store a result and fail @remoulade.actor(store_results=True) def do_work(): raise ValueError() # And this actor is declared stub_broker.declare_actor(do_work) # When I send that actor a message message = do_work.send() # And wait for a result if not block: stub_broker.join(do_work.queue_name) stub_worker.join() # It should raise an error with pytest.raises(ErrorStored) as e: message.result.get(block=block) assert str(e.value) == "ValueError()"
def test_reduce_messages(stub_broker, stub_worker, result_backend): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # Given an actor that stores results @remoulade.actor(store_results=True) def do_work(): return 1 # Given an actor that stores results @remoulade.actor(store_results=True) def merge(results): return sum(results) # And this actor is declared stub_broker.declare_actor(do_work) stub_broker.declare_actor(merge) merged_message = reduce((do_work.message() for _ in range(10)), merge) merged_message.run() result = merged_message.result.get(block=True) assert 10 == result
def test_local_broker_cannot_have_non_local_backend(local_broker, result_backend): # Which is not a LocalBackend assert not isinstance(result_backend, LocalBackend) # Cannot be used with LocalBroker with pytest.raises(RuntimeError): local_broker.add_middleware(Results(backend=result_backend))
def test_group_forget(stub_broker, result_backend, stub_worker, block): # Given a result backend stub_broker.add_middleware(Results(backend=result_backend)) # Given an actor that stores results @remoulade.actor(store_results=True) def do_work(): return 42 # And this actor is declared stub_broker.declare_actor(do_work) # And I've run a group messages = [do_work.message() for _ in range(5)] g = group(messages) g.run() # If i wait for the group to be completed if not block: stub_broker.join(do_work.queue_name) stub_worker.join() # If i forget the results results = g.results.get(block=block, forget=True) assert list(results) == [42] * 5 # All messages have been forgotten results = g.results.get() assert list(results) == [None] * 5
def test_retry_if_increment_group_completion_fail(stub_broker, stub_worker): with patch.object( StubBackend, "increment_group_completion") as mock_increment_group_completion: mock_increment_group_completion.side_effect = Exception( "Cannot increment") middleware = Results(backend=StubBackend()) stub_broker.add_middleware(middleware) attempts = [] # And an actor that stores results @remoulade.actor(store_results=True) def do_work(*args): attempts.append(1) # And this actor is declared stub_broker.declare_actor(do_work) # When I send that actor a message (group([do_work.message(), do_work.message()]) | do_work.message()).run() # And wait for a result stub_broker.join(do_work.queue_name) stub_worker.join() # The actor has been tried 8 times (4 time each do_work and never the last one) assert len(attempts) == 8
def test_pipelines_with_groups(stub_broker, stub_worker, result_backend): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # Given an actor that stores results @remoulade.actor(store_results=True) def do_work(a): return a # Given an actor that stores results @remoulade.actor(store_results=True) def do_sum(results): return sum(results) # And this actor is declared stub_broker.declare_actor(do_work) stub_broker.declare_actor(do_sum) # When I pipe some messages intended for that actor together and run the pipeline g = group([do_work.message(12), do_work.message(15)]) pipe = g | do_sum.message() pipe.build() assert result_backend.get_group_message_ids(g.group_id) == list( g.message_ids) pipe.run() result = pipe.result.get(block=True) assert 12 + 15 == result stub_broker.join(do_work.queue_name) stub_worker.join() # the group result has been forgotten assert list(g.results.get()) == [None, None] # the message_ids has been forgotten with pytest.raises(MessageIdsMissing): result_backend.get_group_message_ids(g.group_id) # Given an actor that stores results @remoulade.actor(store_results=True) def add(a, b): return a + b stub_broker.declare_actor(add) pipe = do_work.message(13) | group([add.message(12), add.message(15)]) pipe.run() result = pipe.result.get(block=True) assert [13 + 12, 13 + 15] == list(result)
def test_inner_groups_forbidden(stub_broker, stub_worker, result_backend): # Given that I have a result backend stub_broker.add_middleware(Results(backend=result_backend)) # And I have an actor @remoulade.actor() def do_work(): return 1 # And this actor is declared stub_broker.declare_actor(do_work) # groups of groups are forbidden with pytest.raises(ValueError): group(group(do_work.message() for _ in range(2)) for _ in range(3))
def test_local_broker_get_result_in_message(local_broker, local_result_backend): local_broker.add_middleware(Results(backend=local_result_backend)) # Given that I have an actor that stores its results @remoulade.actor(store_results=True) def do_work(): return 1 # And this actor is declared local_broker.declare_actor(do_work) # When I send that actor a message message = do_work.send() # I should get the right result assert message.result.get() == 1
def test_local_broker_with_groups(local_broker, local_result_backend): local_broker.add_middleware(Results(backend=local_result_backend)) # Given that I have an actor that stores its results @remoulade.actor(store_results=True) def add(a, b): return a + b # And this actor is declared local_broker.declare_actor(add) # When I run a group g = group([add.message(1, 2), add.message(3, 4), add.message(4, 5)]) g.run() assert list(g.results.get()) == [3, 7, 9]
def test_local_broker_with_pipes(local_broker, local_result_backend): local_broker.add_middleware(Results(backend=local_result_backend)) # Given that I have an actor that stores its results @remoulade.actor(store_results=True) def add(a, b): return a + b # And this actor is declared local_broker.declare_actor(add) # When I run a pipe pipe = add.message(1, 2) | add.message(3) pipe.run() # I should get the right result assert pipe.result.get() == 6
def test_result_default_before_retries(stub_broker, result_backend, stub_worker): # Given a result backend # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) retries_index, results_index = None, None for i, middleware in enumerate(stub_broker.middleware): if isinstance(middleware, Retries): retries_index = i if isinstance(middleware, Results): results_index = i assert results_index is not None assert retries_index is not None # The Results middleware should be before the Retries middleware assert retries_index > results_index
def test_messages_can_get_results_from_inferred_backend(stub_broker, stub_worker, redis_result_backend): # And a broker with the results middleware stub_broker.add_middleware(Results(backend=redis_result_backend)) # And an actor that stores a result @remoulade.actor(store_results=True) def do_work(): return 42 # And this actor is declared stub_broker.declare_actor(do_work) # When I send that actor a message message = do_work.send() # And wait for a result # Then I should get that result back assert message.result.get(block=True) == 42
def test_local_broker_forget(local_broker, local_result_backend): local_broker.add_middleware(Results(backend=local_result_backend)) # Given that I have an actor that stores its results @remoulade.actor(store_results=True) def do_work(): return 1 # And this actor is declared local_broker.declare_actor(do_work) # When I send that actor a message message = do_work.send() # I if forget the result assert message.result.get(forget=True) == 1 # the result is forgotten assert message.result.get() is None
def test_pipelines_store_results_error(stub_broker, result_backend, stub_worker, store_results): # And a broker with the results middleware stub_broker.add_middleware(Results(backend=result_backend)) # Given an actor that fail @remoulade.actor(store_results=store_results) def do_work_fail(): raise ValueError() # Given an actor that stores results @remoulade.actor(store_results=True) def do_work(): return 42 # And these actors are declared stub_broker.declare_actor(do_work_fail) stub_broker.declare_actor(do_work) # And I've run a pipeline g = group([do_work.message(), do_work.message(), do_work.message()]) pipe = do_work_fail.message() | do_work.message() | g | do_work.message() pipe.run() stub_broker.join(do_work.queue_name) stub_worker.join() # I get an error if store_results: with pytest.raises(ErrorStored) as e: pipe.children[0].result.get() assert str(e.value) == "ValueError()" for i in [1, 3]: with pytest.raises(ErrorStored) as e: pipe.children[i].result.get() assert str(e.value).startswith("ParentFailed") for child in g.children: with pytest.raises(ErrorStored) as e: child.result.get() assert str(e.value).startswith("ParentFailed")
def test_pipelines_can_be_incomplete(stub_broker, result_backend): # Given that I am not running a worker # And I have a result backend stub_broker.add_middleware(Results(backend=result_backend)) # And I have an actor that does nothing @remoulade.actor(store_results=True) def do_nothing(): return None # And this actor is declared stub_broker.declare_actor(do_nothing) # And I've run a pipeline pipe = do_nothing.message() | do_nothing.message_with_options(pipe_ignore=True) pipe.run() # When I check if the pipeline has completed # Then it should return False assert not pipe.results.completed