def test_pipelines_expose_completion_stats(stub_broker, stub_worker,
result_backend):
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# And an actor that waits some amount of time
result_backend.store_results, event_result = mock_func(
result_backend.store_results)
event_count = [threading.Event() for _ in range(4)]
@remoulade.actor(store_results=True)
def wait(n):
event_count[n].wait(3)
return n + 1
# And this actor is declared
stub_broker.declare_actor(wait)
# When I pipe some messages intended for that actor together and run the pipeline
pipe = wait.message(0) | wait.message() | wait.message() | wait.message()
pipe.run()
# Then every time a job in the pipeline completes, the completed_count should increase
for count in range(0, len(pipe)):
event_count[count].set()
event_result.wait(2)
event_result.clear()
assert pipe.results.completed_count == count + 1
# Finally, completed should be true
assert pipe.results.completed
def test_complex_pipelines(stub_broker, stub_worker, result_backend):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# Given an actor that stores results
@remoulade.actor(store_results=True)
def do_work():
return 1
# Given an actor that stores results
@remoulade.actor(store_results=True)
def add(a):
return 1 + a
# Given an actor that stores results
@remoulade.actor(store_results=True)
def do_sum(results):
return sum(results)
# And this actor is declared
stub_broker.declare_actor(do_work)
stub_broker.declare_actor(do_sum)
stub_broker.declare_actor(add)
pipe = do_work.message_with_options(pipe_ignore=True) | add.message() | add.message() # return 3 [1, 2, 3] ?
g = group([pipe, add.message(), add.message(), do_work.message_with_options(pipe_ignore=True)]) # return [3,2,2,1]
final_pipe = do_work.message() | g | do_sum.message() | add.message() # return 9
final_pipe.run()
result = final_pipe.result.get(block=True)
assert 9 == result
def test_multiple_groups_pipelines(stub_broker, stub_worker, result_backend):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# Given an actor that stores results
@remoulade.actor(store_results=True)
def do_work():
return 1
# Given an actor that stores results
@remoulade.actor(store_results=True)
def do_sum(results):
return sum(results)
# And this actor is declared
stub_broker.declare_actor(do_work)
stub_broker.declare_actor(do_sum)
pipe = pipeline(
[group([do_work.message(), do_work.message()]), group([do_sum.message(), do_sum.message()]), do_sum.message()]
).run()
result = pipe.result.get(block=True)
assert 4 == result
def test_actors_can_store_results(stub_broker, stub_worker, result_backend,
forget, block):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# And an actor that stores results
@remoulade.actor(store_results=True)
def do_work():
return 42
# And this actor is declared
stub_broker.declare_actor(do_work)
# When I send that actor a message
message = do_work.send()
# And wait for a result
if not block:
stub_broker.join(do_work.queue_name)
stub_worker.join()
result = message.result.get(block=block, forget=forget)
assert isinstance(message.result, Result)
# Then the result should be what the actor returned
assert result == 42
def test_pipelines_expose_completion_stats(stub_broker, stub_worker, result_backend):
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# And an actor that waits some amount of time
condition = Condition()
@remoulade.actor(store_results=True)
def wait(n):
time.sleep(n)
with condition:
condition.notify_all()
return n
# And this actor is declared
stub_broker.declare_actor(wait)
# When I pipe some messages intended for that actor together and run the pipeline
pipe = wait.message(1) | wait.message()
pipe.run()
# Then every time a job in the pipeline completes, the completed_count should increase
for count in range(1, len(pipe) + 1):
with condition:
condition.wait(2)
time.sleep(0.1) # give the worker time to set the result
assert pipe.results.completed_count == count
# Finally, completed should be true
assert pipe.results.completed
def test_redis_backend_keep_ttl_all_time(stub_broker, stub_worker,
redis_result_backend, block, forget):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=redis_result_backend))
# And an actor that stores a result with a result_ttl
result_ttl = 100 * 1000
@remoulade.actor(store_results=True, result_ttl=result_ttl)
def do_work():
return 42
# And this actor is declared
stub_broker.declare_actor(do_work)
# When I send that actor a message
message = do_work.send()
# And wait for a result
if not block:
stub_broker.join(do_work.queue_name)
stub_worker.join()
# The result should have a TTL in redis
assert redis_result_backend.client.ttl(
redis_result_backend.build_message_key(message.message_id)) > 0
# Then I should get that result back
assert message.result.get(block=block, forget=forget) == 42
# The forgotten result should still have a TTL in redis
assert redis_result_backend.client.ttl(
redis_result_backend.build_message_key(message.message_id)) > 0
def test_retry_if_saving_result_fail(stub_broker, stub_worker):
with patch.object(ResultBackend, "store_results") as mock_store_results:
mock_store_results.side_effect = Exception("Cannot save result")
middleware = Results(backend=StubBackend())
stub_broker.add_middleware(middleware)
attempts = []
# And an actor that stores results
@remoulade.actor(store_results=True)
def do_work():
attempts.append(1)
# And this actor is declared
stub_broker.declare_actor(do_work)
# When I send that actor a message
do_work.send()
# And wait for a result
stub_broker.join(do_work.queue_name)
stub_worker.join()
# The actor has been tried 4 times
assert len(attempts) == 4
def test_retrieving_a_result_can_time_out(stub_broker, stub_worker,
result_backend, forget):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# And an actor that sleeps for a long time before it stores a result
@remoulade.actor(store_results=True)
def do_work():
time.sleep(0.5)
return 42
# And this actor is declared
stub_broker.declare_actor(do_work)
# When I send that actor a message
message = do_work.send()
# And wait for a result
# Then a ResultTimeout error should be raised
with pytest.raises(ResultTimeout):
result_backend.get_result(message.message_id,
block=True,
timeout=200,
forget=forget)
def test_messages_can_get_completed(stub_broker, stub_worker, result_backend,
error):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# And an actor that stores results
@remoulade.actor(store_results=True)
def do_work():
if error:
raise ValueError()
else:
return 42
# And this actor is declared
stub_broker.declare_actor(do_work)
# When I send that actor a message
message = do_work.send()
# And wait for a result
stub_broker.join(do_work.queue_name)
stub_worker.join()
result = message.result
# we can get the completion
assert result.completed
result.get(forget=True, raise_on_error=False)
# even after a forget
assert result.completed
def test_result_get_forget_not_store_if_no_result(stub_broker, stub_worker,
result_backend):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
event = threading.Event()
# And an actor that stores results
@remoulade.actor(store_results=True)
def do_work():
event.wait(2)
return 42
# And this actor is declared
stub_broker.declare_actor(do_work)
# When I send that actor a message
message = do_work.send()
result = message.result
# If I get the result and forget it
with pytest.raises(ResultMissing):
result.get(forget=True)
event.set()
# It should not store a forgotten result if there is no key
assert result.get(block=True) == 42
def test_messages_can_get_results_and_forget(stub_broker, stub_worker,
result_backend, block):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# And an actor that stores a result
@remoulade.actor(store_results=True)
def do_work():
return 42
# And this actor is declared
stub_broker.declare_actor(do_work)
# When I send that actor a message
message = do_work.send()
# And wait for a result
if not block:
stub_broker.join(do_work.queue_name)
stub_worker.join()
# Then I should get that result back
assert message.result.get(block=block, forget=True) == 42
# If I ask again for the same result it should have been forgotten
assert message.result.get() is None
def test_groups_expose_completion_stats(stub_broker, stub_worker, result_backend):
# Given that I have a result backend
stub_broker.add_middleware(Results(backend=result_backend))
# And an actor that waits some amount of time
condition = Condition()
@remoulade.actor(store_results=True)
def wait(n):
time.sleep(n)
with condition:
condition.notify_all()
return n
# And this actor is declared
stub_broker.declare_actor(wait)
# When I group messages of varying durations together and run the group
g = group(wait.message(n) for n in range(1, 4))
g.run()
# Then every time a job in the group completes, the completed_count should increase
for count in range(1, len(g) + 1):
with condition:
condition.wait(5)
time.sleep(0.1) # give the worker time to set the result
assert g.results.completed_count == count
# Finally, completed should be true
assert g.results.completed
def test_store_errors(stub_broker, result_backend, stub_worker, block):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# And an actor that store a result and fail
@remoulade.actor(store_results=True)
def do_work():
raise ValueError()
# And this actor is declared
stub_broker.declare_actor(do_work)
# When I send that actor a message
message = do_work.send()
# And wait for a result
if not block:
stub_broker.join(do_work.queue_name)
stub_worker.join()
# Then I should get an ErrorStored
error_stored = message.result.get(block=block, raise_on_error=False)
assert isinstance(error_stored, ErrorStored)
assert str(error_stored) == "ValueError()"
def test_store_errors_after_no_more_retry(stub_broker, result_backend,
stub_worker):
# Given that I have a database
failures = []
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# Given an actor that stores results
@remoulade.actor(max_retries=3,
store_results=True,
min_backoff=10,
max_backoff=100)
def do_work():
failures.append(1)
raise ValueError()
# And this actor is declared
stub_broker.declare_actor(do_work)
# When I send that actor a message,
message = do_work.send()
stub_broker.join(do_work.queue_name)
stub_worker.join()
# I get an error
with pytest.raises(Exception) as e:
message.result.get(block=True)
assert str(e.value) == "ValueError()"
# all the retries have been made
assert sum(failures) == 4
def test_groups_execute_jobs_in_parallel(stub_broker, stub_worker, result_backend):
# Given that I have a result backend
stub_broker.add_middleware(Results(backend=result_backend))
# And I have an actor that sleeps for 100ms
@remoulade.actor(store_results=True)
def wait():
time.sleep(0.1)
# And this actor is declared
stub_broker.declare_actor(wait)
# When I group multiple of these actors together and run them
t = time.monotonic()
g = group([wait.message() for _ in range(5)])
g.run()
# group message_ids are no stored if not needed
with pytest.raises(MessageIdsMissing):
result_backend.get_group_message_ids(g.group_id)
# And wait on the group to complete
results = list(g.results.get(block=True))
# Then the total elapsed time should be less than 500ms
assert time.monotonic() - t <= 0.5
# And I should get back as many results as there were jobs in the group
assert len(results) == len(g)
# And the group should be completed
assert g.results.completed
assert isinstance(g.results, CollectionResults)
def test_raise_on_error(stub_broker, result_backend, stub_worker, block):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# And an actor that store a result and fail
@remoulade.actor(store_results=True)
def do_work():
raise ValueError()
# And this actor is declared
stub_broker.declare_actor(do_work)
# When I send that actor a message
message = do_work.send()
# And wait for a result
if not block:
stub_broker.join(do_work.queue_name)
stub_worker.join()
# It should raise an error
with pytest.raises(ErrorStored) as e:
message.result.get(block=block)
assert str(e.value) == "ValueError()"
def test_reduce_messages(stub_broker, stub_worker, result_backend):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# Given an actor that stores results
@remoulade.actor(store_results=True)
def do_work():
return 1
# Given an actor that stores results
@remoulade.actor(store_results=True)
def merge(results):
return sum(results)
# And this actor is declared
stub_broker.declare_actor(do_work)
stub_broker.declare_actor(merge)
merged_message = reduce((do_work.message() for _ in range(10)), merge)
merged_message.run()
result = merged_message.result.get(block=True)
assert 10 == result
def test_local_broker_cannot_have_non_local_backend(local_broker, result_backend):
# Which is not a LocalBackend
assert not isinstance(result_backend, LocalBackend)
# Cannot be used with LocalBroker
with pytest.raises(RuntimeError):
local_broker.add_middleware(Results(backend=result_backend))
def test_group_forget(stub_broker, result_backend, stub_worker, block):
# Given a result backend
stub_broker.add_middleware(Results(backend=result_backend))
# Given an actor that stores results
@remoulade.actor(store_results=True)
def do_work():
return 42
# And this actor is declared
stub_broker.declare_actor(do_work)
# And I've run a group
messages = [do_work.message() for _ in range(5)]
g = group(messages)
g.run()
# If i wait for the group to be completed
if not block:
stub_broker.join(do_work.queue_name)
stub_worker.join()
# If i forget the results
results = g.results.get(block=block, forget=True)
assert list(results) == [42] * 5
# All messages have been forgotten
results = g.results.get()
assert list(results) == [None] * 5
def test_retry_if_increment_group_completion_fail(stub_broker, stub_worker):
with patch.object(
StubBackend,
"increment_group_completion") as mock_increment_group_completion:
mock_increment_group_completion.side_effect = Exception(
"Cannot increment")
middleware = Results(backend=StubBackend())
stub_broker.add_middleware(middleware)
attempts = []
# And an actor that stores results
@remoulade.actor(store_results=True)
def do_work(*args):
attempts.append(1)
# And this actor is declared
stub_broker.declare_actor(do_work)
# When I send that actor a message
(group([do_work.message(), do_work.message()])
| do_work.message()).run()
# And wait for a result
stub_broker.join(do_work.queue_name)
stub_worker.join()
# The actor has been tried 8 times (4 time each do_work and never the last one)
assert len(attempts) == 8
def test_pipelines_with_groups(stub_broker, stub_worker, result_backend):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# Given an actor that stores results
@remoulade.actor(store_results=True)
def do_work(a):
return a
# Given an actor that stores results
@remoulade.actor(store_results=True)
def do_sum(results):
return sum(results)
# And this actor is declared
stub_broker.declare_actor(do_work)
stub_broker.declare_actor(do_sum)
# When I pipe some messages intended for that actor together and run the pipeline
g = group([do_work.message(12), do_work.message(15)])
pipe = g | do_sum.message()
pipe.build()
assert result_backend.get_group_message_ids(g.group_id) == list(
g.message_ids)
pipe.run()
result = pipe.result.get(block=True)
assert 12 + 15 == result
stub_broker.join(do_work.queue_name)
stub_worker.join()
# the group result has been forgotten
assert list(g.results.get()) == [None, None]
# the message_ids has been forgotten
with pytest.raises(MessageIdsMissing):
result_backend.get_group_message_ids(g.group_id)
# Given an actor that stores results
@remoulade.actor(store_results=True)
def add(a, b):
return a + b
stub_broker.declare_actor(add)
pipe = do_work.message(13) | group([add.message(12), add.message(15)])
pipe.run()
result = pipe.result.get(block=True)
assert [13 + 12, 13 + 15] == list(result)
def test_inner_groups_forbidden(stub_broker, stub_worker, result_backend):
# Given that I have a result backend
stub_broker.add_middleware(Results(backend=result_backend))
# And I have an actor
@remoulade.actor()
def do_work():
return 1
# And this actor is declared
stub_broker.declare_actor(do_work)
# groups of groups are forbidden
with pytest.raises(ValueError):
group(group(do_work.message() for _ in range(2)) for _ in range(3))
def test_local_broker_get_result_in_message(local_broker, local_result_backend):
local_broker.add_middleware(Results(backend=local_result_backend))
# Given that I have an actor that stores its results
@remoulade.actor(store_results=True)
def do_work():
return 1
# And this actor is declared
local_broker.declare_actor(do_work)
# When I send that actor a message
message = do_work.send()
# I should get the right result
assert message.result.get() == 1
def test_local_broker_with_groups(local_broker, local_result_backend):
local_broker.add_middleware(Results(backend=local_result_backend))
# Given that I have an actor that stores its results
@remoulade.actor(store_results=True)
def add(a, b):
return a + b
# And this actor is declared
local_broker.declare_actor(add)
# When I run a group
g = group([add.message(1, 2), add.message(3, 4), add.message(4, 5)])
g.run()
assert list(g.results.get()) == [3, 7, 9]
def test_local_broker_with_pipes(local_broker, local_result_backend):
local_broker.add_middleware(Results(backend=local_result_backend))
# Given that I have an actor that stores its results
@remoulade.actor(store_results=True)
def add(a, b):
return a + b
# And this actor is declared
local_broker.declare_actor(add)
# When I run a pipe
pipe = add.message(1, 2) | add.message(3)
pipe.run()
# I should get the right result
assert pipe.result.get() == 6
def test_result_default_before_retries(stub_broker, result_backend, stub_worker):
# Given a result backend
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
retries_index, results_index = None, None
for i, middleware in enumerate(stub_broker.middleware):
if isinstance(middleware, Retries):
retries_index = i
if isinstance(middleware, Results):
results_index = i
assert results_index is not None
assert retries_index is not None
# The Results middleware should be before the Retries middleware
assert retries_index > results_index
def test_messages_can_get_results_from_inferred_backend(stub_broker, stub_worker, redis_result_backend):
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=redis_result_backend))
# And an actor that stores a result
@remoulade.actor(store_results=True)
def do_work():
return 42
# And this actor is declared
stub_broker.declare_actor(do_work)
# When I send that actor a message
message = do_work.send()
# And wait for a result
# Then I should get that result back
assert message.result.get(block=True) == 42
def test_local_broker_forget(local_broker, local_result_backend):
local_broker.add_middleware(Results(backend=local_result_backend))
# Given that I have an actor that stores its results
@remoulade.actor(store_results=True)
def do_work():
return 1
# And this actor is declared
local_broker.declare_actor(do_work)
# When I send that actor a message
message = do_work.send()
# I if forget the result
assert message.result.get(forget=True) == 1
# the result is forgotten
assert message.result.get() is None
def test_pipelines_store_results_error(stub_broker, result_backend,
stub_worker, store_results):
# And a broker with the results middleware
stub_broker.add_middleware(Results(backend=result_backend))
# Given an actor that fail
@remoulade.actor(store_results=store_results)
def do_work_fail():
raise ValueError()
# Given an actor that stores results
@remoulade.actor(store_results=True)
def do_work():
return 42
# And these actors are declared
stub_broker.declare_actor(do_work_fail)
stub_broker.declare_actor(do_work)
# And I've run a pipeline
g = group([do_work.message(), do_work.message(), do_work.message()])
pipe = do_work_fail.message() | do_work.message() | g | do_work.message()
pipe.run()
stub_broker.join(do_work.queue_name)
stub_worker.join()
# I get an error
if store_results:
with pytest.raises(ErrorStored) as e:
pipe.children[0].result.get()
assert str(e.value) == "ValueError()"
for i in [1, 3]:
with pytest.raises(ErrorStored) as e:
pipe.children[i].result.get()
assert str(e.value).startswith("ParentFailed")
for child in g.children:
with pytest.raises(ErrorStored) as e:
child.result.get()
assert str(e.value).startswith("ParentFailed")
def test_pipelines_can_be_incomplete(stub_broker, result_backend):
# Given that I am not running a worker
# And I have a result backend
stub_broker.add_middleware(Results(backend=result_backend))
# And I have an actor that does nothing
@remoulade.actor(store_results=True)
def do_nothing():
return None
# And this actor is declared
stub_broker.declare_actor(do_nothing)
# And I've run a pipeline
pipe = do_nothing.message() | do_nothing.message_with_options(pipe_ignore=True)
pipe.run()
# When I check if the pipeline has completed
# Then it should return False
assert not pipe.results.completed
