def test_groups_execute_inner_groups(stub_broker, stub_worker, backend, result_backends):
# Given that I have a result backend
backend = result_backends[backend]
stub_broker.add_middleware(Results(backend=backend))
# And I have an actor that sleeps for 100ms
@dramatiq.actor(store_results=True)
def wait():
time.sleep(0.1)
# When I group multiple groups inside one group and run it
t = time.monotonic()
g = group(group(wait.message() for _ in range(2)) for _ in range(3))
g.run()
# And wait on the group to complete
results = list(g.get_results(block=True))
# Then the total elapsed time should be less than 500ms
assert time.monotonic() - t <= 0.5
# And I should get back 3 results each with 2 results inside it
assert results == [[None, None]] * 3
# And the group should be completed
assert g.completed
def test_groups_execute_jobs_in_parallel(stub_broker, stub_worker, backend, result_backends):
# Given that I have a result backend
backend = result_backends[backend]
stub_broker.add_middleware(Results(backend=backend))
# And I have an actor that sleeps for 100ms
@dramatiq.actor(store_results=True)
def wait():
time.sleep(0.1)
# When I group multiple of these actors together and run them
t = time.monotonic()
g = group([wait.message() for _ in range(5)])
g.run()
# And wait on the group to complete
results = list(g.get_results(block=True))
# Then the total elapsed time should be less than 500ms
assert time.monotonic() - t <= 0.5
# And I should get back as many results as there were jobs in the group
assert len(results) == len(g)
# And the group should be completed
assert g.completed
def dispatch_taking_screenshots() -> Dict[int, List[str]]:
"""Checks games on a specified in settings server and dispatches group of
tasks for taking screenshots
Returns:
A dictionary in which keys are game identifiers and values are lists
of screenshot files
"""
logger.debug('Dispatching taking games screenshots tasks')
try:
group = dramatiq.group(
take_sized_screenshots_by_game_id.message(game_id)
for game_id in funcs.get_games_ids())
group.run()
# Dictionary to collect game identifiers as keys and screenshot file
# name lists as values for report
games_screenshots = {}
for game_id, files in group.get_results(block=True):
if files:
logger.debug("Game %s => %s", game_id, files)
games_screenshots[game_id] = files
return games_screenshots
except ValidationError as e:
logger.error('Parse error: %s', e)
except APIError as e:
logger.error('API response error: %s', e)
return {}
def test_groups_expose_completion_stats(stub_broker, stub_worker, backend, result_backends):
# Given that I have a result backend
backend = result_backends[backend]
stub_broker.add_middleware(Results(backend=backend))
# And an actor that waits some amount of time
condition = Condition()
@dramatiq.actor(store_results=True)
def wait(n):
time.sleep(n)
with condition:
condition.notify_all()
return n
# When I group messages of varying durations together and run the group
g = group(wait.message(n) for n in range(1, 4))
g.run()
# Then every time a job in the group completes, the completed_count should increase
for count in range(1, len(g) + 1):
with condition:
condition.wait(5)
time.sleep(0.1) # give the worker time to set the result
assert g.completed_count == count
# Finally, completed should be true
assert g.completed
def wrapper_actorify(*args, **kwargs):
# Set context.
ctx = args[0]
# Encache step.
step = _get_step(ctx, func) if not is_substep else None
# Invoke actor.
result = func(*args, **kwargs)
# Message factories --> dramatiq.group.
if inspect.isfunction(result):
result = dramatiq.group(result())
# Update step.
if step and on_success:
_set_step(step)
# Groups.
if isinstance(result, dramatiq.group):
if on_success:
result.add_completion_callback(on_success().message(ctx))
result.run()
# Continuation.
elif on_success:
on_success().send(ctx)
def top_stories():
story_ids = hn.top_stories()
messages = [actors.hn_story.message(sid) for sid in story_ids]
group = dramatiq.group(messages).run()
# 60 second timeout (in milliseconds)
for result in group.get_results(block=True, timeout=60_000):
print(result)
def do_step_execute(ctx: ExecutionContext, step: WorkflowStep):
"""Performs step execution.
:param ctx: Execution context information.
:param step: Step related execution information.
"""
# Execute step.
step.execute()
# Process errors.
if step.error:
do_step_error.send(ctx, str(step.error))
# Async steps are processed by deploy listeners.
elif step.is_async:
if step.result is not None:
if inspect.isfunction(step.result):
message_factory = step.result()
group = dramatiq.group(message_factory)
group.run()
else:
raise TypeError(
"Expecting either none or a message factory from a step function."
)
logger.log(
f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} :: {ctx.phase_index_label} :: {ctx.step_index_label} :: {step.label} -> listening for chain events"
)
# Sync steps are processed inline.
else:
if step.result is None:
do_step_end.send(ctx)
elif inspect.isfunction(step.result):
message_factory = step.result()
group = dramatiq.group(message_factory)
group.add_completion_callback(do_step_end.message(ctx))
group.run()
else:
raise TypeError(
"Expecting either none or a message factory from a step function."
)
def answer_question(question_id):
quiz = get_image(question_id)
print('題號：', quiz.question_id)
print('文字描述：', quiz.description)
print('Bounding Box:', quiz.bbox)
print('影像物件：', type(quiz.raw_image), quiz.raw_image.dtype, ', 影像大小：',
quiz.raw_image.shape)
# quiz = FoodQuiz(3, raw_image, (2, 2), 'blah balh')
display_image_RGB(quiz.raw_image)
raw_image = convert_RGB_to_BGR(quiz.raw_image)
# labels, scores = mobile_net_clf(raw_image)
# label_scores = {label: score
# for label, score in
# sorted(zip(labels, scores), key=X[1], reverse=True)}
# print(label_scores)
display_cat_id_map()
result_images = {}
for b in range(4):
print(5 * b, 5 * (b + 1))
g = dramatiq.group([
model.predict.send(raw_image)
for model in model_list[5 * b:5 * (b + 1)]
]).run()
for i, result_image in enumerate(
g.get_results(block=True, timeout=50000)):
cat_id = b * 5 + i
print(cat_id)
result_images[cat_id] = result_image
# for i, result_image in enumerate(g.get_results(block=True, timeout=20000)):
# try:
# model = Inpainting.from_cat_id(i)
# result_images[i] = model.predict(raw_image)
# except Exception as err:
# logger.exception(f'cat_id {i} not found')
# continue
# result_images[i] = result_image
# print(f'{i} {cat_id_to_cat_map[i]}')
# display_image_BGR(result_images[i])
display_grid_plot(result_images)
cat_id = int(input())
result_image = convert_BGR_to_RGB(result_images[cat_id])
display('final choice')
display_image_RGB(result_image)
return submit_image(result_image, question_id)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("uri", nargs="+", help="A website URI.")
arguments = parser.parse_args()
jobs = group(request.message(uri) | count_words.message() for uri in arguments.uri).run()
for uri, count in zip(arguments.uri, jobs.get_results(block=True)):
print(f" * {uri} has {count} words")
return 0
def test_groups_with_completion_callbacks_fail_unless_group_callbacks_is_set_up(
stub_broker, stub_worker):
# Given that I haven't set up GroupCallbacks
@dramatiq.actor
def do_nothing():
pass
@dramatiq.actor
def finalize(n):
pass
# When I group together some messages with a completion callback
g = group(do_nothing.message() for n in range(5))
g.add_completion_callback(finalize.message(42))
# And run the group
# Then a RuntimeError should be raised
with pytest.raises(RuntimeError):
g.run()
def test_groups_can_time_out(stub_broker, stub_worker, result_backend):
# Given that I have a result backend
stub_broker.add_middleware(Results(backend=result_backend))
# And I have an actor that sleeps for 300ms
@dramatiq.actor(store_results=True)
def wait():
time.sleep(0.3)
# When I group a few jobs together and run it
g = group(wait.message() for _ in range(2))
g.run()
# And wait for the group to complete with a timeout
# Then a ResultTimeout error should be raised
with pytest.raises(ResultTimeout):
g.wait(timeout=100)
# And the group should not be completed
assert not g.completed
def find_smallest_in_big_array(data):
"""
Lets distribute calculations on different workers
:param data:
:return:
"""
workers_count = 4
task_list = []
slice_start = 0
slice_end = None
len_per_worker = len(data) // workers_count
for i in range(1, workers_count):
slice_end = slice_start + len_per_worker
task_list.append(
find_smallest_in_small_array.message(data[slice_start:slice_end]))
slice_start = slice_end
task_list.append(find_smallest_in_small_array.message(data[slice_end:]))
g = dramatiq.group(task_list).run()
result = list(g.get_results(block=True))
smallest = sorted(result)[0]
return smallest
def test_groups_of_pipelines_can_have_completion_callbacks(
stub_broker, stub_worker, rate_limiter_backend):
# Given that I have a rate limiter backend
# And I've added the GroupCallbacks middleware to my broker
stub_broker.add_middleware(GroupCallbacks(rate_limiter_backend))
do_nothing_times = []
finalize_times = []
finalized = Event()
@dramatiq.actor
def do_nothing(_):
do_nothing_times.append(time.monotonic())
@dramatiq.actor
def finalize(n):
assert n == 42
finalize_times.append(time.monotonic())
finalized.set()
# When I group together some messages with a completion callback
g = group(
[do_nothing.message(1) | do_nothing.message(),
do_nothing.message(1)])
g.add_completion_callback(finalize.message(42))
g.run()
# And wait for the callback to be callled
finalized.wait(timeout=30)
# Then all the messages in the group should run
assert len(do_nothing_times) == 3
# And the callback
assert len(finalize_times) == 1
# And the callback should run after all the messages
assert sorted(do_nothing_times)[-1] <= finalize_times[0]
def test_groups_execute_jobs_in_parallel_get_any_results(
stub_broker, stub_worker, result_backend):
stub_broker.middleware = [
x for x in stub_broker.middleware if 'Prometheus' not in str(x)
]
# Given that I have a result backend
stub_broker.add_middleware(Results(backend=result_backend))
# And I have an actor that sleeps for 100ms
@dramatiq.actor(store_results=True)
def wait(ms):
time.sleep(ms / 1000)
return dict(a=1, b='c')
# When I group multiple of these actors together and run them
g = group([wait.message(ms=100) for _ in range(5)])
g.run()
t = time.monotonic()
# And wait on the group to complete
results = list(g.get_any_results(block=True, with_task=True))
# Then the total elapsed time should be less than 500ms in ideal conditions
assert time.monotonic() - t <= 1.5
# And I should get back as many results as there were jobs in the group
assert len(results) == len(g)
# And the group should be completed
assert g.completed
assert [x[0] for x in results] == list(g.get_results())
g = group([
wait.message(ms=100),
])
g.run()
# Test group with single actor
results = list(g.get_any_results(block=True, with_task=True))
assert len(results) == len(g)
assert g.completed
assert [x[0] for x in results] == list(g.get_results())
g = group([wait.message(ms=100) for _ in range(5)])
g.run(delay=4000)
with pytest.raises(ResultMissing):
list(g.get_any_results(block=False))
with pytest.raises(ResultTimeout):
list(g.get_any_results(block=True, timeout=2000))
t = time.monotonic()
results = list(g.get_any_results(block=True, timeout=2000, with_task=True))
assert time.monotonic() - t <= 3
assert len(results) == len(g)
assert g.completed
assert [x[0] for x in results] == list(g.get_results())
def falcon_pages(fetch_number):
page_messages = [actors.falcon_page.message() for x in range(fetch_number)]
group = dramatiq.group(page_messages).run()
# 60 second timeout (in milliseconds)
for result in group.get_results(block=True, timeout=60_000):
print(result)
redis_broker = RedisBroker(host="redis", port=6379)
redis_broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(redis_broker)
# Using Spacy for entity extraction.
nlp = spacy.load("en_core_web_sm")
@dramatiq.actor(store_results=True)
def extract_ents(paragraph: str):
"""Extract entities from the paragraph."""
doc = nlp(paragraph)
ents = []
for ent in doc.ents:
if ent.label_ == "PERSON":
ents.append(ent.text)
return ents
if __name__ == "__main__":
# Moby Dick - Split into paragraphs.
data = requests.get("https://www.gutenberg.org/files/2701/2701-0.txt").content.decode('utf-8')
paragraphs = data.split("\r\n\r\n")
print("Staring Jobs")
group = dramatiq.group([extract_ents.message(x) for x in paragraphs]).run()
results = itertools.chain(*[x for x in group.get_results(block=True, timeout=100_000)])
print(set(results))
#!/usr/bin/env python
# vim: set fileencoding=utf-8
import time
import dramatiq
from dramatiq.brokers.redis import RedisBroker
from test_worker_8 import test_worker_A, test_worker_B, test_worker_C, setup_broker_and_backend
setup_broker_and_backend()
for i in range(1, 6):
print(i)
g = dramatiq.group([
test_worker_A.message(str(i)),
test_worker_B.message(str(i)),
test_worker_C.message(str(i))
]).run()
g.wait(timeout=20000)
