def test_groups_execute_inner_groups(stub_broker, stub_worker, backend, result_backends): # Given that I have a result backend backend = result_backends[backend] stub_broker.add_middleware(Results(backend=backend)) # And I have an actor that sleeps for 100ms @dramatiq.actor(store_results=True) def wait(): time.sleep(0.1) # When I group multiple groups inside one group and run it t = time.monotonic() g = group(group(wait.message() for _ in range(2)) for _ in range(3)) g.run() # And wait on the group to complete results = list(g.get_results(block=True)) # Then the total elapsed time should be less than 500ms assert time.monotonic() - t <= 0.5 # And I should get back 3 results each with 2 results inside it assert results == [[None, None]] * 3 # And the group should be completed assert g.completed
def test_groups_execute_jobs_in_parallel(stub_broker, stub_worker, backend, result_backends): # Given that I have a result backend backend = result_backends[backend] stub_broker.add_middleware(Results(backend=backend)) # And I have an actor that sleeps for 100ms @dramatiq.actor(store_results=True) def wait(): time.sleep(0.1) # When I group multiple of these actors together and run them t = time.monotonic() g = group([wait.message() for _ in range(5)]) g.run() # And wait on the group to complete results = list(g.get_results(block=True)) # Then the total elapsed time should be less than 500ms assert time.monotonic() - t <= 0.5 # And I should get back as many results as there were jobs in the group assert len(results) == len(g) # And the group should be completed assert g.completed
def dispatch_taking_screenshots() -> Dict[int, List[str]]: """Checks games on a specified in settings server and dispatches group of tasks for taking screenshots Returns: A dictionary in which keys are game identifiers and values are lists of screenshot files """ logger.debug('Dispatching taking games screenshots tasks') try: group = dramatiq.group( take_sized_screenshots_by_game_id.message(game_id) for game_id in funcs.get_games_ids()) group.run() # Dictionary to collect game identifiers as keys and screenshot file # name lists as values for report games_screenshots = {} for game_id, files in group.get_results(block=True): if files: logger.debug("Game %s => %s", game_id, files) games_screenshots[game_id] = files return games_screenshots except ValidationError as e: logger.error('Parse error: %s', e) except APIError as e: logger.error('API response error: %s', e) return {}
def test_groups_expose_completion_stats(stub_broker, stub_worker, backend, result_backends): # Given that I have a result backend backend = result_backends[backend] stub_broker.add_middleware(Results(backend=backend)) # And an actor that waits some amount of time condition = Condition() @dramatiq.actor(store_results=True) def wait(n): time.sleep(n) with condition: condition.notify_all() return n # When I group messages of varying durations together and run the group g = group(wait.message(n) for n in range(1, 4)) g.run() # Then every time a job in the group completes, the completed_count should increase for count in range(1, len(g) + 1): with condition: condition.wait(5) time.sleep(0.1) # give the worker time to set the result assert g.completed_count == count # Finally, completed should be true assert g.completed
def wrapper_actorify(*args, **kwargs): # Set context. ctx = args[0] # Encache step. step = _get_step(ctx, func) if not is_substep else None # Invoke actor. result = func(*args, **kwargs) # Message factories --> dramatiq.group. if inspect.isfunction(result): result = dramatiq.group(result()) # Update step. if step and on_success: _set_step(step) # Groups. if isinstance(result, dramatiq.group): if on_success: result.add_completion_callback(on_success().message(ctx)) result.run() # Continuation. elif on_success: on_success().send(ctx)
def top_stories(): story_ids = hn.top_stories() messages = [actors.hn_story.message(sid) for sid in story_ids] group = dramatiq.group(messages).run() # 60 second timeout (in milliseconds) for result in group.get_results(block=True, timeout=60_000): print(result)
def do_step_execute(ctx: ExecutionContext, step: WorkflowStep): """Performs step execution. :param ctx: Execution context information. :param step: Step related execution information. """ # Execute step. step.execute() # Process errors. if step.error: do_step_error.send(ctx, str(step.error)) # Async steps are processed by deploy listeners. elif step.is_async: if step.result is not None: if inspect.isfunction(step.result): message_factory = step.result() group = dramatiq.group(message_factory) group.run() else: raise TypeError( "Expecting either none or a message factory from a step function." ) logger.log( f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} :: {ctx.phase_index_label} :: {ctx.step_index_label} :: {step.label} -> listening for chain events" ) # Sync steps are processed inline. else: if step.result is None: do_step_end.send(ctx) elif inspect.isfunction(step.result): message_factory = step.result() group = dramatiq.group(message_factory) group.add_completion_callback(do_step_end.message(ctx)) group.run() else: raise TypeError( "Expecting either none or a message factory from a step function." )
def answer_question(question_id): quiz = get_image(question_id) print('題號:', quiz.question_id) print('文字描述:', quiz.description) print('Bounding Box:', quiz.bbox) print('影像物件:', type(quiz.raw_image), quiz.raw_image.dtype, ', 影像大小:', quiz.raw_image.shape) # quiz = FoodQuiz(3, raw_image, (2, 2), 'blah balh') display_image_RGB(quiz.raw_image) raw_image = convert_RGB_to_BGR(quiz.raw_image) # labels, scores = mobile_net_clf(raw_image) # label_scores = {label: score # for label, score in # sorted(zip(labels, scores), key=X[1], reverse=True)} # print(label_scores) display_cat_id_map() result_images = {} for b in range(4): print(5 * b, 5 * (b + 1)) g = dramatiq.group([ model.predict.send(raw_image) for model in model_list[5 * b:5 * (b + 1)] ]).run() for i, result_image in enumerate( g.get_results(block=True, timeout=50000)): cat_id = b * 5 + i print(cat_id) result_images[cat_id] = result_image # for i, result_image in enumerate(g.get_results(block=True, timeout=20000)): # try: # model = Inpainting.from_cat_id(i) # result_images[i] = model.predict(raw_image) # except Exception as err: # logger.exception(f'cat_id {i} not found') # continue # result_images[i] = result_image # print(f'{i} {cat_id_to_cat_map[i]}') # display_image_BGR(result_images[i]) display_grid_plot(result_images) cat_id = int(input()) result_image = convert_BGR_to_RGB(result_images[cat_id]) display('final choice') display_image_RGB(result_image) return submit_image(result_image, question_id)
def main(): parser = argparse.ArgumentParser() parser.add_argument("uri", nargs="+", help="A website URI.") arguments = parser.parse_args() jobs = group(request.message(uri) | count_words.message() for uri in arguments.uri).run() for uri, count in zip(arguments.uri, jobs.get_results(block=True)): print(f" * {uri} has {count} words") return 0
def test_groups_with_completion_callbacks_fail_unless_group_callbacks_is_set_up( stub_broker, stub_worker): # Given that I haven't set up GroupCallbacks @dramatiq.actor def do_nothing(): pass @dramatiq.actor def finalize(n): pass # When I group together some messages with a completion callback g = group(do_nothing.message() for n in range(5)) g.add_completion_callback(finalize.message(42)) # And run the group # Then a RuntimeError should be raised with pytest.raises(RuntimeError): g.run()
def test_groups_can_time_out(stub_broker, stub_worker, result_backend): # Given that I have a result backend stub_broker.add_middleware(Results(backend=result_backend)) # And I have an actor that sleeps for 300ms @dramatiq.actor(store_results=True) def wait(): time.sleep(0.3) # When I group a few jobs together and run it g = group(wait.message() for _ in range(2)) g.run() # And wait for the group to complete with a timeout # Then a ResultTimeout error should be raised with pytest.raises(ResultTimeout): g.wait(timeout=100) # And the group should not be completed assert not g.completed
def find_smallest_in_big_array(data): """ Lets distribute calculations on different workers :param data: :return: """ workers_count = 4 task_list = [] slice_start = 0 slice_end = None len_per_worker = len(data) // workers_count for i in range(1, workers_count): slice_end = slice_start + len_per_worker task_list.append( find_smallest_in_small_array.message(data[slice_start:slice_end])) slice_start = slice_end task_list.append(find_smallest_in_small_array.message(data[slice_end:])) g = dramatiq.group(task_list).run() result = list(g.get_results(block=True)) smallest = sorted(result)[0] return smallest
def test_groups_of_pipelines_can_have_completion_callbacks( stub_broker, stub_worker, rate_limiter_backend): # Given that I have a rate limiter backend # And I've added the GroupCallbacks middleware to my broker stub_broker.add_middleware(GroupCallbacks(rate_limiter_backend)) do_nothing_times = [] finalize_times = [] finalized = Event() @dramatiq.actor def do_nothing(_): do_nothing_times.append(time.monotonic()) @dramatiq.actor def finalize(n): assert n == 42 finalize_times.append(time.monotonic()) finalized.set() # When I group together some messages with a completion callback g = group( [do_nothing.message(1) | do_nothing.message(), do_nothing.message(1)]) g.add_completion_callback(finalize.message(42)) g.run() # And wait for the callback to be callled finalized.wait(timeout=30) # Then all the messages in the group should run assert len(do_nothing_times) == 3 # And the callback assert len(finalize_times) == 1 # And the callback should run after all the messages assert sorted(do_nothing_times)[-1] <= finalize_times[0]
def test_groups_execute_jobs_in_parallel_get_any_results( stub_broker, stub_worker, result_backend): stub_broker.middleware = [ x for x in stub_broker.middleware if 'Prometheus' not in str(x) ] # Given that I have a result backend stub_broker.add_middleware(Results(backend=result_backend)) # And I have an actor that sleeps for 100ms @dramatiq.actor(store_results=True) def wait(ms): time.sleep(ms / 1000) return dict(a=1, b='c') # When I group multiple of these actors together and run them g = group([wait.message(ms=100) for _ in range(5)]) g.run() t = time.monotonic() # And wait on the group to complete results = list(g.get_any_results(block=True, with_task=True)) # Then the total elapsed time should be less than 500ms in ideal conditions assert time.monotonic() - t <= 1.5 # And I should get back as many results as there were jobs in the group assert len(results) == len(g) # And the group should be completed assert g.completed assert [x[0] for x in results] == list(g.get_results()) g = group([ wait.message(ms=100), ]) g.run() # Test group with single actor results = list(g.get_any_results(block=True, with_task=True)) assert len(results) == len(g) assert g.completed assert [x[0] for x in results] == list(g.get_results()) g = group([wait.message(ms=100) for _ in range(5)]) g.run(delay=4000) with pytest.raises(ResultMissing): list(g.get_any_results(block=False)) with pytest.raises(ResultTimeout): list(g.get_any_results(block=True, timeout=2000)) t = time.monotonic() results = list(g.get_any_results(block=True, timeout=2000, with_task=True)) assert time.monotonic() - t <= 3 assert len(results) == len(g) assert g.completed assert [x[0] for x in results] == list(g.get_results())
def falcon_pages(fetch_number): page_messages = [actors.falcon_page.message() for x in range(fetch_number)] group = dramatiq.group(page_messages).run() # 60 second timeout (in milliseconds) for result in group.get_results(block=True, timeout=60_000): print(result)
redis_broker = RedisBroker(host="redis", port=6379) redis_broker.add_middleware(Results(backend=result_backend)) dramatiq.set_broker(redis_broker) # Using Spacy for entity extraction. nlp = spacy.load("en_core_web_sm") @dramatiq.actor(store_results=True) def extract_ents(paragraph: str): """Extract entities from the paragraph.""" doc = nlp(paragraph) ents = [] for ent in doc.ents: if ent.label_ == "PERSON": ents.append(ent.text) return ents if __name__ == "__main__": # Moby Dick - Split into paragraphs. data = requests.get("https://www.gutenberg.org/files/2701/2701-0.txt").content.decode('utf-8') paragraphs = data.split("\r\n\r\n") print("Staring Jobs") group = dramatiq.group([extract_ents.message(x) for x in paragraphs]).run() results = itertools.chain(*[x for x in group.get_results(block=True, timeout=100_000)]) print(set(results))
#!/usr/bin/env python # vim: set fileencoding=utf-8 import time import dramatiq from dramatiq.brokers.redis import RedisBroker from test_worker_8 import test_worker_A, test_worker_B, test_worker_C, setup_broker_and_backend setup_broker_and_backend() for i in range(1, 6): print(i) g = dramatiq.group([ test_worker_A.message(str(i)), test_worker_B.message(str(i)), test_worker_C.message(str(i)) ]).run() g.wait(timeout=20000)