async def test_cannot_create_empty_group(self, ray_context): builder = ParallelPipeline() builder.add_module(ModuleA('m1', group='g1')) builder.add_module(ModuleB('m2', group='g2').depends_on(builder.get_module('m1'))) builder.add_module(ModuleC('m3', group='g3').depends_on(builder.get_module('m2'))) builder.add_group(builder.Group('g4')) with pytest.raises(Exception): await builder.build()
async def test_should_fail_on_adding_to_group_non_callable_hooks(self, ray_context): builder = ParallelPipeline() def callable_1(): pass with pytest.raises(Exception): builder.add_group(builder.Group('g1', after_created=[callable_1, "test"]))
def _add_group_options(cls, group_options, pipeline): if group_options and isinstance(pipeline, ParallelPipeline): for name, params in group_options.items(): group = ParallelPipeline.Group(name) group.set_replicas(params['replicas'] if 'replicas' in params else 1) params.pop('replicas', None) if params.keys(): group.set_options(**params) pipeline.add_group(group) return pipeline
async def test_should_call_hooks_in_groups(self, ray_context): builder = ParallelPipeline() callable_1_counter = Queue() callable_2_counter = Queue() def callable_1(): callable_1_counter.put(1) def callable_2(): callable_2_counter.put(2) builder.add_group(builder.Group('g1', after_created=[callable_1, callable_2])) builder.add_group(builder.Group('g2', after_created=[callable_2])) builder.add_module(ModuleA('m1', group='g1')) builder.add_module(ModuleB('m2', group='g2').depends_on(builder.get_module('m1'))) pipeline = await builder.build() assert isinstance(pipeline, ParallelPipeline.Runtime) assert len(pipeline.groups) == 2 assert set([g.name for g in pipeline.groups]) == {'g1', 'g2'} assert callable_1_counter.qsize() == 1 assert callable_2_counter.qsize() == 2
def _add_group_options(cls, group_options, pipeline, hooks): if group_options and isinstance(pipeline, ParallelPipeline): for name, params in group_options.items(): if hooks: group_hooks = hooks[name] if isinstance(hooks, dict) else hooks else: group_hooks = None group = ParallelPipeline.Group(name, after_created=group_hooks) group.set_replicas(params['replicas'] if 'replicas' in params else 1) params.pop('replicas', None) if params.keys(): group.set_options(**params) pipeline.add_group(group) return pipeline
async def test_should_fail_on_adding_to_group_hooks_with_incorrect_type(self, ray_context): builder = ParallelPipeline() with pytest.raises(Exception): builder.add_group(builder.Group('g1', after_created="test"))