async def test_cannot_create_empty_group(self, ray_context):
builder = ParallelPipeline()
builder.add_module(ModuleA('m1', group='g1'))
builder.add_module(ModuleB('m2', group='g2').depends_on(builder.get_module('m1')))
builder.add_module(ModuleC('m3', group='g3').depends_on(builder.get_module('m2')))
builder.add_group(builder.Group('g4'))
with pytest.raises(Exception):
await builder.build()
async def test_should_fail_on_adding_to_group_non_callable_hooks(self, ray_context):
builder = ParallelPipeline()
def callable_1():
pass
with pytest.raises(Exception):
builder.add_group(builder.Group('g1', after_created=[callable_1, "test"]))
def _add_group_options(cls, group_options, pipeline):
if group_options and isinstance(pipeline, ParallelPipeline):
for name, params in group_options.items():
group = ParallelPipeline.Group(name)
group.set_replicas(params['replicas'] if 'replicas' in params else 1)
params.pop('replicas', None)
if params.keys():
group.set_options(**params)
pipeline.add_group(group)
return pipeline
async def test_should_call_hooks_in_groups(self, ray_context):
builder = ParallelPipeline()
callable_1_counter = Queue()
callable_2_counter = Queue()
def callable_1():
callable_1_counter.put(1)
def callable_2():
callable_2_counter.put(2)
builder.add_group(builder.Group('g1', after_created=[callable_1, callable_2]))
builder.add_group(builder.Group('g2', after_created=[callable_2]))
builder.add_module(ModuleA('m1', group='g1'))
builder.add_module(ModuleB('m2', group='g2').depends_on(builder.get_module('m1')))
pipeline = await builder.build()
assert isinstance(pipeline, ParallelPipeline.Runtime)
assert len(pipeline.groups) == 2
assert set([g.name for g in pipeline.groups]) == {'g1', 'g2'}
assert callable_1_counter.qsize() == 1
assert callable_2_counter.qsize() == 2
def _add_group_options(cls, group_options, pipeline, hooks):
if group_options and isinstance(pipeline, ParallelPipeline):
for name, params in group_options.items():
if hooks:
group_hooks = hooks[name] if isinstance(hooks,
dict) else hooks
else:
group_hooks = None
group = ParallelPipeline.Group(name, after_created=group_hooks)
group.set_replicas(params['replicas'] if 'replicas' in
params else 1)
params.pop('replicas', None)
if params.keys():
group.set_options(**params)
pipeline.add_group(group)
return pipeline
async def test_should_fail_on_adding_to_group_hooks_with_incorrect_type(self, ray_context):
builder = ParallelPipeline()
with pytest.raises(Exception):
builder.add_group(builder.Group('g1', after_created="test"))