async def test_callable_context(self, context, context_type):
builder = SequentialPipeline()
m1 = MockModule('m1')
m2 = MockModuleReturningContext('m2').depends_on(m1)
builder.add_module(m1)
builder.add_module(m2)
pipeline = await builder.build(context)
assert pipeline.context == context
assert isinstance(pipeline.modules[0].context, context_type)
results, _ = await pipeline.run()
assert isinstance(results['m2'], context_type)
async def test_should_accept_noncallable_context(self, context):
builder = SequentialPipeline()
m1 = MockModule('m1')
m2 = MockModuleReturningContext('m2').depends_on(m1)
builder.add_module(m1)
builder.add_module(m2)
pipeline = await builder.build(context)
assert pipeline.context == context
assert pipeline.modules[0].context == context
assert pipeline.modules[0].context == pipeline.modules[1].context
results = await pipeline.run()
assert results['m2'] == context
async def test_object_context(self, context, param):
builder = SequentialPipeline()
m1 = MockModule('m1')
m2 = MockModuleReturningContext('m2').depends_on(m1)
builder.add_module(m1)
builder.add_module(m2)
pipeline = await builder.build(context)
assert pipeline.context == context
assert pipeline.modules[0].context == context
assert pipeline.modules[0].context == pipeline.modules[1].context
assert pipeline.modules[0].context.param == param
results = await pipeline.run()
assert results['m2'] == context
async def test_exposing_under_the_same_name(self):
@expose('a')
@accept(MockModuleA)
@finalize
class MockModuleC(Module.Runtime):
def run(self, **kwargs):
pass
ModuleFactory.register('MockModuleC', MockModuleC)
builder = SequentialPipeline()
builder.add_module(MockModuleA('m1'))
builder.add_module(MockModuleC('m2'))
with pytest.raises(Exception):
pipeline = await builder.build()
async def build(self, prefix: str = '{CTX}'):
builder = SequentialPipeline()
builder.add_module(ModuleC('m1'))
builder.add_module(ModuleB('m2'))
builder.add_module(
ModuleB('m3')
.depends_on(builder.get_module('m2'))
)
builder.add_module(
ModuleA('m4')
.depends_on(builder.get_module('m3'))
.depends_on(builder.get_module('m1'))
.expose_result('final')
)
self.pipeline = await builder.build(lambda: Context(prefix))
async def read(
cls,
config: str,
module_factory: ModuleFactory,
config_parameters: Optional[Dict] = None,
context: Optional[Any] = None,
shared_parameters: Optional[Dict] = None
):
if config_parameters:
cls._validate_config_parameters_structure(config_parameters)
config = cls._check_and_substitute_declared_variables(config, config_parameters)
parsed_yaml = yaml.safe_load(config)
modules, shared_parameters, group_options = \
cls._extract_information_from_yaml(parsed_yaml, shared_parameters)
pipeline = (
ParallelPipeline()
if any([m.group is not None for m in modules])
else SequentialPipeline()
)
pipeline = cls._add_modules_to_pipeline(modules, pipeline, module_factory)
pipeline = cls._add_group_options(group_options, pipeline)
# connect modules
for mod in modules:
curr_mod_obj = pipeline.get_module(mod.name)
for dependent_mod_name in mod.depends_on:
dependent_mod_obj = pipeline.get_module(dependent_mod_name)
if dependent_mod_obj:
curr_mod_obj.depends_on(dependent_mod_obj)
else:
raise AttributeError(
f"Module '{dependent_mod_name}' hasn't been defined in the config file, "
"whereas it's used as a dependency."
)
runtime = await pipeline.build(context, shared_parameters)
return runtime
class TestResults:
def setup_method(self, method):
ModuleFactory.unregister()
ModuleFactory.register('MockModuleContext', MockModuleContext)
ModuleFactory.register('MockModuleData', MockModuleData)
self.builder = SequentialPipeline()
self.mod_c = MockModuleContext('mod_c')
self.mod_d1 = MockModuleData('mod_d1')
self.mod_d2 = MockModuleData('mod_d2')
def teardown_method(self, method):
del self.mod_c
del self.mod_d1
del self.mod_d2
ModuleFactory.unregister()
@pytest.mark.asyncio
async def test_string_results(self):
self.builder.add_module(self.mod_c)
self.builder.add_module(self.mod_d1)
self.builder.add_module(
self.mod_d2.depends_on(self.mod_c).depends_on(self.mod_d1))
pipeline = await self.builder.build([1, 2, 3])
runtime, _ = await pipeline.run()
assert runtime['mod_d1'].of('mod_c') == []
assert runtime['mod_d2'].of('mod_c') == [[1, 2, 3]]
assert runtime['mod_d2'].get('mod_c') == [1, 2, 3]
assert not runtime['mod_d1'].has('mod_c')
assert runtime['mod_d2'].has('mod_c')
assert len(runtime['mod_d1']) == 0
assert len(runtime['mod_d2']) == 2
@pytest.mark.asyncio
async def test_module_results(self):
self.builder.add_module(self.mod_c)
self.builder.add_module(self.mod_d1)
self.builder.add_module(
self.mod_d2.depends_on(self.mod_c).depends_on(self.mod_d1))
pipeline = await self.builder.build([1, 2, 3])
runtime, _ = await pipeline.run()
assert runtime['mod_d1'].of(MockModuleContext) == []
assert runtime['mod_d2'].of(MockModuleContext) == [[1, 2, 3]]
assert runtime['mod_d2'].get(MockModuleContext) == [1, 2, 3]
assert not runtime['mod_d1'].has(MockModuleContext)
assert runtime['mod_d2'].has(MockModuleContext)
@pytest.mark.asyncio
async def test_collection_copy(self):
self.builder.add_module(self.mod_c)
self.builder.add_module(self.mod_d1.depends_on(self.mod_c))
pipeline = await self.builder.build([1, 2, 3])
runtime, _ = await pipeline.run()
assert runtime['mod_d1'].collection
assert runtime['mod_d1'].collection is not runtime[
'mod_d1']._collection
assert runtime['mod_d1'].collection[0] is runtime[
'mod_d1']._collection[0]
@pytest.mark.asyncio
async def test_module_invalid_has(self):
class InvalidModuleClass():
pass
self.builder.add_module(self.mod_c)
self.builder.add_module(self.mod_d1.depends_on(self.mod_c))
pipeline = await self.builder.build('abc')
runtime, _ = await pipeline.run()
assert runtime['mod_d1'].has(MockModuleContext)
with pytest.raises(KeyError):
assert runtime['mod_d1'].has(InvalidModuleClass)
@pytest.mark.asyncio
async def test_module_invalid_filter_of(self):
class InvalidModuleClass():
pass
self.builder.add_module(self.mod_c)
self.builder.add_module(self.mod_d1.depends_on(self.mod_c))
pipeline = await self.builder.build('abc')
runtime, _ = await pipeline.run()
assert runtime['mod_d1'].of(MockModuleContext) == ['abc']
with pytest.raises(KeyError):
assert runtime['mod_d1'].of(InvalidModuleClass) == []
@pytest.mark.asyncio
async def test_module_invalid_get(self):
self.builder.add_module(self.mod_c)
self.builder.add_module(self.mod_d1.depends_on(self.mod_c))
self.builder.add_module(self.mod_d2.depends_on(self.mod_d1))
self.builder.add_module(
MockModuleData('mod_d3').depends_on(self.mod_d1).depends_on(
self.mod_d2))
pipeline = await self.builder.build('abc')
runtime, _ = await pipeline.run()
assert runtime['mod_d3'].has(MockModuleData)
with pytest.raises(Exception):
assert runtime['mod_d3'].get(MockModuleData)
@pytest.mark.asyncio
async def test_module_interface_results(self):
class MockCorrectInterface(Module.Interface):
pass
@expose()
@produce(MockCorrectInterface)
@accept(self=True)
@finalize
class MockModuleDataInterface(Module.Runtime):
def run(self, data, **kwargs):
data.result = 'abc'
return data
self.builder.add_module(MockModuleDataInterface('mod_i1'))
self.builder.add_module(
MockModuleDataInterface('mod_i2').depends_on(
self.builder.get_module('mod_i1')))
pipeline = await self.builder.build()
runtime, _ = await pipeline.run()
assert not runtime['mod_i1'].has(MockCorrectInterface)
assert runtime['mod_i2'].has(MockCorrectInterface)
assert runtime['mod_i2'].get(MockCorrectInterface).result == 'abc'
def setup_method(self, method):
ModuleFactory.unregister()
ModuleFactory.register('MockModule', MockModule)
self.pipeline = SequentialPipeline()
self.context = MockContext()
class TestSequentialPipeline:
def setup_method(self, method):
ModuleFactory.unregister()
ModuleFactory.register('MockModule', MockModule)
self.pipeline = SequentialPipeline()
self.context = MockContext()
def teardown_method(self, method):
ModuleFactory.unregister()
def test_pipeline_add_module(self):
module = MockModule('MockName')
self.pipeline.add_module(module)
assert len(self.pipeline.modules) == 1
def test_pipeline_add_not_submodule(self):
class MockNotSubmodule:
pass
module = MockNotSubmodule()
with pytest.raises(TypeError):
self.pipeline.add_module(module)
def test_pipeline_get_module_by_name(self):
module = MockModule('MockName')
self.pipeline.add_module(module)
assert self.pipeline.get_module('MockName') == module
def test_pipeline_get_module_by_name_not_existing(self):
assert self.pipeline.get_module('MockName') is None
@pytest.mark.asyncio
async def test_pipeline_build_no_modules(self):
runtime = await self.pipeline.build(MockContext())
assert len(runtime.modules) == 0
@pytest.mark.asyncio
async def test_pipeline_run_no_modules(self):
assert len(self.pipeline.modules) == 0
runtime = await self.pipeline.build(MockContext())
result, _ = await runtime.run()
assert isinstance(result, dict)
assert len(result) == 0
@pytest.mark.asyncio
async def test_pipeline_run_single_module(self):
module = MockModule('MockName').expose_result()
self.pipeline.add_module(module)
runtime = await self.pipeline.build(MockContext())
results, _ = await runtime.run()
assert isinstance(results, dict)
assert len(results) == 1
assert 'MockName' in results
assert results['MockName'] == 'output'
@pytest.mark.asyncio
async def test_pipeline_run_multiple_depending_modules(self):
module_1 = MockModule('module_1')
module_2 = MockModule('module_2')
module_2.depends_on(module_1)
self.pipeline.add_module(module_1)
self.pipeline.add_module(module_2)
runtime = await self.pipeline.build(MockContext())
assert len(runtime.modules) == 2
assert runtime.modules[0].name == 'module_1'
assert runtime.modules[1].name == 'module_2'
module_3 = MockModule('module_3')
module_3.depends_on(module_2)
self.pipeline.add_module(module_3)
runtime = await self.pipeline.build(MockContext())
assert len(runtime.modules) == 3
assert runtime.modules[2].name == 'module_3'
@pytest.mark.asyncio
async def test_pipeline_is_regular_flag_shallow(self):
@finalize
class RegularModule(Module.Runtime):
def run(self, **kwargs):
return 'run'
@accept(RegularModule)
@finalize
class AggregatorModule(Module.Aggregate):
def process(self, **kwargs):
return 'process'
@accept(AggregatorModule)
@finalize
class RegularModuleAfterAgg(Module.Runtime):
def run(self, **kwargs):
return 'run'
module_aggregator = AggregatorModule('agg_module')
module_regular = RegularModule('reg_module')
module_regular_after_agg = RegularModuleAfterAgg(
'reg_module_after_agg')
module_aggregator.depends_on(module_regular)
module_regular_after_agg.depends_on(module_aggregator)
self.pipeline.add_module(module_regular)
self.pipeline.add_module(module_aggregator)
self.pipeline.add_module(module_regular_after_agg)
runtime = await self.pipeline.build(MockContext())
assert len(runtime.modules) == 3
assert runtime.modules[0].name == 'reg_module' and runtime.modules[
0].is_regular_module
assert runtime.modules[1].name == 'agg_module' and not runtime.modules[
1].is_regular_module
assert runtime.modules[2].name == 'reg_module_after_agg' and \
not runtime.modules[2].is_regular_module
@pytest.mark.asyncio
async def test_pipeline_is_regular_flag_deep(self):
@accept(self=True)
@finalize
class RegularModule(Module.Runtime):
def run(self, **kwargs):
return 'run'
@accept(RegularModule)
@finalize
class AggregatorModule(Module.Aggregate):
def process(self, **kwargs):
return 'process'
@accept(AggregatorModule, self=True)
@finalize
class RegularModuleAfterAgg(Module.Runtime):
def run(self, **kwargs):
return 'run'
module_aggregator = AggregatorModule('agg_module')
module_regular1 = RegularModule('reg_module1')
module_regular11 = RegularModule('reg_module11')
module_regular12 = RegularModule('reg_module12')
module_regular_after_agg1 = RegularModuleAfterAgg(
'reg_module_after_agg1')
module_regular_after_agg11 = RegularModuleAfterAgg(
'reg_module_after_agg11')
module_regular_after_agg12 = RegularModuleAfterAgg(
'reg_module_after_agg12')
module_regular_after_agg121 = RegularModuleAfterAgg(
'reg_module_after_agg121')
module_regular_after_agg2 = RegularModuleAfterAgg(
'reg_module_after_agg2')
module_aggregator.depends_on(module_regular1)
module_regular11.depends_on(module_regular1)
module_regular12.depends_on(module_regular1)
module_regular_after_agg1.depends_on(module_aggregator)
module_regular_after_agg2.depends_on(module_aggregator)
module_regular_after_agg11.depends_on(module_regular_after_agg1)
module_regular_after_agg12.depends_on(module_regular_after_agg1)
module_regular_after_agg121.depends_on(module_regular_after_agg12)
self.pipeline.add_module(module_regular1)
self.pipeline.add_module(module_regular11)
self.pipeline.add_module(module_regular12)
self.pipeline.add_module(module_aggregator)
self.pipeline.add_module(module_regular_after_agg1)
self.pipeline.add_module(module_regular_after_agg2)
self.pipeline.add_module(module_regular_after_agg11)
self.pipeline.add_module(module_regular_after_agg12)
self.pipeline.add_module(module_regular_after_agg121)
runtime = await self.pipeline.build(MockContext())
assert len(runtime.modules) == 9
assert runtime.modules[0].name == 'reg_module1' and runtime.modules[
0].is_regular_module
assert runtime.modules[1].name == 'reg_module12' and runtime.modules[
1].is_regular_module
assert runtime.modules[2].name == 'reg_module11' and runtime.modules[
2].is_regular_module
assert runtime.modules[3].name == 'agg_module' and not runtime.modules[
3].is_regular_module
assert runtime.modules[4].name == 'reg_module_after_agg2' and \
not runtime.modules[4].is_regular_module
assert runtime.modules[5].name == 'reg_module_after_agg1' and \
not runtime.modules[5].is_regular_module
assert runtime.modules[6].name == 'reg_module_after_agg12' and \
not runtime.modules[6].is_regular_module
assert runtime.modules[7].name == 'reg_module_after_agg121' and \
not runtime.modules[7].is_regular_module
assert runtime.modules[8].name == 'reg_module_after_agg11' and \
not runtime.modules[8].is_regular_module
@pytest.mark.asyncio
async def test_pipeline_module_depends_on_aggregate_and_regular_error(
self):
@accept(self=True)
@finalize
class RegularModule(Module.Runtime):
def run(self, **kwargs):
return 'run'
@accept(RegularModule)
@finalize
class AggregatorModule(Module.Aggregate):
def process(self, **kwargs):
return 'process'
@accept(AggregatorModule, RegularModule, self=True)
@finalize
class RegularModuleFurther(Module.Runtime):
def run(self, **kwargs):
return 'run'
module_regular = RegularModule('1')
module_aggregator = AggregatorModule('2')
module_after_agg = RegularModuleFurther('3')
module_after_regular_and_agg = RegularModuleFurther('4')
module_after_agg.depends_on(module_aggregator)
module_after_regular_and_agg.depends_on(module_after_agg)
module_after_regular_and_agg.depends_on(module_regular)
module_aggregator.depends_on(module_regular)
self.pipeline.add_module(module_aggregator)
self.pipeline.add_module(module_regular)
self.pipeline.add_module(module_after_regular_and_agg)
self.pipeline.add_module(module_after_agg)
with pytest.raises(Exception):
await self.pipeline.build()
@pytest.mark.asyncio
async def test_pipeline_modules_run_only_regular(self):
@accept(self=True)
@finalize
class RegularModule(Module.Runtime):
def run(self, **kwargs):
return 'regular run'
@accept(RegularModule)
@finalize
class AggregatorModule(Module.Aggregate):
def process(self, **kwargs):
return 'aggregator run'
@accept(AggregatorModule, RegularModule, self=True)
@finalize
class ModuleAfterAgg(Module.Runtime):
def run(self, **kwargs):
return 'after aggregator run'
module_regular = RegularModule('regular').expose_result()
module_aggregator = AggregatorModule('aggregator').expose_result()
module_after_agg = ModuleAfterAgg('after_agg').expose_result()
module_after_regular = RegularModule('after_reg').expose_result()
module_aggregator.depends_on(module_regular)
module_after_agg.depends_on(module_aggregator)
module_after_regular.depends_on(module_regular)
self.pipeline.add_module(module_regular)
self.pipeline.add_module(module_aggregator)
self.pipeline.add_module(module_after_agg)
self.pipeline.add_module(module_after_regular)
runtime = await self.pipeline.build()
results, _ = await runtime.run()
assert len(results) == 2
assert 'regular' in results
assert 'aggregator' not in results
assert 'after_agg' not in results
assert 'after_reg' in results
@pytest.mark.asyncio
async def test_pipeline_modules_run_only_aggregate(self):
@accept(self=True)
@finalize
class RegularModule(Module.Runtime):
def run(self, **kwargs):
return 'regular run'
@accept(RegularModule)
@finalize
class AggregatorModule(Module.Aggregate):
def process(self, **kwargs):
return 'aggregator run'
@accept(AggregatorModule, RegularModule, self=True)
@finalize
class ModuleAfterAgg(Module.Runtime):
def run(self, **kwargs):
return 'after aggregator run'
module_regular = RegularModule('regular').expose_result()
module_aggregator = AggregatorModule('aggregator').expose_result()
module_after_agg = ModuleAfterAgg('after_agg').expose_result()
module_after_regular = RegularModule('after_reg').expose_result()
module_aggregator.depends_on(module_regular)
module_after_agg.depends_on(module_aggregator)
module_after_regular.depends_on(module_regular)
self.pipeline.add_module(module_regular)
self.pipeline.add_module(module_aggregator)
self.pipeline.add_module(module_after_agg)
self.pipeline.add_module(module_after_regular)
runtime = await self.pipeline.build()
results, _ = await runtime.process()
assert len(results) == 2
assert 'regular' not in results
assert 'aggregator' in results
assert 'after_agg' in results
assert 'after_reg' not in results
@pytest.mark.asyncio
async def test_pipeline_module_aggregate_keeps_state(self):
@accept(self=True)
@finalize
class RegularModule(Module.Runtime):
def run(self, **kwargs):
return 'regular run'
@accept(RegularModule)
@finalize
class AggregatorModule(Module.Aggregate):
def process(self, **kwargs):
return self.state
def aggregate(self, **kwargs):
self.add_data('state')
return self.state
regular = RegularModule('regular').expose_result()
aggregate = AggregatorModule('aggregate').expose_result()
aggregate.depends_on(regular)
self.pipeline.add_module(regular)
self.pipeline.add_module(aggregate)
runtime = await self.pipeline.build()
aggregate_runtime_module = runtime.modules[1]
assert aggregate_runtime_module.name == 'aggregate'
assert aggregate_runtime_module.state_size == 0
assert aggregate_runtime_module.state == []
await runtime.run()
assert aggregate_runtime_module.state_size == 1
assert aggregate_runtime_module.state == ['state']
await runtime.run()
assert aggregate_runtime_module.state_size == 2
assert aggregate_runtime_module.state == ['state', 'state']
@pytest.mark.asyncio
async def test_pipeline_module_aggregate_clears_state(self):
@accept(self=True)
@finalize
class RegularModule(Module.Runtime):
def run(self, **kwargs):
return 'regular run'
@accept(RegularModule)
@finalize
class AggregatorModule(Module.Aggregate):
def process(self, **kwargs):
state_copy = self.state.copy()
self.clear_state()
return state_copy
def aggregate(self, **kwargs):
self.add_data('state')
return self.state
regular = RegularModule('regular').expose_result()
aggregate = AggregatorModule('aggregate').expose_result()
aggregate.depends_on(regular)
self.pipeline.add_module(regular)
self.pipeline.add_module(aggregate)
runtime = await self.pipeline.build()
aggregate_runtime_module = runtime.modules[1]
assert aggregate_runtime_module.name == 'aggregate'
await runtime.run()
await runtime.run()
assert aggregate_runtime_module.state_size == 2
assert aggregate_runtime_module.state == ['state', 'state']
await runtime.process()
assert aggregate_runtime_module.state_size == 0
assert aggregate_runtime_module.state == []
@pytest.mark.asyncio
async def test_pipeline_module_after_gets_aggregated_results(self):
@accept(self=True)
@finalize
class RegularModule(Module.Runtime):
def run(self, **kwargs):
return 'regular run'
@accept(RegularModule)
@finalize
class AggregatorModule(Module.Aggregate):
def aggregate(self, **kwargs):
self.add_data('state')
return self.state
@accept(AggregatorModule, self=True)
@finalize
class ModuleAfterAgg(Module.Runtime):
def run(self, data, **kwargs):
return data.of('aggregate')[0] * 2
regular = RegularModule('regular').expose_result()
aggregate = AggregatorModule('aggregate').expose_result()
module_after_agg = ModuleAfterAgg('after_agg').expose_result()
aggregate.depends_on(regular)
module_after_agg.depends_on(aggregate)
self.pipeline.add_module(regular)
self.pipeline.add_module(aggregate)
self.pipeline.add_module(module_after_agg)
runtime = await self.pipeline.build()
aggregate_runtime_module = runtime.modules[1]
assert aggregate_runtime_module.name == 'aggregate'
await runtime.run()
await runtime.run()
assert aggregate_runtime_module.state_size == 2
assert aggregate_runtime_module.state == ['state', 'state']
results, _ = await runtime.process()
assert 'after_agg' in results
assert len(results['after_agg']) == 4
assert results['after_agg'] == ['state', 'state', 'state', 'state']
@pytest.mark.asyncio
async def test_pipeline_set_parameters(self):
parameters = {'param1': 1}
module = MockModule('MockName').set_parameters(parameters)
self.pipeline.add_module(module)
runtime = await self.pipeline.build()
module_runtime = runtime.modules[0]
assert len(module_runtime.parameters) == 1
assert 'param1' in module_runtime.parameters
@pytest.mark.asyncio
async def test_pipeline_runtime_readonly_parameters(self):
parameters = {'param1': 1}
module = MockModule('MockName').set_parameters(parameters)
self.pipeline.add_module(module)
runtime = await self.pipeline.build()
module_runtime = runtime.modules[0]
with pytest.raises(AttributeError):
module_runtime.parameters = {}
@pytest.mark.asyncio
async def test_pipeline_parameters_available_in_run(self):
@finalize
class ModuleWithParams(Module.Runtime):
def run(self, **kwargs):
return self.parameters
parameters = {'param1': 1}
module = ModuleWithParams(
'module_with_params').expose_result().set_parameters(parameters)
self.pipeline.add_module(module)
runtime = await self.pipeline.build()
results, _ = await runtime.run()
assert 'module_with_params' in results
assert results['module_with_params'] == parameters
@pytest.mark.asyncio
async def test_pipeline_shared_parameters_available_in_runtime(self):
shared_parameters = {'shared_param1': 1}
runtime = await self.pipeline.build(shared_parameters=shared_parameters
)
assert len(runtime.shared_parameters) == 1
assert runtime.shared_parameters == shared_parameters
@pytest.mark.asyncio
async def test_pipeline_same_shared_parameters_in_modules(self):
@finalize
class ModuleWithParams(Module.Runtime):
def run(self, **kwargs):
return self.shared_parameters
@accept(ModuleWithParams)
@finalize
class ModuleWithParamsOther(Module.Runtime):
def run(self, **kwargs):
return self.shared_parameters
module = ModuleWithParams('module_with_params').expose_result()
module_other = ModuleWithParamsOther(
'module_with_params_other').expose_result()
module_other.depends_on(module)
self.pipeline.add_module(module)
self.pipeline.add_module(module_other)
shared_parameters = {'shared_param1': 1}
runtime = await self.pipeline.build(shared_parameters=shared_parameters
)
results, _ = await runtime.run()
assert len(results) == 2
assert 'module_with_params' in results
assert 'module_with_params_other' in results
assert results['module_with_params'] == results[
'module_with_params_other']
@pytest.mark.asyncio
async def test_pipeline_build_run_with_defaults(self):
module = MockModule('mock_name').expose_result()
self.pipeline.add_module(module)
runtime = await self.pipeline.build()
results, _ = await runtime.run()
assert 'mock_name' in results
assert results['mock_name'] == 'output'
@pytest.mark.asyncio
async def test_pipeline_with_groups(self):
self.pipeline.add_module(MockModule('m1', group='g1'))
self.pipeline.add_module(
MockModule('m2').depends_on(self.pipeline.get_module('m1')))
with pytest.warns(UserWarning):
await self.pipeline.build(MockContext())
@pytest.mark.asyncio
async def test_close_pipeline(self):
@accept(self=True)
@finalize
class TestModule(Module.Runtime):
def bootstrap(self):
self.state = 'opened'
def run(self, **kwargs):
return 'regular run'
def teardown(self):
self.state = 'closed'
module_1 = TestModule('module_1')
module_2 = TestModule('module_2')
module_3 = TestModule('module_3')
module_2.depends_on(module_1)
module_3.depends_on(module_2)
self.pipeline.add_module(module_1)
self.pipeline.add_module(module_2)
self.pipeline.add_module(module_3)
runtime = await self.pipeline.build()
await runtime.run()
await runtime.close()
assert runtime._is_closed
assert runtime.modules[0].state == 'closed'
assert runtime.modules[1].state == 'closed'
assert runtime.modules[2].state == 'closed'
with pytest.raises(ClosedPipelineException):
await runtime.run()
with pytest.raises(ClosedPipelineException):
await runtime.process()
@pytest.mark.asyncio
async def test_should_catch_module_exception(self):
@accept(self=True)
@finalize
class TestModuleError(Module.Runtime):
def run(self, **kwargs):
raise Exception()
@accept(TestModuleError, self=True)
@finalize
class TestModuleOk(Module.Runtime):
def run(self, **kwargs):
return 'ok'
module_1_ok = TestModuleOk('module_1_ok').expose_result()
module_2_error = TestModuleError('module_2_error').expose_result()
module_3_ok = TestModuleOk('module_3_ok').expose_result()
module_4_ok = TestModuleOk('module_4_ok').expose_result()
module_3_ok.depends_on(module_1_ok)
module_3_ok.depends_on(module_2_error)
module_4_ok.depends_on(module_3_ok)
self.pipeline.add_module(module_1_ok)
self.pipeline.add_module(module_2_error)
self.pipeline.add_module(module_3_ok)
self.pipeline.add_module(module_4_ok)
runtime = await self.pipeline.build()
result, error = await runtime.run()
assert result is None
assert isinstance(error, Exception)
async def read(
cls: Type[ConfigReader],
config: str,
module_factory: ModuleFactory,
config_parameters: Optional[Dict] = None,
name: Optional[str] = None,
context: Optional[Any] = None,
shared_parameters: Optional[Dict] = None,
*,
logger: Optional[MagdaLogger.Config] = None,
) -> BasePipeline.Runtime:
if config_parameters:
cls._validate_config_parameters_structure(config_parameters)
config = cls._check_and_substitute_declared_variables(
config, config_parameters)
parsed_yaml = yaml.safe_load(config)
config_pipeline_name, modules, shared_parameters, group_options = \
cls._extract_information_from_yaml(parsed_yaml, shared_parameters)
cls._check_expose_settings(modules)
if name and config_pipeline_name:
warnings.warn(
'The pipeline name specified in config wil be overriden '
'by ConfigReader.read parameter')
name = name or config_pipeline_name
if name:
cls._check_pipeline_name(name)
pipeline = (ParallelPipeline(name=name) if any(
[m.group is not None
for m in modules]) else SequentialPipeline(name=name))
pipeline = cls._add_modules_to_pipeline(modules, pipeline,
module_factory)
pipeline = cls._add_group_options(group_options, pipeline)
# connect modules
for mod in modules:
curr_mod_obj = pipeline.get_module(mod.name)
for dependent_mod_name in mod.depends_on:
dependent_mod_obj = pipeline.get_module(dependent_mod_name)
if dependent_mod_obj:
curr_mod_obj.depends_on(dependent_mod_obj)
else:
raise AttributeError(
f"Module '{dependent_mod_name}' hasn't been defined in the config file, "
"whereas it's used as a dependency.")
runtime = await pipeline.build(
context=context,
shared_parameters=shared_parameters,
logger=logger,
)
return runtime
async def build(self, prefix: str = '{CTX}'):
builder = SequentialPipeline()
builder.add_module(ModuleC('m1'))
builder.add_module(ModuleB('m2'))
builder.add_module(ModuleB('m3').depends_on(builder.get_module('m2')))
builder.add_module(
ModuleA('m4').depends_on(builder.get_module('m3')).depends_on(
builder.get_module('m1')).expose_result('final'))
self.pipeline = await builder.build(
context=lambda: Context(prefix),
logger=MagdaLogger.Config(
output=MagdaLogger.Config.Output.LOGGING, ),
)
async def test_should_fail_on_long_cycle(self):
builder = SequentialPipeline()
module_a = SkipModule('A')
module_side_a = SkipModule('side_A')
module_b = SkipModule('B')
module_c = SkipModule('C')
module_d = SkipModule('D')
module_side_d = SkipModule('side_D')
module_e = SkipModule('E')
module_f = SkipModule('F')
module_g = SkipModule('G')
module_side_g = SkipModule('side_G')
# circular dependency
module_b.depends_on(module_a)
module_c.depends_on(module_b)
module_d.depends_on(module_c)
module_e.depends_on(module_d)
module_f.depends_on(module_e)
module_g.depends_on(module_f)
module_a.depends_on(module_g)
module_side_a.depends_on(module_a)
module_side_d.depends_on(module_d)
module_side_g.depends_on(module_g)
builder.add_module(module_a)
builder.add_module(module_b)
builder.add_module(module_c)
builder.add_module(module_d)
builder.add_module(module_e)
builder.add_module(module_f)
builder.add_module(module_g)
builder.add_module(module_side_a)
builder.add_module(module_side_d)
builder.add_module(module_side_g)
with pytest.raises(CyclicDependenciesException):
await builder.build()
async def test_should_not_fail_on_branched_graph_without_cycle(self):
builder = SequentialPipeline()
builder.add_module(SkipModule('A'))
builder.add_module(SkipModule('B').depends_on(builder.get_module('A')))
builder.add_module(SkipModule('C').depends_on(builder.get_module('A')))
builder.add_module(
SkipModule('D').depends_on(builder.get_module('B')).depends_on(
builder.get_module('C')))
pipeline = await builder.build()
assert 4 == len(pipeline.modules)
async def test_should_fail_on_not_connected_graph(self):
builder = SequentialPipeline()
builder.add_module(SkipModule('A'))
builder.add_module(SkipModule('B').depends_on(builder.get_module('A')))
builder.add_module(
SkipModule('C').depends_on(builder.get_module('A')).depends_on(
builder.get_module('B')))
builder.add_module(SkipModule('D'))
with pytest.raises(DisjointGraphException):
results = await builder.build()