async def build(self, prefix: str = '{CTX}'):
builder = SequentialPipeline()
builder.add_module(ModuleC('m1'))
builder.add_module(ModuleB('m2'))
builder.add_module(ModuleB('m3').depends_on(builder.get_module('m2')))
builder.add_module(
ModuleA('m4').depends_on(builder.get_module('m3')).depends_on(
builder.get_module('m1')).expose_result('final'))
self.pipeline = await builder.build(lambda: Context(prefix))
async def build(self, prefix: str = '{CTX}'):
builder = SequentialPipeline()
builder.add_module(ModuleC('m1'))
builder.add_module(ModuleB('m2'))
builder.add_module(ModuleB('m3').depends_on(builder.get_module('m2')))
builder.add_module(
ModuleA('m4').depends_on(builder.get_module('m3')).depends_on(
builder.get_module('m1')).expose_result('final'))
self.pipeline = await builder.build(
context=lambda: Context(prefix),
logger=MagdaLogger.Config(
output=MagdaLogger.Config.Output.LOGGING, ),
)
async def build(self, prefix: str = '{CTX}'):
ModuleFactory.register('ModuleA', ModuleA)
ModuleFactory.register('ModuleB', ModuleB)
ModuleFactory.register('ModuleC', ModuleC)
config_file = self.get_config_file('sample_config_parallel.yaml')
with open(config_file, 'r') as config:
config = config.read()
self.pipeline = await ConfigReader.read(
config,
ModuleFactory,
{'GROUP_M1': 'g1', 'GROUP_M4': 'g3'},
context=lambda: Context(prefix)
)
async def build(self, prefix: str = '{CTX}'):
ModuleFactory.register('ModuleA', ModuleA)
ModuleFactory.register('ModuleD', ModuleD)
ModuleFactory.register('ModuleC', ModuleC)
config_file = self.get_config_file('sample_config_sequential.yaml')
with open(config_file, 'r') as config:
config = config.read()
self.pipeline = await ConfigReader.read(
config,
ModuleFactory, {
'THRESHOLD_1': 0.2,
'THRESHOLD_2': 0.5
},
context=lambda: Context(prefix))
async def build(self, prefix: str = '{CTX}'):
builder = ParallelPipeline()
builder.add_group(ParallelPipeline.Group('g1'))
builder.add_group(ParallelPipeline.Group('g2'))
builder.add_group(ParallelPipeline.Group('g3'))
builder.add_module(ModuleC('m1', group='g1'))
builder.add_module(ModuleB('m2', group='g2'))
builder.add_module(
ModuleB('m3', group='g2').depends_on(builder.get_module('m2')))
builder.add_module(
ModuleA('m4', group='g3').depends_on(
builder.get_module('m3')).depends_on(
builder.get_module('m1')).expose_result('final'))
self.pipeline = await builder.build(lambda: Context(prefix))
async def build(self, prefix: str = '{CTX}'):
config_file = self.get_config_file('sample_config_parallel.yaml')
config_params = {
'GROUP_M1': 'g1',
'GROUP_M4': 'g3',
}
with open(config_file, 'r') as config:
config = config.read()
self.pipeline = await ConfigReader.read(
config,
ModuleFactory,
config_parameters=config_params,
context=lambda: Context(prefix),
logger=MagdaLogger.Config(),
)
async def build(self, prefix: str = '{CTX}'):
config_file = self.get_config_file('sample_config_sequential.yaml')
with open(config_file, 'r') as config:
config = config.read()
config_params = {
'THRESHOLD_1': 0.2,
'THRESHOLD_2': 0.5,
}
self.pipeline = await ConfigReader.read(
config,
ModuleFactory,
name='CustomName',
config_parameters=config_params,
context=lambda: Context(prefix),
logger=MagdaLogger.Config(),
)
async def build(self, prefix: str = '{CTX}'):
# The pipeline name is optional but we can directly set it to 'Example' here
builder = ParallelPipeline(name='Example')
builder.add_group(ParallelPipeline.Group('g1'))
builder.add_group(ParallelPipeline.Group('g2'))
builder.add_group(ParallelPipeline.Group('g3'))
builder.add_module(ModuleC('m1', group='g1'))
builder.add_module(ModuleB('m2', group='g2'))
builder.add_module(
ModuleB('m3', group='g2')
.depends_on(builder.get_module('m2'))
)
builder.add_module(
ModuleA('m4', group='g3')
.depends_on(builder.get_module('m3'))
.depends_on(builder.get_module('m1'))
.expose_result('final')
)
self.pipeline = await builder.build(
context=lambda: Context(prefix),
logger=MagdaLogger.Config(),
)