def bootstrap(self, logger: MagdaLogger):
ctx: Context = self.context
logger.info(f'Context.timer = {ctx.timer} to demonstrate INFO level')
logger.debug(f'Context.timer = {ctx.timer} to demonstrate DEBUG level')
logger.warn(f'Context.timer = {ctx.timer} to demonstrate WARN level')
logger.error(f'Context.timer = {ctx.timer} to demonstrate ERROR level')
logger.critical(
f'Context.timer = {ctx.timer} to demonstrate CRITICAL level')
async def bootstrap(self, logger: MagdaLogger) -> None:
self._logger = logger
for module in self._modules:
logger = self._logger.chain(module=MagdaLogger.Parts.Module(
name=module.name,
kind=module.__class__.__name__,
), )
logger.event(self.BOOTSTRAP_EVENT)
await module._on_bootstrap(logger=logger)
async def test_should_log_pipeline_from_config(self):
mock = MagicMock()
module1_name, module2_name = 'TestModuleA1', 'TestModuleA2'
config_path = Path(__file__).parent / 'configs' / 'pipeline.yml'
ModuleFactory.register('LoggerModuleA', ModuleA)
with open(config_path) as config:
pipe = await ConfigReader.read(
config=config.read(),
module_factory=ModuleFactory,
config_parameters={
"name1": module1_name,
"name2": module2_name
},
logger=MagdaLogger.Config(colors=False, output=mock),
)
await pipe.run()
assert mock.call_count == 6
mock.assert_any_call(
PartialOutput(f'{pipe.name}.*{module1_name}.*BOOTSTRAP'))
mock.assert_any_call(
PartialOutput(f'{pipe.name}.*{module2_name}.*BOOTSTRAP'))
mock.assert_any_call(
PartialOutput(f'{pipe.name}.*{module1_name}.*RUN'))
mock.assert_any_call(
PartialOutput(f'{pipe.name}.*{module2_name}.*RUN'))
mock.assert_any_call(
PartialOutput(f'{pipe.name}.*{module1_name}.*{MESSAGE}'))
mock.assert_any_call(
PartialOutput(f'{pipe.name}.*{module2_name}.*{MESSAGE}'))
async def test_should_log_pipeline(self):
mock = MagicMock()
pipe_name = 'TestPipeline1'
module1, module2 = ModuleA('TestModuleA1'), ModuleA('TestModuleA2')
builder = SequentialPipeline(pipe_name)
builder.add_module(module1)
builder.add_module(module2.depends_on(module1))
pipe = await builder.build(
logger=MagdaLogger.Config(colors=False, output=mock))
await pipe.run()
assert mock.call_count == 6
mock.assert_any_call(
PartialOutput(f'{pipe_name}.*{module1.name}.*BOOTSTRAP'))
mock.assert_any_call(
PartialOutput(f'{pipe_name}.*{module2.name}.*BOOTSTRAP'))
mock.assert_any_call(
PartialOutput(f'{pipe_name}.*{module1.name}.*RUN'))
mock.assert_any_call(
PartialOutput(f'{pipe_name}.*{module2.name}.*RUN'))
mock.assert_any_call(
PartialOutput(f'{pipe_name}.*{module1.name}.*{MESSAGE}'))
mock.assert_any_call(
PartialOutput(f'{pipe_name}.*{module2.name}.*{MESSAGE}'))
def __init__(self, *, graph: Graph, logger_config: MagdaLogger.Config,
**kwargs):
super().__init__(**kwargs)
self.graph = graph
self._is_closed = False
self._logger = MagdaLogger.of(
logger_config,
pipeline=MagdaLogger.Parts.Pipeline(
name=self.name,
kind='SequentialPipeline',
),
)
def __init__(
self,
*,
groups: List[Group.Runtime],
logger_config: MagdaLogger.Config,
**kwargs,
):
super().__init__(**kwargs)
self.groups = groups
self._jobs: List[Job] = []
self._is_closed = False
self._idle_flag = asyncio.Event()
self._idle_flag.set()
self._logger = MagdaLogger.of(logger_config,
pipeline=MagdaLogger.Parts.Pipeline(
name=self.name,
kind='ParallelPipeline',
))
async def build(self, prefix: str = '{CTX}'):
builder = ParallelPipeline()
builder.add_group(ParallelPipeline.Group('g1', replicas=3))
builder.add_group(ParallelPipeline.Group('g2', replicas=3))
builder.add_group(ParallelPipeline.Group('g3', replicas=3))
builder.add_module(ModuleC('m1', group='g1'))
builder.add_module(ModuleB('m2', group='g2'))
builder.add_module(
ModuleB('m3', group='g2').depends_on(builder.get_module('m2')))
builder.add_module(
ModuleA('m4', group='g3').depends_on(
builder.get_module('m3')).depends_on(
builder.get_module('m1')).expose_result('final'))
self.pipeline = await builder.build(
context=lambda: Context(prefix),
logger=MagdaLogger.Config(),
)
async def build(self, prefix: str = '{CTX}'):
# The pipeline name is optional but we can directly set it to 'Example' here
builder = ParallelPipeline(name='Example')
builder.add_group(ParallelPipeline.Group('g1'))
builder.add_group(ParallelPipeline.Group('g2'))
builder.add_group(ParallelPipeline.Group('g3'))
builder.add_module(ModuleC('m1', group='g1'))
builder.add_module(ModuleB('m2', group='g2'))
builder.add_module(
ModuleB('m3', group='g2')
.depends_on(builder.get_module('m2'))
)
builder.add_module(
ModuleA('m4', group='g3')
.depends_on(builder.get_module('m3'))
.depends_on(builder.get_module('m1'))
.expose_result('final')
)
self.pipeline = await builder.build(
context=lambda: Context(prefix),
logger=MagdaLogger.Config(),
)
async def bootstrap(self, logger: MagdaLogger):
self._logger = logger.chain(group=MagdaLogger.Parts.Group(
name=self.name,
replica=self.index,
), )
await self.graph.bootstrap(self._logger)
def run(self, logger: MagdaLogger, *args, **kwargs):
logger.info(MESSAGE)
return None
async def bootstrap(self, logger: MagdaLogger):
self._logger = logger.chain(group=MagdaLogger.Parts.Group(
name=self.name))
await self.pool.bootstrap(self._logger, self._hooks)
async def teardown(self, logger: MagdaLogger):
ctx: Context = self.context
logger.info(f'Long... | Context.timer = {ctx.timer}')
await asyncio.sleep(1)
logger.info(f'...Teardown | Context.timer = {ctx.timer}')