def __init__(self,
config: PipelineConfig,
dry_run: bool = False,
preview_mode: bool = False):
self.config = config
self.dry_run = dry_run
self.preview_mode = preview_mode
self.ctx = PipelineContext(
run_id=self.config.run_id,
datahub_api=self.config.datahub_api,
pipeline_name=self.config.pipeline_name,
dry_run=dry_run,
preview_mode=preview_mode,
)
source_type = self.config.source.type
source_class = source_registry.get(source_type)
self.source: Source = source_class.create(
self.config.source.dict().get("config", {}), self.ctx)
logger.debug(f"Source type:{source_type},{source_class} configured")
sink_type = self.config.sink.type
sink_class = sink_registry.get(sink_type)
sink_config = self.config.sink.dict().get("config", {})
self.sink: Sink = sink_class.create(sink_config, self.ctx)
logger.debug(
f"Sink type:{self.config.sink.type},{sink_class} configured")
self.extractor_class = extractor_registry.get(
self.config.source.extractor)
self._configure_transforms()
self._configure_reporting()
def __init__(self, config: PipelineConfig):
self.config = config
self.ctx = PipelineContext(run_id=self.config.run_id)
source_type = self.config.source.type
source_class = source_registry.get(source_type)
self.source: Source = source_class.create(
self.config.source.dict().get("config", {}), self.ctx
)
logger.debug(f"Source type:{source_type},{source_class} configured")
sink_type = self.config.sink.type
sink_class = sink_registry.get(sink_type)
sink_config = self.config.sink.dict().get("config", {})
self.sink: Sink = sink_class.create(sink_config, self.ctx)
logger.debug(f"Sink type:{self.config.sink.type},{sink_class} configured")
self.extractor_class = extractor_registry.get(self.config.source.extractor)