def processing_pipeline(
processing_cfg: dict,
shared_processors: dict,
shared_uploaders: dict,
):
name = processing_cfg.get('name', '<no name>')
filter_cfgs = processing_cfg['filter']
if isinstance(filter_cfgs, dict):
filter_cfgs = [filter_cfgs]
filters = [_filter(filter_cfg=filter_cfg) for filter_cfg in filter_cfgs]
downloader = p_downloaders.Downloader()
if 'processor' in processing_cfg:
processor_cfg = processing_cfg['processor']
if isinstance(processor_cfg, str):
proc = shared_processors[processor_cfg]
else:
proc = _processor(processor_cfg=processor_cfg)
else:
proc = p_processors.NoOpProcessor()
upload_cfgs = processing_cfg['upload']
if not isinstance(upload_cfgs, list):
upload_cfgs = [upload_cfgs] # normalise to list
def instantiate_uploader(upload_cfg):
if isinstance(upload_cfg, str):
return shared_uploaders[upload_cfg]
return _uploader(upload_cfg)
uploaders = [
instantiate_uploader(upload_cfg) for upload_cfg in upload_cfgs
]
pipeline = ProcessingPipeline(
name=name,
filters=filters,
downloader=downloader,
processor=proc,
uploaders=uploaders,
)
return pipeline
def test_noop_processor(job):
examinee = processors.NoOpProcessor()
result = examinee.process(job)
assert result is job