def datafiles_to_db_by_source(**kwargs):
"""
Populate tables with source files
"""
if not any(kwargs.values()):
ctx = click.get_current_context()
click.echo(ctx.get_help())
else:
manager = PipelineManager(storage=get_source_folder(), dbi=app.dbi)
for _arg, pipeline_info_list in arg_to_pipeline_config_list.items():
arg = _arg.replace(".", "__")
if kwargs['all'] or kwargs[arg]:
for pipeline, sub_dir, options in pipeline_info_list:
required_flag = options.get('required_flag', None)
if not required_flag or kwargs.get(required_flag, False):
manager.pipeline_register(
pipeline=pipeline,
sub_directory=sub_dir,
force=kwargs['force'],
continue_transform=kwargs['continue'],
products=kwargs['products'],
**options,
)
manager.pipeline_process_all()
def test_pipeline_process_all(self, mock_pipeline_process):
mock_pipeline_process.return_value = None
manager = Manager()
manager._pipelines['fake_pipeline'] = None
manager._pipelines['fake_pipeline_2'] = None
manager.pipeline_process_all()
assert mock_pipeline_process.call_args_list[0][0][0] == 'fake_pipeline'
assert mock_pipeline_process.call_args_list[1][0][
0] == 'fake_pipeline_2'