def run(self, pipeline: Pipeline, catalog: DataCatalog) -> Dict[str, Any]:
"""Run the ``Pipeline`` using the ``DataSet``s provided by ``catalog``
and save results back to the same objects.
Args:
pipeline: The ``Pipeline`` to run.
catalog: The ``DataCatalog`` from which to fetch data.
Raises:
ValueError: Raised when ``Pipeline`` inputs cannot be satisfied.
Returns:
Any node outputs that cannot be processed by the ``DataCatalog``.
These are returned in a dictionary, where the keys are defined
by the node outputs.
"""
catalog = catalog.shallow_copy()
unsatisfied = pipeline.inputs() - set(catalog.list())
if unsatisfied:
raise ValueError("Pipeline input(s) {} not found in the "
"DataCatalog".format(unsatisfied))
free_outputs = pipeline.outputs() - set(catalog.list())
unregistered_ds = pipeline.data_sets() - set(catalog.list())
for ds_name in unregistered_ds:
catalog.add(ds_name, self.create_default_data_set(ds_name))
self._run(pipeline, catalog)
self._logger.info("Pipeline execution completed successfully.")
return {ds_name: catalog.load(ds_name) for ds_name in free_outputs}
def test_all_before_copy_and_add(self, fake_data_set, fake_transformer):
catalog = DataCatalog()
catalog.add_transformer(fake_transformer)
catalog = catalog.shallow_copy()
catalog.add("test", fake_data_set)
catalog.save("test", 42)
assert catalog.load("test") == 44
assert fake_data_set.log == [("save", 43), ("load", 43)]
assert fake_transformer.log == [("save", 42), ("load", 43)]
