def extract_pipeline_artifacts(
self, parameters_saving_folder: Optional[Path] = None):
artifacts = {}
for name, dataset in self.initial_catalog._data_sets.items():
if name != self.input_name:
if name.startswith("params:"):
# we need to persist it locally for mlflow access
absolute_param_path = (parameters_saving_folder /
f"params_{name[7:]}.pkl")
persisted_dataset = PickleDataSet(
filepath=absolute_param_path.as_posix())
persisted_dataset.save(dataset.load())
artifact_path = absolute_param_path.as_uri()
self._logger.info((
f"The parameter '{name[7:]}' is persisted (as pickle) "
"at the following location: '{artifact_path}'"))
else:
# In this second case, we know it cannot be a MemoryDataSet
# weird bug when directly converting PurePosixPath to windows: it is considered as relative
artifact_path = (Path(
dataset._filepath.as_posix()).resolve().as_uri())
artifacts[name] = artifact_path
return artifacts
def extract_pipeline_artifacts(self, catalog: DataCatalog,
temp_folder: Path):
pipeline_catalog = self._extract_pipeline_catalog(catalog)
artifacts = {}
for name, dataset in pipeline_catalog._data_sets.items():
if name != self.input_name:
if name.startswith("params:"):
# we need to persist it locally for mlflow access
absolute_param_path = temp_folder / f"params_{name[7:]}.pkl"
persisted_dataset = PickleDataSet(
filepath=absolute_param_path.as_posix())
persisted_dataset.save(dataset.load())
artifact_path = absolute_param_path.as_uri()
else:
# In this second case, we know it cannot be a MemoryDataSet
# weird bug when directly converting PurePosixPath to windows: it is considered as relative
artifact_path = (Path(
dataset._filepath.as_posix()).resolve().as_uri())
artifacts[name] = artifact_path
return artifacts