def extract_pipeline_artifacts( self, parameters_saving_folder: Optional[Path] = None): artifacts = {} for name, dataset in self.initial_catalog._data_sets.items(): if name != self.input_name: if name.startswith("params:"): # we need to persist it locally for mlflow access absolute_param_path = (parameters_saving_folder / f"params_{name[7:]}.pkl") persisted_dataset = PickleDataSet( filepath=absolute_param_path.as_posix()) persisted_dataset.save(dataset.load()) artifact_path = absolute_param_path.as_uri() self._logger.info(( f"The parameter '{name[7:]}' is persisted (as pickle) " "at the following location: '{artifact_path}'")) else: # In this second case, we know it cannot be a MemoryDataSet # weird bug when directly converting PurePosixPath to windows: it is considered as relative artifact_path = (Path( dataset._filepath.as_posix()).resolve().as_uri()) artifacts[name] = artifact_path return artifacts
def extract_pipeline_artifacts(self, catalog: DataCatalog, temp_folder: Path): pipeline_catalog = self._extract_pipeline_catalog(catalog) artifacts = {} for name, dataset in pipeline_catalog._data_sets.items(): if name != self.input_name: if name.startswith("params:"): # we need to persist it locally for mlflow access absolute_param_path = temp_folder / f"params_{name[7:]}.pkl" persisted_dataset = PickleDataSet( filepath=absolute_param_path.as_posix()) persisted_dataset.save(dataset.load()) artifact_path = absolute_param_path.as_uri() else: # In this second case, we know it cannot be a MemoryDataSet # weird bug when directly converting PurePosixPath to windows: it is considered as relative artifact_path = (Path( dataset._filepath.as_posix()).resolve().as_uri()) artifacts[name] = artifact_path return artifacts