def to_config(path: Text, artifact_store_path: Text = None, metadata_store: Optional[Type[ZenMLMetadataStore]] = None, pipelines_dir: Text = None): """ Creates a default .zenml config at path/zenml/.zenml_config. Args: path (str): path to a directory. metadata_store: metadata store definition. artifact_store_path (str): path where to store artifacts. pipelines_dir (str): path where to store pipeline configs. """ config_dir_path = os.path.join(path, ZENML_DIR_NAME) config_path = os.path.join(config_dir_path, ZENML_CONFIG_NAME) if path_utils.file_exists(config_path): raise AssertionError(f'.zenml file already exists at ' f'{config_path}. ' f'Cannot replace. Please delete the ' f'{config_dir_path} directory first.') # Create config dir path_utils.create_dir_if_not_exists(config_dir_path) if artifact_store_path is None: artifact_store_path = \ os.path.join(config_dir_path, ARTIFACT_STORE_DEFAULT_DIR) else: # if provided, then resolve it absolutely artifact_store_path = path_utils.resolve_relative_path( artifact_store_path) # create artifact_store path path_utils.create_dir_if_not_exists(artifact_store_path) if metadata_store is None: uri = os.path.join(artifact_store_path, ML_METADATA_SQLITE_DEFAULT_NAME) from zenml.metadata import \ SQLiteMetadataStore metadata_dict = SQLiteMetadataStore(uri).to_config() else: metadata_dict = metadata_store.to_config() if pipelines_dir is None: pipelines_dir = os.path.join(path, PIPELINES_DEFAULT_DIR_NAME) else: # if provided, still resolve pipelines_dir = path_utils.resolve_relative_path(pipelines_dir) path_utils.create_dir_if_not_exists(pipelines_dir) config_dict = { ARTIFACT_STORE_KEY: artifact_store_path, METADATA_KEY: metadata_dict, PIPELINES_DIR_KEY: pipelines_dir, } # Write initial config yaml_utils.write_yaml(config_path, config_dict)
def save(self): config_dict = { ARTIFACT_STORE_KEY: self.artifact_store.path, METADATA_KEY: self.metadata_store.to_config(), PIPELINES_DIR_KEY: self.pipelines_dir, } # Write initial config yaml_utils.write_yaml(self.config_path, config_dict)
def register_pipeline(self, file_name: Text, config: Dict[Text, Any]): """ Registers a pipeline in the artifact store as a YAML file. Args: file_name (str): file name of pipeline config (dict): dict representation of ZenML config """ self._check_if_initialized() pipelines_dir = self.zenml_config.get_pipelines_dir() # Create dir path_utils.create_dir_if_not_exists(pipelines_dir) # Write yaml_utils.write_yaml(os.path.join(pipelines_dir, file_name), config)
def test_yaml_configuration_with_invalid_step_name(tmp_path): """Test that a config yaml with an invalid step name raises an exception""" pipeline_instance = create_pipeline_with_config_value(0) yaml_path = os.path.join(tmp_path, "config.yaml") write_yaml( yaml_path, {"steps": { "WRONG_STEP_NAME": { "parameters": { "value": 0 } } }}, ) with pytest.raises(PipelineConfigurationError): _ = pipeline_instance.with_config(yaml_path)
def test_dont_overwrite_step_parameter_with_config_yaml(tmp_path): """Test that step parameters don't get overwritten by yaml file if not forced.""" config_value = 0 pipeline_instance = create_pipeline_with_config_value(config_value) yaml_path = os.path.join(tmp_path, "config.yaml") yaml_config_value = 1 write_yaml( yaml_path, {"steps": { "step_": { "parameters": { "value": yaml_config_value } } }}, ) pipeline_instance = pipeline_instance.with_config(yaml_path) step_instance = pipeline_instance.steps["step_"] assert step_instance.PARAM_SPEC["value"] == config_value
def test_overwrite_step_parameter_with_config_yaml(tmp_path): """Test whether step parameters can be overwritten using a config yaml.""" config_value = 0 pipeline_instance = create_pipeline_with_config_value(config_value) yaml_path = os.path.join(tmp_path, "config.yaml") yaml_config_value = 1 write_yaml( yaml_path, {"steps": { "step_": { "parameters": { "value": yaml_config_value } } }}, ) pipeline_instance = pipeline_instance.with_config( yaml_path, overwrite_step_parameters=True) step_instance = pipeline_instance.steps["step_"] assert step_instance.PARAM_SPEC["value"] == yaml_config_value