def to_config(path: Text,
artifact_store_path: Text = None,
metadata_store: Optional[Type[ZenMLMetadataStore]] = None,
pipelines_dir: Text = None):
"""
Creates a default .zenml config at path/zenml/.zenml_config.
Args:
path (str): path to a directory.
metadata_store: metadata store definition.
artifact_store_path (str): path where to store artifacts.
pipelines_dir (str): path where to store pipeline configs.
"""
config_dir_path = os.path.join(path, ZENML_DIR_NAME)
config_path = os.path.join(config_dir_path, ZENML_CONFIG_NAME)
if path_utils.file_exists(config_path):
raise AssertionError(f'.zenml file already exists at '
f'{config_path}. '
f'Cannot replace. Please delete the '
f'{config_dir_path} directory first.')
# Create config dir
path_utils.create_dir_if_not_exists(config_dir_path)
if artifact_store_path is None:
artifact_store_path = \
os.path.join(config_dir_path, ARTIFACT_STORE_DEFAULT_DIR)
else:
# if provided, then resolve it absolutely
artifact_store_path = path_utils.resolve_relative_path(
artifact_store_path)
# create artifact_store path
path_utils.create_dir_if_not_exists(artifact_store_path)
if metadata_store is None:
uri = os.path.join(artifact_store_path,
ML_METADATA_SQLITE_DEFAULT_NAME)
from zenml.metadata import \
SQLiteMetadataStore
metadata_dict = SQLiteMetadataStore(uri).to_config()
else:
metadata_dict = metadata_store.to_config()
if pipelines_dir is None:
pipelines_dir = os.path.join(path, PIPELINES_DEFAULT_DIR_NAME)
else:
# if provided, still resolve
pipelines_dir = path_utils.resolve_relative_path(pipelines_dir)
path_utils.create_dir_if_not_exists(pipelines_dir)
config_dict = {
ARTIFACT_STORE_KEY: artifact_store_path,
METADATA_KEY: metadata_dict,
PIPELINES_DIR_KEY: pipelines_dir,
}
# Write initial config
yaml_utils.write_yaml(config_path, config_dict)
def save(self):
config_dict = {
ARTIFACT_STORE_KEY: self.artifact_store.path,
METADATA_KEY: self.metadata_store.to_config(),
PIPELINES_DIR_KEY: self.pipelines_dir,
}
# Write initial config
yaml_utils.write_yaml(self.config_path, config_dict)
def register_pipeline(self, file_name: Text, config: Dict[Text, Any]):
"""
Registers a pipeline in the artifact store as a YAML file.
Args:
file_name (str): file name of pipeline
config (dict): dict representation of ZenML config
"""
self._check_if_initialized()
pipelines_dir = self.zenml_config.get_pipelines_dir()
# Create dir
path_utils.create_dir_if_not_exists(pipelines_dir)
# Write
yaml_utils.write_yaml(os.path.join(pipelines_dir, file_name), config)
def test_yaml_configuration_with_invalid_step_name(tmp_path):
"""Test that a config yaml with an invalid step name raises an exception"""
pipeline_instance = create_pipeline_with_config_value(0)
yaml_path = os.path.join(tmp_path, "config.yaml")
write_yaml(
yaml_path,
{"steps": {
"WRONG_STEP_NAME": {
"parameters": {
"value": 0
}
}
}},
)
with pytest.raises(PipelineConfigurationError):
_ = pipeline_instance.with_config(yaml_path)
def test_dont_overwrite_step_parameter_with_config_yaml(tmp_path):
"""Test that step parameters don't get overwritten by yaml file
if not forced."""
config_value = 0
pipeline_instance = create_pipeline_with_config_value(config_value)
yaml_path = os.path.join(tmp_path, "config.yaml")
yaml_config_value = 1
write_yaml(
yaml_path,
{"steps": {
"step_": {
"parameters": {
"value": yaml_config_value
}
}
}},
)
pipeline_instance = pipeline_instance.with_config(yaml_path)
step_instance = pipeline_instance.steps["step_"]
assert step_instance.PARAM_SPEC["value"] == config_value
def test_overwrite_step_parameter_with_config_yaml(tmp_path):
"""Test whether step parameters can be overwritten using a config yaml."""
config_value = 0
pipeline_instance = create_pipeline_with_config_value(config_value)
yaml_path = os.path.join(tmp_path, "config.yaml")
yaml_config_value = 1
write_yaml(
yaml_path,
{"steps": {
"step_": {
"parameters": {
"value": yaml_config_value
}
}
}},
)
pipeline_instance = pipeline_instance.with_config(
yaml_path, overwrite_step_parameters=True)
step_instance = pipeline_instance.steps["step_"]
assert step_instance.PARAM_SPEC["value"] == yaml_config_value