def from_config(self, config_dict: Dict):
"""
Sets metadata and artifact_store variables
Args:
config_dict (dict): .zenml config object in dict format.
"""
assert METADATA_KEY in config_dict
assert ARTIFACT_STORE_KEY in config_dict
assert PIPELINES_DIR_KEY in config_dict
self.artifact_store = ArtifactStore(config_dict[ARTIFACT_STORE_KEY])
self.metadata_store = ZenMLMetadataStore.from_config(
config=config_dict[METADATA_KEY])
self.pipelines_dir = config_dict[PIPELINES_DIR_KEY]
def from_config(cls, config: Dict):
"""
Convert from pipeline config to ZenML Pipeline object.
All steps are also populated and configuration set to parameters set
in the config file.
Args:
config: a ZenML config in dict-form (probably loaded from YAML).
"""
# start with artifact store
artifact_store = ArtifactStore(config[keys.GlobalKeys.ARTIFACT_STORE])
# metadata store
metadata_store = ZenMLMetadataStore.from_config(
config=config[keys.GlobalKeys.METADATA_STORE])
# orchestration backend
backend = OrchestratorBaseBackend.from_config(
config[keys.GlobalKeys.BACKEND])
# pipeline configuration
p_config = config[keys.GlobalKeys.PIPELINE]
kwargs = p_config[keys.PipelineKeys.ARGS]
pipeline_name = kwargs.pop(keys.PipelineDetailKeys.NAME)
pipeline_source = p_config[keys.PipelineKeys.SOURCE]
# populate steps
steps_dict: Dict = {}
for step_key, step_config in p_config[keys.PipelineKeys.STEPS].items():
steps_dict[step_key] = BaseStep.from_config(step_config)
# datasource
datasource = BaseDatasource.from_config(
config[keys.GlobalKeys.PIPELINE])
class_ = source_utils.load_source_path_class(pipeline_source)
obj = class_(steps_dict=steps_dict,
backend=backend,
artifact_store=artifact_store,
metadata_store=metadata_store,
datasource=datasource,
pipeline_name=pipeline_name,
name=cls.get_name_from_pipeline_name(pipeline_name),
**kwargs)
obj._immutable = True
return obj
def test_from_config():
config = {MLMetadataKeys.TYPE: None, MLMetadataKeys.ARGS: {}}
# throws because base MDStore is not in the factory
with pytest.raises(AssertionError):
_ = ZenMLMetadataStore.from_config(config)