def test_get_components_status(repo): mds1 = ZenMLMetadataStore() random_pipeline = random.choice(repo.get_pipelines()) with pytest.raises(expected_query_error): _ = mds1.get_components_status(random_pipeline)
def test_to_config(): mds1 = ZenMLMetadataStore() # disallow to/from_config for the base class by checking against # factory keys with pytest.raises(AssertionError): mds1.to_config()
def test_get_artifacts_by_execution(): mds1 = ZenMLMetadataStore() # no execution possible fake_id = "abcdefg" with pytest.raises(ValueError): _ = mds1.get_artifacts_by_execution(fake_id)
def test_get_component_execution(repo): mds1 = ZenMLMetadataStore() random_pipeline = random.choice(repo.get_pipelines()) component_name = GDPComponent.SplitGen.name with pytest.raises(expected_query_error): _ = mds1.get_component_execution(random_pipeline, component_name)
def test_get_pipeline_executions(repo): mds1 = ZenMLMetadataStore() random_pipeline = random.choice(repo.get_pipelines()) # if we query a different metadata store for the pipeline, # there should be no executions, i.e. an empty list with pytest.raises(expected_query_error): _ = mds1.get_pipeline_executions(random_pipeline)
def test_get_artifacts_by_component(repo): mds1 = ZenMLMetadataStore() random_pipeline = random.choice(repo.get_pipelines()) # pick a component guaranteed to be present component_name = GDPComponent.SplitGen.name with pytest.raises(expected_query_error): _ = mds1.get_artifacts_by_component(random_pipeline, component_name)
def test_get_pipeline_status(repo): random_pipeline = random.choice(repo.get_pipelines()) mds1 = ZenMLMetadataStore() # TODO: This returns a NotStarted enum, which may be misleading as the # associated store does not even exist # with pytest.raises(expected_query_error): assert mds1.get_pipeline_status(random_pipeline) == \ PipelineStatusTypes.NotStarted.name
def from_config(self, config_dict: Dict): """ Sets metadata and artifact_store variables Args: config_dict (dict): .zenml config object in dict format. """ assert METADATA_KEY in config_dict assert ARTIFACT_STORE_KEY in config_dict assert PIPELINES_DIR_KEY in config_dict self.artifact_store = ArtifactStore(config_dict[ARTIFACT_STORE_KEY]) self.metadata_store = ZenMLMetadataStore.from_config( config=config_dict[METADATA_KEY]) self.pipelines_dir = config_dict[PIPELINES_DIR_KEY]
def from_config(cls, config: Dict): """ Convert from pipeline config to ZenML Pipeline object. All steps are also populated and configuration set to parameters set in the config file. Args: config: a ZenML config in dict-form (probably loaded from YAML). """ # start with artifact store artifact_store = ArtifactStore(config[keys.GlobalKeys.ARTIFACT_STORE]) # metadata store metadata_store = ZenMLMetadataStore.from_config( config=config[keys.GlobalKeys.METADATA_STORE]) # orchestration backend backend = OrchestratorBaseBackend.from_config( config[keys.GlobalKeys.BACKEND]) # pipeline configuration p_config = config[keys.GlobalKeys.PIPELINE] kwargs = p_config[keys.PipelineKeys.ARGS] pipeline_name = kwargs.pop(keys.PipelineDetailKeys.NAME) pipeline_source = p_config[keys.PipelineKeys.SOURCE] # populate steps steps_dict: Dict = {} for step_key, step_config in p_config[keys.PipelineKeys.STEPS].items(): steps_dict[step_key] = BaseStep.from_config(step_config) # datasource datasource = BaseDatasource.from_config( config[keys.GlobalKeys.PIPELINE]) class_ = source_utils.load_source_path_class(pipeline_source) obj = class_(steps_dict=steps_dict, backend=backend, artifact_store=artifact_store, metadata_store=metadata_store, datasource=datasource, pipeline_name=pipeline_name, name=cls.get_name_from_pipeline_name(pipeline_name), **kwargs) obj._immutable = True return obj
def test_from_config(): config = {MLMetadataKeys.TYPE: None, MLMetadataKeys.ARGS: {}} # throws because base MDStore is not in the factory with pytest.raises(AssertionError): _ = ZenMLMetadataStore.from_config(config)
def test_metadata_init(): mds1 = ZenMLMetadataStore() with pytest.raises(ValueError): _ = mds1.store