def test_builder_metadata(raw_model_config): """ Ensure the builder works with various model configs and that each has expected/valid metadata results. """ model_config = yaml.load(raw_model_config, Loader=yaml.FullLoader) data_config = get_random_data() machine = Machine( name="model-name", dataset=data_config, model=model_config, project_name="test" ) model, machine_out = ModelBuilder(machine).build() # Check metadata, and only verify 'history' if it's a *Keras* type model machine_check(machine_out, "Keras" in raw_model_config)
def test_provide_saved_model_simple_happy_path(tmpdir): """ Test provide_saved_model with no caching """ model_config = {"sklearn.decomposition.PCA": {"svd_solver": "auto"}} data_config = get_random_data() output_dir = os.path.join(tmpdir, "model") machine = Machine( name="model-name", dataset=data_config, model=model_config, project_name="test" ) ModelBuilder(machine).build(output_dir=output_dir) # Assert the model was saved at the location # Should be model file, and the metadata assert len(os.listdir(output_dir)) == 2
def test_output_scores_metadata(): data_config = get_random_data() raw_model_config = f""" gordo.machine.model.anomaly.diff.DiffBasedAnomalyDetector: scaler: sklearn.preprocessing.MinMaxScaler base_estimator: sklearn.compose.TransformedTargetRegressor: transformer: sklearn.preprocessing.MinMaxScaler regressor: sklearn.pipeline.Pipeline: steps: - sklearn.preprocessing.MinMaxScaler - gordo.machine.model.models.KerasAutoEncoder: kind: feedforward_hourglass batch_size: 3 compression_factor: 0.5 encoding_layers: 1 func: tanh out_func: linear epochs: 1 """ model_config = yaml.load(raw_model_config, Loader=yaml.FullLoader) machine = Machine(name="model-name", dataset=data_config, model=model_config, project_name="test") model, machine_out = ModelBuilder(machine).build() scores_metadata = machine_out.metadata.build_metadata.model.cross_validation.scores assert (scores_metadata["explained-variance-score-Tag-1"]["fold-mean"] + scores_metadata["explained-variance-score-Tag-2"]["fold-mean"] ) / 2 == pytest.approx( scores_metadata["explained-variance-score"]["fold-mean"]) assert ( scores_metadata["r2-score-Tag-1"]["fold-mean"] + scores_metadata["r2-score-Tag-2"]["fold-mean"]) / 2 == pytest.approx( scores_metadata["r2-score"]["fold-mean"]) assert (scores_metadata["mean-squared-error-Tag-1"]["fold-mean"] + scores_metadata["mean-squared-error-Tag-2"]["fold-mean"] ) / 2 == pytest.approx( scores_metadata["mean-squared-error"]["fold-mean"]) assert (scores_metadata["mean-absolute-error-Tag-1"]["fold-mean"] + scores_metadata["mean-absolute-error-Tag-2"]["fold-mean"] ) / 2 == pytest.approx( scores_metadata["mean-absolute-error"]["fold-mean"])
def test_get_metadata_helper(model: BaseEstimator, expect_empty_dict: bool): """ Ensure the builder works with various model configs and that each has expected/valid metadata results. """ X, y = np.random.random((1000, 4)), np.random.random((1000, )) model.fit(X, y) metadata = ModelBuilder._extract_metadata_from_model(model) # All the metadata we've implemented so far is 'history', so we'll check that if not expect_empty_dict: assert "history" in metadata assert all(name in metadata["history"] for name in ("params", "loss", "accuracy")) else: assert dict() == metadata
def test_provide_saved_model_caching_handle_existing_different_register(tmpdir): """If the model exists in the model register, but the output_dir is not where the model is, the model is copied to the new location, unless the new location already exists. If it does then return it""" model_config = {"sklearn.decomposition.PCA": {"svd_solver": "auto"}} data_config = get_random_data() output_dir1 = os.path.join(tmpdir, "model1") output_dir2 = os.path.join(tmpdir, "model2") registry_dir = os.path.join(tmpdir, "registry") machine = Machine( name="model-name", dataset=data_config, model=model_config, project_name="test" ) builder = ModelBuilder(machine) builder.build(output_dir=output_dir1, model_register_dir=registry_dir) builder.build(output_dir=output_dir2, model_register_dir=registry_dir) assert builder.cached_model_path == output_dir2 builder.build(output_dir=output_dir2, model_register_dir=registry_dir) assert builder.cached_model_path == output_dir2
def test_get_metrics_dict_scaler(scaler, mock): mock_model = mock metrics_list = [sklearn.metrics.mean_squared_error] # make the features in y be in different scales y = pd.DataFrame( np.array([[1, 1], [2, 2], [3, 3], [4, 4], [5, 5]]) * [1, 100], columns=["Tag 1", "Tag 2"], ) metrics_dict = ModelBuilder.build_metrics_dict(metrics_list, y, scaler=scaler) metric_func = metrics_dict["mean-squared-error"] mock_model.predict = lambda _y: _y * [0.8, 1] mse_feature_one_wrong = metric_func(mock_model, y, y) mock_model.predict = lambda _y: _y * [1, 0.8] mse_feature_two_wrong = metric_func(mock_model, y, y) if scaler: assert np.isclose(mse_feature_one_wrong, mse_feature_two_wrong) else: assert not np.isclose(mse_feature_one_wrong, mse_feature_two_wrong)
def test_provide_saved_model_caching_handle_existing_same_dir(tmpdir): """If the model exists in the model register, and the path there is the same as output_dir, output_dir is returned""" model_config = {"sklearn.decomposition.PCA": {"svd_solver": "auto"}} data_config = get_random_data() output_dir = os.path.join(tmpdir, "model") registry_dir = os.path.join(tmpdir, "registry") machine = Machine( name="model-name", dataset=data_config, model=model_config, project_name="test" ) builder = ModelBuilder(machine) builder.build(output_dir=output_dir, model_register_dir=registry_dir) assert builder.cached_model_path == output_dir # Saving to same output_dir as the one saved in the registry just returns the output_dir builder.build(output_dir=output_dir, model_register_dir=registry_dir) assert builder.cached_model_path == output_dir
def test_provide_saved_model_caching( should_be_equal: bool, metadata: Optional[Metadata], tag_list: Optional[List[SensorTag]], replace_cache, tmpdir, ): """ Test provide_saved_model with caching and possible cache busting if tag_list, or replace_cache is set. Builds two models and checks if their model-creation-date's are the same, which will be if and only if there is caching. Parameters ---------- should_be_equal : bool Do we expect the two generated models to be at the same location or not? I.e. do we expect caching. metadata: Metadata Optional metadata which will be used as metadata for the second model. tag_list Optional list of strings which be used as the taglist in the dataset for the second model. replace_cache: bool Should we force a model cache replacement? """ if tag_list is None: tag_list = [] if metadata is None: metadata = Metadata() model_config = {"sklearn.decomposition.pca.PCA": {"svd_solver": "auto"}} data_config = get_random_data() output_dir = os.path.join(tmpdir, "model") registry_dir = os.path.join(tmpdir, "registry") machine = Machine(name="model-name", dataset=data_config, model=model_config, project_name="test") _, first_machine = ModelBuilder(machine).build( output_dir=output_dir, model_register_dir=registry_dir) if tag_list: data_config["tag_list"] = tag_list new_output_dir = os.path.join(tmpdir, "model2") _, second_machine = ModelBuilder(machine=Machine( name="model-name", dataset=data_config, model=model_config, metadata=metadata, project_name="test", runtime={"something": True}, )).build( output_dir=new_output_dir, model_register_dir=registry_dir, replace_cache=replace_cache, ) model1_creation_date = ( first_machine.metadata.build_metadata.model.model_creation_date) model2_creation_date = ( second_machine.metadata.build_metadata.model.model_creation_date) assert "something" in second_machine.runtime if should_be_equal: assert model1_creation_date == model2_creation_date else: assert model1_creation_date != model2_creation_date if metadata is not None: assert metadata.user_defined == second_machine.metadata.user_defined