def test_embed_dim(estimator, build_dataset): # Checks that the the dimension of the output space is as expected input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) assert model.transform(X).shape == X.shape # assert that ValueError is thrown if input shape is 1D context = make_context(estimator) err_msg = ("2D array of formed points expected{}. Found 1D array " "instead:\ninput={}. Reshape your data and/or use a " "preprocessor.\n".format(context, X[0])) with pytest.raises(ValueError) as raised_error: model.score_pairs(model.transform(X[0, :])) assert str(raised_error.value) == err_msg # we test that the shape is also OK when doing dimensionality reduction if hasattr(model, 'n_components'): model.set_params(n_components=2) model.fit(*remove_y(estimator, input_data, labels)) assert model.transform(X).shape == (X.shape[0], 2) # assert that ValueError is thrown if input shape is 1D with pytest.raises(ValueError) as raised_error: model.transform(model.transform(X[0, :])) assert str(raised_error.value) == err_msg
def test_components_is_2D(estimator, build_dataset): """Tests that the transformation matrix of metric learners is 2D""" input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) # test that it works for X.shape[1] features model.fit(*remove_y(estimator, input_data, labels)) assert model.components_.shape == (X.shape[1], X.shape[1]) # test that it works for 1 feature trunc_data = input_data[..., :1] # we drop duplicates that might have been formed, i.e. of the form # aabc or abcc or aabb for quadruplets, and aa for pairs. if isinstance(estimator, _QuadrupletsClassifierMixin): pairs_idx = [[0, 1], [2, 3]] elif isinstance(estimator, _TripletsClassifierMixin): pairs_idx = [[0, 1], [0, 2]] elif isinstance(estimator, _PairsClassifierMixin): pairs_idx = [[0, 1]] else: pairs_idx = [] for pair_idx in pairs_idx: pairs = trunc_data[:, pair_idx, :] diffs = pairs[:, 1, :] - pairs[:, 0, :] to_keep = np.abs(diffs.ravel()) > 1e-9 trunc_data = trunc_data[to_keep] labels = labels[to_keep] model.fit(*remove_y(estimator, trunc_data, labels)) assert model.components_.shape == (1, 1) # the components must be 2D
def test_n_components(estimator, build_dataset): """Check that estimators that have a n_components parameters can use it and that it actually works as expected""" input_data, labels, _, X = build_dataset() model = clone(estimator) if hasattr(model, 'n_components'): set_random_state(model) model.set_params(n_components=None) model.fit(*remove_y(model, input_data, labels)) assert model.components_.shape == (X.shape[1], X.shape[1]) model = clone(estimator) set_random_state(model) model.set_params(n_components=X.shape[1] - 1) model.fit(*remove_y(model, input_data, labels)) assert model.components_.shape == (X.shape[1] - 1, X.shape[1]) model = clone(estimator) set_random_state(model) model.set_params(n_components=X.shape[1] + 1) with pytest.raises(ValueError) as expected_err: model.fit(*remove_y(model, input_data, labels)) assert (str(expected_err.value) == 'Invalid n_components, must be in [1, {}]'.format(X.shape[1])) model = clone(estimator) set_random_state(model) model.set_params(n_components=0) with pytest.raises(ValueError) as expected_err: model.fit(*remove_y(model, input_data, labels)) assert (str(expected_err.value) == 'Invalid n_components, must be in [1, {}]'.format(X.shape[1]))
def test_cross_validation_is_finite(estimator, build_dataset): """Tests that validation on metric-learn estimators returns something finite """ input_data, labels, preprocessor, _ = build_dataset() estimator = clone(estimator) estimator.set_params(preprocessor=preprocessor) set_random_state(estimator) assert np.isfinite( cross_val_score(estimator, *remove_y(estimator, input_data, labels))).all() assert np.isfinite( cross_val_predict(estimator, *remove_y(estimator, input_data, labels))).all()
def test_array_like_inputs(estimator, build_dataset, with_preprocessor): """Test that metric-learners can have as input (of all functions that are applied on data) any array-like object.""" input_data, labels, preprocessor, X = build_dataset(with_preprocessor) # we subsample the data for the test to be more efficient input_data, _, labels, _ = train_test_split(input_data, labels, train_size=40, random_state=42) X = X[:10] estimator = clone(estimator) estimator.set_params(preprocessor=preprocessor) set_random_state(estimator) input_variants, label_variants = generate_array_like(input_data, labels) for input_variant in input_variants: for label_variant in label_variants: estimator.fit(*remove_y(estimator, input_variant, label_variant)) if hasattr(estimator, "predict"): estimator.predict(input_variant) if hasattr(estimator, "predict_proba"): estimator.predict_proba(input_variant) # anticipation in case some # time we have that, or if ppl want to contribute with new algorithms # it will be checked automatically if hasattr(estimator, "decision_function"): estimator.decision_function(input_variant) if hasattr(estimator, "score"): for label_variant in label_variants: estimator.score( *remove_y(estimator, input_variant, label_variant)) X_variants, _ = generate_array_like(X) for X_variant in X_variants: estimator.transform(X_variant) pairs = np.array([[X[0], X[1]], [X[0], X[2]]]) pairs_variants, _ = generate_array_like(pairs) not_implemented_msg = "" # Todo in 0.7.0: Change 'not_implemented_msg' for the message that says # "This learner does not have pair_distance" for pairs_variant in pairs_variants: estimator.pair_score(pairs_variant) # All learners have pair_score # But not all of them will have pair_distance try: estimator.pair_distance(pairs_variant) except Exception as raised_exception: assert raised_exception.value.args[0] == not_implemented_msg
def test_get_metric_works_does_not_raise(estimator, build_dataset): """Tests that the metric returned by get_metric does not raise errors (or warnings) similarly to the distance functions in scipy.spatial.distance""" input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) model.fit(*remove_y(model, input_data, labels)) metric = model.get_metric() list_test_get_metric_doesnt_raise = [(X[0], X[1]), (X[0].tolist(), X[1].tolist()), (X[0][None], X[1][None])] for u, v in list_test_get_metric_doesnt_raise: with pytest.warns(None) as record: metric(u, v) assert len(record) == 0 # Test that the scalar case works model.components_ = np.array([3.1]) metric = model.get_metric() for u, v in [(5, 6.7), ([5], [6.7]), ([[5]], [[6.7]])]: with pytest.warns(None) as record: metric(u, v) assert len(record) == 0
def test_embed_finite(estimator, build_dataset): # Checks that embed returns vectors with finite values input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) assert np.isfinite(model.transform(X)).all()
def test_deterministic_initialization(estimator, build_dataset): """Test that estimators that have a prior or an init are deterministic when it is set to to random and when the random_state is fixed.""" input_data, labels, _, X = build_dataset() model = clone(estimator) if hasattr(estimator, 'init'): model.set_params(init='random') if hasattr(estimator, 'prior'): model.set_params(prior='random') model1 = clone(model) set_random_state(model1, 42) model1 = model1.fit(*remove_y(model, input_data, labels)) model2 = clone(model) set_random_state(model2, 42) model2 = model2.fit(*remove_y(model, input_data, labels)) np.testing.assert_allclose(model1.get_mahalanobis_matrix(), model2.get_mahalanobis_matrix())
def test_estimators_fit_returns_self(estimator, build_dataset, with_preprocessor): """Check if self is returned when calling fit""" # Adapted from scikit-learn input_data, labels, preprocessor, _ = build_dataset(with_preprocessor) estimator = clone(estimator) estimator.set_params(preprocessor=preprocessor) assert estimator.fit(*remove_y(estimator, input_data, labels)) is estimator
def test_score_pairs_finite(estimator, build_dataset): # tests that the score is finite input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) pairs = np.array(list(product(X, X))) assert np.isfinite(model.score_pairs(pairs)).all()
def test_get_metric_compatible_with_scikit_learn(estimator, build_dataset): """Check that the metric returned by get_metric is compatible with scikit-learn's algorithms using a custom metric, DBSCAN for instance""" input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) clustering = DBSCAN(metric=model.get_metric()) clustering.fit(X)
def test_embed_toy_example(estimator, build_dataset): # Checks that embed works on a toy example input_data, labels, _, X = build_dataset() n_samples = 20 X = X[:n_samples] model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) embedded_points = X.dot(model.components_.T) assert_array_almost_equal(model.transform(X), embedded_points)
def test_embed_is_linear(estimator, build_dataset): # Checks that the embedding is linear input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) assert_array_almost_equal( model.transform(X[:10] + X[10:20]), model.transform(X[:10]) + model.transform(X[10:20])) assert_array_almost_equal(model.transform(5 * X[:10]), 5 * model.transform(X[:10]))
def test_get_metric_is_independent_from_metric_learner(estimator, build_dataset): """Tests that the get_metric method returns a function that is independent from the original metric learner""" input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) # we fit the metric learner on it and then we compute the metric on some # points model.fit(*remove_y(model, input_data, labels)) metric = model.get_metric() score = metric(X[0], X[1]) # then we refit the estimator on another dataset model.fit(*remove_y(model, np.sin(input_data), labels)) # we recompute the distance between the two points: it should be the same score_bis = metric(X[0], X[1]) assert score_bis == score
def test_cross_validation_manual_vs_scikit(estimator, build_dataset, with_preprocessor): """Tests that if we make a manual cross-validation, the result will be the same as scikit-learn's cross-validation (some code for generating the folds is taken from scikit-learn). """ if any(hasattr(estimator, method) for method in ["predict", "score"]): input_data, labels, preprocessor, _ = build_dataset(with_preprocessor) estimator = clone(estimator) estimator.set_params(preprocessor=preprocessor) set_random_state(estimator) n_splits = 3 kfold = KFold(shuffle=False, n_splits=n_splits) n_samples = input_data.shape[0] fold_sizes = (n_samples // n_splits) * np.ones(n_splits, dtype=np.int64) fold_sizes[:n_samples % n_splits] += 1 current = 0 scores, predictions = [], np.zeros(input_data.shape[0]) for fold_size in fold_sizes: start, stop = current, current + fold_size current = stop test_slice = slice(start, stop) train_mask = np.ones(input_data.shape[0], bool) train_mask[test_slice] = False y_train, y_test = labels[train_mask], labels[test_slice] estimator.fit( *remove_y(estimator, input_data[train_mask], y_train)) if hasattr(estimator, "score"): scores.append( estimator.score( *remove_y(estimator, input_data[test_slice], y_test))) if hasattr(estimator, "predict"): predictions[test_slice] = estimator.predict( input_data[test_slice]) if hasattr(estimator, "score"): assert all(scores == cross_val_score( estimator, *remove_y(estimator, input_data, labels), cv=kfold)) if hasattr(estimator, "predict"): assert all(predictions == cross_val_predict( estimator, *remove_y(estimator, input_data, labels), cv=kfold))
def test_array_like_inputs(estimator, build_dataset, with_preprocessor): """Test that metric-learners can have as input (of all functions that are applied on data) any array-like object.""" input_data, labels, preprocessor, X = build_dataset(with_preprocessor) # we subsample the data for the test to be more efficient input_data, _, labels, _ = train_test_split(input_data, labels, train_size=40, random_state=42) X = X[:10] estimator = clone(estimator) estimator.set_params(preprocessor=preprocessor) set_random_state(estimator) input_variants, label_variants = generate_array_like(input_data, labels) for input_variant in input_variants: for label_variant in label_variants: estimator.fit(*remove_y(estimator, input_variant, label_variant)) if hasattr(estimator, "predict"): estimator.predict(input_variant) if hasattr(estimator, "predict_proba"): estimator.predict_proba(input_variant) # anticipation in case some # time we have that, or if ppl want to contribute with new algorithms # it will be checked automatically if hasattr(estimator, "decision_function"): estimator.decision_function(input_variant) if hasattr(estimator, "score"): for label_variant in label_variants: estimator.score( *remove_y(estimator, input_variant, label_variant)) X_variants, _ = generate_array_like(X) for X_variant in X_variants: estimator.transform(X_variant) pairs = np.array([[X[0], X[1]], [X[0], X[2]]]) pairs_variants, _ = generate_array_like(pairs) for pairs_variant in pairs_variants: estimator.score_pairs(pairs_variant)
def test_score_pairs_toy_example(estimator, build_dataset): # Checks that score_pairs works on a toy example input_data, labels, _, X = build_dataset() n_samples = 20 X = X[:n_samples] model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) pairs = np.stack([X[:10], X[10:20]], axis=1) embedded_pairs = pairs.dot(model.components_.T) distances = np.sqrt( np.sum((embedded_pairs[:, 1] - embedded_pairs[:, 0])**2, axis=-1)) assert_array_almost_equal(model.score_pairs(pairs), distances)
def test_simple_estimator(estimator, build_dataset, with_preprocessor): """Tests that fit, predict and scoring works. """ if any(hasattr(estimator, method) for method in ["predict", "score"]): input_data, labels, preprocessor, _ = build_dataset(with_preprocessor) (tuples_train, tuples_test, y_train, y_test) = train_test_split(input_data, labels, random_state=RNG) estimator = clone(estimator) estimator.set_params(preprocessor=preprocessor) set_random_state(estimator) estimator.fit(*remove_y(estimator, tuples_train, y_train)) check_score(estimator, tuples_test, y_test) check_predict(estimator, tuples_test)
def test_pair_distance_pair_score_equivalent(estimator, build_dataset): """ For Mahalanobis learners, pair_score should be equivalent to the opposite of the pair_distance result. """ input_data, labels, _, X = build_dataset() n_samples = 20 X = X[:n_samples] model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) distances = model.pair_distance(np.array(list(product(X, X)))) scores = model.pair_score(np.array(list(product(X, X)))) assert_array_equal(distances, -1 * scores)
def test_get_squared_metric(estimator, build_dataset): """Test that the squared metric returned is indeed the square of the metric""" input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) metric = model.get_metric() n_features = X.shape[1] for seed in range(10): rng = np.random.RandomState(seed) a, b = (rng.randn(n_features) for _ in range(2)) assert_allclose(metric(a, b, squared=True), metric(a, b, squared=False)**2, rtol=1e-15)
def test_get_metric_equivalent_to_explicit_mahalanobis(estimator, build_dataset): """Tests that using the get_metric method of mahalanobis metric learners is equivalent to explicitely calling scipy's mahalanobis metric """ rng = np.random.RandomState(42) input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) metric = model.get_metric() n_features = X.shape[1] a, b = (rng.randn(n_features), rng.randn(n_features)) expected_dist = mahalanobis(a[None], b[None], VI=model.get_mahalanobis_matrix()) assert_allclose(metric(a, b), expected_dist, rtol=1e-13)
def test_score_pairs_dim(estimator, build_dataset): # scoring of 3D arrays should return 1D array (several tuples), # and scoring of 2D arrays (one tuple) should return an error (like # scikit-learn's error when scoring 1D arrays) input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) tuples = np.array(list(product(X, X))) assert model.score_pairs(tuples).shape == (tuples.shape[0], ) context = make_context(estimator) msg = ("3D array of formed tuples expected{}. Found 2D array " "instead:\ninput={}. Reshape your data and/or use a preprocessor.\n" .format(context, tuples[1])) with pytest.raises(ValueError) as raised_error: model.score_pairs(tuples[1]) assert str(raised_error.value) == msg
def test_dont_overwrite_parameters(estimator, build_dataset, with_preprocessor): # Adapted from scikit-learn # check that fit method only changes or sets private attributes input_data, labels, preprocessor, _ = build_dataset(with_preprocessor) estimator = clone(estimator) estimator.set_params(preprocessor=preprocessor) if hasattr(estimator, "n_components"): estimator.n_components = 1 dict_before_fit = estimator.__dict__.copy() estimator.fit(*remove_y(estimator, input_data, labels)) dict_after_fit = estimator.__dict__ public_keys_after_fit = [ key for key in dict_after_fit.keys() if is_public_parameter(key) ] attrs_added_by_fit = [ key for key in public_keys_after_fit if key not in dict_before_fit.keys() ] # check that fit doesn't add any public attribute assert not attrs_added_by_fit, ( "Estimator adds public attribute(s) during" " the fit method." " Estimators are only allowed to add private " "attributes" " either started with _ or ended" " with _ but %s added" % ', '.join(attrs_added_by_fit)) # check that fit doesn't change any public attribute attrs_changed_by_fit = [ key for key in public_keys_after_fit if (dict_before_fit[key] is not dict_after_fit[key]) ] assert not attrs_changed_by_fit, ( "Estimator changes public attribute(s) during" " the fit method. Estimators are only allowed" " to change attributes started" " or ended with _, but" " %s changed" % ', '.join(attrs_changed_by_fit))
def test_get_metric_raises_error(estimator, build_dataset): """Tests that the metric returned by get_metric raises errors similar to the distance functions in scipy.spatial.distance""" input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) model.fit(*remove_y(model, input_data, labels)) metric = model.get_metric() list_test_get_metric_raises = [(X[0].tolist() + [5.2], X[1]), # vectors with # different dimensions (X[0:4], X[1:5]), # 2D vectors (X[0].tolist() + [5.2], X[1] + [7.2])] # vectors of same dimension but incompatible with what the metric learner # was trained on for u, v in list_test_get_metric_raises: with pytest.raises(ValueError): metric(u, v)
def test_score_pairs_warning(estimator, build_dataset): """Tests that score_pairs returns a FutureWarning regarding deprecation. Also that score_pairs and pair_distance have the same behaviour""" input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) # We fit the metric learner on it and then we call score_pairs on some # points model.fit(*remove_y(model, input_data, labels)) msg = ("score_pairs will be deprecated in release 0.7.0. " "Use pair_score to compute similarity scores, or " "pair_distances to compute distances.") with pytest.warns(FutureWarning) as raised_warning: score = model.score_pairs([[X[0], X[1]], ]) dist = model.pair_distance([[X[0], X[1]], ]) assert array_equal(score, dist) assert any([str(warning.message) == msg for warning in raised_warning])
def test_score_pairs_pairwise(estimator, build_dataset): # Computing pairwise scores should return a euclidean distance matrix. input_data, labels, _, X = build_dataset() n_samples = 20 X = X[:n_samples] model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) pairwise = model.score_pairs(np.array(list(product(X, X))))\ .reshape(n_samples, n_samples) check_is_distance_matrix(pairwise) # a necessary condition for euclidean distance matrices: (see # https://en.wikipedia.org/wiki/Euclidean_distance_matrix) assert np.linalg.matrix_rank(pairwise**2) <= min(X.shape) + 2 # assert that this distance is coherent with pdist on embeddings assert_array_almost_equal(squareform(pairwise), pdist(model.transform(X)))
def test_get_metric_is_pseudo_metric(estimator, build_dataset): """Tests that the get_metric method of mahalanobis metric learners returns a pseudo-metric (metric but without one side of the equivalence of the identity of indiscernables property) """ input_data, labels, _, X = build_dataset() model = clone(estimator) set_random_state(model) model.fit(*remove_y(estimator, input_data, labels)) metric = model.get_metric() n_features = X.shape[1] for seed in range(10): rng = np.random.RandomState(seed) a, b, c = (rng.randn(n_features) for _ in range(3)) assert metric(a, b) >= 0 # positivity assert metric(a, b) == metric(b, a) # symmetry # one side of identity indiscernables: x == y => d(x, y) == 0. The other # side of the equivalence is not always true for Mahalanobis distances. assert metric(a, a) == 0 # triangular inequality assert (metric(a, c) < metric(a, b) + metric(b, c) or np.isclose( metric(a, c), metric(a, b) + metric(b, c), rtol=1e-20))
def test_dict_unchanged(estimator, build_dataset, with_preprocessor): # Adapted from scikit-learn (input_data, labels, preprocessor, to_transform) = build_dataset(with_preprocessor) estimator = clone(estimator) estimator.set_params(preprocessor=preprocessor) if hasattr(estimator, "n_components"): estimator.n_components = 1 estimator.fit(*remove_y(estimator, input_data, labels)) def check_dict(): assert estimator.__dict__ == dict_before, ( "Estimator changes __dict__ during %s" % method) for method in ["predict", "decision_function", "predict_proba"]: if hasattr(estimator, method): dict_before = estimator.__dict__.copy() getattr(estimator, method)(input_data) check_dict() if hasattr(estimator, "transform"): dict_before = estimator.__dict__.copy() # we transform only dataset of points estimator.transform(to_transform) check_dict()
def check_score(estimator, tuples, y): if hasattr(estimator, "score"): score = estimator.score(*remove_y(estimator, tuples, y)) assert np.isfinite(score)