def test_singular_covariance_init_of_non_strict_pd(estimator, build_dataset):
"""Tests that when using the 'covariance' init or prior, it returns the
appropriate warning if the covariance matrix is singular, for algorithms
that don't need a strictly PD init. Also checks that the returned
inverse matrix has finite values
"""
input_data, labels, _, X = build_dataset()
model = clone(estimator)
set_random_state(model)
# We create a feature that is a linear combination of the first two
# features:
input_data = np.concatenate(
[input_data, input_data[:, ..., :2].dot([[2], [3]])], axis=-1)
model.set_params(init='covariance')
msg = ('The covariance matrix is not invertible: '
'using the pseudo-inverse instead.'
'To make the covariance matrix invertible'
' you can remove any linearly dependent features and/or '
'reduce the dimensionality of your input, '
'for instance using `sklearn.decomposition.PCA` as a '
'preprocessing step.')
with pytest.warns(UserWarning) as raised_warning:
model.fit(input_data, labels)
assert np.any([str(warning.message) == msg for warning in raised_warning])
M, _ = _initialize_metric_mahalanobis(X,
init='covariance',
random_state=RNG,
return_inverse=True,
strict_pd=False)
assert np.isfinite(M).all()
def test_raise_not_fitted_error_if_not_fitted(estimator, build_dataset,
with_preprocessor):
"""Test that a NotFittedError is raised if someone tries to use
pair_score, score_pairs, decision_function, get_metric, transform or
get_mahalanobis_matrix on input data and the metric learner
has not been fitted."""
input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
estimator = clone(estimator)
estimator.set_params(preprocessor=preprocessor)
set_random_state(estimator)
with pytest.raises(NotFittedError): # Remove in 0.8.0
estimator.score_pairs(input_data)
with pytest.raises(NotFittedError):
estimator.pair_score(input_data)
with pytest.raises(NotFittedError):
estimator.decision_function(input_data)
with pytest.raises(NotFittedError):
estimator.get_metric()
with pytest.raises(NotFittedError):
estimator.transform(input_data)
with pytest.raises(NotFittedError):
estimator.get_mahalanobis_matrix()
with pytest.raises(NotFittedError):
estimator.calibrate_threshold(input_data, labels)
with pytest.raises(NotFittedError):
estimator.set_threshold(0.5)
with pytest.raises(NotFittedError):
estimator.predict(input_data)
def test_get_metric_works_does_not_raise(estimator, build_dataset):
"""Tests that the metric returned by get_metric does not raise errors (or
warnings) similarly to the distance functions in scipy.spatial.distance"""
input_data, labels, _, X = build_dataset()
model = clone(estimator)
set_random_state(model)
model.fit(*remove_y(model, input_data, labels))
metric = model.get_metric()
list_test_get_metric_doesnt_raise = [(X[0], X[1]),
(X[0].tolist(), X[1].tolist()),
(X[0][None], X[1][None])]
for u, v in list_test_get_metric_doesnt_raise:
with pytest.warns(None) as record:
metric(u, v)
assert len(record) == 0
# Test that the scalar case works
model.components_ = np.array([3.1])
metric = model.get_metric()
for u, v in [(5, 6.7), ([5], [6.7]), ([[5]], [[6.7]])]:
with pytest.warns(None) as record:
metric(u, v)
assert len(record) == 0
def test_singular_covariance_init_or_prior_strictpd(estimator, build_dataset):
"""Tests that when using the 'covariance' init or prior, it returns the
appropriate error if the covariance matrix is singular, for algorithms
that need a strictly PD prior or init (see
https://github.com/scikit-learn-contrib/metric-learn/issues/202 and
https://github.com/scikit-learn-contrib/metric-learn/pull/195#issuecomment
-492332451)
"""
matrices_to_set = []
if hasattr(estimator, 'init'):
matrices_to_set.append('init')
if hasattr(estimator, 'prior'):
matrices_to_set.append('prior')
input_data, labels, _, X = build_dataset()
for param in matrices_to_set:
model = clone(estimator)
set_random_state(model)
# We create a feature that is a linear combination of the first two
# features:
input_data = np.concatenate(
[input_data, input_data[:, ..., :2].dot([[2], [3]])], axis=-1)
model.set_params(**{param: 'covariance'})
msg = ("Unable to get a true inverse of the covariance "
"matrix since it is not definite. Try another "
"`{}`, or an algorithm that does not "
"require the `{}` to be strictly positive definite.".format(
param, param))
with pytest.raises(LinAlgError) as raised_err:
model.fit(input_data, labels)
assert str(raised_err.value) == msg
def test_various_scoring_on_tuples_learners(estimator, build_dataset,
with_preprocessor):
"""Tests that scikit-learn's scoring returns something finite,
for other scoring than default scoring. (List of scikit-learn's scores can be
found in sklearn.metrics._scorer). For each type of output (predict,
predict_proba, decision_function), we test a bunch of scores.
We only test on pairs learners because quadruplets don't have a y argument.
"""
input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
estimator = clone(estimator)
estimator.set_params(preprocessor=preprocessor)
set_random_state(estimator)
# scores that need a predict function: every tuples learner should have a
# predict function (whether the pair is of positive samples or negative
# samples)
for scoring in ['accuracy', 'f1']:
check_score_is_finite(scoring, estimator, input_data, labels)
# scores that need a predict_proba:
if hasattr(estimator, "predict_proba"):
for scoring in ['neg_log_loss', 'brier_score']:
check_score_is_finite(scoring, estimator, input_data, labels)
# scores that need a decision_function: every tuples learner should have a
# decision function (the metric between points)
for scoring in ['roc_auc', 'average_precision', 'precision', 'recall']:
check_score_is_finite(scoring, estimator, input_data, labels)
def test_components_is_2D(estimator, build_dataset):
"""Tests that the transformation matrix of metric learners is 2D"""
input_data, labels, _, X = build_dataset()
model = clone(estimator)
set_random_state(model)
# test that it works for X.shape[1] features
model.fit(*remove_y(estimator, input_data, labels))
assert model.components_.shape == (X.shape[1], X.shape[1])
# test that it works for 1 feature
trunc_data = input_data[..., :1]
# we drop duplicates that might have been formed, i.e. of the form
# aabc or abcc or aabb for quadruplets, and aa for pairs.
if isinstance(estimator, _QuadrupletsClassifierMixin):
pairs_idx = [[0, 1], [2, 3]]
elif isinstance(estimator, _TripletsClassifierMixin):
pairs_idx = [[0, 1], [0, 2]]
elif isinstance(estimator, _PairsClassifierMixin):
pairs_idx = [[0, 1]]
else:
pairs_idx = []
for pair_idx in pairs_idx:
pairs = trunc_data[:, pair_idx, :]
diffs = pairs[:, 1, :] - pairs[:, 0, :]
to_keep = np.abs(diffs.ravel()) > 1e-9
trunc_data = trunc_data[to_keep]
labels = labels[to_keep]
model.fit(*remove_y(estimator, trunc_data, labels))
assert model.components_.shape == (1, 1) # the components must be 2D
def test_embed_finite(estimator, build_dataset):
# Checks that embed returns vectors with finite values
input_data, labels, _, X = build_dataset()
model = clone(estimator)
set_random_state(model)
model.fit(*remove_y(estimator, input_data, labels))
assert np.isfinite(model.transform(X)).all()
def test_embed_dim(estimator, build_dataset):
# Checks that the the dimension of the output space is as expected
input_data, labels, _, X = build_dataset()
model = clone(estimator)
set_random_state(model)
model.fit(*remove_y(estimator, input_data, labels))
assert model.transform(X).shape == X.shape
# assert that ValueError is thrown if input shape is 1D
context = make_context(estimator)
err_msg = ("2D array of formed points expected{}. Found 1D array "
"instead:\ninput={}. Reshape your data and/or use a "
"preprocessor.\n".format(context, X[0]))
with pytest.raises(ValueError) as raised_error:
model.score_pairs(model.transform(X[0, :]))
assert str(raised_error.value) == err_msg
# we test that the shape is also OK when doing dimensionality reduction
if hasattr(model, 'n_components'):
model.set_params(n_components=2)
model.fit(*remove_y(estimator, input_data, labels))
assert model.transform(X).shape == (X.shape[0], 2)
# assert that ValueError is thrown if input shape is 1D
with pytest.raises(ValueError) as raised_error:
model.transform(model.transform(X[0, :]))
assert str(raised_error.value) == err_msg
def test_accuracy_toy_example(estimator, build_dataset):
"""Test that the accuracy works on some toy example (hence that the
prediction is OK)"""
input_data, labels, preprocessor, X = build_dataset(
with_preprocessor=False)
estimator = clone(estimator)
estimator.set_params(preprocessor=preprocessor)
set_random_state(estimator)
estimator.fit(input_data, labels)
# we force the transformation to be identity so that we control what it does
estimator.components_ = np.eye(X.shape[1])
# the threshold for similar or dissimilar pairs is half of the distance
# between X[0] and X[1]
estimator.set_threshold(euclidean(X[0], X[1]) / 2)
# We take the two first points and we build 4 regularly spaced points on the
# line they define, so that it's easy to build quadruplets of different
# similarities.
X_test = X[0] + np.arange(4)[:, np.newaxis] * (X[0] - X[1]) / 4
pairs_test = np.array([
[X_test[0], X_test[1]], # similar
[X_test[0], X_test[3]], # dissimilar
[X_test[1], X_test[2]], # similar
[X_test[2], X_test[3]]
]) # similar
y = np.array([-1, 1, 1, -1]) # [F, F, T, F]
assert accuracy_score(estimator.predict(pairs_test), y) == 0.25
def test_score_pairs_finite(estimator, build_dataset):
# tests that the score is finite
input_data, labels, _, X = build_dataset()
model = clone(estimator)
set_random_state(model)
model.fit(*remove_y(estimator, input_data, labels))
pairs = np.array(list(product(X, X)))
assert np.isfinite(model.score_pairs(pairs)).all()
def test_get_metric_compatible_with_scikit_learn(estimator, build_dataset):
"""Check that the metric returned by get_metric is compatible with
scikit-learn's algorithms using a custom metric, DBSCAN for instance"""
input_data, labels, _, X = build_dataset()
model = clone(estimator)
set_random_state(model)
model.fit(*remove_y(estimator, input_data, labels))
clustering = DBSCAN(metric=model.get_metric())
clustering.fit(X)
def test_embed_toy_example(estimator, build_dataset):
# Checks that embed works on a toy example
input_data, labels, _, X = build_dataset()
n_samples = 20
X = X[:n_samples]
model = clone(estimator)
set_random_state(model)
model.fit(*remove_y(estimator, input_data, labels))
embedded_points = X.dot(model.components_.T)
assert_array_almost_equal(model.transform(X), embedded_points)
def test_raise_not_fitted_error_if_not_fitted(estimator, build_dataset,
with_preprocessor):
"""Test that a NotFittedError is raised if someone tries to predict and
the metric learner has not been fitted."""
input_data, _, preprocessor, _ = build_dataset(with_preprocessor)
estimator = clone(estimator)
estimator.set_params(preprocessor=preprocessor)
set_random_state(estimator)
with pytest.raises(NotFittedError):
estimator.predict(input_data)
def test_fit_with_valid_threshold_params(estimator, build_dataset,
with_preprocessor,
calibration_params):
"""Tests that fitting `calibration_params` with appropriate parameters works
as expected"""
pairs, y, preprocessor, _ = build_dataset(with_preprocessor)
estimator = clone(estimator)
estimator.set_params(preprocessor=preprocessor)
set_random_state(estimator)
estimator.fit(pairs, y, calibration_params=calibration_params)
estimator.predict(pairs)
def test_embed_is_linear(estimator, build_dataset):
# Checks that the embedding is linear
input_data, labels, _, X = build_dataset()
model = clone(estimator)
set_random_state(model)
model.fit(*remove_y(estimator, input_data, labels))
assert_array_almost_equal(
model.transform(X[:10] + X[10:20]),
model.transform(X[:10]) + model.transform(X[10:20]))
assert_array_almost_equal(model.transform(5 * X[:10]),
5 * model.transform(X[:10]))
def test_threshold_different_scores_is_finite(estimator, build_dataset,
with_preprocessor, kwargs):
# test that calibrating the threshold works for every metric learner
input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
estimator = clone(estimator)
estimator.set_params(preprocessor=preprocessor)
set_random_state(estimator)
estimator.fit(input_data, labels)
with pytest.warns(None) as record:
estimator.calibrate_threshold(input_data, labels, **kwargs)
assert len(record) == 0
def test_raise_big_number_of_features():
triplets, _, _, X = build_triplets(with_preprocessor=False)
triplets = triplets[:3, :, :]
estimator = SCML(n_basis=320)
set_random_state(estimator)
with pytest.raises(ValueError) as exc_info:
estimator.fit(triplets)
assert exc_info.value.args[0] == \
"Number of features (4) is greater than the number of triplets(3)." \
"\nConsider using dimensionality reduction or using another basis " \
"generation scheme."
def test_score_pairs_toy_example(estimator, build_dataset):
# Checks that score_pairs works on a toy example
input_data, labels, _, X = build_dataset()
n_samples = 20
X = X[:n_samples]
model = clone(estimator)
set_random_state(model)
model.fit(*remove_y(estimator, input_data, labels))
pairs = np.stack([X[:10], X[10:20]], axis=1)
embedded_pairs = pairs.dot(model.components_.T)
distances = np.sqrt(
np.sum((embedded_pairs[:, 1] - embedded_pairs[:, 0])**2, axis=-1))
assert_array_almost_equal(model.score_pairs(pairs), distances)
def test_predict_only_one_or_minus_one(estimator, build_dataset,
with_preprocessor):
"""Test that all predicted values are either +1 or -1"""
input_data, _, preprocessor, _ = build_dataset(with_preprocessor)
estimator = clone(estimator)
estimator.set_params(preprocessor=preprocessor)
set_random_state(estimator)
triplets_train, triplets_test = train_test_split(input_data)
estimator.fit(triplets_train)
predictions = estimator.predict(triplets_test)
not_valid = [e for e in predictions if e not in [-1, 1]]
assert len(not_valid) == 0
def test_cross_validation_is_finite(estimator, build_dataset):
"""Tests that validation on metric-learn estimators returns something finite
"""
input_data, labels, preprocessor, _ = build_dataset()
estimator = clone(estimator)
estimator.set_params(preprocessor=preprocessor)
set_random_state(estimator)
assert np.isfinite(
cross_val_score(estimator, *remove_y(estimator, input_data,
labels))).all()
assert np.isfinite(
cross_val_predict(estimator, *remove_y(estimator, input_data,
labels))).all()
def test_array_like_inputs(estimator, build_dataset, with_preprocessor):
"""Test that metric-learners can have as input (of all functions that are
applied on data) any array-like object."""
input_data, labels, preprocessor, X = build_dataset(with_preprocessor)
# we subsample the data for the test to be more efficient
input_data, _, labels, _ = train_test_split(input_data,
labels,
train_size=40,
random_state=42)
X = X[:10]
estimator = clone(estimator)
estimator.set_params(preprocessor=preprocessor)
set_random_state(estimator)
input_variants, label_variants = generate_array_like(input_data, labels)
for input_variant in input_variants:
for label_variant in label_variants:
estimator.fit(*remove_y(estimator, input_variant, label_variant))
if hasattr(estimator, "predict"):
estimator.predict(input_variant)
if hasattr(estimator, "predict_proba"):
estimator.predict_proba(input_variant) # anticipation in case some
# time we have that, or if ppl want to contribute with new algorithms
# it will be checked automatically
if hasattr(estimator, "decision_function"):
estimator.decision_function(input_variant)
if hasattr(estimator, "score"):
for label_variant in label_variants:
estimator.score(
*remove_y(estimator, input_variant, label_variant))
X_variants, _ = generate_array_like(X)
for X_variant in X_variants:
estimator.transform(X_variant)
pairs = np.array([[X[0], X[1]], [X[0], X[2]]])
pairs_variants, _ = generate_array_like(pairs)
not_implemented_msg = ""
# Todo in 0.7.0: Change 'not_implemented_msg' for the message that says
# "This learner does not have pair_distance"
for pairs_variant in pairs_variants:
estimator.pair_score(pairs_variant) # All learners have pair_score
# But not all of them will have pair_distance
try:
estimator.pair_distance(pairs_variant)
except Exception as raised_exception:
assert raised_exception.value.args[0] == not_implemented_msg
def test_simple_estimator(estimator, build_dataset, with_preprocessor):
"""Tests that fit, predict and scoring works.
"""
if any(hasattr(estimator, method) for method in ["predict", "score"]):
input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
(tuples_train, tuples_test, y_train,
y_test) = train_test_split(input_data, labels, random_state=RNG)
estimator = clone(estimator)
estimator.set_params(preprocessor=preprocessor)
set_random_state(estimator)
estimator.fit(*remove_y(estimator, tuples_train, y_train))
check_score(estimator, tuples_test, y_test)
check_predict(estimator, tuples_test)
def test_n_components(estimator, build_dataset):
"""Check that estimators that have a n_components parameters can use it
and that it actually works as expected"""
input_data, labels, _, X = build_dataset()
model = clone(estimator)
if hasattr(model, 'n_components'):
set_random_state(model)
model.set_params(n_components=None)
model.fit(*remove_y(model, input_data, labels))
assert model.components_.shape == (X.shape[1], X.shape[1])
model = clone(estimator)
set_random_state(model)
model.set_params(n_components=X.shape[1] - 1)
model.fit(*remove_y(model, input_data, labels))
assert model.components_.shape == (X.shape[1] - 1, X.shape[1])
model = clone(estimator)
set_random_state(model)
model.set_params(n_components=X.shape[1] + 1)
with pytest.raises(ValueError) as expected_err:
model.fit(*remove_y(model, input_data, labels))
assert (str(expected_err.value) ==
'Invalid n_components, must be in [1, {}]'.format(X.shape[1]))
model = clone(estimator)
set_random_state(model)
model.set_params(n_components=0)
with pytest.raises(ValueError) as expected_err:
model.fit(*remove_y(model, input_data, labels))
assert (str(expected_err.value) ==
'Invalid n_components, must be in [1, {}]'.format(X.shape[1]))
def test_get_squared_metric(estimator, build_dataset):
"""Test that the squared metric returned is indeed the square of the
metric"""
input_data, labels, _, X = build_dataset()
model = clone(estimator)
set_random_state(model)
model.fit(*remove_y(estimator, input_data, labels))
metric = model.get_metric()
n_features = X.shape[1]
for seed in range(10):
rng = np.random.RandomState(seed)
a, b = (rng.randn(n_features) for _ in range(2))
assert_allclose(metric(a, b, squared=True),
metric(a, b, squared=False)**2,
rtol=1e-15)
def test_pair_distance_pair_score_equivalent(estimator, build_dataset):
"""
For Mahalanobis learners, pair_score should be equivalent to the
opposite of the pair_distance result.
"""
input_data, labels, _, X = build_dataset()
n_samples = 20
X = X[:n_samples]
model = clone(estimator)
set_random_state(model)
model.fit(*remove_y(estimator, input_data, labels))
distances = model.pair_distance(np.array(list(product(X, X))))
scores = model.pair_score(np.array(list(product(X, X))))
assert_array_equal(distances, -1 * scores)
def test_get_metric_equivalent_to_explicit_mahalanobis(estimator,
build_dataset):
"""Tests that using the get_metric method of mahalanobis metric learners is
equivalent to explicitely calling scipy's mahalanobis metric
"""
rng = np.random.RandomState(42)
input_data, labels, _, X = build_dataset()
model = clone(estimator)
set_random_state(model)
model.fit(*remove_y(estimator, input_data, labels))
metric = model.get_metric()
n_features = X.shape[1]
a, b = (rng.randn(n_features), rng.randn(n_features))
expected_dist = mahalanobis(a[None],
b[None],
VI=model.get_mahalanobis_matrix())
assert_allclose(metric(a, b), expected_dist, rtol=1e-13)
def test_score_pairs_dim(estimator, build_dataset):
# scoring of 3D arrays should return 1D array (several tuples),
# and scoring of 2D arrays (one tuple) should return an error (like
# scikit-learn's error when scoring 1D arrays)
input_data, labels, _, X = build_dataset()
model = clone(estimator)
set_random_state(model)
model.fit(*remove_y(estimator, input_data, labels))
tuples = np.array(list(product(X, X)))
assert model.score_pairs(tuples).shape == (tuples.shape[0], )
context = make_context(estimator)
msg = ("3D array of formed tuples expected{}. Found 2D array "
"instead:\ninput={}. Reshape your data and/or use a preprocessor.\n"
.format(context, tuples[1]))
with pytest.raises(ValueError) as raised_error:
model.score_pairs(tuples[1])
assert str(raised_error.value) == msg
def test_deterministic_initialization(estimator, build_dataset):
"""Test that estimators that have a prior or an init are deterministic
when it is set to to random and when the random_state is fixed."""
input_data, labels, _, X = build_dataset()
model = clone(estimator)
if hasattr(estimator, 'init'):
model.set_params(init='random')
if hasattr(estimator, 'prior'):
model.set_params(prior='random')
model1 = clone(model)
set_random_state(model1, 42)
model1 = model1.fit(*remove_y(model, input_data, labels))
model2 = clone(model)
set_random_state(model2, 42)
model2 = model2.fit(*remove_y(model, input_data, labels))
np.testing.assert_allclose(model1.get_mahalanobis_matrix(),
model2.get_mahalanobis_matrix())
def test_predict_monotonous(estimator, build_dataset, with_preprocessor):
"""Test that there is a threshold distance separating points labeled as
similar and points labeled as dissimilar """
input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
estimator = clone(estimator)
estimator.set_params(preprocessor=preprocessor)
set_random_state(estimator)
pairs_train, pairs_test, y_train, y_test = train_test_split(
input_data, labels)
estimator.fit(pairs_train, y_train)
distances = estimator.score_pairs(pairs_test)
predictions = estimator.predict(pairs_test)
min_dissimilar = np.min(distances[predictions == -1])
max_similar = np.max(distances[predictions == 1])
assert max_similar <= min_dissimilar
separator = np.mean([min_dissimilar, max_similar])
assert (predictions[distances > separator] == -1).all()
assert (predictions[distances < separator] == 1).all()
def test_accuracy_toy_example(estimator, build_dataset):
"""Test that the default scoring for triplets (accuracy) works on some
toy example"""
triplets, _, _, X = build_dataset(with_preprocessor=False)
estimator = clone(estimator)
set_random_state(estimator)
estimator.fit(triplets)
# We take the two first points and we build 4 regularly spaced points on the
# line they define, so that it's easy to build triplets of different
# similarities.
X_test = X[0] + np.arange(4)[:, np.newaxis] * (X[0] - X[1]) / 4
triplets_test = np.array([[X_test[0], X_test[2], X_test[1]],
[X_test[1], X_test[3], X_test[0]],
[X_test[1], X_test[2], X_test[3]],
[X_test[3], X_test[0], X_test[2]]])
# we force the transformation to be identity so that we control what it does
estimator.components_ = np.eye(X.shape[1])
assert estimator.score(triplets_test) == 0.25
