def test_no_data_conversion_warning():
# No warnings issued if metric is not a boolean distance function
rng = np.random.RandomState(0)
X = rng.randn(5, 4)
with pytest.warns(None) as records:
pairwise_distances(X, metric="minkowski")
assert len(records) == 0
def test_pairwise_boolean_distance(metric):
# test that we convert to boolean arrays for boolean distances
rng = np.random.RandomState(0)
X = rng.randn(5, 4)
Y = X.copy()
Y[0, 0] = 1 - Y[0, 0]
# ignore conversion to boolean in pairwise_distances
with ignore_warnings(category=DataConversionWarning):
for Z in [Y, None]:
res = pairwise_distances(X, Z, metric=metric)
res[np.isnan(res)] = 0
assert np.sum(res != 0) == 0
# non-boolean arrays are converted to boolean for boolean
# distance metrics with a data conversion warning
msg = "Data was converted to boolean for metric %s" % metric
with pytest.warns(DataConversionWarning, match=msg):
pairwise_distances(X, metric=metric)
# Check that the warning is raised if X is boolean by Y is not boolean:
with pytest.warns(DataConversionWarning, match=msg):
pairwise_distances(X.astype(bool), Y=Y, metric=metric)
# Check that no warning is raised if X is already boolean and Y is None:
with pytest.warns(None) as records:
pairwise_distances(X.astype(bool), metric=metric)
assert len(records) == 0
def test_precomputed_dists():
redX = X[::2]
dists = pairwise_distances(redX, metric='euclidean')
clust1 = OPTICS(min_samples=10, algorithm='brute',
metric='precomputed').fit(dists)
clust2 = OPTICS(min_samples=10, algorithm='brute',
metric='euclidean').fit(redX)
assert_allclose(clust1.reachability_, clust2.reachability_)
assert_array_equal(clust1.labels_, clust2.labels_)
def test_pairwise_distances_chunked():
# Test the pairwise_distance helper function.
rng = np.random.RandomState(0)
# Euclidean distance should be equivalent to calling the function.
X = rng.random_sample((200, 4))
check_pairwise_distances_chunked(X,
None,
working_memory=1,
metric='euclidean')
# Test small amounts of memory
for power in range(-16, 0):
check_pairwise_distances_chunked(X,
None,
working_memory=2**power,
metric='euclidean')
# X as list
check_pairwise_distances_chunked(X.tolist(),
None,
working_memory=1,
metric='euclidean')
# Euclidean distance, with Y != X.
Y = rng.random_sample((100, 4))
check_pairwise_distances_chunked(X,
Y,
working_memory=1,
metric='euclidean')
check_pairwise_distances_chunked(X.tolist(),
Y.tolist(),
working_memory=1,
metric='euclidean')
# absurdly large working_memory
check_pairwise_distances_chunked(X,
Y,
working_memory=10000,
metric='euclidean')
# "cityblock" uses scikit-learn metric, cityblock (function) is
# scipy.spatial.
check_pairwise_distances_chunked(X,
Y,
working_memory=1,
metric='cityblock')
# Test that a value error is raised if the metric is unknown
assert_raises(ValueError, next,
pairwise_distances_chunked(X, Y, metric="blah"))
# Test precomputed returns all at once
D = pairwise_distances(X)
gen = pairwise_distances_chunked(D,
working_memory=2**-16,
metric='precomputed')
assert isinstance(gen, GeneratorType)
assert next(gen) is D
assert_raises(StopIteration, next, gen)
def test_pairwise_distances_chunked_reduce():
rng = np.random.RandomState(0)
X = rng.random_sample((400, 4))
# Reduced Euclidean distance
S = pairwise_distances(X)[:, :100]
S_chunks = pairwise_distances_chunked(X,
None,
reduce_func=_reduce_func,
working_memory=2**-16)
assert isinstance(S_chunks, GeneratorType)
S_chunks = list(S_chunks)
assert len(S_chunks) > 1
# atol is for diagonal where S is explicitly zeroed on the diagonal
assert_allclose(np.vstack(S_chunks), S, atol=1e-7)
def test_small_distance_threshold():
rng = np.random.RandomState(0)
n_samples = 10
X = rng.randint(-300, 300, size=(n_samples, 3))
# this should result in all data in their own clusters, given that
# their pairwise distances are bigger than .1 (which may not be the case
# with a different random seed).
clustering = AgglomerativeClustering(n_clusters=None,
distance_threshold=1.,
linkage="single").fit(X)
# check that the pairwise distances are indeed all larger than .1
all_distances = pairwise_distances(X, metric='minkowski', p=2)
np.fill_diagonal(all_distances, np.inf)
assert np.all(all_distances > .1)
assert clustering.n_clusters_ == n_samples
def test_dbscan_sparse_precomputed(include_self):
D = pairwise_distances(X)
nn = NearestNeighbors(radius=.9).fit(X)
X_ = X if include_self else None
D_sparse = nn.radius_neighbors_graph(X=X_, mode='distance')
# Ensure it is sparse not merely on diagonals:
assert D_sparse.nnz < D.shape[0] * (D.shape[0] - 1)
core_sparse, labels_sparse = dbscan(D_sparse,
eps=.8,
min_samples=10,
metric='precomputed')
core_dense, labels_dense = dbscan(D,
eps=.8,
min_samples=10,
metric='precomputed')
assert_array_equal(core_dense, core_sparse)
assert_array_equal(labels_dense, labels_sparse)
def check_pairwise_distances_chunked(X, Y, working_memory, metric='euclidean'):
gen = pairwise_distances_chunked(X,
Y,
working_memory=working_memory,
metric=metric)
assert isinstance(gen, GeneratorType)
blockwise_distances = list(gen)
Y = X if Y is None else Y
min_block_mib = len(Y) * 8 * 2**-20
for block in blockwise_distances:
memory_used = block.nbytes
assert memory_used <= max(working_memory, min_block_mib) * 2**20
blockwise_distances = np.vstack(blockwise_distances)
S = pairwise_distances(X, Y, metric=metric)
assert_array_almost_equal(blockwise_distances, S)
def test_dbscan_balltree():
# Tests the DBSCAN algorithm with balltree for neighbor calculation.
eps = 0.8
min_samples = 10
D = pairwise_distances(X)
core_samples, labels = dbscan(D,
metric="precomputed",
eps=eps,
min_samples=min_samples)
# number of clusters, ignoring noise if present
n_clusters_1 = len(set(labels)) - int(-1 in labels)
assert n_clusters_1 == n_clusters
db = DBSCAN(p=2.0, eps=eps, min_samples=min_samples, algorithm='ball_tree')
labels = db.fit(X).labels_
n_clusters_2 = len(set(labels)) - int(-1 in labels)
assert n_clusters_2 == n_clusters
db = DBSCAN(p=2.0, eps=eps, min_samples=min_samples, algorithm='kd_tree')
labels = db.fit(X).labels_
n_clusters_3 = len(set(labels)) - int(-1 in labels)
assert n_clusters_3 == n_clusters
db = DBSCAN(p=1.0, eps=eps, min_samples=min_samples, algorithm='ball_tree')
labels = db.fit(X).labels_
n_clusters_4 = len(set(labels)) - int(-1 in labels)
assert n_clusters_4 == n_clusters
db = DBSCAN(leaf_size=20,
eps=eps,
min_samples=min_samples,
algorithm='ball_tree')
labels = db.fit(X).labels_
n_clusters_5 = len(set(labels)) - int(-1 in labels)
assert n_clusters_5 == n_clusters
def test_cluster_distances_with_distance_threshold():
rng = np.random.RandomState(0)
n_samples = 100
X = rng.randint(-10, 10, size=(n_samples, 3))
# check the distances within the clusters and with other clusters
distance_threshold = 4
clustering = AgglomerativeClustering(n_clusters=None,
distance_threshold=distance_threshold,
linkage="single").fit(X)
labels = clustering.labels_
D = pairwise_distances(X, metric="minkowski", p=2)
# to avoid taking the 0 diagonal in min()
np.fill_diagonal(D, np.inf)
for label in np.unique(labels):
in_cluster_mask = labels == label
max_in_cluster_distance = (D[in_cluster_mask][:, in_cluster_mask].min(
axis=0).max())
min_out_cluster_distance = (D[in_cluster_mask][:,
~in_cluster_mask].min(
axis=0).min())
# single data point clusters only have that inf diagonal here
if in_cluster_mask.sum() > 1:
assert max_in_cluster_distance < distance_threshold
assert min_out_cluster_distance >= distance_threshold
def test_agglomerative_clustering():
# Check that we obtain the correct number of clusters with
# agglomerative clustering.
rng = np.random.RandomState(0)
mask = np.ones([10, 10], dtype=np.bool)
n_samples = 100
X = rng.randn(n_samples, 50)
connectivity = grid_to_graph(*mask.shape)
for linkage in ("ward", "complete", "average", "single"):
clustering = AgglomerativeClustering(n_clusters=10,
connectivity=connectivity,
linkage=linkage)
clustering.fit(X)
# test caching
try:
tempdir = mkdtemp()
clustering = AgglomerativeClustering(n_clusters=10,
connectivity=connectivity,
memory=tempdir,
linkage=linkage)
clustering.fit(X)
labels = clustering.labels_
assert np.size(np.unique(labels)) == 10
finally:
shutil.rmtree(tempdir)
# Turn caching off now
clustering = AgglomerativeClustering(n_clusters=10,
connectivity=connectivity,
linkage=linkage)
# Check that we obtain the same solution with early-stopping of the
# tree building
clustering.compute_full_tree = False
clustering.fit(X)
assert_almost_equal(
normalized_mutual_info_score(clustering.labels_, labels), 1)
clustering.connectivity = None
clustering.fit(X)
assert np.size(np.unique(clustering.labels_)) == 10
# Check that we raise a TypeError on dense matrices
clustering = AgglomerativeClustering(
n_clusters=10,
connectivity=sparse.lil_matrix(connectivity.toarray()[:10, :10]),
linkage=linkage)
with pytest.raises(ValueError):
clustering.fit(X)
# Test that using ward with another metric than euclidean raises an
# exception
clustering = AgglomerativeClustering(n_clusters=10,
connectivity=connectivity.toarray(),
affinity="manhattan",
linkage="ward")
with pytest.raises(ValueError):
clustering.fit(X)
# Test using another metric than euclidean works with linkage complete
for affinity in PAIRED_DISTANCES.keys():
# Compare our (structured) implementation to scipy
clustering = AgglomerativeClustering(n_clusters=10,
connectivity=np.ones(
(n_samples, n_samples)),
affinity=affinity,
linkage="complete")
clustering.fit(X)
clustering2 = AgglomerativeClustering(n_clusters=10,
connectivity=None,
affinity=affinity,
linkage="complete")
clustering2.fit(X)
assert_almost_equal(
normalized_mutual_info_score(clustering2.labels_,
clustering.labels_), 1)
# Test that using a distance matrix (affinity = 'precomputed') has same
# results (with connectivity constraints)
clustering = AgglomerativeClustering(n_clusters=10,
connectivity=connectivity,
linkage="complete")
clustering.fit(X)
X_dist = pairwise_distances(X)
clustering2 = AgglomerativeClustering(n_clusters=10,
connectivity=connectivity,
affinity='precomputed',
linkage="complete")
clustering2.fit(X_dist)
assert_array_equal(clustering.labels_, clustering2.labels_)
def test_weighted_dbscan():
# ensure sample_weight is validated
with pytest.raises(ValueError):
dbscan([[0], [1]], sample_weight=[2])
with pytest.raises(ValueError):
dbscan([[0], [1]], sample_weight=[2, 3, 4])
# ensure sample_weight has an effect
assert_array_equal([],
dbscan([[0], [1]], sample_weight=None,
min_samples=6)[0])
assert_array_equal([],
dbscan([[0], [1]], sample_weight=[5, 5],
min_samples=6)[0])
assert_array_equal([0],
dbscan([[0], [1]], sample_weight=[6, 5],
min_samples=6)[0])
assert_array_equal([0, 1],
dbscan([[0], [1]], sample_weight=[6, 6],
min_samples=6)[0])
# points within eps of each other:
assert_array_equal([0, 1],
dbscan([[0], [1]],
eps=1.5,
sample_weight=[5, 1],
min_samples=6)[0])
# and effect of non-positive and non-integer sample_weight:
assert_array_equal([],
dbscan([[0], [1]],
sample_weight=[5, 0],
eps=1.5,
min_samples=6)[0])
assert_array_equal([0, 1],
dbscan([[0], [1]],
sample_weight=[5.9, 0.1],
eps=1.5,
min_samples=6)[0])
assert_array_equal([0, 1],
dbscan([[0], [1]],
sample_weight=[6, 0],
eps=1.5,
min_samples=6)[0])
assert_array_equal([],
dbscan([[0], [1]],
sample_weight=[6, -1],
eps=1.5,
min_samples=6)[0])
# for non-negative sample_weight, cores should be identical to repetition
rng = np.random.RandomState(42)
sample_weight = rng.randint(0, 5, X.shape[0])
core1, label1 = dbscan(X, sample_weight=sample_weight)
assert len(label1) == len(X)
X_repeated = np.repeat(X, sample_weight, axis=0)
core_repeated, label_repeated = dbscan(X_repeated)
core_repeated_mask = np.zeros(X_repeated.shape[0], dtype=bool)
core_repeated_mask[core_repeated] = True
core_mask = np.zeros(X.shape[0], dtype=bool)
core_mask[core1] = True
assert_array_equal(np.repeat(core_mask, sample_weight), core_repeated_mask)
# sample_weight should work with precomputed distance matrix
D = pairwise_distances(X)
core3, label3 = dbscan(D,
sample_weight=sample_weight,
metric='precomputed')
assert_array_equal(core1, core3)
assert_array_equal(label1, label3)
# sample_weight should work with estimator
est = DBSCAN().fit(X, sample_weight=sample_weight)
core4 = est.core_sample_indices_
label4 = est.labels_
assert_array_equal(core1, core4)
assert_array_equal(label1, label4)
est = DBSCAN()
label5 = est.fit_predict(X, sample_weight=sample_weight)
core5 = est.core_sample_indices_
assert_array_equal(core1, core5)
assert_array_equal(label1, label5)
assert_array_equal(label1, est.labels_)
def test_pairwise_distances():
# Test the pairwise_distance helper function.
rng = np.random.RandomState(0)
# Euclidean distance should be equivalent to calling the function.
X = rng.random_sample((5, 4))
S = pairwise_distances(X, metric="euclidean")
S2 = euclidean_distances(X)
assert_array_almost_equal(S, S2)
# Euclidean distance, with Y != X.
Y = rng.random_sample((2, 4))
S = pairwise_distances(X, Y, metric="euclidean")
S2 = euclidean_distances(X, Y)
assert_array_almost_equal(S, S2)
# Test with tuples as X and Y
X_tuples = tuple([tuple([v for v in row]) for row in X])
Y_tuples = tuple([tuple([v for v in row]) for row in Y])
S2 = pairwise_distances(X_tuples, Y_tuples, metric="euclidean")
assert_array_almost_equal(S, S2)
# Test haversine distance
# The data should be valid latitude and longitude
X = rng.random_sample((5, 2))
X[:, 0] = (X[:, 0] - 0.5) * 2 * np.pi / 2
X[:, 1] = (X[:, 1] - 0.5) * 2 * np.pi
S = pairwise_distances(X, metric="haversine")
S2 = haversine_distances(X)
assert_array_almost_equal(S, S2)
# Test haversine distance, with Y != X
Y = rng.random_sample((2, 2))
Y[:, 0] = (Y[:, 0] - 0.5) * 2 * np.pi / 2
Y[:, 1] = (Y[:, 1] - 0.5) * 2 * np.pi
S = pairwise_distances(X, Y, metric="haversine")
S2 = haversine_distances(X, Y)
assert_array_almost_equal(S, S2)
# "cityblock" uses scikit-learn metric, cityblock (function) is
# scipy.spatial.
S = pairwise_distances(X, metric="cityblock")
S2 = pairwise_distances(X, metric=cityblock)
assert S.shape[0] == S.shape[1]
assert S.shape[0] == X.shape[0]
assert_array_almost_equal(S, S2)
# The manhattan metric should be equivalent to cityblock.
S = pairwise_distances(X, Y, metric="manhattan")
S2 = pairwise_distances(X, Y, metric=cityblock)
assert S.shape[0] == X.shape[0]
assert S.shape[1] == Y.shape[0]
assert_array_almost_equal(S, S2)
# Test cosine as a string metric versus cosine callable
# The string "cosine" uses mrex.metric,
# while the function cosine is scipy.spatial
S = pairwise_distances(X, Y, metric="cosine")
S2 = pairwise_distances(X, Y, metric=cosine)
assert S.shape[0] == X.shape[0]
assert S.shape[1] == Y.shape[0]
assert_array_almost_equal(S, S2)
# Test with sparse X and Y,
# currently only supported for Euclidean, L1 and cosine.
X_sparse = csr_matrix(X)
Y_sparse = csr_matrix(Y)
S = pairwise_distances(X_sparse, Y_sparse, metric="euclidean")
S2 = euclidean_distances(X_sparse, Y_sparse)
assert_array_almost_equal(S, S2)
S = pairwise_distances(X_sparse, Y_sparse, metric="cosine")
S2 = cosine_distances(X_sparse, Y_sparse)
assert_array_almost_equal(S, S2)
S = pairwise_distances(X_sparse, Y_sparse.tocsc(), metric="manhattan")
S2 = manhattan_distances(X_sparse.tobsr(), Y_sparse.tocoo())
assert_array_almost_equal(S, S2)
S2 = manhattan_distances(X, Y)
assert_array_almost_equal(S, S2)
# Test with scipy.spatial.distance metric, with a kwd
kwds = {"p": 2.0}
S = pairwise_distances(X, Y, metric="minkowski", **kwds)
S2 = pairwise_distances(X, Y, metric=minkowski, **kwds)
assert_array_almost_equal(S, S2)
# same with Y = None
kwds = {"p": 2.0}
S = pairwise_distances(X, metric="minkowski", **kwds)
S2 = pairwise_distances(X, metric=minkowski, **kwds)
assert_array_almost_equal(S, S2)
# Test that scipy distance metrics throw an error if sparse matrix given
assert_raises(TypeError, pairwise_distances, X_sparse, metric="minkowski")
assert_raises(TypeError,
pairwise_distances,
X,
Y_sparse,
metric="minkowski")
# Test that a value error is raised if the metric is unknown
assert_raises(ValueError, pairwise_distances, X, Y, metric="blah")
def test_pairwise_distances_argmin_min():
# Check pairwise minimum distances computation for any metric
X = [[0], [1]]
Y = [[-2], [3]]
Xsp = dok_matrix(X)
Ysp = csr_matrix(Y, dtype=np.float32)
expected_idx = [0, 1]
expected_vals = [2, 2]
expected_vals_sq = [4, 4]
# euclidean metric
idx, vals = pairwise_distances_argmin_min(X, Y, metric="euclidean")
idx2 = pairwise_distances_argmin(X, Y, metric="euclidean")
assert_array_almost_equal(idx, expected_idx)
assert_array_almost_equal(idx2, expected_idx)
assert_array_almost_equal(vals, expected_vals)
# sparse matrix case
idxsp, valssp = pairwise_distances_argmin_min(Xsp, Ysp, metric="euclidean")
assert_array_almost_equal(idxsp, expected_idx)
assert_array_almost_equal(valssp, expected_vals)
# We don't want np.matrix here
assert type(idxsp) == np.ndarray
assert type(valssp) == np.ndarray
# euclidean metric squared
idx, vals = pairwise_distances_argmin_min(X,
Y,
metric="euclidean",
metric_kwargs={"squared": True})
assert_array_almost_equal(idx, expected_idx)
assert_array_almost_equal(vals, expected_vals_sq)
# Non-euclidean scikit-learn metric
idx, vals = pairwise_distances_argmin_min(X, Y, metric="manhattan")
idx2 = pairwise_distances_argmin(X, Y, metric="manhattan")
assert_array_almost_equal(idx, expected_idx)
assert_array_almost_equal(idx2, expected_idx)
assert_array_almost_equal(vals, expected_vals)
# sparse matrix case
idxsp, valssp = pairwise_distances_argmin_min(Xsp, Ysp, metric="manhattan")
assert_array_almost_equal(idxsp, expected_idx)
assert_array_almost_equal(valssp, expected_vals)
# Non-euclidean Scipy distance (callable)
idx, vals = pairwise_distances_argmin_min(X,
Y,
metric=minkowski,
metric_kwargs={"p": 2})
assert_array_almost_equal(idx, expected_idx)
assert_array_almost_equal(vals, expected_vals)
# Non-euclidean Scipy distance (string)
idx, vals = pairwise_distances_argmin_min(X,
Y,
metric="minkowski",
metric_kwargs={"p": 2})
assert_array_almost_equal(idx, expected_idx)
assert_array_almost_equal(vals, expected_vals)
# Compare with naive implementation
rng = np.random.RandomState(0)
X = rng.randn(97, 149)
Y = rng.randn(111, 149)
dist = pairwise_distances(X, Y, metric="manhattan")
dist_orig_ind = dist.argmin(axis=0)
dist_orig_val = dist[dist_orig_ind, range(len(dist_orig_ind))]
dist_chunked_ind, dist_chunked_val = pairwise_distances_argmin_min(
X, Y, axis=0, metric="manhattan")
np.testing.assert_almost_equal(dist_orig_ind, dist_chunked_ind, decimal=7)
np.testing.assert_almost_equal(dist_orig_val, dist_chunked_val, decimal=7)
def test_pairwise_callable_nonstrict_metric():
# paired_distances should allow callable metric where metric(x, x) != 0
# Knowing that the callable is a strict metric would allow the diagonal to
# be left uncalculated and set to 0.
assert pairwise_distances([[1.]], metric=lambda x, y: 5)[0, 0] == 5
def test_parallel_pairwise_distances_diagonal(metric):
rng = np.random.RandomState(0)
X = rng.normal(size=(1000, 10), scale=1e10)
distances = pairwise_distances(X, metric=metric, n_jobs=2)
assert_allclose(np.diag(distances), 0, atol=1e-10)
def euclidean_distances(X, n_jobs):
return pairwise_distances(X, metric="euclidean", n_jobs=n_jobs)
