def pearson_correlation(X, Y): """ Considering the rows of X (and Y=X) as vectors, compute the distance matrix between each pair of vectors. This correlation implementation is equivalent to the cosine similarity since the data it receives is assumed to be centered -- mean is 0. The correlation may be interpreted as the cosine of the angle between the two vectors defined by the users' preference values. Parameters ---------- X : {array-like, sparse matrix}, shape = [n_samples_1, n_features] Y : {array-like, sparse matrix}, shape = [n_samples_2, n_features] Returns ------- distances : {array, sparse matrix}, shape = [n_samples_1, n_samples_2] Examples -------- >>> from crab.metrics.pairwise import pearson_correlation >>> X = [[2.5, 3.5, 3.0, 3.5, 2.5, 3.0],[2.5, 3.5, 3.0, 3.5, 2.5, 3.0]] >>> # distance between rows of X >>> pearson_correlation(X, X) array([[ 1., 1.], [ 1., 1.]]) >>> pearson_correlation(X, [[3.0, 3.5, 1.5, 5.0, 3.5,3.0]]) array([[ 0.39605902], [ 0.39605902]]) """ # should not need X_norm_squared because if you could precompute that as # well as Y, then you should just pre-compute the output and not even # call this function. from distmetrics import DistanceMetric X, Y = check_pairwise_arrays(X, Y) n_samples_X, n_features_X = X.shape n_samples_Y, n_features_Y = Y.shape if n_features_X != n_features_Y: raise Exception("X and Y should have the same number of features!") if X is Y: X = Y = np.asanyarray(X) else: X = np.asanyarray(X) Y = np.asanyarray(Y) dm = DistanceMetric(metric='correlation') D = dm.pdist(X, squareform=True) return 1 - D
def bench_float(m1=200, m2=200, rseed=0): print 79 * '_' print " real valued distance metrics" print np.random.seed(rseed) X1 = np.random.random((m1, DTEST)) X2 = np.random.random((m2, DTEST)) for (metric, argdict) in METRIC_DICT.iteritems(): keys = argdict.keys() for vals in itertools.product(*argdict.values()): kwargs = dict(zip(keys, vals)) print metric, param_info(kwargs) t0 = time() try: dist_metric = DistanceMetric(metric, **kwargs) Yc1 = dist_metric.cdist(X1, X2) except Exception as inst: print " >>>>>>>>>> error in pyDistances cdist:" print " ", inst t1 = time() try: Yc2 = cdist(X1, X2, metric, **kwargs) except Exception as inst: print " >>>>>>>>>> error in scipy cdist:" print " ", inst t2 = time() try: dist_metric = DistanceMetric(metric, **kwargs) Yp1 = dist_metric.pdist(X1) except Exception as inst: print " >>>>>>>>>> error in pyDistances pdist:" print " ", inst t3 = time() try: Yp2 = pdist(X1, metric, **kwargs) except Exception as inst: print " >>>>>>>>>> error in scipy pdist:" print " ", inst t4 = time() if not np.allclose(Yc1, Yc2): print " >>>>>>>>>> FAIL: cdist results don't match" if not np.allclose(Yp1, Yp2): print " >>>>>>>>>> FAIL: pdist results don't match" print " - pyDistances: c: %.4f sec p: %.4f sec" % (t1 - t0, t3 - t2) print " - scipy: c: %.4f sec p: %.4f sec" % (t2 - t1, t4 - t3) print ''
def bench_float(m1=100, m2=100, rseed=0): print 79 * '_' print " real valued distance metrics" print np.random.seed(rseed) X1 = np.random.random((m1, DTEST)) X2 = np.random.random((m2, DTEST)) for (metric, argdict) in METRIC_DICT.iteritems(): keys = argdict.keys() for vals in itertools.product(*argdict.values()): kwargs = dict(zip(keys, vals)) print metric, param_info(kwargs) t0 = time() try: dist_metric = DistanceMetric(metric, **kwargs) Yc1 = dist_metric.cdist(X1, X2) except Exception as inst: print " >>>>>>>>>> error in pyDistances cdist:" print " ", inst t1 = time() try: Yc2 = cdist(X1, X2, metric, **kwargs) except Exception as inst: print " >>>>>>>>>> error in scipy cdist:" print " ", inst t2 = time() try: dist_metric = DistanceMetric(metric, **kwargs) Yp1 = dist_metric.pdist(X1) except Exception as inst: print " >>>>>>>>>> error in pyDistances pdist:" print " ", inst t3 = time() try: Yp2 = pdist(X1, metric, **kwargs) except Exception as inst: print " >>>>>>>>>> error in scipy pdist:" print " ", inst t4 = time() if not np.allclose(Yc1, Yc2): print " >>>>>>>>>> FAIL: cdist results don't match" if not np.allclose(Yp1, Yp2): print " >>>>>>>>>> FAIL: pdist results don't match" print " - pyDistances: c: %.2g sec p: %.2g sec" % (t1 - t0, t3 - t2) print " - scipy: c: %.2g sec p: %.2g sec" % (t2 - t1, t4 - t3) print ''
def test_ball_tree_query_radius_count(n_samples=100, n_features=10): X = 2 * np.random.random(size=(n_samples, n_features)) - 1 dm = DistanceMetric() D = dm.pdist(X, squareform=True) r = np.mean(D) bt = BallTree(X) count1 = bt.query_radius(X, r, count_only=True) count2 = (D <= r).sum(1) assert_array_almost_equal(count1, count2)
def _check_metrics_float(self, k, metric, kwargs): bt = BallTree(self.X, metric=metric, **kwargs) dist_bt, ind_bt = bt.query(self.X, k=k) dm = DistanceMetric(metric=metric, **kwargs) D = dm.pdist(self.X, squareform=True) ind_dm = np.argsort(D, 1)[:, :k] dist_dm = D[np.arange(self.X.shape[0])[:, None], ind_dm] # we don't check the indices here because if there is a tie for # nearest neighbor, then the test may fail. Distances will reflect # whether the search was successful assert_array_almost_equal(dist_bt, dist_dm)
def test_ball_tree_query_radius_count(n_samples=100, n_features=10): X = 2 * np.random.random(size=(n_samples, n_features)) - 1 dm = DistanceMetric() D = dm.pdist(X, squareform=True) r = np.mean(D) bt = BallTree(X) count1 = bt.query_radius(X, r, count_only=True) count2 = (D <= r).sum(1) assert_array_almost_equal(count1, count2)
def _check_metrics_float(self, k, metric, kwargs): bt = BallTree(self.X, metric=metric, **kwargs) dist_bt, ind_bt = bt.query(self.X, k=k) dm = DistanceMetric(metric=metric, **kwargs) D = dm.pdist(self.X, squareform=True) ind_dm = np.argsort(D, 1)[:, :k] dist_dm = D[np.arange(self.X.shape[0])[:, None], ind_dm] # we don't check the indices here because if there is a tie for # nearest neighbor, then the test may fail. Distances will reflect # whether the search was successful assert_array_almost_equal(dist_bt, dist_dm)
def test_query_radius_count(self): # center the data X = 2 * self.X - 1 dm = DistanceMetric() D = dm.pdist(X, squareform=True) r = np.mean(D) bt = BallTree(X) count1 = bt.query_radius(X, r, count_only=True) count2 = (D <= r).sum(1) assert_array_almost_equal(count1, count2)
def test_query_radius_count(self): # center the data X = 2 * self.X - 1 dm = DistanceMetric() D = dm.pdist(X, squareform=True) r = np.mean(D) bt = BallTree(X) count1 = bt.query_radius(X, r, count_only=True) count2 = (D <= r).sum(1) assert_array_almost_equal(count1, count2)
def test_pdist(m=15, rseed=0): """Compare DistanceMetric.pdist to scipy.spatial.distance.pdist""" np.random.seed(rseed) X = np.random.random((m, DTEST)) for (metric, argdict) in METRIC_DICT.iteritems(): keys = argdict.keys() for vals in itertools.product(*argdict.values()): kwargs = dict(zip(keys, vals)) dist_metric = DistanceMetric(metric, **kwargs) Y1 = dist_metric.pdist(X) Y2 = pdist(X, metric, **kwargs) if not np.allclose(Y1, Y2): print metric, keys, vals print Y1[:5, :5] print Y2[:5, :5] assert np.allclose(Y1, Y2)
def test_pdist(m=15, rseed=0): """Compare DistanceMetric.pdist to scipy.spatial.distance.pdist""" np.random.seed(rseed) X = np.random.random((m, DTEST)) for (metric, argdict) in METRIC_DICT.iteritems(): keys = argdict.keys() for vals in itertools.product(*argdict.values()): kwargs = dict(zip(keys, vals)) dist_metric = DistanceMetric(metric, **kwargs) Y1 = dist_metric.pdist(X) Y2 = squareform(pdist(X, metric, **kwargs)) if not np.allclose(Y1, Y2): print metric, keys, vals print Y1[:5, :5] print Y2[:5, :5] assert np.allclose(Y1, Y2)
def test_pdist_squareform(m=10, d=3, rseed=0): X = np.random.random((m, d)) dist_metric = DistanceMetric() Y1 = squareform(dist_metric.pdist(X, squareform=False)) Y2 = dist_metric.pdist(X, squareform=True) assert np.allclose(Y1, Y2)
def test_pdist_squareform(m=10, d=3, rseed=0): X = np.random.random((m, d)) dist_metric = DistanceMetric() Y1 = squareform(dist_metric.pdist(X, squareform=False)) Y2 = dist_metric.pdist(X, squareform=True) assert np.allclose(Y1, Y2)