class NCA: def __init__(self): self.metric_model = NCA_ml() self.X_tr = None self.y_train = None self.X_te = None def fit(self, X_tr, y_train): """Fits the model to the prescribed data.""" self.X_tr = X_tr self.y_train = y_train return self.metric_model.fit(X_tr, y_train) def transform(self, X): """Transforms the test data according to the model""" return self.metric_model.transform(X) def predict_proba(self, X_te): """Predicts the probabilities of each of the test samples""" test_samples = X_te.shape[0] self.X_tr = self.transform(self.X_tr) clf = NearestCentroid() clf.fit(self.X_tr, self.y_train) centroids = clf.centroids_ probabilities = np.zeros((test_samples, centroids.shape[0])) for sample in xrange(test_samples): probabilities[sample] = sk_nearest_neighbour_proba( centroids, X_te[sample, :]) return probabilities
def process_nca(self, **option): '''Metric Learning algorithm: NCA''' GeneExp = self.GeneExp_train Label = self.Label_train nca = NCA(**option) nca.fit(GeneExp, Label) self.Trans['NCA'] = nca.transformer()
def NCA(self): print "Warning the features will be transformed" lmnn = NCA() NCA.fit(self.features, targets) self.features = NCA.transform(self.features) self.prepare_for_testing() #Evaluate with nn self.nearest_neighbors("NCA + KNN")
def test_one_class(self): # if there is only one class the gradient is null, so the final matrix # must stay like the initialization X = self.iris_points[self.iris_labels == 0] y = self.iris_labels[self.iris_labels == 0] A = make_spd_matrix(X.shape[1], X.shape[1]) nca = NCA(init=A, max_iter=30, n_components=X.shape[1]) nca.fit(X, y) assert_array_equal(nca.components_, A)
def test_one_class(self): # if there is only one class the gradient is null, so the final matrix # must stay like the initialization X = self.iris_points[self.iris_labels == 0] y = self.iris_labels[self.iris_labels == 0] EPS = np.finfo(float).eps A = np.zeros((X.shape[1], X.shape[1])) np.fill_diagonal(A, 1. / (np.maximum(X.max(axis=0) - X.min(axis=0), EPS))) nca = NCA(max_iter=30, num_dims=X.shape[1]) nca.fit(X, y) assert_array_equal(nca.transformer_, A)
def test_iris(self): n = self.iris_points.shape[0] nca = NCA(max_iter=(100000//n), learning_rate=0.01) nca.fit(self.iris_points, self.iris_labels) # Result copied from Iris example at # https://github.com/vomjom/nca/blob/master/README.mkd expected = [[-0.09935, -0.2215, 0.3383, 0.443], [+0.2532, 0.5835, -0.8461, -0.8915], [-0.729, -0.6386, 1.767, 1.832], [-0.9405, -0.8461, 2.281, 2.794]] assert_array_almost_equal(expected, nca.transformer(), decimal=3)
def test_iris(self): n = self.iris_points.shape[0] nca = NCA(max_iter=(100000 // n), learning_rate=0.01) nca.fit(self.iris_points, self.iris_labels) # Result copied from Iris example at # https://github.com/vomjom/nca/blob/master/README.mkd expected = [[-0.09935, -0.2215, 0.3383, 0.443], [+0.2532, 0.5835, -0.8461, -0.8915], [-0.729, -0.6386, 1.767, 1.832], [-0.9405, -0.8461, 2.281, 2.794]] assert_array_almost_equal(expected, nca.transformer(), decimal=3)
def test_iris(self): n = self.iris_points.shape[0] # Without dimension reduction nca = NCA(max_iter=(100000 // n)) nca.fit(self.iris_points, self.iris_labels) csep = class_separation(nca.transform(), self.iris_labels) self.assertLess(csep, 0.15) # With dimension reduction nca = NCA(max_iter=(100000 // n), num_dims=2, tol=1e-9) nca.fit(self.iris_points, self.iris_labels) csep = class_separation(nca.transform(), self.iris_labels) self.assertLess(csep, 0.20)
def test_simple_example(self): """Test on a simple example. Puts four points in the input space where the opposite labels points are next to each other. After transform the same labels points should be next to each other. """ X = np.array([[0, 0], [0, 1], [2, 0], [2, 1]]) y = np.array([1, 0, 1, 0]) nca = NCA(n_components=2,) nca.fit(X, y) Xansformed = nca.transform(X) np.testing.assert_equal(pairwise_distances(Xansformed).argsort()[:, 1], np.array([2, 3, 0, 1]))
def test_simple_example(self): """Test on a simple example. Puts four points in the input space where the opposite labels points are next to each other. After transform the same labels points should be next to each other. """ X = np.array([[0, 0], [0, 1], [2, 0], [2, 1]]) y = np.array([1, 0, 1, 0]) nca = NCA(num_dims=2,) nca.fit(X, y) Xansformed = nca.transform(X) np.testing.assert_equal(pairwise_distances(Xansformed).argsort()[:, 1], np.array([2, 3, 0, 1]))
def test_nca(self): n = self.X.shape[0] nca = NCA(max_iter=(100000 // n)) nca.fit(self.X, self.y) res_1 = nca.transform(self.X) nca = NCA(max_iter=(100000 // n)) res_2 = nca.fit_transform(self.X, self.y) assert_array_almost_equal(res_1, res_2)
def test_deprecation(self): # test that the right deprecation message is thrown. # TODO: remove in v.0.5 X = np.array([[0, 0], [0, 1], [2, 0], [2, 1]]) y = np.array([1, 0, 1, 0]) nca = NCA(num_dims=2, learning_rate=0.01) msg = ('"learning_rate" parameter is not used.' ' It has been deprecated in version 0.4 and will be' 'removed in 0.5') assert_warns_message(DeprecationWarning, msg, nca.fit, X, y)
def test_iris(self): n = self.iris_points.shape[0] # Without dimension reduction nca = NCA(max_iter=(100000//n)) nca.fit(self.iris_points, self.iris_labels) csep = class_separation(nca.transform(self.iris_points), self.iris_labels) self.assertLess(csep, 0.15) # With dimension reduction nca = NCA(max_iter=(100000//n), num_dims=2) nca.fit(self.iris_points, self.iris_labels) csep = class_separation(nca.transform(self.iris_points), self.iris_labels) self.assertLess(csep, 0.20)
def test_finite_differences(self): """Test gradient of loss function Assert that the gradient is almost equal to its finite differences approximation. """ # Initialize the transformation `M`, as well as `X` and `y` and `NCA` X, y = make_classification() M = np.random.randn(np.random.randint(1, X.shape[1] + 1), X.shape[1]) mask = y[:, np.newaxis] == y[np.newaxis, :] nca = NCA() nca.n_iter_ = 0 def fun(M): return nca._loss_grad_lbfgs(M, X, mask)[0] def grad(M): return nca._loss_grad_lbfgs(M, X, mask)[1].ravel() # compute relative error rel_diff = check_grad(fun, grad, M.ravel()) / np.linalg.norm(grad(M)) np.testing.assert_almost_equal(rel_diff, 0., decimal=6)
def runNCA(X_train, X_test, y_train, y_test): transformer = NCA(max_iter=100, verbose=True) transformer.fit(X_train, y_train) X_train_proj = transformer.transform(X_train) X_test_proj = transformer.transform(X_test) np.save('X_train_NCA', X_train_proj) np.save('X_test_NCA', X_test_proj) return X_train_proj, X_test_proj
def test_finite_differences(self): """Test gradient of loss function Assert that the gradient is almost equal to its finite differences approximation. """ # Initialize the transformation `M`, as well as `X` and `y` and `NCA` X, y = make_classification() M = np.random.randn(np.random.randint(1, X.shape[1] + 1), X.shape[1]) mask = y[:, np.newaxis] == y[np.newaxis, :] nca = NCA() nca.n_iter_ = 0 def fun(M): return nca._loss_grad_lbfgs(M, X, mask)[0] def grad(M): return nca._loss_grad_lbfgs(M, X, mask)[1].ravel() # compute relative error epsilon = np.sqrt(np.finfo(float).eps) rel_diff = (check_grad(fun, grad, M.ravel()) / np.linalg.norm(approx_fprime(M.ravel(), fun, epsilon))) np.testing.assert_almost_equal(rel_diff, 0., decimal=6)
def test_nca(self): n = self.X.shape[0] nca = NCA(max_iter=(100000//n)) nca.fit(self.X, self.y) res_1 = nca.transform(self.X) nca = NCA(max_iter=(100000//n)) res_2 = nca.fit_transform(self.X, self.y) assert_array_almost_equal(res_1, res_2)
def test_singleton_class(self): X = self.iris_points y = self.iris_labels # one singleton class: test fitting works singleton_class = 1 ind_singleton, = np.where(y == singleton_class) y[ind_singleton] = 2 y[ind_singleton[0]] = singleton_class nca = NCA(max_iter=30) nca.fit(X, y) # One non-singleton class: test fitting works ind_1, = np.where(y == 1) ind_2, = np.where(y == 2) y[ind_1] = 0 y[ind_1[0]] = 1 y[ind_2] = 0 y[ind_2[0]] = 2 nca = NCA(max_iter=30) nca.fit(X, y) # Only singleton classes: test fitting does nothing (the gradient # must be null in this case, so the final matrix must stay like # the initialization) ind_0, = np.where(y == 0) ind_1, = np.where(y == 1) ind_2, = np.where(y == 2) X = X[[ind_0[0], ind_1[0], ind_2[0]]] y = y[[ind_0[0], ind_1[0], ind_2[0]]] EPS = np.finfo(float).eps A = np.zeros((X.shape[1], X.shape[1])) np.fill_diagonal(A, 1. / (np.maximum(X.max(axis=0) - X.min(axis=0), EPS))) nca = NCA(max_iter=30, num_dims=X.shape[1]) nca.fit(X, y) assert_array_equal(nca.transformer_, A)
def __init__(self, init_with=0): """Initialize the transformation pipleine, with an optional list of transformation modes. init_with can be any bitwise-or'd combination of the TRANSFORM_* modes.""" self.pipeline = [] self.supervised_pipeline = [] self.mink = 4 if init_with & self.TRANSFORM_SCALE: self.add_standard_scaler() if init_with & self.TRANSFORM_PCA: print "Adding PCA" self.add_pca() if init_with & self.TRANSFORM_RANDOM_PCA: print "Adding RandomPCA" self.add_random_pca() if init_with & self.TRANSFORM_KERNEL_PCA: self.add_kernel_pca() if init_with & self.TRANSFORM_LMNN: self.supervised_pipeline.append(LMNN(k=self.mink)) if init_with & self.TRANSFORM_NCA: self.supervised_pipeline.append(NCA())
def get_dist_func( data: Array[np.float64], target: Array[int] ) -> Callable[[Callable[[np.float64, np.float64], np.float64], int, int], np.float64]: """ Get function that returns distances between examples in learned space. Args: data : Array[np.float64] - training data_trans target : Array[int] - target variable values (classes of training examples) Returns: Callable[[Callable[[np.float64, np.float64], np.float64], np.float64, np.float64], np.float64] - function that takes indices of training examples and returns the distance between them in learned metric space using specified metric. """ # Get transformed data. data_trans: Array[np.float64] = NCA().fit_transform( StandardScaler().fit_transform(data), target) # Computing distance: def dist_func_res(metric: Callable[[np.float64, np.float64], np.float64], i1: int, i2: int) -> np.float64: """ distance function that takes metric function and indices of examples in training set and returns distance in learned space using specified distance metric. Args: metric: Callable[[np.flaot64, np.float64], np.float64] - metric to use in learned metric space. i1 : int - index of first training example i2 : int - index of second training example Returns: np.float64 - distance in learned metric space using specified metric between specified training examples. """ # Compute distance in learned metric space using specified metric. return metric(data_trans[i1, :], data_trans[i2, :]) return dist_func_res # Return distance function.
def nca_fit(X_train, Y_train, X_test, Y_test, color_map): nca = NCA(init='pca', max_iter=5000) nca.fit(X_train, Y_train) X_train_transformed = nca.transform(X_train) if (X_train.shape[1] == 2): plt.figure() plt.scatter(X_train_transformed[:, 0], X_train_transformed[:, 1], c=color_map[Y_train], s=2) plt.savefig("after_nca_transform_train.png", dpi=300) X_test_transformed = nca.transform(X_test) if (X_test.shape[1] == 2): plt.figure() plt.scatter(X_test_transformed[:, 0], X_test_transformed[:, 1], c=color_map[Y_test], s=2) plt.savefig("after_nca_transform_test.png", dpi=300) return (X_train_transformed, X_test_transformed)
ids_quadruplets_learners = list( map(lambda x: x.__class__.__name__, [learner for (learner, _) in quadruplets_learners])) pairs_learners = [ (ITML(), build_pairs), (MMC(max_iter=2), build_pairs), # max_iter=2 for faster (SDML(), build_pairs), ] ids_pairs_learners = list( map(lambda x: x.__class__.__name__, [learner for (learner, _) in pairs_learners])) classifiers = [(Covariance(), build_classification), (LFDA(), build_classification), (LMNN(), build_classification), (NCA(), build_classification), (RCA(), build_classification), (ITML_Supervised(max_iter=5), build_classification), (LSML_Supervised(), build_classification), (MMC_Supervised(max_iter=5), build_classification), (RCA_Supervised(num_chunks=10), build_classification), (SDML_Supervised(), build_classification)] ids_classifiers = list( map(lambda x: x.__class__.__name__, [learner for (learner, _) in classifiers])) regressors = [(MLKR(), build_regression)] ids_regressors = list( map(lambda x: x.__class__.__name__, [learner for (learner, _) in regressors])) WeaklySupervisedClasses = (_PairsClassifierMixin, _QuadrupletsClassifierMixin)
def test_nca(self): check_estimator(NCA())
M_array = [16,32,64,128,256] scores_NCA_A = np.zeros([5,3]) row = 0 for M in M_array: U = norm_eigenvec_ld[:, 0:M] W = np.matmul(U.T, A) W_testing = np.matmul(U.T, A_testing) index_training_array = np.array(index_training[0,:]) index_testing_array = np.array(index_testing[0,:]) nca = NCA() nca.fit(W.T, index_training_array) new_indices, rank_1, rank_10 = KNN(W_testing.T, index_testing_array, nca.get_metric()) mAP = mAP_calculation(new_indices, index_testing_array) scores_NCA_A[row,0] = rank_1 scores_NCA_A[row,1] = rank_10 scores_NCA_A[row,2] = mAP row = row+1 # NCA B_testing mean_training_B = B_training.mean(axis=1, keepdims=True) B = B_training - mean_training_B norm_eigenvec_ld_B = PCA_eigenvec(B)
def test_singleton_class(self): X = self.iris_points y = self.iris_labels # one singleton class: test fitting works singleton_class = 1 ind_singleton, = np.where(y == singleton_class) y[ind_singleton] = 2 y[ind_singleton[0]] = singleton_class nca = NCA(max_iter=30) nca.fit(X, y) # One non-singleton class: test fitting works ind_1, = np.where(y == 1) ind_2, = np.where(y == 2) y[ind_1] = 0 y[ind_1[0]] = 1 y[ind_2] = 0 y[ind_2[0]] = 2 nca = NCA(max_iter=30) nca.fit(X, y) # Only singleton classes: test fitting does nothing (the gradient # must be null in this case, so the final matrix must stay like # the initialization) ind_0, = np.where(y == 0) ind_1, = np.where(y == 1) ind_2, = np.where(y == 2) X = X[[ind_0[0], ind_1[0], ind_2[0]]] y = y[[ind_0[0], ind_1[0], ind_2[0]]] A = make_spd_matrix(X.shape[1], X.shape[1]) nca = NCA(init=A, max_iter=30, n_components=X.shape[1]) nca.fit(X, y) assert_array_equal(nca.components_, A)
parser = argparse.ArgumentParser() parser.add_argument('--data', default="/data/hp/dpr-c.embd_cn") parser.add_argument('--seed', type=int, default=0) args = parser.parse_args() data = read_pickle(args.data) dataNew = np.array(np.concatenate((data["docs"], data["queries"]))) dataNew -= dataNew.mean() print("Computing similarities") # similarities = euclidean_distances(dataNew) print("Preparing model") model = NCA( n_components=128, max_iter=10, eps=1e-9, random_state=args.seed, # dissimilarity="precomputed", n_jobs=6) print("Fitting model") # dataNew = model.fit_transform(similarities) dataNew = model.fit_transform(dataNew) dataNew = { "docs": dataNew[:len(data["docs"])].copy(), "queries": dataNew[len(data["docs"]):].copy(), } print(len(dataNew["docs"])) print(len(dataNew["queries"]))
def test_nca(self): n = self.X.shape[0] nca = NCA(max_iter=(100000 // n)) nca.fit(self.X, self.y) L = nca.components_ assert_array_almost_equal(L.T.dot(L), nca.get_mahalanobis_matrix())
quadruplets_learners = [(LSML(), build_quadruplets)] ids_quadruplets_learners = list(map(lambda x: x.__class__.__name__, [learner for (learner, _) in quadruplets_learners])) pairs_learners = [(ITML(), build_pairs), (MMC(max_iter=2), build_pairs), # max_iter=2 for faster (SDML(use_cov=False, balance_param=1e-5), build_pairs)] ids_pairs_learners = list(map(lambda x: x.__class__.__name__, [learner for (learner, _) in pairs_learners])) classifiers = [(Covariance(), build_classification), (LFDA(), build_classification), (LMNN(), build_classification), (NCA(), build_classification), (RCA(), build_classification), (ITML_Supervised(max_iter=5), build_classification), (LSML_Supervised(), build_classification), (MMC_Supervised(max_iter=5), build_classification), (RCA_Supervised(num_chunks=10), build_classification), (SDML_Supervised(use_cov=False, balance_param=1e-5), build_classification)] ids_classifiers = list(map(lambda x: x.__class__.__name__, [learner for (learner, _) in classifiers])) regressors = [(MLKR(), build_regression)] ids_regressors = list(map(lambda x: x.__class__.__name__, [learner for (learner, _) in regressors]))
def fun(M): return NCA._loss_grad_lbfgs(M, X, mask)[0]
majority_roc = None return { 'cv_score': top_score, 'accuracy': accuracy, 'roc': roc, 'majority_accuracy': majority_accuracy, 'majority_roc': majority_roc } for d in range(len(dataset_collection)): print("Metric learning") nca1 = NCA(max_iter=1000, learning_rate=0.01) nca1.fit(x_train, y_train) t_x_train = nca1.transform() nca2 = NCA(max_iter=1000, learning_rate=0.01) nca2.fit(x_test, np.array(y_test)) t_x_test = nca2.transform() dat = [t_x_train, t_x_test] nn_metric = KNeighborsClassifier() nn_metric_params = { "n_neighbors": range(5, max(6, len(data) / 10)), 'leaf_size': range(30, 100) } classifier_stats['nn_metric'] = test_classifier(nn_metric, nn_metric_params, dat)
def test_nca(self): n = self.X.shape[0] nca = NCA(max_iter=(100000//n)) nca.fit(self.X, self.y) L = nca.transformer_ assert_array_almost_equal(L.T.dot(L), nca.get_mahalanobis_matrix())
def grad(M): return NCA._loss_grad_lbfgs(M, X, mask)[1].ravel()
elif Method == 'LFDA': print("Method: LFDA", '\n') lfda = LFDA(k=4, dim=1) x = lfda.fit(FSTrainData, TrainLabels) TFSTestData = x.transform(FSTestData) print('Transformation Done', '\n') elif Method == 'NCA': print("Method: NCA", '\n') #print('Max', TrainData.max(axis=0)) #print('sssssssss', len(TrainData[0])) #print('sssssssss', len(TrainData.max(axis=0))) #print('Min', TrainData.min(axis=0)) nca = NCA(max_iter=500, learning_rate=0.01) # print('ssssssss', TrainData) x = nca.fit(FSTrainData, TrainLabels) TFSTestData = x.transform(FSTestData) print('Transformation Done', '\n') elif Method == 'SDML': print("Method: SDML", '\n') sdml = SDML_Supervised(num_constraints=200) x = sdml.fit(FSTrainData, TrainLabels) TFSTestData = x.transform(FSTestData) print('Transformation Done', '\n') elif Method == 'RCA': print("Method: RCA", '\n')
# http://contrib.scikit-learn.org/metric-learn/generated/metric_learn.NCA.html from metric_learn import NCA from sklearn.datasets import make_classification from sklearn.neighbors import KNeighborsClassifier nca = NCA() X, y = make_classification() nca.fit(X, y) knn = KNeighborsClassifier(metric=nca.get_metric()) knn.fit(X, y) print(knn.predict(X[0:2, :])) print(y[0:2])
if __name__ == '__main__': parser = argparse.ArgumentParser("NCA") parser.add_argument('--data-root', default='./data/raw_split') parser.add_argument('--n-components', type=int, default=2) parser.add_argument('--max-iter', type=int, default=100) args = parser.parse_args() name = f"{args.n_components}_{args.max_iter}" data_save_folder = f"./data/NCA/{name}" makedirs(data_save_folder) X_train, X_test, y_train, y_test = load_split(args) print(X_train.shape) t = time.time() nca = NCA(n_components=args.n_components, max_iter=args.max_iter, verbose=1) nca.fit(X_train, y_train) print(" # NCA fit done.") np.save(osp.join(data_save_folder, "feature_train.npy"), nca.transform(X_train)) np.save(osp.join(data_save_folder, "label_train.npy"), y_train) np.save(osp.join(data_save_folder, "feature_test.npy"), nca.transform(X_test)) np.save(osp.join(data_save_folder, "label_test.npy"), y_test)
def test_nca(self): n = self.X.shape[0] nca = NCA(max_iter=(100000//n), learning_rate=0.01) nca.fit(self.X, self.y) L = nca.transformer() assert_array_almost_equal(L.T.dot(L), nca.metric())
def main(params): initialize_results_dir(params.get('results_dir')) backup_params(params, params.get('results_dir')) print('>>> loading data...') X_train, y_train, X_test, y_test = LoaderFactory().create( name=params.get('dataset'), root=params.get('dataset_dir'), random=True, seed=params.getint('split_seed'))() print('<<< data loaded') print('>>> computing psd matrix...') if params.get('algorithm') == 'identity': psd_matrix = np.identity(X_train.shape[1], dtype=X_train.dtype) elif params.get('algorithm') == 'nca': nca = NCA(init='auto', verbose=True, random_state=params.getint('algorithm_seed')) nca.fit(X_train, y_train) psd_matrix = nca.get_mahalanobis_matrix() elif params.get('algorithm') == 'lmnn': lmnn = LMNN(init='auto', verbose=True, random_state=params.getint('algorithm_seed')) lmnn.fit(X_train, y_train) psd_matrix = lmnn.get_mahalanobis_matrix() elif params.get('algorithm') == 'itml': itml = ITML_Supervised(verbose=True, random_state=params.getint('algorithm_seed')) itml.fit(X_train, y_train) psd_matrix = itml.get_mahalanobis_matrix() elif params.get('algorithm') == 'lfda': lfda = LFDA() lfda.fit(X_train, y_train) psd_matrix = lfda.get_mahalanobis_matrix() elif params.get('algorithm') == 'arml': learner = TripleLearner( optimizer=params.get('optimizer'), optimizer_params={ 'lr': params.getfloat('lr'), 'momentum': params.getfloat('momentum'), 'weight_decay': params.getfloat('weight_decay'), }, criterion=params.get('criterion'), criterion_params={'calibration': params.getfloat('calibration')}, n_epochs=params.getint('n_epochs'), batch_size=params.getint('batch_size'), random_initialization=params.getboolean('random_initialization', fallback=False), update_triple=params.getboolean('update_triple', fallback=False), device=params.get('device'), seed=params.getint('learner_seed')) psd_matrix = learner(X_train, y_train, n_candidate_mins=params.getint('n_candidate_mins', fallback=1)) else: raise Exception('unsupported algorithm') print('<<< psd matrix got') np.savetxt(os.path.join(params.get('results_dir'), 'psd_matrix.txt'), psd_matrix)