def testModel(self): Y = np.random.random((10)) X = np.random.random((10, 100)) kwargs = {} kwargs["Y"] = Y kwargs["X"] = X kwargs["regparam"] = 1 learner = RLS(**kwargs) model = learner.predictor print #print 'Ten data points, single label ' model = mod.LinearPredictor(np.random.random((100))) self.all_pred_cases(model) model = mod.LinearPredictor(np.random.random((100, 2))) self.all_pred_cases(model) #model = mod.LinearPredictor(np.random.random((1, 2))) #self.all_pred_cases(model) kwargs["kernel"] = "GaussianKernel" Y = np.random.random((10)) kwargs["Y"] = Y learner = RLS(**kwargs) model = learner.predictor self.all_pred_cases(model) kwargs["kernel"] = "GaussianKernel" Y = np.random.random((10, 2)) kwargs["Y"] = Y learner = RLS(**kwargs) model = learner.predictor self.all_pred_cases(model)
def out_of_sample_kfold_cv(self, rowfolds, colfolds): """ Computes the out-of-sample cross-validation predictions with given subset of rows and columns. By out-of-sample we denote the setting, where when leaving out an entry (a,b) in Y, we also remove from training set all instances of type (a,x) and (x,b). Returns ------- F : array, shape = [n_samples1*n_samples2] Training set labels. Label for (X1[i], X2[j]) maps to F[i + j*n_samples1] (column order). Notes ----- Computational complexity [TODO] """ rlsparams = {} rlsparams["regparam"] = self.regparam1 rlsparams["Y"] = self.Y rlsparams["bias"] = 0. if self.kernelmode: rlsparams["X"] = np.array(self.K1) rlsparams['kernel'] = 'PrecomputedKernel' else: rlsparams["X"] = np.array(self.X1) ordinary_rls_for_rows = RLS(**rlsparams) allrowhopreds = np.zeros(self.Y.shape) for fold in rowfolds: Pfold = ordinary_rls_for_rows.holdout(fold) if len(fold) == 1: Pfold = Pfold.reshape((Pfold.shape[0], 1)) allrowhopreds[fold] = Pfold rlsparams = {} rlsparams["regparam"] = self.regparam2 rlsparams["Y"] = allrowhopreds.T rlsparams["bias"] = 0. if self.kernelmode: rlsparams["X"] = np.array(self.K2) rlsparams['kernel'] = 'PrecomputedKernel' else: rlsparams["X"] = np.array(self.X2) ordinary_rls_for_columns = RLS(**rlsparams) allcolhopreds = np.zeros(self.Y.shape) for fold in colfolds: Pfold = ordinary_rls_for_columns.holdout(fold) #print(allhopreds.shape, Pfold.shape) if len(fold) == 1: Pfold = Pfold.reshape((1, Pfold.shape[0])) allcolhopreds[:, fold] = Pfold.T return allcolhopreds.ravel(order = 'F')
def train_rls(): X_train, Y_train, X_test, Y_test = load_housing() #select randomly 100 basis vectors indices = range(X_train.shape[0]) indices = random.sample(indices, 100) basis_vectors = X_train[indices] kernel = GaussianKernel(basis_vectors, gamma=0.00003) K_train = kernel.getKM(X_train) K_rr = kernel.getKM(basis_vectors) K_test = kernel.getKM(X_test) learner = RLS(K_train, Y_train, basis_vectors=K_rr, kernel="PrecomputedKernel", regparam=0.0003) #Leave-one-out cross-validation predictions, this is fast due to #computational short-cut P_loo = learner.leave_one_out() #Test set predictions P_test = learner.predict(K_test) print("leave-one-out error %f" % sqerror(Y_train, P_loo)) print("test error %f" % sqerror(Y_test, P_test)) #Sanity check, can we do better than predicting mean of training labels? print("mean predictor %f" % sqerror(Y_test, np.ones(Y_test.shape) * np.mean(Y_train)))
def leave_x1_out(self): """ Computes the leave-row-out cross-validation predictions. Here, all instances related to a single object from domain 1 are left out together at a time. Returns ------- F : array, shape = [n_samples1*n_samples2] Training set labels. Label for (X1[i], X2[j]) maps to F[i + j*n_samples1] (column order). """ YU = self.Y * self.U filteredevals2 = self.evals2 / (self.evals2 + self.regparam2) foo = np.multiply(YU, filteredevals2.T) foo = foo * self.U.T foo = np.array(foo) rlsparams = {} rlsparams["regparam"] = self.regparam1 rlsparams["Y"] = foo rlsparams["bias"] = 0. if self.kernelmode: rlsparams["X"] = np.array(self.K1) rlsparams['kernel'] = 'PrecomputedKernel' else: rlsparams["X"] = np.array(self.X1) ordinary_rls_for_rows = RLS(**rlsparams) lro = ordinary_rls_for_rows.leave_one_out().ravel(order = 'F') return lro
def leave_x2_out(self): """ Computes the leave-column-out cross-validation predictions. Here, all instances related to a single object from domain 2 are left out together at a time. Returns ------- F : array, shape = [n_samples1*n_samples2] Training set labels. Label for (X1[i], X2[j]) maps to F[i + j*n_samples1] (column order). """ VTY = self.V.T * self.Y filteredevals1 = self.evals1 / (self.evals1 + self.regparam1) foo = np.multiply(VTY, filteredevals1) foo = self.V * foo foo = np.array(foo) rlsparams = {} rlsparams["regparam"] = self.regparam2 rlsparams["Y"] = foo.T rlsparams["bias"] = 0. if self.kernelmode: rlsparams["X"] = np.array(self.K2) rlsparams['kernel'] = 'PrecomputedKernel' else: rlsparams["X"] = np.array(self.X2) ordinary_rls_for_columns = RLS(**rlsparams) lco = ordinary_rls_for_columns.leave_one_out().T.ravel(order = 'F') return lco
def train_rls(): X_train, Y_train, X_test, Y_test = load_housing() #select randomly 20 basis vectors indices = range(X_train.shape[0]) indices = random.sample(indices, 20) basis_vectors = X_train[indices] learner = RLS(X_train, Y_train, basis_vectors=basis_vectors, kernel="GaussianKernel", regparam=0.0003, gamma=0.00003) #Test set predictions P_test = learner.predict(X_test) #We can separate the predictor from learner predictor = learner.predictor #And do the same predictions P_test = predictor.predict(X_test) #Let's get the coefficients of the predictor A = predictor.A print("A-coefficients " + str(A)) print("number of coefficients %d" % len(A))
def x2_kfold_cv(self, folds): """ Computes the leave-column-out cross-validation predictions. Here, all instances related to a single object from domain 2 are left out together at a time. Returns ------- F : array, shape = [n_samples1*n_samples2] Training set labels. Label for (X1[i], X2[j]) maps to F[i + j*n_samples1] (column order). """ VTY = self.V.T @ self.Y filteredevals1 = self.evals1 / (self.evals1 + self.regparam1) foo = np.multiply(VTY, filteredevals1) foo = self.V @ foo foo = np.array(foo) rlsparams = {} rlsparams["regparam"] = self.regparam2 rlsparams["Y"] = foo.T rlsparams["bias"] = 0. if self.kernelmode: rlsparams["X"] = np.array(self.K2) rlsparams['kernel'] = 'PrecomputedKernel' else: rlsparams["X"] = np.array(self.X2) ordinary_rls_for_columns = RLS(**rlsparams) allhopreds = np.zeros(foo.shape) for fold in folds: Pfold = ordinary_rls_for_columns.holdout(fold) #print(allhopreds.shape, Pfold.shape) if len(fold) == 1: Pfold = Pfold.reshape((1, Pfold.shape[0])) allhopreds[:, fold] = Pfold.T return allhopreds.ravel(order = 'F')
def x1_kfold_cv(self, folds): """ Computes the leave-row-out cross-validation predictions. Here, all instances related to a single object from domain 1 are left out together at a time. Returns ------- F : array, shape = [n_samples1*n_samples2] Training set labels. Label for (X1[i], X2[j]) maps to F[i + j*n_samples1] (column order). """ YU = self.Y @ self.U filteredevals2 = self.evals2 / (self.evals2 + self.regparam2) foo = np.multiply(YU, filteredevals2.T) foo = foo @ self.U.T foo = np.array(foo) rlsparams = {} rlsparams["regparam"] = self.regparam1 rlsparams["Y"] = foo rlsparams["bias"] = 0. if self.kernelmode: rlsparams["X"] = np.array(self.K1) rlsparams['kernel'] = 'PrecomputedKernel' else: rlsparams["X"] = np.array(self.X1) ordinary_rls_for_rows = RLS(**rlsparams) allhopreds = np.zeros(foo.shape) for fold in folds: Pfold = ordinary_rls_for_rows.holdout(fold) if len(fold) == 1: Pfold = Pfold.reshape((Pfold.shape[0], 1)) allhopreds[fold] = Pfold return allhopreds.ravel(order = 'F')
def test_two_step_rls(self): regparam1 = 0.001 regparam2 = 10 #regparam1 = 1 #regparam2 = 1 K_train1, K_train2, Y_train, K_test1, K_test2, Y_test, X_train1, X_train2, X_test1, X_test2 \ = self.generate_xortask() Y_train = Y_train.ravel(order = 'F') Y_test = Y_test.ravel(order = 'F') train_rows, train_columns = K_train1.shape[0], K_train2.shape[0] #print K_train1.shape, K_train2.shape, K_test1.shape, K_test2.shape, train_rows, train_columns trainlabelcount = train_rows * train_columns #Train linear two-step RLS with data-matrices params = {} params["regparam1"] = regparam1 params["regparam2"] = regparam2 params["X1"] = X_train1 params["X2"] = X_train2 params["Y"] = Y_train linear_two_step_learner = TwoStepRLS(**params) linear_twostepoutofsampleloo = linear_two_step_learner.out_of_sample_loo().reshape((train_rows, train_columns), order = 'F') linear_lro = linear_two_step_learner.leave_x1_out() linear_lco = linear_two_step_learner.leave_x2_out() #Train kernel two-step RLS with pre-computed kernel matrices params = {} params["regparam1"] = regparam1 params["regparam2"] = regparam2 params["K1"] = K_train1 params["K2"] = K_train2 params["Y"] = Y_train kernel_two_step_learner = TwoStepRLS(**params) kernel_twostepoutofsampleloo = kernel_two_step_learner.out_of_sample_loo().reshape((train_rows, train_columns), order = 'F') kernel_lro = kernel_two_step_learner.leave_x1_out() kernel_lco = kernel_two_step_learner.leave_x2_out() tspred = kernel_two_step_learner.predict(K_train1, K_train2) #Train ordinary linear RLS in two steps for a reference params = {} params["regparam"] = regparam2 params["X"] = X_train2 params["Y"] = Y_train.reshape((train_rows, train_columns), order = 'F').T params['bias'] = 0 ordinary_linear_rls_first_step = RLS(**params) firststeploo = ordinary_linear_rls_first_step.leave_one_out().T params = {} params["regparam"] = regparam1 params["X"] = X_train1 params["Y"] = firststeploo.reshape((train_rows, train_columns), order = 'F') params['bias'] = 0 ordinary_linear_rls_second_step = RLS(**params) secondsteploo_linear_rls = ordinary_linear_rls_second_step.leave_one_out() #Train ordinary kernel RLS in two steps for a reference params = {} params["regparam"] = regparam2 params["X"] = K_train2 params['kernel'] = 'PrecomputedKernel' params["Y"] = Y_train.reshape((train_rows, train_columns), order = 'F').T ordinary_kernel_rls_first_step = RLS(**params) firststeploo = ordinary_kernel_rls_first_step.leave_one_out().T params = {} params["regparam"] = regparam1 params["X"] = K_train1 params["kernel"] = "PrecomputedKernel" params["Y"] = firststeploo.reshape((train_rows, train_columns), order = 'F') ordinary_kernel_rls_second_step = RLS(**params) secondsteploo_kernel_rls = ordinary_kernel_rls_second_step.leave_one_out() #Train ordinary kernel RLS in one step with the crazy kernel for a reference params = {} params["regparam"] = 1. crazykernel = la.inv(regparam1 * regparam2 * np.kron(la.inv(K_train2), la.inv(K_train1)) + regparam1 * np.kron(np.eye(K_train2.shape[0]), la.inv(K_train1)) + regparam2 * np.kron(la.inv(K_train2), np.eye(K_train1.shape[0]))) params["X"] = crazykernel params['kernel'] = 'PrecomputedKernel' params["Y"] = Y_train ordinary_one_step_kernel_rls_with_crazy_kernel = RLS(**params) fooloo = ordinary_one_step_kernel_rls_with_crazy_kernel.leave_one_out()[0] allinds = np.arange(trainlabelcount) allinds_fortran_shaped = allinds.reshape((train_rows, train_columns), order = 'F') hoinds = sorted(allinds_fortran_shaped[0].tolist() + allinds_fortran_shaped[1:, 0].tolist()) hocompl = sorted(list(set(allinds)-set(hoinds))) fooholdout = ordinary_one_step_kernel_rls_with_crazy_kernel.holdout(hoinds)[0] params = {} params["regparam"] = 1. params["X"] = crazykernel[np.ix_(hocompl, hocompl)] params['kernel'] = 'PrecomputedKernel' params["Y"] = Y_train[hocompl] ordinary_one_step_kernel_rls_with_crazy_kernel = RLS(**params) barholdout = ordinary_one_step_kernel_rls_with_crazy_kernel.predict(crazykernel[np.ix_([0], hocompl)]) params = {} params["regparam"] = 1. K_train1_cut = K_train1[np.ix_(range(1, K_train1.shape[0]), range(1, K_train1.shape[1]))] K_train2_cut = K_train2[np.ix_(range(1, K_train2.shape[0]), range(1, K_train2.shape[1]))] crazykernel_cut = la.inv(regparam1 * regparam2 * np.kron(la.inv(K_train2_cut), la.inv(K_train1_cut)) + regparam1 * np.kron(np.eye(K_train2_cut.shape[0]), la.inv(K_train1_cut)) + regparam2 * np.kron(la.inv(K_train2_cut), np.eye(K_train1_cut.shape[0]))) params["X"] = crazykernel_cut params['kernel'] = 'PrecomputedKernel' #params["Y"] = Y_train[hocompl] params["Y"] = Y_train.reshape((train_rows, train_columns), order = 'F')[np.ix_(range(1, train_rows), range(1, train_columns))].ravel(order = 'F') ordinary_one_step_kernel_rls_with_crazy_kernel = RLS(**params) bazholdout = ordinary_one_step_kernel_rls_with_crazy_kernel.predict( np.dot(np.dot(np.kron(K_train2[np.ix_([0], range(1, K_train2.shape[1]))], K_train1[np.ix_([0], range(1, K_train1.shape[1]))]), la.inv(np.kron(K_train2_cut, K_train1_cut))), crazykernel_cut)) #print fooholdout, 'fooholdout', barholdout, bazholdout #Train linear two-step RLS without out-of-sample rows or columns for [0,0] params = {} params["regparam1"] = regparam1 params["regparam2"] = regparam2 params["X1"] = X_train1[range(1, X_train1.shape[0])] params["X2"] = X_train2[range(1, X_train2.shape[0])] params["Y"] = Y_train.reshape((train_rows, train_columns), order = 'F')[np.ix_(range(1, train_rows), range(1, train_columns))].ravel(order = 'F') linear_two_step_learner_00 = TwoStepRLS(**params) linear_two_step_testpred_00 = linear_two_step_learner_00.predict(X_train1[0], X_train2[0]) #Train linear two-step RLS without out-of-sample rows or columns for [2,4] params = {} params["regparam1"] = regparam1 params["regparam2"] = regparam2 params["X1"] = X_train1[[0, 1] + range(3, K_train1.shape[0])] params["X2"] = X_train2[[0, 1, 2, 3] + range(5, K_train2.shape[0])] params["Y"] = Y_train.reshape((train_rows, train_columns), order = 'F')[np.ix_([0, 1] + range(3, train_rows), [0, 1, 2, 3] + range(5, train_columns))].ravel(order = 'F') linear_two_step_learner_24 = TwoStepRLS(**params) linear_two_step_testpred_24 = linear_two_step_learner_24.predict(X_train1[2], X_train2[4]) #Train kernel two-step RLS without out-of-sample rows or columns for [0,0] params = {} params["regparam1"] = regparam1 params["regparam2"] = regparam2 params["K1"] = K_train1[np.ix_(range(1, K_train1.shape[0]), range(1, K_train1.shape[1]))] params["K2"] = K_train2[np.ix_(range(1, K_train2.shape[0]), range(1, K_train2.shape[1]))] params["Y"] = Y_train.reshape((train_rows, train_columns), order = 'F')[np.ix_(range(1, train_rows), range(1, train_columns))].ravel(order = 'F') kernel_two_step_learner_00 = TwoStepRLS(**params) kernel_two_step_testpred_00 = kernel_two_step_learner_00.predict(K_train1[range(1, K_train1.shape[0]), 0], K_train2[0, range(1, K_train2.shape[0])]) #Train kernel two-step RLS without out-of-sample rows or columns for [2,4] params = {} params["regparam1"] = regparam1 params["regparam2"] = regparam2 params["K1"] = K_train1[np.ix_([0, 1] + range(3, K_train1.shape[0]), [0, 1] + range(3, K_train1.shape[0]))] params["K2"] = K_train2[np.ix_([0, 1, 2, 3] + range(5, K_train2.shape[0]), [0, 1, 2, 3] + range(5, K_train2.shape[0]))] params["Y"] = Y_train.reshape((train_rows, train_columns), order = 'F')[np.ix_([0, 1] + range(3, train_rows), [0, 1, 2, 3] + range(5, train_columns))].ravel(order = 'F') kernel_two_step_learner_24 = TwoStepRLS(**params) kernel_two_step_testpred_24 = kernel_two_step_learner_24.predict(K_train1[[0, 1] + range(3, K_train1.shape[0]), 2], K_train2[4, [0, 1, 2, 3] + range(5, K_train2.shape[0])]) #Train kernel two-step RLS without out-of-sample row 0 params = {} params["regparam1"] = regparam1 params["regparam2"] = regparam2 params["K1"] = K_train1[np.ix_(range(1, K_train1.shape[0]), range(1, K_train1.shape[1]))] params["K2"] = K_train2 params["Y"] = Y_train.reshape((train_rows, train_columns), order = 'F')[range(1, train_rows)].ravel(order = 'F') kernel_two_step_learner_lro_0 = TwoStepRLS(**params) kernel_two_step_testpred_lro_0 = kernel_two_step_learner_lro_0.predict(K_train1[range(1, K_train1.shape[0]), 0], K_train2) print('') print('Leave-row-out with linear two-step RLS:') print(linear_lro.reshape((train_rows, train_columns), order = 'F')[0]) print('Leave-row-out with kernel two-step RLS:') print(kernel_lro.reshape((train_rows, train_columns), order = 'F')[0]) print('Two-step RLS trained without the held-out row predictions for the row:') print(kernel_two_step_testpred_lro_0) np.testing.assert_almost_equal(linear_lro.reshape((train_rows, train_columns), order = 'F')[0], kernel_two_step_testpred_lro_0) np.testing.assert_almost_equal(kernel_lro.reshape((train_rows, train_columns), order = 'F')[0], kernel_two_step_testpred_lro_0) #Train kernel two-step RLS without out-of-sample column 0 params = {} params["regparam1"] = regparam1 params["regparam2"] = regparam2 params["K1"] = K_train1 params["K2"] = K_train2[np.ix_(range(1, K_train2.shape[0]), range(1, K_train2.shape[1]))] params["Y"] = Y_train.reshape((train_rows, train_columns), order = 'F')[:, range(1, train_columns)].ravel(order = 'F') kernel_two_step_learner_lco_0 = TwoStepRLS(**params) kernel_two_step_testpred_lco_0 = kernel_two_step_learner_lco_0.predict(K_train1, K_train2[range(1, K_train2.shape[0]), 0]) print('') print('Leave-column-out with linear two-step RLS:') print(linear_lco[range(train_rows)]) print('Leave-column-out with kernel two-step RLS:') print(kernel_lco[range(train_rows)]) print('Two-step RLS trained without the held-out column predictions for the column:') print(kernel_two_step_testpred_lco_0) np.testing.assert_almost_equal(linear_lco[range(train_rows)], kernel_two_step_testpred_lco_0) np.testing.assert_almost_equal(kernel_lco[range(train_rows)], kernel_two_step_testpred_lco_0) print('') print('Out-of-sample LOO: Stacked ordinary linear RLS LOO, Stacked ordinary kernel RLS LOO, linear two-step RLS OOSLOO, kernel two-step RLS OOSLOO, linear two-step RLS OOS-pred, kernel two-step RLS OOS-pred') print('[0, 0]: ' + str(secondsteploo_linear_rls[0, 0]) + ' ' + str(secondsteploo_kernel_rls[0, 0]) + ' ' + str(linear_two_step_testpred_00) + ' ' + str(kernel_two_step_testpred_00) + ' ' + str(linear_twostepoutofsampleloo[0, 0]) + ' ' + str(kernel_twostepoutofsampleloo[0, 0])) print('[2, 4]: ' + str(secondsteploo_linear_rls[2, 4]) + ' ' + str(secondsteploo_kernel_rls[2, 4]) + ' ' + str(linear_two_step_testpred_24) + ' ' + str(kernel_two_step_testpred_24) + ' ' + str(linear_twostepoutofsampleloo[2, 4]) + ' ' + str(kernel_twostepoutofsampleloo[2, 4])) np.testing.assert_almost_equal(secondsteploo_linear_rls, secondsteploo_kernel_rls) np.testing.assert_almost_equal(secondsteploo_linear_rls, linear_twostepoutofsampleloo) np.testing.assert_almost_equal(secondsteploo_linear_rls, kernel_twostepoutofsampleloo) np.testing.assert_almost_equal(secondsteploo_linear_rls[0, 0], linear_two_step_testpred_00) np.testing.assert_almost_equal(secondsteploo_linear_rls[0, 0], kernel_two_step_testpred_00) #Train kernel two-step RLS with pre-computed kernel matrices and with output at position [2, 4] changed Y_24 = Y_train.copy() Y_24 = Y_24.reshape((train_rows, train_columns), order = 'F') Y_24[2, 4] = 55. Y_24 = Y_24.ravel(order = 'F') params = {} params["regparam1"] = regparam1 params["regparam2"] = regparam2 params["K1"] = K_train1 params["K2"] = K_train2 params["Y"] = Y_24 kernel_two_step_learner_Y_24 = TwoStepRLS(**params) kernel_two_step_testpred_Y_24 = kernel_two_step_learner_Y_24.predict(K_test1, K_test2) kernel_two_step_learner_inSampleLOO_24a = kernel_two_step_learner.in_sample_loo().reshape((train_rows, train_columns), order = 'F')[2, 4] kernel_two_step_learner_inSampleLOO_24b = kernel_two_step_learner_Y_24.in_sample_loo().reshape((train_rows, train_columns), order = 'F')[2, 4] print('') print('In-sample LOO: Kernel two-step RLS ISLOO with original outputs, Kernel two-step RLS ISLOO with modified output at [2, 4]') print('[2, 4] ' + str(kernel_two_step_learner_inSampleLOO_24a) + ' ' + str(kernel_two_step_learner_inSampleLOO_24b)) np.testing.assert_almost_equal(kernel_two_step_learner_inSampleLOO_24a, kernel_two_step_learner_inSampleLOO_24b) #Train kernel two-step RLS with pre-computed kernel matrices and with output at position [1, 1] changed Y_00 = Y_train.copy() Y_00 = Y_00.reshape((train_rows, train_columns), order = 'F') Y_00[0, 0] = 55. Y_00 = Y_00.ravel(order = 'F') params = {} params["regparam1"] = regparam1 params["regparam2"] = regparam2 params["K1"] = K_train1 params["K2"] = K_train2 params["Y"] = Y_00 kernel_two_step_learner_Y_00 = TwoStepRLS(**params) kernel_two_step_testpred_Y_00 = kernel_two_step_learner_Y_00.predict(K_test1, K_test2) kernel_two_step_learner_inSampleLOO_00a = kernel_two_step_learner.in_sample_loo()[0] kernel_two_step_learner_inSampleLOO_00b = kernel_two_step_learner_Y_00.in_sample_loo()[0] print('') print('In-sample LOO: Kernel two-step RLS ISLOO with original outputs, Kernel two-step RLS ISLOO with modified output at [0, 0]') print('[0, 0] ' + str(kernel_two_step_learner_inSampleLOO_00a) + ' ' + str(kernel_two_step_learner_inSampleLOO_00b)) np.testing.assert_almost_equal(kernel_two_step_learner_inSampleLOO_00a, kernel_two_step_learner_inSampleLOO_00b) #Create symmetric data K_train1, K_train2, Y_train, K_test1, K_test2, Y_test, X_train1, X_train2, X_test1, X_test2 \ = self.generate_xortask( trainpos1 = 6, trainneg1 = 7, trainpos2 = 6, trainneg2 = 7, testpos1 = 26, testneg1 = 27, testpos2 = 25, testneg2 = 25 ) K_train1 = K_train2 K_test1 = K_test2 Y_train = 0.5 * (Y_train + Y_train.T) Y_train = Y_train.ravel(order = 'F') Y_test = Y_test.ravel(order = 'F') train_rows, train_columns = K_train1.shape[0], K_train2.shape[0] test_rows, test_columns = K_test1.shape[0], K_test2.shape[0] trainlabelcount = train_rows * train_columns #Train symmetric kernel two-step RLS with pre-computed kernel matrices params = {} params["regparam1"] = regparam2 params["regparam2"] = regparam2 params["K1"] = K_train1 params["K2"] = K_train2 params["Y"] = Y_train kernel_two_step_learner = TwoStepRLS(**params) kernel_two_step_testpred = kernel_two_step_learner.predict(K_test1, K_test2).reshape((test_rows, test_columns), order = 'F') #Train two-step RLS without out-of-sample rows or columns rowind, colind = 2, 4 trainrowinds = range(K_train1.shape[0]) trainrowinds.remove(rowind) trainrowinds.remove(colind) traincolinds = range(K_train2.shape[0]) traincolinds.remove(rowind) traincolinds.remove(colind) params = {} params["regparam1"] = regparam2 params["regparam2"] = regparam2 params["K1"] = K_train1[np.ix_(trainrowinds, trainrowinds)] params["K2"] = K_train2[np.ix_(traincolinds, traincolinds)] params["Y"] = Y_train.reshape((train_rows, train_columns), order = 'F')[np.ix_(trainrowinds, traincolinds)].ravel(order = 'F') kernel_kron_learner = TwoStepRLS(**params) kernel_kron_testpred = kernel_kron_learner.predict(K_train1[np.ix_([rowind], trainrowinds)], K_train2[np.ix_([colind], traincolinds)]).reshape((1, 1), order = 'F') fcsho = kernel_two_step_learner.out_of_sample_loo_symmetric().reshape((train_rows, train_columns), order = 'F') print('') print('Symmetric double out-of-sample LOO: Test prediction, LOO') print('[2, 4]: ' + str(kernel_kron_testpred[0, 0]) + ' ' + str(fcsho[2, 4])) np.testing.assert_almost_equal(kernel_kron_testpred[0, 0], fcsho[2, 4])
def test_cg_kron_rls(self): regparam = 0.0001 K_train1, K_train2, Y_train, K_test1, K_test2, Y_test, X_train1, X_train2, X_test1, X_test2 = self.generate_xortask( ) #K_train1, K_train2, Y_train, K_test1, K_test2, Y_test, X_train1, X_train2, X_test1, X_test2 = self.generate_xortask(trainpos1 = 1, trainneg1 = 1, trainpos2 = 1, trainneg2 = 1, testpos1 = 1, testneg1 = 1, testpos2 = 1, testneg2 = 1) Y_train = Y_train.ravel(order='F') Y_test = Y_test.ravel(order='F') train_rows, train_columns = K_train1.shape[0], K_train2.shape[0] test_rows, test_columns = K_test1.shape[0], K_test2.shape[0] rowstimescols = train_rows * train_columns allindices = np.arange(rowstimescols) all_label_row_inds, all_label_col_inds = np.unravel_index( allindices, (train_rows, train_columns), order='F') #incinds = np.random.permutation(allindices) #incinds = np.random.choice(allindices, 50, replace = False) incinds = np.random.choice(allindices, 40, replace=False) label_row_inds, label_col_inds = all_label_row_inds[ incinds], all_label_col_inds[incinds] Y_train_known_outputs = Y_train.reshape(rowstimescols, order='F')[incinds] alltestindices = np.arange(test_rows * test_columns) all_test_label_row_inds, all_test_label_col_inds = np.unravel_index( alltestindices, (test_rows, test_columns), order='F') #Train an ordinary RLS regressor for reference params = {} params["X"] = np.kron(K_train2, K_train1)[np.ix_(incinds, incinds)] params["kernel"] = "PrecomputedKernel" params["Y"] = Y_train_known_outputs params["regparam"] = regparam ordrls_learner = RLS(**params) ordrls_model = ordrls_learner.predictor K_Kron_test = np.kron(K_test2, K_test1)[:, incinds] ordrls_testpred = ordrls_model.predict(K_Kron_test) ordrls_testpred = ordrls_testpred.reshape((test_rows, test_columns), order='F') #Train linear Kronecker RLS class TestCallback(): def __init__(self): self.round = 0 def callback(self, learner): self.round = self.round + 1 tp = LinearPairwisePredictor(learner.W).predict( X_test1, X_test2) print( str(self.round) + ' ' + str(np.mean(np.abs(tp - ordrls_testpred.ravel(order='F'))))) def finished(self, learner): print('finished') params = {} params["regparam"] = regparam params["X1"] = X_train1 params["X2"] = X_train2 params["Y"] = Y_train_known_outputs params["label_row_inds"] = label_row_inds params["label_col_inds"] = label_col_inds tcb = TestCallback() params['callback'] = tcb linear_kron_learner = CGKronRLS(**params) linear_kron_testpred = linear_kron_learner.predict( X_test1, X_test2).reshape((test_rows, test_columns), order='F') linear_kron_testpred_alt = linear_kron_learner.predict( X_test1, X_test2, [0, 0, 1], [0, 1, 0]) #Train kernel Kronecker RLS params = {} params["regparam"] = regparam params["K1"] = K_train1 params["K2"] = K_train2 params["Y"] = Y_train_known_outputs params["label_row_inds"] = label_row_inds params["label_col_inds"] = label_col_inds class KernelCallback(): def __init__(self): self.round = 0 def callback(self, learner): self.round = self.round + 1 tp = KernelPairwisePredictor(learner.A, learner.input1_inds, learner.input2_inds).predict( K_test1, K_test2) print( str(self.round) + ' ' + str(np.mean(np.abs(tp - ordrls_testpred.ravel(order='F'))))) def finished(self, learner): print('finished') tcb = KernelCallback() params['callback'] = tcb kernel_kron_learner = CGKronRLS(**params) kernel_kron_testpred = kernel_kron_learner.predict( K_test1, K_test2).reshape((test_rows, test_columns), order='F') kernel_kron_testpred_alt = kernel_kron_learner.predict( K_test1, K_test2, [0, 0, 1], [0, 1, 0]) print('Predictions: Linear CgKronRLS, Kernel CgKronRLS, ordinary RLS') print('[0, 0]: ' + str(linear_kron_testpred[0, 0]) + ' ' + str(kernel_kron_testpred[0, 0]) + ' ' + str(ordrls_testpred[0, 0]) ) #, linear_kron_testpred_alt[0], kernel_kron_testpred_alt[0] print('[0, 1]: ' + str(linear_kron_testpred[0, 1]) + ' ' + str(kernel_kron_testpred[0, 1]) + ' ' + str(ordrls_testpred[0, 1]) ) #, linear_kron_testpred_alt[1], kernel_kron_testpred_alt[1] print('[1, 0]: ' + str(linear_kron_testpred[1, 0]) + ' ' + str(kernel_kron_testpred[1, 0]) + ' ' + str(ordrls_testpred[1, 0]) ) #, linear_kron_testpred_alt[2], kernel_kron_testpred_alt[2] print( 'Meanabsdiff: linear KronRLS - ordinary RLS, kernel KronRLS - ordinary RLS' ) print( str(np.mean(np.abs(linear_kron_testpred - ordrls_testpred))) + ' ' + str(np.mean(np.abs(kernel_kron_testpred - ordrls_testpred)))) np.testing.assert_almost_equal(linear_kron_testpred, ordrls_testpred, decimal=5) np.testing.assert_almost_equal(kernel_kron_testpred, ordrls_testpred, decimal=4) #Train multiple kernel Kronecker RLS params = {} params["regparam"] = regparam params["K1"] = [K_train1, K_train1] params["K2"] = [K_train2, K_train2] params["weights"] = [1. / 3, 2. / 3] params["Y"] = Y_train_known_outputs params["label_row_inds"] = [label_row_inds, label_row_inds] params["label_col_inds"] = [label_col_inds, label_col_inds] class KernelCallback(): def __init__(self): self.round = 0 def callback(self, learner): self.round = self.round + 1 tp = KernelPairwisePredictor( learner.A, learner.input1_inds, learner.input2_inds, params["weights"]).predict([K_test1, K_test1], [K_test2, K_test2]) print( str(self.round) + ' ' + str(np.mean(np.abs(tp - ordrls_testpred.ravel(order='F'))))) def finished(self, learner): print('finished') tcb = KernelCallback() params['callback'] = tcb mkl_kernel_kron_learner = CGKronRLS(**params) mkl_kernel_kron_testpred = mkl_kernel_kron_learner.predict( [K_test1, K_test1], [K_test2, K_test2]).reshape( (test_rows, test_columns), order='F') #kernel_kron_testpred_alt = kernel_kron_learner.predict(K_test1, K_test2, [0, 0, 1], [0, 1, 0]) print( 'Predictions: Linear CgKronRLS, MKL Kernel CgKronRLS, ordinary RLS' ) print('[0, 0]: ' + str(linear_kron_testpred[0, 0]) + ' ' + str(mkl_kernel_kron_testpred[0, 0]) + ' ' + str(ordrls_testpred[0, 0]) ) #, linear_kron_testpred_alt[0], kernel_kron_testpred_alt[0] print('[0, 1]: ' + str(linear_kron_testpred[0, 1]) + ' ' + str(mkl_kernel_kron_testpred[0, 1]) + ' ' + str(ordrls_testpred[0, 1]) ) #, linear_kron_testpred_alt[1], kernel_kron_testpred_alt[1] print('[1, 0]: ' + str(linear_kron_testpred[1, 0]) + ' ' + str(mkl_kernel_kron_testpred[1, 0]) + ' ' + str(ordrls_testpred[1, 0]) ) #, linear_kron_testpred_alt[2], kernel_kron_testpred_alt[2] print('Meanabsdiff: MKL kernel KronRLS - ordinary RLS') print(str(np.mean(np.abs(mkl_kernel_kron_testpred - ordrls_testpred)))) np.testing.assert_almost_equal(mkl_kernel_kron_testpred, ordrls_testpred, decimal=4) #Train polynomial kernel Kronecker RLS params = {} params["regparam"] = regparam params["K1"] = [K_train1, K_train1, K_train2] params["K2"] = [K_train1, K_train2, K_train2] params["weights"] = [1., 2., 1.] params["Y"] = Y_train_known_outputs params["label_row_inds"] = [ label_row_inds, label_row_inds, label_col_inds ] params["label_col_inds"] = [ label_row_inds, label_col_inds, label_col_inds ] class KernelCallback(): def __init__(self): self.round = 0 def callback(self, learner): self.round = self.round + 1 #tp = KernelPairwisePredictor(learner.A, learner.input1_inds, learner.input2_inds, params["weights"]).predict([K_test1, K_test1], [K_test2, K_test2]) #print(str(self.round) + ' ' + str(np.mean(np.abs(tp - ordrls_testpred.ravel(order = 'F'))))) def finished(self, learner): print('finished') tcb = KernelCallback() params['callback'] = tcb poly_kernel_kron_learner = CGKronRLS(**params) poly_kernel_kron_testpred = poly_kernel_kron_learner.predict( [K_test1, K_test1, K_test2], [K_test1, K_test2, K_test2], [ all_test_label_row_inds, all_test_label_row_inds, all_test_label_col_inds ], [ all_test_label_row_inds, all_test_label_col_inds, all_test_label_col_inds ]) #print(poly_kernel_kron_testpred, 'Polynomial kernel via CGKronRLS') #Train an ordinary RLS regressor with polynomial kernel for reference params = {} params["X"] = np.hstack([ np.kron(np.ones((X_train2.shape[0], 1)), X_train1), np.kron(X_train2, np.ones((X_train1.shape[0], 1))) ])[incinds] #params["X"] = np.hstack([np.kron(X_train1, np.ones((X_train2.shape[0], 1))), np.kron(np.ones((X_train1.shape[0], 1)), X_train2)])[incinds] params["kernel"] = "PolynomialKernel" params["Y"] = Y_train_known_outputs params["regparam"] = regparam ordrls_poly_kernel_learner = RLS(**params) X_dir_test = np.hstack([ np.kron(np.ones((X_test2.shape[0], 1)), X_test1), np.kron(X_test2, np.ones((X_test1.shape[0], 1))) ]) #X_dir_test = np.hstack([np.kron(X_test1, np.ones((X_test2.shape[0], 1))), np.kron(np.ones((X_test1.shape[0], 1)), X_test2)]) ordrls_poly_kernel_testpred = ordrls_poly_kernel_learner.predict( X_dir_test) #print(ordrls_poly_kernel_testpred, 'Ord. poly RLS') print( 'Meanabsdiff: Polynomial kernel KronRLS - Ordinary polynomial kernel RLS' ) print( str( np.mean( np.abs(poly_kernel_kron_testpred - ordrls_poly_kernel_testpred)))) '''
import numpy as np from rlscore.learner.rls import RLS from rlscore.utilities.reader import read_sparse from rlscore.measure import auc train_labels = np.loadtxt("./legacy_tests/data/class_train.labels") test_labels = np.loadtxt("./legacy_tests/data/class_test.labels") train_features = read_sparse("./legacy_tests/data/class_train.features") test_features = read_sparse("./legacy_tests/data/class_test.features") kwargs = {} kwargs["Y"] = train_labels kwargs["X"] = train_features kwargs["regparam"] = 1 learner = RLS(**kwargs) P = learner.predict(test_features) test_perf = auc(test_labels, P) print("test set performance: %f" %test_perf)
def test_kron_rls(self): regparam = 0.001 K_train1, K_train2, Y_train, K_test1, K_test2, Y_test, X_train1, X_train2, X_test1, X_test2 = self.generate_xortask() Y_train = Y_train.ravel(order = 'F') Y_test = Y_test.ravel(order = 'F') train_rows, train_columns = K_train1.shape[0], K_train2.shape[0] test_rows, test_columns = K_test1.shape[0], K_test2.shape[0] trainlabelcount = train_rows * train_columns #Train linear Kronecker RLS with data-matrices params = {} params["regparam"] = regparam params["X1"] = X_train1 params["X2"] = X_train2 params["Y"] = Y_train linear_kron_learner = KronRLS(**params) linear_kron_testpred = linear_kron_learner.predict(X_test1, X_test2).reshape((test_rows, test_columns), order = 'F') #Train kernel Kronecker RLS with pre-computed kernel matrices params = {} params["regparam"] = regparam params["K1"] = K_train1 params["K2"] = K_train2 params["Y"] = Y_train kernel_kron_learner = KronRLS(**params) kernel_kron_testpred = kernel_kron_learner.predict(K_test1, K_test2).reshape((test_rows, test_columns), order = 'F') #Train an ordinary RLS regressor for reference K_Kron_train_x = np.kron(K_train2, K_train1) params = {} params["X"] = K_Kron_train_x params["kernel"] = "PrecomputedKernel" params["Y"] = Y_train.reshape(trainlabelcount, 1, order = 'F') ordrls_learner = RLS(**params) ordrls_learner.solve(regparam) K_Kron_test_x = np.kron(K_test2, K_test1) ordrls_testpred = ordrls_learner.predict(K_Kron_test_x) ordrls_testpred = ordrls_testpred.reshape((test_rows, test_columns), order = 'F') print('') print('Prediction: linear KronRLS, kernel KronRLS, ordinary RLS') print('[0, 0] ' + str(linear_kron_testpred[0, 0]) + ' ' + str(kernel_kron_testpred[0, 0]) + ' ' + str(ordrls_testpred[0, 0])) print('[0, 1] ' + str(linear_kron_testpred[0, 1]) + ' ' + str(kernel_kron_testpred[0, 1]) + ' ' + str(ordrls_testpred[0, 1])) print('[1, 0] ' + str(linear_kron_testpred[1, 0]) + ' ' + str(kernel_kron_testpred[1, 0]) + ' ' + str(ordrls_testpred[1, 0])) print('Meanabsdiff: linear KronRLS - ordinary RLS, kernel KronRLS - ordinary RLS') print(str(np.mean(np.abs(linear_kron_testpred - ordrls_testpred))) + ' ' + str(np.mean(np.abs(kernel_kron_testpred - ordrls_testpred)))) np.testing.assert_almost_equal(linear_kron_testpred, ordrls_testpred) np.testing.assert_almost_equal(kernel_kron_testpred, ordrls_testpred) print('') ordrls_loopred = ordrls_learner.leave_one_out().reshape((train_rows, train_columns), order = 'F') linear_kron_loopred = linear_kron_learner.in_sample_loo().reshape((train_rows, train_columns), order = 'F') kernel_kron_loopred = kernel_kron_learner.in_sample_loo().reshape((train_rows, train_columns), order = 'F') print('In-sample LOO: linear KronRLS, kernel KronRLS, ordinary RLS') print('[0, 0] ' + str(linear_kron_loopred[0, 0]) + ' ' + str(kernel_kron_loopred[0, 0]) + ' ' + str(ordrls_loopred[0, 0])) print('[0, 1] ' + str(linear_kron_loopred[0, 1]) + ' ' + str(kernel_kron_loopred[0, 1]) + ' ' + str(ordrls_loopred[0, 1])) print('[1, 0] ' + str(linear_kron_loopred[1, 0]) + ' ' + str(kernel_kron_loopred[1, 0]) + ' ' + str(ordrls_loopred[1, 0])) print('Meanabsdiff: linear KronRLS - ordinary RLS, kernel KronRLS - ordinary RLS') print(str(np.mean(np.abs(linear_kron_loopred - ordrls_loopred))) + ' ' + str(np.mean(np.abs(kernel_kron_loopred - ordrls_loopred)))) np.testing.assert_almost_equal(linear_kron_loopred, ordrls_loopred) np.testing.assert_almost_equal(kernel_kron_loopred, ordrls_loopred)
def test_cg_kron_rls(self): regparam = 0.0001 K_train1, K_train2, Y_train, K_test1, K_test2, Y_test, X_train1, X_train2, X_test1, X_test2 = self.generate_xortask( ) Y_train = Y_train.ravel(order='F') Y_test = Y_test.ravel(order='F') train_rows, train_columns = K_train1.shape[0], K_train2.shape[0] test_rows, test_columns = K_test1.shape[0], K_test2.shape[0] rowstimescols = train_rows * train_columns allindices = np.arange(rowstimescols) all_label_row_inds, all_label_col_inds = np.unravel_index( allindices, (train_rows, train_columns), order='F') incinds = pyrandom.sample(allindices, 50) label_row_inds, label_col_inds = all_label_row_inds[ incinds], all_label_col_inds[incinds] Y_train_known_outputs = Y_train.reshape(rowstimescols, order='F')[incinds] #Train an ordinary RLS regressor for reference params = {} params["X"] = np.kron(K_train2, K_train1)[np.ix_(incinds, incinds)] params["kernel"] = "PrecomputedKernel" params["Y"] = Y_train_known_outputs params["regparam"] = regparam ordrls_learner = RLS(**params) ordrls_model = ordrls_learner.predictor K_Kron_test = np.kron(K_test2, K_test1)[:, incinds] ordrls_testpred = ordrls_model.predict(K_Kron_test) ordrls_testpred = ordrls_testpred.reshape((test_rows, test_columns), order='F') #Train linear Kronecker RLS class TestCallback(): def __init__(self): self.round = 0 def callback(self, learner): self.round = self.round + 1 tp = LinearPairwisePredictor(learner.W).predict( X_test1, X_test2) print( str(self.round) + ' ' + str(np.mean(np.abs(tp - ordrls_testpred.ravel(order='F'))))) def finished(self, learner): print('finished') params = {} params["regparam"] = regparam params["X1"] = X_train1 params["X2"] = X_train2 params["Y"] = Y_train_known_outputs params["label_row_inds"] = label_row_inds params["label_col_inds"] = label_col_inds tcb = TestCallback() params['callback'] = tcb linear_kron_learner = CGKronRLS(**params) linear_kron_testpred = linear_kron_learner.predict( X_test1, X_test2).reshape((test_rows, test_columns), order='F') linear_kron_testpred_alt = linear_kron_learner.predict( X_test1, X_test2, [0, 0, 1], [0, 1, 0]) #Train kernel Kronecker RLS params = {} params["regparam"] = regparam params["K1"] = K_train1 params["K2"] = K_train2 params["Y"] = Y_train_known_outputs params["label_row_inds"] = label_row_inds params["label_col_inds"] = label_col_inds class KernelCallback(): def __init__(self): self.round = 0 def callback(self, learner): self.round = self.round + 1 tp = KernelPairwisePredictor(learner.A, learner.input1_inds, learner.input2_inds).predict( K_test1, K_test2) print( str(self.round) + ' ' + str(np.mean(np.abs(tp - ordrls_testpred.ravel(order='F'))))) def finished(self, learner): print('finished') tcb = KernelCallback() params['callback'] = tcb kernel_kron_learner = CGKronRLS(**params) kernel_kron_testpred = kernel_kron_learner.predict( K_test1, K_test2).reshape((test_rows, test_columns), order='F') kernel_kron_testpred_alt = kernel_kron_learner.predict( K_test1, K_test2, [0, 0, 1], [0, 1, 0]) print('Predictions: Linear CgKronRLS, Kernel CgKronRLS, ordinary RLS') print('[0, 0]: ' + str(linear_kron_testpred[0, 0]) + ' ' + str(kernel_kron_testpred[0, 0]) + ' ' + str(ordrls_testpred[0, 0]) ) #, linear_kron_testpred_alt[0], kernel_kron_testpred_alt[0] print('[0, 1]: ' + str(linear_kron_testpred[0, 1]) + ' ' + str(kernel_kron_testpred[0, 1]) + ' ' + str(ordrls_testpred[0, 1]) ) #, linear_kron_testpred_alt[1], kernel_kron_testpred_alt[1] print('[1, 0]: ' + str(linear_kron_testpred[1, 0]) + ' ' + str(kernel_kron_testpred[1, 0]) + ' ' + str(ordrls_testpred[1, 0]) ) #, linear_kron_testpred_alt[2], kernel_kron_testpred_alt[2] print( 'Meanabsdiff: linear KronRLS - ordinary RLS, kernel KronRLS - ordinary RLS' ) print( str(np.mean(np.abs(linear_kron_testpred - ordrls_testpred))) + ' ' + str(np.mean(np.abs(kernel_kron_testpred - ordrls_testpred)))) np.testing.assert_almost_equal(linear_kron_testpred, ordrls_testpred, decimal=5) np.testing.assert_almost_equal(kernel_kron_testpred, ordrls_testpred, decimal=5)