def test_leave_pair_out(self): #compares holdout and leave-pair-out start = [0, 2, 3, 5] end = [1, 3, 6, 8] for X in [self.Xtrain1, self.Xtrain2]: for Y in [self.Ytrain1, self.Ytrain2]: #LPO with linear kernel rls1 = RLS(X, Y, regparam=7.0, bias=3.0) lpo_start, lpo_end = rls1.leave_pair_out(start, end) ho_start, ho_end = [], [] for i in range(len(start)): P = rls1.holdout([start[i], end[i]]) ho_start.append(P[0]) ho_end.append(P[1]) ho_start = np.array(ho_start) ho_end = np.array(ho_end) assert_allclose(ho_start, lpo_start) assert_allclose(ho_end, lpo_end) #LPO Gaussian kernel rls1 = RLS(X, Y, regparam=11.0, kenerl="PolynomialKernel", coef0=1, degree=3) lpo_start, lpo_end = rls1.leave_pair_out(start, end) ho_start, ho_end = [], [] for i in range(len(start)): P = rls1.holdout([start[i], end[i]]) ho_start.append(P[0]) ho_end.append(P[1]) ho_start = np.array(ho_start) ho_end = np.array(ho_end) assert_allclose(ho_start, lpo_start) assert_allclose(ho_end, lpo_end)
def test_leave_pair_out(self): #compares holdout and leave-pair-out start = [0, 2, 3, 5] end = [1, 3, 6, 8] for X in [self.Xtrain1, self.Xtrain2]: for Y in [self.Ytrain1, self.Ytrain2]: #LPO with linear kernel rls1 = RLS(X, Y, regparam = 7.0, bias=3.0) lpo_start, lpo_end = rls1.leave_pair_out(start, end) ho_start, ho_end = [], [] for i in range(len(start)): P = rls1.holdout([start[i], end[i]]) ho_start.append(P[0]) ho_end.append(P[1]) ho_start = np.array(ho_start) ho_end = np.array(ho_end) assert_allclose(ho_start, lpo_start) assert_allclose(ho_end, lpo_end) #LPO Gaussian kernel rls1 = RLS(X, Y, regparam = 11.0, kenerl="PolynomialKernel", coef0=1, degree=3) lpo_start, lpo_end = rls1.leave_pair_out(start, end) ho_start, ho_end = [], [] for i in range(len(start)): P = rls1.holdout([start[i], end[i]]) ho_start.append(P[0]) ho_end.append(P[1]) ho_start = np.array(ho_start) ho_end = np.array(ho_end) assert_allclose(ho_start, lpo_start) assert_allclose(ho_end, lpo_end)
def lposcv_rls(coordinates, Xdata, Ydata, dzradii, regparam, visualization): print("Starting lposcv-rls analysis...") # Calculate sorted pairwise distance matrix and indexes performanceTable = np.zeros([len(dzradii), 2]) data_distances, data_distance_indexes = distanceMatrix(coordinates) negcount = len(np.where(Ydata == 0)[0]) folds = lpo_folds(Ydata, negcount) for rind, dzradius in enumerate(dzradii): print("Analysis on going, dead zone radius: " + str(dzradius) + "m / " + str(dzradii[len(dzradii) - 1]) + "m") # Calculate dead zone folds dz_folds = dzfolds(dzradius, folds, data_distances, data_distance_indexes) learner = RLS(Xdata, Ydata, regparam=regparam) perfs = [] for dz_fold in dz_folds: preds = learner.holdout(dz_fold) if preds[0] > preds[1]: perfs.append(1.) elif preds[0] == preds[1]: perfs.append(0.5) else: perfs.append(0.) if visualization: # Check for visualization testcoords = coordinates[dz_fold[0:2], :] dzcoords = coordinates[dz_fold, :] visualize_lposcv(coordinates, testcoords, dzcoords, dzradius) perf = np.mean(perfs) performanceTable[rind, 0] = dzradius performanceTable[rind, 1] = perf plotRes_lposcv(performanceTable, rind, len(folds), 'rls') print("Analysis done.") return performanceTable
def skcv_rls(coordinates, Xdata, Ydata, number_of_folds, dzradii, regparam, visualization): print("Starting skcv-rls analysis...") # Calculate sorted pairwise distance matrix and indexes performanceTable = np.zeros([len(dzradii),2]) data_distances, data_distance_indexes = distanceMatrix(coordinates) folds = random_folds(len(Ydata), number_of_folds) for rind, dzradius in enumerate(dzradii): print("Analysis ongoing, dead zone radius: " + str(dzradius) + "m / " + str(dzradii[len(dzradii)-1]) + "m") # Calculate dead zone folds dz_folds = dzfolds(dzradius, folds, data_distances, data_distance_indexes) learner = RLS(Xdata, Ydata, regparam=regparam) P = np.zeros(Ydata.shape) for fold_id, dz_fold in enumerate(dz_folds): preds = learner.holdout(dz_fold) if preds.ndim == 0: P[folds[fold_id]] = preds else: P[folds[fold_id]] = preds[0:len(folds[fold_id])] if visualization: # Check for visualization testcoords = coordinates[folds[fold_id],:] dzcoords = coordinates[dz_fold, :] visualize_skcv(coordinates, testcoords, dzcoords, dzradius) perf = cindex(Ydata, P) performanceTable[rind,0] = dzradius performanceTable[rind, 1] = perf plotRes_skcv(performanceTable, rind, number_of_folds, "rls") print("Analysis done.") return performanceTable
def test_holdout(self): for X in [self.Xtrain1, self.Xtrain2]: for Y in [self.Ytrain1, self.Ytrain2]: m = X.shape[0] hoindices = [3, 5, 8, 10, 17, 21] hocompl = list(set(range(m)) - set(hoindices)) #Holdout with linear kernel rls1 = RLS(X, Y) rls2 = RLS(X[hocompl], Y[hocompl]) P1 = rls1.holdout(hoindices) P2 = rls2.predict(X[hoindices]) assert_allclose(P1, P2) #Holdout with bias rls1 = RLS(X, Y, bias=3.0) rls2 = RLS(X[hocompl], Y[hocompl], bias=3.0) P1 = rls1.holdout(hoindices) P2 = rls2.predict(X[hoindices]) assert_allclose(P1, P2) #Fast regularization for i in range(-15, 15): rls1.solve(2**i) rls2.solve(2**i) P1 = rls1.holdout(hoindices) P2 = rls2.predict(X[hoindices]) assert_allclose(P1, P2) #Kernel holdout rls1 = RLS(X, Y, kernel="GaussianKernel", gamma=0.01) rls2 = RLS(X[hocompl], Y[hocompl], kernel="GaussianKernel", gamma=0.01) P1 = rls1.holdout(hoindices) P2 = rls2.predict(X[hoindices]) assert_allclose(P1, P2) for i in range(-15, 15): rls1.solve(2**i) rls2.solve(2**i) P1 = rls1.holdout(hoindices) P2 = rls2.predict(X[hoindices]) assert_allclose(P1, P2) #Incorrect indices I = [0, 3, 100] self.assertRaises(IndexError, rls1.holdout, I) I = [-1, 0, 2] self.assertRaises(IndexError, rls1.holdout, I) I = [1, 1, 2] self.assertRaises(IndexError, rls1.holdout, I)
def lgo_core(X, y, groups, regparam): logo = LeaveOneGroupOut() rls = RLS(X, y, regparam=regparam, kernel="GaussianKernel", gamma=0.01) errors = [] for train, test in logo.split(X, y, groups=groups): p = rls.holdout(test) e = sqerror(y[test], p) errors.append(e) return np.mean(errors)
def lgo_core(X,y, groups, regparam): logo = LeaveOneGroupOut() rls = RLS(X,y, regparam=regparam, kernel="GaussianKernel", gamma=0.01) errors = [] for train, test in logo.split(X, y, groups=groups): p = rls.holdout(test) e = sqerror(y[test], p) errors.append(e) return np.mean(errors)
def test_holdout(self): for X in [self.Xtrain1, self.Xtrain2]: for Y in [self.Ytrain1, self.Ytrain2]: m = X.shape[0] hoindices = [3, 5, 8, 10, 17, 21] hocompl = list(set(range(m)) - set(hoindices)) #Holdout with linear kernel rls1 = RLS(X, Y) rls2 = RLS(X[hocompl], Y[hocompl]) P1 = rls1.holdout(hoindices) P2 = rls2.predict(X[hoindices]) assert_allclose(P1, P2) #Holdout with bias rls1 = RLS(X, Y, bias = 3.0) rls2 = RLS(X[hocompl], Y[hocompl], bias = 3.0) P1 = rls1.holdout(hoindices) P2 = rls2.predict(X[hoindices]) assert_allclose(P1, P2) #Fast regularization for i in range(-15, 15): rls1.solve(2**i) rls2.solve(2**i) P1 = rls1.holdout(hoindices) P2 = rls2.predict(X[hoindices]) assert_allclose(P1, P2) #Kernel holdout rls1 = RLS(X, Y, kernel = "GaussianKernel", gamma = 0.01) rls2 = RLS(X[hocompl], Y[hocompl], kernel = "GaussianKernel", gamma = 0.01) P1 = rls1.holdout(hoindices) P2 = rls2.predict(X[hoindices]) assert_allclose(P1, P2) for i in range(-15, 15): rls1.solve(2**i) rls2.solve(2**i) P1 = rls1.holdout(hoindices) P2 = rls2.predict(X[hoindices]) assert_allclose(P1, P2) #Incorrect indices I = [0, 3, 100] self.assertRaises(IndexError, rls1.holdout, I) I = [-1, 0, 2] self.assertRaises(IndexError, rls1.holdout, I) I = [1,1,2] self.assertRaises(IndexError, rls1.holdout, I)
def plot_rls(): #Select regparam with k-fold cross-validation, #where instances related to a single sentence form #together a fold X_train = read_sparse("train_2000_x.txt") Y_train = np.loadtxt("train_2000_y.txt") X_test = read_sparse("test_2000_x.txt", X_train.shape[1]) Y_test = np.loadtxt("test_2000_y.txt") #list of sentence ids ids = np.loadtxt("train_2000_qids.txt") #mapped to a list of lists, where each list #contains indices for one fold folds = map_ids(ids) learner = RLS(X_train, Y_train) best_regparam = None best_error = float("inf") #exponential grid of possible regparam values log_regparams = range(-15, 16) kfold_errors = [] loo_errors = [] test_errors = [] for log_regparam in log_regparams: regparam = 2.**log_regparam #RLS is re-trained with the new regparam, this #is very fast due to computational short-cut learner.solve(regparam) #K-fold cross-validation perfs = [] for fold in folds: #computes holdout predictions, where instances #in fold are left out of training set P = learner.holdout(fold) perfs.append(sqerror(Y_train[fold], P)) e_kfold = np.mean(perfs) kfold_errors.append(e_kfold) P_loo = learner.leave_one_out() e_loo = sqerror(Y_train, P_loo) loo_errors.append(e_loo) P_test = learner.predict(X_test) e_test = sqerror(Y_test, P_test) test_errors.append(e_test) plt.semilogy(log_regparams, loo_errors, label="leave-one-out") plt.semilogy(log_regparams, kfold_errors, label="leave-sentence-out") plt.semilogy(log_regparams, test_errors, label="test error") plt.xlabel("$log_2(\lambda)$") plt.ylabel("mean squared error") plt.legend(loc=3) plt.show()
def plot_rls(): #Select regparam with k-fold cross-validation, #where instances related to a single sentence form #together a fold X_train = read_sparse("train_2000_x.txt") Y_train = np.loadtxt("train_2000_y.txt") X_test = read_sparse("test_2000_x.txt", X_train.shape[1]) Y_test = np.loadtxt("test_2000_y.txt") #list of sentence ids ids = np.loadtxt("train_2000_qids.txt") #mapped to a list of lists, where each list #contains indices for one fold folds = map_ids(ids) learner = RLS(X_train, Y_train) best_regparam = None best_error = float("inf") #exponential grid of possible regparam values log_regparams = range(-15, 16) kfold_errors = [] loo_errors = [] test_errors = [] for log_regparam in log_regparams: regparam = 2.**log_regparam #RLS is re-trained with the new regparam, this #is very fast due to computational short-cut learner.solve(regparam) #K-fold cross-validation perfs = [] for fold in folds: #computes holdout predictions, where instances #in fold are left out of training set P = learner.holdout(fold) perfs.append(sqerror(Y_train[fold], P)) e_kfold = np.mean(perfs) kfold_errors.append(e_kfold) P_loo = learner.leave_one_out() e_loo = sqerror(Y_train, P_loo) loo_errors.append(e_loo) P_test = learner.predict(X_test) e_test = sqerror(Y_test, P_test) test_errors.append(e_test) plt.semilogy(log_regparams, loo_errors, label = "leave-one-out") plt.semilogy(log_regparams, kfold_errors, label = "leave-sentence-out") plt.semilogy(log_regparams, test_errors, label = "test error") plt.xlabel("$log_2(\lambda)$") plt.ylabel("mean squared error") plt.legend(loc=3) plt.show()
def train_rls(): #Select regparam with k-fold cross-validation, #where instances related to a single sentence form #together a fold X_train = read_sparse("train_2000_x.txt") Y_train = np.loadtxt("train_2000_y.txt") X_test = read_sparse("test_2000_x.txt", X_train.shape[1]) Y_test = np.loadtxt("test_2000_y.txt") #list of sentence ids ids = np.loadtxt("train_2000_qids.txt") #mapped to a list of lists, where each list #contains indices for one fold folds = map_ids(ids) learner = RLS(X_train, Y_train) best_regparam = None best_error = float("inf") #exponential grid of possible regparam values log_regparams = range(-15, 16) for log_regparam in log_regparams: regparam = 2.**log_regparam #RLS is re-trained with the new regparam, this #is very fast due to computational short-cut learner.solve(regparam) #K-fold cross-validation P = np.zeros(Y_train.shape) for fold in folds: #computes holdout predictions, where instances #in fold are left out of training set P[fold] = learner.holdout(fold) e = sqerror(Y_train, P) print("regparam 2**%d, k-fold error %f" % (log_regparam, e)) if e < best_error: best_error = e best_regparam = regparam learner.solve(best_regparam) P_test = learner.predict(X_test) print("best regparam %f k-fold error %f" % (best_regparam, best_error)) print("test error %f" % sqerror(Y_test, P_test))
def train_rls(): #Select regparam with k-fold cross-validation, #where instances related to a single sentence form #together a fold X_train = read_sparse("train_2000_x.txt") Y_train = np.loadtxt("train_2000_y.txt") X_test = read_sparse("test_2000_x.txt", X_train.shape[1]) Y_test = np.loadtxt("test_2000_y.txt") #list of sentence ids ids = np.loadtxt("train_2000_qids.txt") #mapped to a list of lists, where each list #contains indices for one fold folds = map_ids(ids) learner = RLS(X_train, Y_train) best_regparam = None best_error = float("inf") #exponential grid of possible regparam values log_regparams = range(-15, 16) for log_regparam in log_regparams: regparam = 2.**log_regparam #RLS is re-trained with the new regparam, this #is very fast due to computational short-cut learner.solve(regparam) #K-fold cross-validation P = np.zeros(Y_train.shape) for fold in folds: #computes holdout predictions, where instances #in fold are left out of training set P[fold] = learner.holdout(fold) e = sqerror(Y_train, P) print("regparam 2**%d, k-fold error %f" %(log_regparam, e)) if e < best_error: best_error = e best_regparam = regparam learner.solve(best_regparam) P_test = learner.predict(X_test) print("best regparam %f k-fold error %f" %(best_regparam, best_error)) print("test error %f" %sqerror(Y_test, P_test))
def testRLS(self): print print print print print("Testing the cross-validation routines of the RLS module.") print print floattype = np.float64 m, n = 400, 100 Xtrain = np.random.rand(m, n) K = np.dot(Xtrain, Xtrain.T) ylen = 2 Y = np.zeros((m, ylen), dtype=floattype) Y = np.random.rand(m, ylen) hoindices = [45] hoindices2 = [45, 50] hoindices3 = [45, 50, 55] hocompl = list(set(range(m)) - set(hoindices)) Kho = K[np.ix_(hocompl, hocompl)] Yho = Y[hocompl] kwargs = {} kwargs['Y'] = Y kwargs['X'] = K kwargs['kernel'] = 'PrecomputedKernel' dualrls = RLS(**kwargs) kwargs = {} kwargs["X"] = Xtrain kwargs["Y"] = Y kwargs["bias"] = 0. primalrls = RLS(**kwargs) kwargs = {} kwargs['Y'] = Yho kwargs['X'] = Kho kwargs['kernel'] = 'PrecomputedKernel' dualrls_naive = RLS(**kwargs) testkm = K[np.ix_(hocompl, hoindices)] trainX = Xtrain[hocompl] testX = Xtrain[hoindices] kwargs = {} kwargs['Y'] = Yho kwargs['X'] = trainX kwargs["bias"] = 0. primalrls_naive = RLS(**kwargs) loglambdas = range(-5, 5) for j in range(0, len(loglambdas)): regparam = 2. ** loglambdas[j] print print("Regparam 2^%1d" % loglambdas[j]) dumbho = np.dot(testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho)) dumbho = np.squeeze(dumbho) print(str(dumbho) + ' Dumb HO (dual)') dualrls_naive.solve(regparam) predho1 = dualrls_naive.predictor.predict(testkm.T) print(str(predho1) + ' Naive HO (dual)') dualrls.solve(regparam) predho2 = dualrls.holdout(hoindices) print(str(predho2) + ' Fast HO (dual)') dualrls.solve(regparam) predho = dualrls.leave_one_out()[hoindices[0]] print(str(predho) + ' Fast LOO (dual)') primalrls_naive.solve(regparam) predho3 = primalrls_naive.predictor.predict(testX) print(str(predho3) + ' Naive HO (primal)') primalrls.solve(regparam) predho4 = primalrls.holdout(hoindices) print(str(predho4) + ' Fast HO (primal)') for predho in [predho1, predho2, predho3, predho4]: self.assertEqual(dumbho.shape, predho.shape) assert_allclose(dumbho, predho) #for row in range(predho.shape[0]): # for col in range(predho.shape[1]): # self.assertAlmostEqual(dumbho[row,col],predho[row,col]) primalrls.solve(regparam) predho = primalrls.leave_one_out()[hoindices[0]] print(str(predho) + ' Fast LOO (primal)') print() hoindices = range(100, 300) hocompl = list(set(range(m)) - set(hoindices)) Kho = K[np.ix_(hocompl, hocompl)] Yho = Y[hocompl] testkm = K[np.ix_(hocompl, hoindices)] dumbho = np.dot(testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho)) kwargs = {} kwargs['Y'] = Y kwargs['X'] = Xtrain dualrls.solve(regparam) predho2 = dualrls.holdout(hoindices2) print(str(predho2) + ' Fast HO') hopred = dualrls.leave_pair_out(np.array([hoindices2[0], 4, 6]), np.array([hoindices2[1], 5, 7])) print(str(hopred[0][0]) + '\n' + str(hopred[1][0]) + ' Fast LPO')
def testRLS(self): print print print print print("Testing the cross-validation routines of the RLS module.") print print floattype = np.float64 m, n = 400, 100 Xtrain = np.random.rand(m, n) K = np.dot(Xtrain, Xtrain.T) ylen = 2 Y = np.zeros((m, ylen), dtype=floattype) Y = np.random.rand(m, ylen) hoindices = [45] hoindices2 = [45, 50] hoindices3 = [45, 50, 55] hocompl = list(set(range(m)) - set(hoindices)) Kho = K[np.ix_(hocompl, hocompl)] Yho = Y[hocompl] kwargs = {} kwargs['Y'] = Y kwargs['X'] = K kwargs['kernel'] = 'PrecomputedKernel' dualrls = RLS(**kwargs) kwargs = {} kwargs["X"] = Xtrain kwargs["Y"] = Y kwargs["bias"] = 0. primalrls = RLS(**kwargs) kwargs = {} kwargs['Y'] = Yho kwargs['X'] = Kho kwargs['kernel'] = 'PrecomputedKernel' dualrls_naive = RLS(**kwargs) testkm = K[np.ix_(hocompl, hoindices)] trainX = Xtrain[hocompl] testX = Xtrain[hoindices] kwargs = {} kwargs['Y'] = Yho kwargs['X'] = trainX kwargs["bias"] = 0. primalrls_naive = RLS(**kwargs) loglambdas = range(-5, 5) for j in range(0, len(loglambdas)): regparam = 2.**loglambdas[j] print print("Regparam 2^%1d" % loglambdas[j]) dumbho = np.dot( testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho)) dumbho = np.squeeze(dumbho) print(str(dumbho) + ' Dumb HO (dual)') dualrls_naive.solve(regparam) predho1 = dualrls_naive.predictor.predict(testkm.T) print(str(predho1) + ' Naive HO (dual)') dualrls.solve(regparam) predho2 = dualrls.holdout(hoindices) print(str(predho2) + ' Fast HO (dual)') dualrls.solve(regparam) predho = dualrls.leave_one_out()[hoindices[0]] print(str(predho) + ' Fast LOO (dual)') primalrls_naive.solve(regparam) predho3 = primalrls_naive.predictor.predict(testX) print(str(predho3) + ' Naive HO (primal)') primalrls.solve(regparam) predho4 = primalrls.holdout(hoindices) print(str(predho4) + ' Fast HO (primal)') for predho in [predho1, predho2, predho3, predho4]: self.assertEqual(dumbho.shape, predho.shape) assert_allclose(dumbho, predho) #for row in range(predho.shape[0]): # for col in range(predho.shape[1]): # self.assertAlmostEqual(dumbho[row,col],predho[row,col]) primalrls.solve(regparam) predho = primalrls.leave_one_out()[hoindices[0]] print(str(predho) + ' Fast LOO (primal)') print() hoindices = range(100, 300) hocompl = list(set(range(m)) - set(hoindices)) Kho = K[np.ix_(hocompl, hocompl)] Yho = Y[hocompl] testkm = K[np.ix_(hocompl, hoindices)] dumbho = np.dot( testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho)) kwargs = {} kwargs['Y'] = Y kwargs['X'] = Xtrain dualrls.solve(regparam) predho2 = dualrls.holdout(hoindices2) print(str(predho2) + ' Fast HO') hopred = dualrls.leave_pair_out(np.array([hoindices2[0], 4, 6]), np.array([hoindices2[1], 5, 7])) print(str(hopred[0][0]) + '\n' + str(hopred[1][0]) + ' Fast LPO')