def train_rls(): #Selects both the gamma parameter for Gaussian kernel, and regparam with loocv X_train, Y_train, X_test, Y_test = load_housing() regparams = [2.**i for i in range(-15, 16)] gammas = regparams best_regparam = None best_gamma = None best_error = float("inf") for gamma in gammas: #New RLS is initialized for each kernel parameter learner = RLS(X_train, Y_train, kernel="GaussianKernel", gamma=gamma) for regparam in regparams: #RLS is re-trained with the new regparam, this #is very fast due to computational short-cut learner.solve(regparam) #Leave-one-out cross-validation predictions, this is fast due to #computational short-cut P_loo = learner.leave_one_out() e = sqerror(Y_train, P_loo) #print "regparam", regparam, "gamma", gamma, "loo-error", e if e < best_error: best_error = e best_regparam = regparam best_gamma = gamma learner = RLS(X_train, Y_train, regparam = best_regparam, kernel="GaussianKernel", gamma=best_gamma) P_test = learner.predict(X_test) print("best parameters gamma %f regparam %f" %(best_gamma, best_regparam)) print("best leave-one-out error %f" %best_error) print("test error %f" %sqerror(Y_test, P_test))
def train_rls(): X_train, Y_train, X_test, Y_test = load_housing() #select randomly 100 basis vectors indices = range(X_train.shape[0]) indices = random.sample(indices, 100) basis_vectors = X_train[indices] kernel = GaussianKernel(basis_vectors, gamma=0.00003) K_train = kernel.getKM(X_train) K_rr = kernel.getKM(basis_vectors) K_test = kernel.getKM(X_test) learner = RLS(K_train, Y_train, basis_vectors=K_rr, kernel="PrecomputedKernel", regparam=0.0003) #Leave-one-out cross-validation predictions, this is fast due to #computational short-cut P_loo = learner.leave_one_out() #Test set predictions P_test = learner.predict(K_test) print("leave-one-out error %f" % sqerror(Y_train, P_loo)) print("test error %f" % sqerror(Y_test, P_test)) #Sanity check, can we do better than predicting mean of training labels? print("mean predictor %f" % sqerror(Y_test, np.ones(Y_test.shape) * np.mean(Y_train)))
def train_rls(): #Selects both the gamma parameter for Gaussian kernel, and regparam with loocv X_train, Y_train, X_test, Y_test = load_housing() regparams = [2.**i for i in range(-15, 16)] gammas = regparams best_regparam = None best_gamma = None best_error = float("inf") for gamma in gammas: #New RLS is initialized for each kernel parameter learner = RLS(X_train, Y_train, kernel="GaussianKernel", gamma=gamma) for regparam in regparams: #RLS is re-trained with the new regparam, this #is very fast due to computational short-cut learner.solve(regparam) #Leave-one-out cross-validation predictions, this is fast due to #computational short-cut P_loo = learner.leave_one_out() e = sqerror(Y_train, P_loo) #print "regparam", regparam, "gamma", gamma, "loo-error", e if e < best_error: best_error = e best_regparam = regparam best_gamma = gamma learner = RLS(X_train, Y_train, regparam=best_regparam, kernel="GaussianKernel", gamma=best_gamma) P_test = learner.predict(X_test) print("best parameters gamma %f regparam %f" % (best_gamma, best_regparam)) print("best leave-one-out error %f" % best_error) print("test error %f" % sqerror(Y_test, P_test))
def train_rls(): #Select regparam with leave-one-out cross-validation X_train, Y_train, X_test, Y_test = load_housing() learner = RLS(X_train, Y_train) best_regparam = None best_error = float("inf") #exponential grid of possible regparam values log_regparams = range(-15, 16) for log_regparam in log_regparams: regparam = 2.**log_regparam #RLS is re-trained with the new regparam, this #is very fast due to computational short-cut learner.solve(regparam) #Leave-one-out cross-validation predictions, this is fast due to #computational short-cut P_loo = learner.leave_one_out() e = sqerror(Y_train, P_loo) print("regparam 2**%d, loo-error %f" % (log_regparam, e)) if e < best_error: best_error = e best_regparam = regparam learner.solve(best_regparam) P_test = learner.predict(X_test) print("best regparam %f with loo-error %f" % (best_regparam, best_error)) print("test error %f" % sqerror(Y_test, P_test))
def train_rls(): #Select regparam with leave-one-out cross-validation X_train, Y_train, X_test, Y_test = load_housing() learner = RLS(X_train, Y_train) best_regparam = None best_error = float("inf") #exponential grid of possible regparam values log_regparams = range(-15, 16) for log_regparam in log_regparams: regparam = 2.**log_regparam #RLS is re-trained with the new regparam, this #is very fast due to computational short-cut learner.solve(regparam) #Leave-one-out cross-validation predictions, this is fast due to #computational short-cut P_loo = learner.leave_one_out() e = sqerror(Y_train, P_loo) print("regparam 2**%d, loo-error %f" %(log_regparam, e)) if e < best_error: best_error = e best_regparam = regparam learner.solve(best_regparam) P_test = learner.predict(X_test) print("best regparam %f with loo-error %f" %(best_regparam, best_error)) print("test error %f" %sqerror(Y_test, P_test))
def train_rls(): X_train, Y_train, X_test, Y_test = load_housing() learner = RLS(X_train, Y_train, kernel="GaussianKernel", regparam=1, gamma=1) #Leave-one-out cross-validation predictions, this is fast due to #computational short-cut P_loo = learner.leave_one_out() #Test set predictions P_test = learner.predict(X_test) print("leave-one-out error %f" %sqerror(Y_train, P_loo)) print("test error %f" %sqerror(Y_test, P_test)) #Sanity check, can we do better than predicting mean of training labels? print("mean predictor %f" %sqerror(Y_test, np.ones(Y_test.shape)*np.mean(Y_train)))
def train_rls(): X_train, Y_train, X_test, Y_test = load_housing() learner = RLS(X_train, Y_train, kernel="LinearKernel", bias=1, regparam=1) #Leave-one-out cross-validation predictions, this is fast due to #computational short-cut P_loo = learner.leave_one_out() #Test set predictions P_test = learner.predict(X_test) print("leave-one-out error %f" % sqerror(Y_train, P_loo)) print("test error %f" % sqerror(Y_test, P_test)) #Sanity check, can we do better than predicting mean of training labels? print("mean predictor %f" % sqerror(Y_test, np.ones(Y_test.shape) * np.mean(Y_train)))
def plot_rls(): #Select regparam with k-fold cross-validation, #where instances related to a single sentence form #together a fold X_train = read_sparse("train_2000_x.txt") Y_train = np.loadtxt("train_2000_y.txt") X_test = read_sparse("test_2000_x.txt", X_train.shape[1]) Y_test = np.loadtxt("test_2000_y.txt") #list of sentence ids ids = np.loadtxt("train_2000_qids.txt") #mapped to a list of lists, where each list #contains indices for one fold folds = map_ids(ids) learner = RLS(X_train, Y_train) best_regparam = None best_error = float("inf") #exponential grid of possible regparam values log_regparams = range(-15, 16) kfold_errors = [] loo_errors = [] test_errors = [] for log_regparam in log_regparams: regparam = 2.**log_regparam #RLS is re-trained with the new regparam, this #is very fast due to computational short-cut learner.solve(regparam) #K-fold cross-validation perfs = [] for fold in folds: #computes holdout predictions, where instances #in fold are left out of training set P = learner.holdout(fold) perfs.append(sqerror(Y_train[fold], P)) e_kfold = np.mean(perfs) kfold_errors.append(e_kfold) P_loo = learner.leave_one_out() e_loo = sqerror(Y_train, P_loo) loo_errors.append(e_loo) P_test = learner.predict(X_test) e_test = sqerror(Y_test, P_test) test_errors.append(e_test) plt.semilogy(log_regparams, loo_errors, label="leave-one-out") plt.semilogy(log_regparams, kfold_errors, label="leave-sentence-out") plt.semilogy(log_regparams, test_errors, label="test error") plt.xlabel("$log_2(\lambda)$") plt.ylabel("mean squared error") plt.legend(loc=3) plt.show()
def plot_rls(): #Select regparam with k-fold cross-validation, #where instances related to a single sentence form #together a fold X_train = read_sparse("train_2000_x.txt") Y_train = np.loadtxt("train_2000_y.txt") X_test = read_sparse("test_2000_x.txt", X_train.shape[1]) Y_test = np.loadtxt("test_2000_y.txt") #list of sentence ids ids = np.loadtxt("train_2000_qids.txt") #mapped to a list of lists, where each list #contains indices for one fold folds = map_ids(ids) learner = RLS(X_train, Y_train) best_regparam = None best_error = float("inf") #exponential grid of possible regparam values log_regparams = range(-15, 16) kfold_errors = [] loo_errors = [] test_errors = [] for log_regparam in log_regparams: regparam = 2.**log_regparam #RLS is re-trained with the new regparam, this #is very fast due to computational short-cut learner.solve(regparam) #K-fold cross-validation perfs = [] for fold in folds: #computes holdout predictions, where instances #in fold are left out of training set P = learner.holdout(fold) perfs.append(sqerror(Y_train[fold], P)) e_kfold = np.mean(perfs) kfold_errors.append(e_kfold) P_loo = learner.leave_one_out() e_loo = sqerror(Y_train, P_loo) loo_errors.append(e_loo) P_test = learner.predict(X_test) e_test = sqerror(Y_test, P_test) test_errors.append(e_test) plt.semilogy(log_regparams, loo_errors, label = "leave-one-out") plt.semilogy(log_regparams, kfold_errors, label = "leave-sentence-out") plt.semilogy(log_regparams, test_errors, label = "test error") plt.xlabel("$log_2(\lambda)$") plt.ylabel("mean squared error") plt.legend(loc=3) plt.show()
def train_rls(): X_train, Y_train, X_test, Y_test = load_housing() #select randomly 100 basis vectors indices = range(X_train.shape[0]) indices = random.sample(indices, 100) basis_vectors = X_train[indices] learner = RLS(X_train, Y_train, basis_vectors = basis_vectors, kernel="GaussianKernel", regparam=0.0003, gamma=0.00003) #Leave-one-out cross-validation predictions, this is fast due to #computational short-cut P_loo = learner.leave_one_out() #Test set predictions P_test = learner.predict(X_test) print("leave-one-out error %f" %sqerror(Y_train, P_loo)) print("test error %f" %sqerror(Y_test, P_test)) #Sanity check, can we do better than predicting mean of training labels? print("mean predictor %f" %sqerror(Y_test, np.ones(Y_test.shape)*np.mean(Y_train)))
def train_rls(): #Selects both the gamma parameter for Gaussian kernel, and regparam with loocv X_train, Y_train, X_test, Y_test = load_housing() regparams = [2.**i for i in range(-15, 16)] gammas = regparams best_regparam = None best_gamma = None best_error = float("inf") best_learner = None for gamma in gammas: #New RLS is initialized for each kernel parameter learner = LeaveOneOutRLS(X_train, Y_train, kernel="GaussianKernel", gamma=gamma, regparams=regparams) e = np.min(learner.cv_performances) if e < best_error: best_error = e best_regparam = learner.regparam best_gamma = gamma best_learner = learner P_test = best_learner.predict(X_test) print("best parameters gamma %f regparam %f" % (best_gamma, best_regparam)) print("best leave-one-out error %f" % best_error) print("test error %f" % sqerror(Y_test, P_test))
def train_rls(): # Trains RLS with a precomputed kernel matrix X_train, Y_train, X_test, Y_test = load_housing() # Minor techincal detail: adding 1.0 simulates the effect of adding a # constant valued bias feature, as is done by 'LinearKernel' by deafault K_train = np.dot(X_train, X_train.T) + 1.0 K_test = np.dot(X_test, X_train.T) + 1.0 learner = RLS(K_train, Y_train, kernel="PrecomputedKernel") # Leave-one-out cross-validation predictions, this is fast due to # computational short-cut P_loo = learner.leave_one_out() # Test set predictions P_test = learner.predict(K_test) print("leave-one-out error %f" % sqerror(Y_train, P_loo)) print("test error %f" % sqerror(Y_test, P_test)) # Sanity check, can we do better than predicting mean of training labels? print("mean predictor %f" % sqerror(Y_test, np.ones(Y_test.shape) * np.mean(Y_train)))
def train_rls(): X_train, Y_train, X_test, Y_test = load_housing() cb = Callback(X_test, Y_test) learner = GreedyRLS(X_train, Y_train, 13, callbackfun=cb) #Test set predictions P_test = learner.predict(X_test) print("test error %f" % sqerror(Y_test, P_test)) print("Selected features " + str(learner.selected))
def train_rls(): X_train, Y_train, X_test, Y_test = load_housing() cb = Callback(X_test, Y_test) learner = GreedyRLS(X_train, Y_train, 13, callbackfun = cb) #Test set predictions P_test = learner.predict(X_test) print("test error %f" %sqerror(Y_test, P_test)) print("Selected features " +str(learner.selected))
def train_rls(): X_train, Y_train, X_test, Y_test = load_housing() #we select 5 features learner = GreedyRLS(X_train, Y_train, 5) #Test set predictions P_test = learner.predict(X_test) print("test error %f" %sqerror(Y_test, P_test)) print("Selected features " +str(learner.selected))
def lgo_core(X, y, groups, regparam): logo = LeaveOneGroupOut() rls = RLS(X, y, regparam=regparam, kernel="GaussianKernel", gamma=0.01) errors = [] for train, test in logo.split(X, y, groups=groups): p = rls.holdout(test) e = sqerror(y[test], p) errors.append(e) return np.mean(errors)
def lgo_core(X,y, groups, regparam): logo = LeaveOneGroupOut() rls = RLS(X,y, regparam=regparam, kernel="GaussianKernel", gamma=0.01) errors = [] for train, test in logo.split(X, y, groups=groups): p = rls.holdout(test) e = sqerror(y[test], p) errors.append(e) return np.mean(errors)
def train_rls(): #Trains RLS with automatically selected regularization parameter X_train, Y_train, X_test, Y_test = load_housing() regparams = [2.**i for i in range(-15, 16)] learner = LeaveOneOutRLS(X_train, Y_train, regparams = regparams) loo_errors = learner.cv_performances P_test = learner.predict(X_test) print("leave-one-out errors " +str(loo_errors)) print("chosen regparam %f" %learner.regparam) print("test error %f" %sqerror(Y_test, P_test))
def train_rls(): #Trains RLS with a precomputed kernel matrix X_train, Y_train, X_test, Y_test = load_housing() #Minor techincal detail: adding 1.0 simulates the effect of adding a #constant valued bias feature, as is done by 'LinearKernel' by deafault K_train = np.dot(X_train, X_train.T) + 1.0 K_test = np.dot(X_test, X_train.T) + 1.0 learner = RLS(K_train, Y_train, kernel="PrecomputedKernel") #Leave-one-out cross-validation predictions, this is fast due to #computational short-cut P_loo = learner.leave_one_out() #Test set predictions P_test = learner.predict(K_test) print("leave-one-out error %f" % sqerror(Y_train, P_loo)) print("test error %f" % sqerror(Y_test, P_test)) #Sanity check, can we do better than predicting mean of training labels? print("mean predictor %f" % sqerror(Y_test, np.ones(Y_test.shape) * np.mean(Y_train)))
def lgo_sklearn(X,y, groups, regparam): logo = LeaveOneGroupOut() errors = [] for train, test in logo.split(X, y, groups=groups): rls = KernelRidge(kernel="rbf", gamma=0.01) rls.fit(X[train], y[train]) p = rls.predict(X[test]) e = sqerror(y[test], p) errors.append(e) return np.mean(errors)
def lgo_sklearn(X, y, groups, regparam): logo = LeaveOneGroupOut() errors = [] for train, test in logo.split(X, y, groups=groups): rls = KernelRidge(kernel="rbf", gamma=0.01) rls.fit(X[train], y[train]) p = rls.predict(X[test]) e = sqerror(y[test], p) errors.append(e) return np.mean(errors)
def train_rls(): X_train, Y_train, X_test, Y_test = load_housing() kernel = GaussianKernel(X_train, gamma=0.00003) K_train = kernel.getKM(X_train) K_test = kernel.getKM(X_test) learner = RLS(K_train, Y_train, kernel="PrecomputedKernel", regparam=0.0003) #Leave-one-out cross-validation predictions, this is fast due to #computational short-cut P_loo = learner.leave_one_out() #Test set predictions P_test = learner.predict(K_test) print("leave-one-out error %f" % sqerror(Y_train, P_loo)) print("test error %f" % sqerror(Y_test, P_test)) #Sanity check, can we do better than predicting mean of training labels? print("mean predictor %f" % sqerror(Y_test, np.ones(Y_test.shape) * np.mean(Y_train)))
def train_rls(): #Select regparam with k-fold cross-validation, #where instances related to a single sentence form #together a fold X_train = read_sparse("train_2000_x.txt") Y_train = np.loadtxt("train_2000_y.txt") X_test = read_sparse("test_2000_x.txt", X_train.shape[1]) Y_test = np.loadtxt("test_2000_y.txt") #list of sentence ids ids = np.loadtxt("train_2000_qids.txt") #mapped to a list of lists, where each list #contains indices for one fold folds = map_ids(ids) learner = RLS(X_train, Y_train) best_regparam = None best_error = float("inf") #exponential grid of possible regparam values log_regparams = range(-15, 16) for log_regparam in log_regparams: regparam = 2.**log_regparam #RLS is re-trained with the new regparam, this #is very fast due to computational short-cut learner.solve(regparam) #K-fold cross-validation P = np.zeros(Y_train.shape) for fold in folds: #computes holdout predictions, where instances #in fold are left out of training set P[fold] = learner.holdout(fold) e = sqerror(Y_train, P) print("regparam 2**%d, k-fold error %f" %(log_regparam, e)) if e < best_error: best_error = e best_regparam = regparam learner.solve(best_regparam) P_test = learner.predict(X_test) print("best regparam %f k-fold error %f" %(best_regparam, best_error)) print("test error %f" %sqerror(Y_test, P_test))
def train_rls(): #Select regparam with k-fold cross-validation, #where instances related to a single sentence form #together a fold X_train = read_sparse("train_2000_x.txt") Y_train = np.loadtxt("train_2000_y.txt") X_test = read_sparse("test_2000_x.txt", X_train.shape[1]) Y_test = np.loadtxt("test_2000_y.txt") #list of sentence ids ids = np.loadtxt("train_2000_qids.txt") #mapped to a list of lists, where each list #contains indices for one fold folds = map_ids(ids) learner = RLS(X_train, Y_train) best_regparam = None best_error = float("inf") #exponential grid of possible regparam values log_regparams = range(-15, 16) for log_regparam in log_regparams: regparam = 2.**log_regparam #RLS is re-trained with the new regparam, this #is very fast due to computational short-cut learner.solve(regparam) #K-fold cross-validation P = np.zeros(Y_train.shape) for fold in folds: #computes holdout predictions, where instances #in fold are left out of training set P[fold] = learner.holdout(fold) e = sqerror(Y_train, P) print("regparam 2**%d, k-fold error %f" % (log_regparam, e)) if e < best_error: best_error = e best_regparam = regparam learner.solve(best_regparam) P_test = learner.predict(X_test) print("best regparam %f k-fold error %f" % (best_regparam, best_error)) print("test error %f" % sqerror(Y_test, P_test))
def train_rls(): #Selects both the gamma parameter for Gaussian kernel, and regparam with loocv X_train, Y_train, X_test, Y_test = load_housing() regparams = [2.**i for i in range(-15, 16)] gammas = regparams best_regparam = None best_gamma = None best_error = float("inf") best_learner = None for gamma in gammas: #New RLS is initialized for each kernel parameter learner = LeaveOneOutRLS(X_train, Y_train, kernel="GaussianKernel", gamma=gamma, regparams=regparams) e = np.min(learner.cv_performances) if e < best_error: best_error = e best_regparam = learner.regparam best_gamma = gamma best_learner = learner P_test = best_learner.predict(X_test) print("best parameters gamma %f regparam %f" %(best_gamma, best_regparam)) print("best leave-one-out error %f" %best_error) print("test error %f" %sqerror(Y_test, P_test))
a=10.0**alpha # Y_train_in=Y(mask_train_in) # Y_test_in = Y(mask_test_in) K_train=K[train_in_index][:,train_in_index] K_test = K[test_in_index][:, train_in_index] Y_train=Y[train_in_index] Y_test = Y[test_in_index] #print(Y_train.shape) #print(Y_test.shape) clf = KernelRidge(alpha=a) clf.fit(K_train, Y_train) pred=clf.predict(K_test) #print(pred.shape) #print(Y_train.shape) mse=np.sqrt(sqerror(np.ravel(Y_test,'F'),np.ravel(pred,'F'))) MSE[i,j]=mse MSE_m = np.mean(MSE, axis=0) opt_regpram = 10. ** (reg_par[np.argmin(MSE_m)]) clf = KernelRidge(alpha=opt_regpram) K_train_o=K[train_out][:,train_out] K_test_o = K[test_out][:, train_out] Y_train_o = Y[train_out] Y_test_o = Y[test_out] clf.fit(K_train_o, Y_train_o) pred_o = clf.predict(K_test_o) mse_o[k] = np.sqrt(sqerror(np.ravel(Y_test_o, 'F'), np.ravel(pred_o, 'F'))) cind_o[k] = cindex(np.ravel(Y_test_o, 'F'), np.ravel(pred_o, 'F')) pear_o[k] = pearsonr(np.ravel(Y_test_o, 'F'), np.ravel(pred_o, 'F'))[0] spear_o[k] = spearmanr(np.ravel(Y_test_o, 'F'), np.ravel(pred_o, 'F'))[0] #plt.scatter(np.ravel(Y_test_o, 'F'), np.ravel(pred_o, 'F'))
import numpy as np from rlscore.learner.rls import LeaveOneOutRLS from rlscore.reader import read_sparse from rlscore.reader import read_sparse from rlscore.measure import sqerror train_labels = np.loadtxt("./examples/data/reg_train.labels") test_labels = np.loadtxt("./examples/data/reg_test.labels") train_features = read_sparse("./examples/data/reg_train.features") test_features = read_sparse("./examples/data/reg_test.features") kwargs = {} kwargs['measure']=sqerror kwargs['regparams'] = [2**i for i in range(-10,11)] kwargs["Y"] = train_labels kwargs["X"] = train_features learner = LeaveOneOutRLS(**kwargs) grid = kwargs['regparams'] perfs = learner.cv_performances for i in range(len(grid)): print "parameter %f cv_performance %f" %(grid[i], perfs[i]) P = learner.predict(test_features) test_perf = sqerror(test_labels, P) print "test set performance: %f" %test_perf
def callback(self, learner): self.iteration += 1 P = learner.predict(self.X_test) e = sqerror(self.Y_test, P) print("Features selected %d, test error %f" %(self.iteration, e))
import numpy as np from rlscore.learner.rls import LeaveOneOutRLS from rlscore.utilities.reader import read_sparse from rlscore.measure import sqerror train_labels = np.loadtxt("./legacy_tests/data/reg_train.labels") test_labels = np.loadtxt("./legacy_tests/data/reg_test.labels") train_features = read_sparse("./legacy_tests/data/reg_train.features") test_features = read_sparse("./legacy_tests/data/reg_test.features") kwargs = {} kwargs['measure']=sqerror kwargs['regparams'] = [2**i for i in range(-10,11)] kwargs["Y"] = train_labels kwargs["X"] = train_features learner = LeaveOneOutRLS(**kwargs) grid = kwargs['regparams'] perfs = learner.cv_performances for i in range(len(grid)): print("parameter %f cv_performance %f" %(grid[i], perfs[i])) P = learner.predict(test_features) test_perf = sqerror(test_labels, P) print("test set performance: %f" %test_perf)
def callback(self, learner): self.iteration += 1 P = learner.predict(self.X_test) e = sqerror(self.Y_test, P) print("Features selected %d, test error %f" % (self.iteration, e))
def loo_core(X,y,regparam): learner = RLS(X,y,regparam, bias=0.) p = learner.leave_one_out() e = sqerror(y, p) return e