def test_leave_pair_out(self): #compares holdout and leave-pair-out start = [0, 2, 3, 5] end = [1, 3, 6, 8] for X in [self.Xtrain1, self.Xtrain2]: for Y in [self.Ytrain1, self.Ytrain2]: #LPO with linear kernel rls1 = RLS(X, Y, regparam = 7.0, bias=3.0) lpo_start, lpo_end = rls1.leave_pair_out(start, end) ho_start, ho_end = [], [] for i in range(len(start)): P = rls1.holdout([start[i], end[i]]) ho_start.append(P[0]) ho_end.append(P[1]) ho_start = np.array(ho_start) ho_end = np.array(ho_end) assert_allclose(ho_start, lpo_start) assert_allclose(ho_end, lpo_end) #LPO Gaussian kernel rls1 = RLS(X, Y, regparam = 11.0, kenerl="PolynomialKernel", coef0=1, degree=3) lpo_start, lpo_end = rls1.leave_pair_out(start, end) ho_start, ho_end = [], [] for i in range(len(start)): P = rls1.holdout([start[i], end[i]]) ho_start.append(P[0]) ho_end.append(P[1]) ho_start = np.array(ho_start) ho_end = np.array(ho_end) assert_allclose(ho_start, lpo_start) assert_allclose(ho_end, lpo_end)
def test_leave_pair_out(self): #compares holdout and leave-pair-out start = [0, 2, 3, 5] end = [1, 3, 6, 8] for X in [self.Xtrain1, self.Xtrain2]: for Y in [self.Ytrain1, self.Ytrain2]: #LPO with linear kernel rls1 = RLS(X, Y, regparam=7.0, bias=3.0) lpo_start, lpo_end = rls1.leave_pair_out(start, end) ho_start, ho_end = [], [] for i in range(len(start)): P = rls1.holdout([start[i], end[i]]) ho_start.append(P[0]) ho_end.append(P[1]) ho_start = np.array(ho_start) ho_end = np.array(ho_end) assert_allclose(ho_start, lpo_start) assert_allclose(ho_end, lpo_end) #LPO Gaussian kernel rls1 = RLS(X, Y, regparam=11.0, kenerl="PolynomialKernel", coef0=1, degree=3) lpo_start, lpo_end = rls1.leave_pair_out(start, end) ho_start, ho_end = [], [] for i in range(len(start)): P = rls1.holdout([start[i], end[i]]) ho_start.append(P[0]) ho_end.append(P[1]) ho_start = np.array(ho_start) ho_end = np.array(ho_end) assert_allclose(ho_start, lpo_start) assert_allclose(ho_end, lpo_end)
def train_rls(): X_train, Y_train, foo = read_svmlight("a1a.t") X_test, Y_test, foo = read_svmlight("a1a", X_train.shape[1]) lpo_aucs = [] test_aucs = [] for i in range(1000): X_small = X_train[i * 30:i * 30 + 30] Y_small = Y_train[i * 30:i * 30 + 30] pairs_start = [] pairs_end = [] for i in range(len(Y_small)): for j in range(len(Y_small)): if Y_small[i] == 1. and Y_small[j] == -1.: pairs_start.append(i) pairs_end.append(j) learner = RLS(X_small, Y_small) pairs_start = np.array(pairs_start) pairs_end = np.array(pairs_end) P_start, P_end = learner.leave_pair_out(pairs_start, pairs_end) lpo_a = np.mean(P_start > P_end + 0.5 * (P_start == P_end)) P_test = learner.predict(X_test) test_a = auc(Y_test, P_test) lpo_aucs.append(lpo_a) test_aucs.append(test_a) print("mean lpo over auc over 1000 repetitions: %f" % np.mean(lpo_aucs)) print("mean test auc over 1000 repetitions %f" % np.mean(test_aucs))
def lpo_core(X,y, regparam): start, end = [], [] for i in range(X.shape[0]-1): for j in range(i+1, X.shape[0]): start.append(i) end.append(j) rls = RLS(X,y, regparam=regparam, kernel="GaussianKernel", gamma=0.01) pred0, pred1 = rls.leave_pair_out(start, end) return pred0, pred1
def lpo_core(X, y, regparam): start, end = [], [] for i in range(X.shape[0] - 1): for j in range(i + 1, X.shape[0]): start.append(i) end.append(j) rls = RLS(X, y, regparam=regparam, kernel="GaussianKernel", gamma=0.01) pred0, pred1 = rls.leave_pair_out(start, end) return pred0, pred1
def testRLS(self): print print print print print("Testing the cross-validation routines of the RLS module.") print print floattype = np.float64 m, n = 400, 100 Xtrain = np.random.rand(m, n) K = np.dot(Xtrain, Xtrain.T) ylen = 2 Y = np.zeros((m, ylen), dtype=floattype) Y = np.random.rand(m, ylen) hoindices = [45] hoindices2 = [45, 50] hoindices3 = [45, 50, 55] hocompl = list(set(range(m)) - set(hoindices)) Kho = K[np.ix_(hocompl, hocompl)] Yho = Y[hocompl] kwargs = {} kwargs['Y'] = Y kwargs['X'] = K kwargs['kernel'] = 'PrecomputedKernel' dualrls = RLS(**kwargs) kwargs = {} kwargs["X"] = Xtrain kwargs["Y"] = Y kwargs["bias"] = 0. primalrls = RLS(**kwargs) kwargs = {} kwargs['Y'] = Yho kwargs['X'] = Kho kwargs['kernel'] = 'PrecomputedKernel' dualrls_naive = RLS(**kwargs) testkm = K[np.ix_(hocompl, hoindices)] trainX = Xtrain[hocompl] testX = Xtrain[hoindices] kwargs = {} kwargs['Y'] = Yho kwargs['X'] = trainX kwargs["bias"] = 0. primalrls_naive = RLS(**kwargs) loglambdas = range(-5, 5) for j in range(0, len(loglambdas)): regparam = 2. ** loglambdas[j] print print("Regparam 2^%1d" % loglambdas[j]) dumbho = np.dot(testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho)) dumbho = np.squeeze(dumbho) print(str(dumbho) + ' Dumb HO (dual)') dualrls_naive.solve(regparam) predho1 = dualrls_naive.predictor.predict(testkm.T) print(str(predho1) + ' Naive HO (dual)') dualrls.solve(regparam) predho2 = dualrls.holdout(hoindices) print(str(predho2) + ' Fast HO (dual)') dualrls.solve(regparam) predho = dualrls.leave_one_out()[hoindices[0]] print(str(predho) + ' Fast LOO (dual)') primalrls_naive.solve(regparam) predho3 = primalrls_naive.predictor.predict(testX) print(str(predho3) + ' Naive HO (primal)') primalrls.solve(regparam) predho4 = primalrls.holdout(hoindices) print(str(predho4) + ' Fast HO (primal)') for predho in [predho1, predho2, predho3, predho4]: self.assertEqual(dumbho.shape, predho.shape) assert_allclose(dumbho, predho) #for row in range(predho.shape[0]): # for col in range(predho.shape[1]): # self.assertAlmostEqual(dumbho[row,col],predho[row,col]) primalrls.solve(regparam) predho = primalrls.leave_one_out()[hoindices[0]] print(str(predho) + ' Fast LOO (primal)') print() hoindices = range(100, 300) hocompl = list(set(range(m)) - set(hoindices)) Kho = K[np.ix_(hocompl, hocompl)] Yho = Y[hocompl] testkm = K[np.ix_(hocompl, hoindices)] dumbho = np.dot(testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho)) kwargs = {} kwargs['Y'] = Y kwargs['X'] = Xtrain dualrls.solve(regparam) predho2 = dualrls.holdout(hoindices2) print(str(predho2) + ' Fast HO') hopred = dualrls.leave_pair_out(np.array([hoindices2[0], 4, 6]), np.array([hoindices2[1], 5, 7])) print(str(hopred[0][0]) + '\n' + str(hopred[1][0]) + ' Fast LPO')
def testRLS(self): print print print print print("Testing the cross-validation routines of the RLS module.") print print floattype = np.float64 m, n = 400, 100 Xtrain = np.random.rand(m, n) K = np.dot(Xtrain, Xtrain.T) ylen = 2 Y = np.zeros((m, ylen), dtype=floattype) Y = np.random.rand(m, ylen) hoindices = [45] hoindices2 = [45, 50] hoindices3 = [45, 50, 55] hocompl = list(set(range(m)) - set(hoindices)) Kho = K[np.ix_(hocompl, hocompl)] Yho = Y[hocompl] kwargs = {} kwargs['Y'] = Y kwargs['X'] = K kwargs['kernel'] = 'PrecomputedKernel' dualrls = RLS(**kwargs) kwargs = {} kwargs["X"] = Xtrain kwargs["Y"] = Y kwargs["bias"] = 0. primalrls = RLS(**kwargs) kwargs = {} kwargs['Y'] = Yho kwargs['X'] = Kho kwargs['kernel'] = 'PrecomputedKernel' dualrls_naive = RLS(**kwargs) testkm = K[np.ix_(hocompl, hoindices)] trainX = Xtrain[hocompl] testX = Xtrain[hoindices] kwargs = {} kwargs['Y'] = Yho kwargs['X'] = trainX kwargs["bias"] = 0. primalrls_naive = RLS(**kwargs) loglambdas = range(-5, 5) for j in range(0, len(loglambdas)): regparam = 2.**loglambdas[j] print print("Regparam 2^%1d" % loglambdas[j]) dumbho = np.dot( testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho)) dumbho = np.squeeze(dumbho) print(str(dumbho) + ' Dumb HO (dual)') dualrls_naive.solve(regparam) predho1 = dualrls_naive.predictor.predict(testkm.T) print(str(predho1) + ' Naive HO (dual)') dualrls.solve(regparam) predho2 = dualrls.holdout(hoindices) print(str(predho2) + ' Fast HO (dual)') dualrls.solve(regparam) predho = dualrls.leave_one_out()[hoindices[0]] print(str(predho) + ' Fast LOO (dual)') primalrls_naive.solve(regparam) predho3 = primalrls_naive.predictor.predict(testX) print(str(predho3) + ' Naive HO (primal)') primalrls.solve(regparam) predho4 = primalrls.holdout(hoindices) print(str(predho4) + ' Fast HO (primal)') for predho in [predho1, predho2, predho3, predho4]: self.assertEqual(dumbho.shape, predho.shape) assert_allclose(dumbho, predho) #for row in range(predho.shape[0]): # for col in range(predho.shape[1]): # self.assertAlmostEqual(dumbho[row,col],predho[row,col]) primalrls.solve(regparam) predho = primalrls.leave_one_out()[hoindices[0]] print(str(predho) + ' Fast LOO (primal)') print() hoindices = range(100, 300) hocompl = list(set(range(m)) - set(hoindices)) Kho = K[np.ix_(hocompl, hocompl)] Yho = Y[hocompl] testkm = K[np.ix_(hocompl, hoindices)] dumbho = np.dot( testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho)) kwargs = {} kwargs['Y'] = Y kwargs['X'] = Xtrain dualrls.solve(regparam) predho2 = dualrls.holdout(hoindices2) print(str(predho2) + ' Fast HO') hopred = dualrls.leave_pair_out(np.array([hoindices2[0], 4, 6]), np.array([hoindices2[1], 5, 7])) print(str(hopred[0][0]) + '\n' + str(hopred[1][0]) + ' Fast LPO')