def test_leave_pair_out(self):
#compares holdout and leave-pair-out
start = [0, 2, 3, 5]
end = [1, 3, 6, 8]
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
#LPO with linear kernel
rls1 = RLS(X, Y, regparam = 7.0, bias=3.0)
lpo_start, lpo_end = rls1.leave_pair_out(start, end)
ho_start, ho_end = [], []
for i in range(len(start)):
P = rls1.holdout([start[i], end[i]])
ho_start.append(P[0])
ho_end.append(P[1])
ho_start = np.array(ho_start)
ho_end = np.array(ho_end)
assert_allclose(ho_start, lpo_start)
assert_allclose(ho_end, lpo_end)
#LPO Gaussian kernel
rls1 = RLS(X, Y, regparam = 11.0, kenerl="PolynomialKernel", coef0=1, degree=3)
lpo_start, lpo_end = rls1.leave_pair_out(start, end)
ho_start, ho_end = [], []
for i in range(len(start)):
P = rls1.holdout([start[i], end[i]])
ho_start.append(P[0])
ho_end.append(P[1])
ho_start = np.array(ho_start)
ho_end = np.array(ho_end)
assert_allclose(ho_start, lpo_start)
assert_allclose(ho_end, lpo_end)
def test_leave_pair_out(self):
#compares holdout and leave-pair-out
start = [0, 2, 3, 5]
end = [1, 3, 6, 8]
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
#LPO with linear kernel
rls1 = RLS(X, Y, regparam=7.0, bias=3.0)
lpo_start, lpo_end = rls1.leave_pair_out(start, end)
ho_start, ho_end = [], []
for i in range(len(start)):
P = rls1.holdout([start[i], end[i]])
ho_start.append(P[0])
ho_end.append(P[1])
ho_start = np.array(ho_start)
ho_end = np.array(ho_end)
assert_allclose(ho_start, lpo_start)
assert_allclose(ho_end, lpo_end)
#LPO Gaussian kernel
rls1 = RLS(X,
Y,
regparam=11.0,
kenerl="PolynomialKernel",
coef0=1,
degree=3)
lpo_start, lpo_end = rls1.leave_pair_out(start, end)
ho_start, ho_end = [], []
for i in range(len(start)):
P = rls1.holdout([start[i], end[i]])
ho_start.append(P[0])
ho_end.append(P[1])
ho_start = np.array(ho_start)
ho_end = np.array(ho_end)
assert_allclose(ho_start, lpo_start)
assert_allclose(ho_end, lpo_end)
def train_rls():
X_train, Y_train, foo = read_svmlight("a1a.t")
X_test, Y_test, foo = read_svmlight("a1a", X_train.shape[1])
lpo_aucs = []
test_aucs = []
for i in range(1000):
X_small = X_train[i * 30:i * 30 + 30]
Y_small = Y_train[i * 30:i * 30 + 30]
pairs_start = []
pairs_end = []
for i in range(len(Y_small)):
for j in range(len(Y_small)):
if Y_small[i] == 1. and Y_small[j] == -1.:
pairs_start.append(i)
pairs_end.append(j)
learner = RLS(X_small, Y_small)
pairs_start = np.array(pairs_start)
pairs_end = np.array(pairs_end)
P_start, P_end = learner.leave_pair_out(pairs_start, pairs_end)
lpo_a = np.mean(P_start > P_end + 0.5 * (P_start == P_end))
P_test = learner.predict(X_test)
test_a = auc(Y_test, P_test)
lpo_aucs.append(lpo_a)
test_aucs.append(test_a)
print("mean lpo over auc over 1000 repetitions: %f" % np.mean(lpo_aucs))
print("mean test auc over 1000 repetitions %f" % np.mean(test_aucs))
def lpo_core(X,y, regparam):
start, end = [], []
for i in range(X.shape[0]-1):
for j in range(i+1, X.shape[0]):
start.append(i)
end.append(j)
rls = RLS(X,y, regparam=regparam, kernel="GaussianKernel", gamma=0.01)
pred0, pred1 = rls.leave_pair_out(start, end)
return pred0, pred1
def lpo_core(X, y, regparam):
start, end = [], []
for i in range(X.shape[0] - 1):
for j in range(i + 1, X.shape[0]):
start.append(i)
end.append(j)
rls = RLS(X, y, regparam=regparam, kernel="GaussianKernel", gamma=0.01)
pred0, pred1 = rls.leave_pair_out(start, end)
return pred0, pred1
def testRLS(self):
print
print
print
print
print("Testing the cross-validation routines of the RLS module.")
print
print
floattype = np.float64
m, n = 400, 100
Xtrain = np.random.rand(m, n)
K = np.dot(Xtrain, Xtrain.T)
ylen = 2
Y = np.zeros((m, ylen), dtype=floattype)
Y = np.random.rand(m, ylen)
hoindices = [45]
hoindices2 = [45, 50]
hoindices3 = [45, 50, 55]
hocompl = list(set(range(m)) - set(hoindices))
Kho = K[np.ix_(hocompl, hocompl)]
Yho = Y[hocompl]
kwargs = {}
kwargs['Y'] = Y
kwargs['X'] = K
kwargs['kernel'] = 'PrecomputedKernel'
dualrls = RLS(**kwargs)
kwargs = {}
kwargs["X"] = Xtrain
kwargs["Y"] = Y
kwargs["bias"] = 0.
primalrls = RLS(**kwargs)
kwargs = {}
kwargs['Y'] = Yho
kwargs['X'] = Kho
kwargs['kernel'] = 'PrecomputedKernel'
dualrls_naive = RLS(**kwargs)
testkm = K[np.ix_(hocompl, hoindices)]
trainX = Xtrain[hocompl]
testX = Xtrain[hoindices]
kwargs = {}
kwargs['Y'] = Yho
kwargs['X'] = trainX
kwargs["bias"] = 0.
primalrls_naive = RLS(**kwargs)
loglambdas = range(-5, 5)
for j in range(0, len(loglambdas)):
regparam = 2. ** loglambdas[j]
print
print("Regparam 2^%1d" % loglambdas[j])
dumbho = np.dot(testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho))
dumbho = np.squeeze(dumbho)
print(str(dumbho) + ' Dumb HO (dual)')
dualrls_naive.solve(regparam)
predho1 = dualrls_naive.predictor.predict(testkm.T)
print(str(predho1) + ' Naive HO (dual)')
dualrls.solve(regparam)
predho2 = dualrls.holdout(hoindices)
print(str(predho2) + ' Fast HO (dual)')
dualrls.solve(regparam)
predho = dualrls.leave_one_out()[hoindices[0]]
print(str(predho) + ' Fast LOO (dual)')
primalrls_naive.solve(regparam)
predho3 = primalrls_naive.predictor.predict(testX)
print(str(predho3) + ' Naive HO (primal)')
primalrls.solve(regparam)
predho4 = primalrls.holdout(hoindices)
print(str(predho4) + ' Fast HO (primal)')
for predho in [predho1, predho2, predho3, predho4]:
self.assertEqual(dumbho.shape, predho.shape)
assert_allclose(dumbho, predho)
#for row in range(predho.shape[0]):
# for col in range(predho.shape[1]):
# self.assertAlmostEqual(dumbho[row,col],predho[row,col])
primalrls.solve(regparam)
predho = primalrls.leave_one_out()[hoindices[0]]
print(str(predho) + ' Fast LOO (primal)')
print()
hoindices = range(100, 300)
hocompl = list(set(range(m)) - set(hoindices))
Kho = K[np.ix_(hocompl, hocompl)]
Yho = Y[hocompl]
testkm = K[np.ix_(hocompl, hoindices)]
dumbho = np.dot(testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho))
kwargs = {}
kwargs['Y'] = Y
kwargs['X'] = Xtrain
dualrls.solve(regparam)
predho2 = dualrls.holdout(hoindices2)
print(str(predho2) + ' Fast HO')
hopred = dualrls.leave_pair_out(np.array([hoindices2[0], 4, 6]), np.array([hoindices2[1], 5, 7]))
print(str(hopred[0][0]) + '\n' + str(hopred[1][0]) + ' Fast LPO')
def testRLS(self):
print
print
print
print
print("Testing the cross-validation routines of the RLS module.")
print
print
floattype = np.float64
m, n = 400, 100
Xtrain = np.random.rand(m, n)
K = np.dot(Xtrain, Xtrain.T)
ylen = 2
Y = np.zeros((m, ylen), dtype=floattype)
Y = np.random.rand(m, ylen)
hoindices = [45]
hoindices2 = [45, 50]
hoindices3 = [45, 50, 55]
hocompl = list(set(range(m)) - set(hoindices))
Kho = K[np.ix_(hocompl, hocompl)]
Yho = Y[hocompl]
kwargs = {}
kwargs['Y'] = Y
kwargs['X'] = K
kwargs['kernel'] = 'PrecomputedKernel'
dualrls = RLS(**kwargs)
kwargs = {}
kwargs["X"] = Xtrain
kwargs["Y"] = Y
kwargs["bias"] = 0.
primalrls = RLS(**kwargs)
kwargs = {}
kwargs['Y'] = Yho
kwargs['X'] = Kho
kwargs['kernel'] = 'PrecomputedKernel'
dualrls_naive = RLS(**kwargs)
testkm = K[np.ix_(hocompl, hoindices)]
trainX = Xtrain[hocompl]
testX = Xtrain[hoindices]
kwargs = {}
kwargs['Y'] = Yho
kwargs['X'] = trainX
kwargs["bias"] = 0.
primalrls_naive = RLS(**kwargs)
loglambdas = range(-5, 5)
for j in range(0, len(loglambdas)):
regparam = 2.**loglambdas[j]
print
print("Regparam 2^%1d" % loglambdas[j])
dumbho = np.dot(
testkm.T,
np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho))
dumbho = np.squeeze(dumbho)
print(str(dumbho) + ' Dumb HO (dual)')
dualrls_naive.solve(regparam)
predho1 = dualrls_naive.predictor.predict(testkm.T)
print(str(predho1) + ' Naive HO (dual)')
dualrls.solve(regparam)
predho2 = dualrls.holdout(hoindices)
print(str(predho2) + ' Fast HO (dual)')
dualrls.solve(regparam)
predho = dualrls.leave_one_out()[hoindices[0]]
print(str(predho) + ' Fast LOO (dual)')
primalrls_naive.solve(regparam)
predho3 = primalrls_naive.predictor.predict(testX)
print(str(predho3) + ' Naive HO (primal)')
primalrls.solve(regparam)
predho4 = primalrls.holdout(hoindices)
print(str(predho4) + ' Fast HO (primal)')
for predho in [predho1, predho2, predho3, predho4]:
self.assertEqual(dumbho.shape, predho.shape)
assert_allclose(dumbho, predho)
#for row in range(predho.shape[0]):
# for col in range(predho.shape[1]):
# self.assertAlmostEqual(dumbho[row,col],predho[row,col])
primalrls.solve(regparam)
predho = primalrls.leave_one_out()[hoindices[0]]
print(str(predho) + ' Fast LOO (primal)')
print()
hoindices = range(100, 300)
hocompl = list(set(range(m)) - set(hoindices))
Kho = K[np.ix_(hocompl, hocompl)]
Yho = Y[hocompl]
testkm = K[np.ix_(hocompl, hoindices)]
dumbho = np.dot(
testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])),
Yho))
kwargs = {}
kwargs['Y'] = Y
kwargs['X'] = Xtrain
dualrls.solve(regparam)
predho2 = dualrls.holdout(hoindices2)
print(str(predho2) + ' Fast HO')
hopred = dualrls.leave_pair_out(np.array([hoindices2[0], 4, 6]),
np.array([hoindices2[1], 5, 7]))
print(str(hopred[0][0]) + '\n' + str(hopred[1][0]) + ' Fast LPO')
