def test_kernel(self):
#tests that learning with kernels works
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
#Basic case
dual_rls = RLS(X, Y, kernel= "GaussianKernel", regparam=5.0, gamma=0.01)
kernel = GaussianKernel(X, gamma = 0.01)
K = kernel.getKM(X)
m = K.shape[0]
A = dual_rls.predictor.A
A2 = np.linalg.solve(K+5.0*np.eye(m), Y)
assert_allclose(A, A2)
#Fast regularization
dual_rls.solve(1000)
A = dual_rls.predictor.A
A2 = np.linalg.solve(K+ 1000 * np.eye(m), Y)
assert_allclose(A, A2)
#Precomputed kernel
dual_rls = RLS(K, Y, kernel="PrecomputedKernel", regparam = 1000)
assert_allclose(dual_rls.predictor.W, A2)
#Reduced set approximation
kernel = PolynomialKernel(X[self.bvectors], gamma=0.5, coef0 = 1.2, degree = 2)
Kr = kernel.getKM(X)
Krr = kernel.getKM(X[self.bvectors])
dual_rls = RLS(X, Y, kernel="PolynomialKernel", basis_vectors = X[self.bvectors], regparam = 200, gamma=0.5, coef0=1.2, degree = 2)
A = dual_rls.predictor.A
A2 = np.linalg.solve(np.dot(Kr.T, Kr)+ 200 * Krr, np.dot(Kr.T, Y))
assert_allclose(A, A2)
dual_rls = RLS(Kr, Y, kernel="PrecomputedKernel", basis_vectors = Krr, regparam=200)
A = dual_rls.predictor.W
assert_allclose(A, A2)
def test_compare(self):
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
#No bias
greedy_rls = GreedyRLS(X,
Y,
subsetsize=10,
regparam=12,
bias=0.)
selected = greedy_rls.selected
s_complement = list(
set(range(X.shape[1])).difference(selected))
X_cut = X[:, selected]
rls = RLS(X_cut, Y, regparam=12., bias=0.)
W = greedy_rls.predictor.W[selected]
W2 = rls.predictor.W
assert_allclose(W, W2)
assert_array_equal(greedy_rls.predictor.W[s_complement], 0)
assert_array_equal(greedy_rls.predictor.b, 0)
#Bias
greedy_rls = GreedyRLS(X,
Y,
subsetsize=10,
regparam=12,
bias=2.)
selected = greedy_rls.selected
X_cut = X[:, selected]
rls = RLS(X_cut, Y, regparam=12., bias=2.)
W = greedy_rls.predictor.W[selected]
W2 = rls.predictor.W
assert_allclose(W, W2)
assert_allclose(greedy_rls.predictor.b, rls.predictor.b)
def test_leave_pair_out(self):
#compares holdout and leave-pair-out
start = [0, 2, 3, 5]
end = [1, 3, 6, 8]
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
#LPO with linear kernel
rls1 = RLS(X, Y, regparam = 7.0, bias=3.0)
lpo_start, lpo_end = rls1.leave_pair_out(start, end)
ho_start, ho_end = [], []
for i in range(len(start)):
P = rls1.holdout([start[i], end[i]])
ho_start.append(P[0])
ho_end.append(P[1])
ho_start = np.array(ho_start)
ho_end = np.array(ho_end)
assert_allclose(ho_start, lpo_start)
assert_allclose(ho_end, lpo_end)
#LPO Gaussian kernel
rls1 = RLS(X, Y, regparam = 11.0, kenerl="PolynomialKernel", coef0=1, degree=3)
lpo_start, lpo_end = rls1.leave_pair_out(start, end)
ho_start, ho_end = [], []
for i in range(len(start)):
P = rls1.holdout([start[i], end[i]])
ho_start.append(P[0])
ho_end.append(P[1])
ho_start = np.array(ho_start)
ho_end = np.array(ho_end)
assert_allclose(ho_start, lpo_start)
assert_allclose(ho_end, lpo_end)
def test_sparse(self):
#mix of linear and kernel learning testing that learning works also with
#sparse data matrices
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
Xsp = csc_matrix(X)
#linear kernel without bias
primal_rls = RLS(Xsp, Y, regparam=2.0, bias=0.)
W = primal_rls.predictor.W
d = X.shape[1]
W2 = np.linalg.solve(np.dot(X.T, X) + 2.0 * np.eye(d), np.dot(X.T, Y))
assert_allclose(W, W2)
#linear kernel with bias
primal_rls = RLS(Xsp, Y, regparam=1.0, bias=2.)
O = np.sqrt(2.) * np.ones((X.shape[0],1))
X_new = np.hstack((X, O))
W = primal_rls.predictor.W
W2 = np.linalg.solve(np.dot(X_new.T, X_new) + np.eye(d+1), np.dot(X_new.T, Y))
b = primal_rls.predictor.b
b2 = W2[-1]
W2 = W2[:-1]
assert_allclose(W, W2)
assert_allclose(b, np.sqrt(2) * b2)
#reduced set approximation
primal_rls = RLS(Xsp, Y, basis_vectors = Xsp[self.bvectors], regparam=5.0, bias=2.)
W = primal_rls.predictor.W
b = primal_rls.predictor.b
K = np.dot(X_new, X_new.T)
Kr = K[:, self.bvectors]
Krr = K[np.ix_(self.bvectors, self.bvectors)]
A = np.linalg.solve(np.dot(Kr.T, Kr)+ 5.0 * Krr, np.dot(Kr.T, Y))
W2 = np.dot(X_new[self.bvectors].T, A)
b2 = W2[-1]
W2 = W2[:-1]
assert_allclose(W, W2)
assert_allclose(b, np.sqrt(2) * b2)
#Kernels
dual_rls = RLS(Xsp, Y, kernel= "GaussianKernel", regparam=5.0, gamma=0.01)
kernel = GaussianKernel(X, gamma = 0.01)
K = kernel.getKM(X)
m = K.shape[0]
A = dual_rls.predictor.A
A2 = np.linalg.solve(K+5.0*np.eye(m), Y)
assert_allclose(A, A2)
def testCGRLS(self):
m, n = 100, 300
for regparam in [0.00000001, 1, 100000000]:
Xtrain = np.mat(np.random.rand(m, n))
Y = np.mat(np.random.rand(m, 1))
rpool = {}
rpool['X'] = Xtrain
rpool['Y'] = Y
rpool['regparam'] = regparam
rpool["bias"] = 2.0
rls = RLS(**rpool)
rls.solve(regparam)
model = rls.predictor
W = model.W
b = model.b
rls = CGRLS(**rpool)
model = rls.predictor
W2 = model.W
b2 = model.b
for i in range(W.shape[0]):
self.assertAlmostEqual(W[i], W2[i], places=5)
self.assertAlmostEqual(b, b2, places=5)
def testRLS(self):
print("\n\n\n\nTesting the cross-validation routines of the RLS module.\n\n")
m, n = 100, 300
Xtrain = random.rand(m, n)
Y = mat(random.rand(m, 1))
basis_vectors = [0,3,7,8]
#hoindices = [45, 50, 55]
hoindices = [0, 1, 2]
hocompl = list(set(range(m)) - set(hoindices))
bk = GaussianKernel(**{'X':Xtrain[basis_vectors], 'gamma':0.001})
rpool = {}
rpool['X'] = Xtrain
bk2 = GaussianKernel(**{'X':Xtrain, 'gamma':0.001})
K = np.mat(bk2.getKM(Xtrain))
Yho = Y[hocompl]
rpool = {}
rpool['Y'] = Y
rpool['X'] = Xtrain
rpool['basis_vectors'] = Xtrain[basis_vectors]
Xhocompl = Xtrain[hocompl]
testX = Xtrain[hoindices]
rpool = {}
rpool['Y'] = Yho
rpool['X'] = Xhocompl
rpool["kernel"] = "RsetKernel"
rpool["base_kernel"] = bk
rpool["basis_features"] = Xtrain[basis_vectors]
#rk = RsetKernel(**{'base_kernel':bk, 'basis_features':Xtrain[basis_vectors], 'X':Xhocompl})
dualrls_naive = RLS(**rpool)
rpool = {}
rpool['Y'] = Yho
rpool['X'] = Xhocompl
rsaK = K[:, basis_vectors] * la.inv(K[ix_(basis_vectors, basis_vectors)]) * K[basis_vectors]
rsaKho = rsaK[ix_(hocompl, hocompl)]
rsa_testkm = rsaK[ix_(hocompl, hoindices)]
loglambdas = range(-5, 5)
for j in range(0, len(loglambdas)):
regparam = 2. ** loglambdas[j]
print("\nRegparam 2^%1d" % loglambdas[j])
print((rsa_testkm.T * la.inv(rsaKho + regparam * eye(rsaKho.shape[0])) * Yho).T, 'Dumb HO (dual)')
dumbho = np.squeeze(np.array(rsa_testkm.T * la.inv(rsaKho + regparam * eye(rsaKho.shape[0])) * Yho))
dualrls_naive.solve(regparam)
predho1 = np.squeeze(dualrls_naive.predictor.predict(testX))
print(predho1.T, 'Naive HO (dual)')
#dualrls.solve(regparam)
#predho2 = np.squeeze(dualrls.computeHO(hoindices))
#print predho2.T, 'Fast HO (dual)'
for predho in [dumbho, predho1]:#, predho2]:
self.assertEqual(dumbho.shape, predho.shape)
for row in range(predho.shape[0]):
#for col in range(predho.shape[1]):
# self.assertAlmostEqual(dumbho[row,col],predho[row,col])
self.assertAlmostEqual(dumbho[row],predho[row])
def testRLS(self):
print
print
print
print
print("Testing the cross-validation routines of the RLS module.")
print
print
floattype = np.float64
m, n = 400, 100
Xtrain = np.random.rand(m, n)
K = np.dot(Xtrain, Xtrain.T)
ylen = 2
Y = np.zeros((m, ylen), dtype=floattype)
Y = np.random.rand(m, ylen)
hoindices = [45]
hoindices2 = [45, 50]
hoindices3 = [45, 50, 55]
hocompl = list(set(range(m)) - set(hoindices))
Kho = K[np.ix_(hocompl, hocompl)]
Yho = Y[hocompl]
kwargs = {}
kwargs['Y'] = Y
kwargs['X'] = K
kwargs['kernel'] = 'PrecomputedKernel'
dualrls = RLS(**kwargs)
kwargs = {}
kwargs["X"] = Xtrain
kwargs["Y"] = Y
kwargs["bias"] = 0.
primalrls = RLS(**kwargs)
kwargs = {}
kwargs['Y'] = Yho
kwargs['X'] = Kho
kwargs['kernel'] = 'PrecomputedKernel'
dualrls_naive = RLS(**kwargs)
testkm = K[np.ix_(hocompl, hoindices)]
trainX = Xtrain[hocompl]
testX = Xtrain[hoindices]
kwargs = {}
kwargs['Y'] = Yho
kwargs['X'] = trainX
kwargs["bias"] = 0.
primalrls_naive = RLS(**kwargs)
loglambdas = range(-5, 5)
for j in range(0, len(loglambdas)):
regparam = 2. ** loglambdas[j]
print
print("Regparam 2^%1d" % loglambdas[j])
dumbho = np.dot(testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho))
dumbho = np.squeeze(dumbho)
print(str(dumbho) + ' Dumb HO (dual)')
dualrls_naive.solve(regparam)
predho1 = dualrls_naive.predictor.predict(testkm.T)
print(str(predho1) + ' Naive HO (dual)')
dualrls.solve(regparam)
predho2 = dualrls.holdout(hoindices)
print(str(predho2) + ' Fast HO (dual)')
dualrls.solve(regparam)
predho = dualrls.leave_one_out()[hoindices[0]]
print(str(predho) + ' Fast LOO (dual)')
primalrls_naive.solve(regparam)
predho3 = primalrls_naive.predictor.predict(testX)
print(str(predho3) + ' Naive HO (primal)')
primalrls.solve(regparam)
predho4 = primalrls.holdout(hoindices)
print(str(predho4) + ' Fast HO (primal)')
for predho in [predho1, predho2, predho3, predho4]:
self.assertEqual(dumbho.shape, predho.shape)
assert_allclose(dumbho, predho)
#for row in range(predho.shape[0]):
# for col in range(predho.shape[1]):
# self.assertAlmostEqual(dumbho[row,col],predho[row,col])
primalrls.solve(regparam)
predho = primalrls.leave_one_out()[hoindices[0]]
print(str(predho) + ' Fast LOO (primal)')
print()
hoindices = range(100, 300)
hocompl = list(set(range(m)) - set(hoindices))
Kho = K[np.ix_(hocompl, hocompl)]
Yho = Y[hocompl]
testkm = K[np.ix_(hocompl, hoindices)]
dumbho = np.dot(testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho))
kwargs = {}
kwargs['Y'] = Y
kwargs['X'] = Xtrain
dualrls.solve(regparam)
predho2 = dualrls.holdout(hoindices2)
print(str(predho2) + ' Fast HO')
hopred = dualrls.leave_pair_out(np.array([hoindices2[0], 4, 6]), np.array([hoindices2[1], 5, 7]))
print(str(hopred[0][0]) + '\n' + str(hopred[1][0]) + ' Fast LPO')
def test_linear(self):
#Test that learning with linear kernel works correctly both
#with low and high-dimensional data
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
#Basic case
primal_rls = RLS(X, Y, regparam=1.0, bias=0.)
W = primal_rls.predictor.W
d = X.shape[1]
W2 = np.linalg.solve(np.dot(X.T, X) + np.eye(d), np.dot(X.T, Y))
assert_allclose(W, W2)
#Fast regularization algorithm
primal_rls.solve(10.)
W = primal_rls.predictor.W
W2 = np.linalg.solve(np.dot(X.T, X) + 10.*np.eye(d), np.dot(X.T, Y))
assert_allclose(W, W2)
#Bias term included
primal_rls = RLS(X, Y, regparam=1.0, bias=2.)
O = np.sqrt(2.) * np.ones((X.shape[0],1))
X_new = np.hstack((X, O))
W = primal_rls.predictor.W
W2 = np.linalg.solve(np.dot(X_new.T, X_new) + np.eye(d+1), np.dot(X_new.T, Y))
b = primal_rls.predictor.b
b2 = W2[-1]
W2 = W2[:-1]
assert_allclose(W, W2)
assert_allclose(b, np.sqrt(2) * b2)
#reduced set approximation
primal_rls = RLS(X, Y, basis_vectors = X[self.bvectors], regparam=5.0, bias=2.)
W = primal_rls.predictor.W
b = primal_rls.predictor.b
K = np.dot(X_new, X_new.T)
Kr = K[:, self.bvectors]
Krr = K[np.ix_(self.bvectors, self.bvectors)]
A = np.linalg.solve(np.dot(Kr.T, Kr)+ 5.0 * Krr, np.dot(Kr.T, Y))
W2 = np.dot(X_new[self.bvectors].T, A)
b2 = W2[-1]
W2 = W2[:-1]
assert_allclose(W, W2)
assert_allclose(b, np.sqrt(2) * b2)
#Using pre-computed linear kernel matrix
kernel = LinearKernel(X, bias = 2.)
K = kernel.getKM(X)
dual_rls = RLS(K, Y, kernel = "PrecomputedKernel", regparam=0.01)
W = np.dot(X_new.T, dual_rls.predictor.W)
b = W[-1]
W = W[:-1]
W2 = np.linalg.solve(np.dot(X_new.T, X_new) + 0.01 * np.eye(d+1), np.dot(X_new.T, Y))
b2 = W2[-1]
W2 = W2[:-1]
assert_allclose(W, W2)
assert_allclose(b, b2)
#Pre-computed linear kernel, reduced set approximation
kernel = LinearKernel(X[self.bvectors], bias = 2.)
dual_rls = RLS(kernel.getKM(X), Y, kernel="PrecomputedKernel", basis_vectors = kernel.getKM(X[self.bvectors]), regparam=5.0)
W = np.dot(X_new[self.bvectors].T, dual_rls.predictor.W)
b = W[-1]
W = W[:-1]
K = np.dot(X_new, X_new.T)
Kr = K[:, self.bvectors]
Krr = K[np.ix_(self.bvectors, self.bvectors)]
A = np.linalg.solve(np.dot(Kr.T, Kr)+ 5.0 * Krr, np.dot(Kr.T, Y))
W2 = np.dot(X_new[self.bvectors].T, A)
b2 = W2[-1]
W2 = W2[:-1]
assert_allclose(W, W2)
assert_allclose(b, b2)
def test_loo(self):
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
m = X.shape[0]
#LOO with linear kernel
rls1 = RLS(X, Y, regparam = 7.0, bias=3.0)
P1 = rls1.leave_one_out()
P2 = []
for i in range(X.shape[0]):
X_train = np.delete(X, i, axis=0)
Y_train = np.delete(Y, i, axis=0)
X_test = X[i]
rls2 = RLS(X_train, Y_train, regparam = 7.0, bias = 3.0)
P2.append(rls2.predict(X_test))
P2 = np.array(P2)
assert_allclose(P1, P2)
#Fast regularization
rls1.solve(1024)
P1 = rls1.leave_one_out()
P2 = []
for i in range(X.shape[0]):
X_train = np.delete(X, i, axis=0)
Y_train = np.delete(Y, i, axis=0)
X_test = X[i]
rls2 = RLS(X_train, Y_train, regparam = 1024, bias = 3.0)
P2.append(rls2.predict(X_test))
P2 = np.array(P2)
assert_allclose(P1, P2)
#kernels
rls1 = RLS(X, Y, kernel = "GaussianKernel", gamma = 0.01)
P1 = rls1.leave_one_out()
P2 = []
for i in range(X.shape[0]):
X_train = np.delete(X, i, axis=0)
Y_train = np.delete(Y, i, axis=0)
X_test = X[i]
rls2 = RLS(X_train, Y_train, kernel = "GaussianKernel", gamma = 0.01)
P2.append(rls2.predict(X_test))
P2 = np.array(P2)
assert_allclose(P1, P2)
def test_holdout(self):
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
m = X.shape[0]
hoindices = [3, 5, 8, 10, 17, 21]
hocompl = list(set(range(m)) - set(hoindices))
#Holdout with linear kernel
rls1 = RLS(X, Y)
rls2 = RLS(X[hocompl], Y[hocompl])
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
#Holdout with bias
rls1 = RLS(X, Y, bias = 3.0)
rls2 = RLS(X[hocompl], Y[hocompl], bias = 3.0)
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
#Fast regularization
for i in range(-15, 15):
rls1.solve(2**i)
rls2.solve(2**i)
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
#Kernel holdout
rls1 = RLS(X, Y, kernel = "GaussianKernel", gamma = 0.01)
rls2 = RLS(X[hocompl], Y[hocompl], kernel = "GaussianKernel", gamma = 0.01)
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
for i in range(-15, 15):
rls1.solve(2**i)
rls2.solve(2**i)
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
#Incorrect indices
I = [0, 3, 100]
self.assertRaises(IndexError, rls1.holdout, I)
I = [-1, 0, 2]
self.assertRaises(IndexError, rls1.holdout, I)
I = [1,1,2]
self.assertRaises(IndexError, rls1.holdout, I)