def test_leave_pair_out(self):
#compares holdout and leave-pair-out
start = [0, 2, 3, 5]
end = [1, 3, 6, 8]
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
#LPO with linear kernel
rls1 = RLS(X, Y, regparam=7.0, bias=3.0)
lpo_start, lpo_end = rls1.leave_pair_out(start, end)
ho_start, ho_end = [], []
for i in range(len(start)):
P = rls1.holdout([start[i], end[i]])
ho_start.append(P[0])
ho_end.append(P[1])
ho_start = np.array(ho_start)
ho_end = np.array(ho_end)
assert_allclose(ho_start, lpo_start)
assert_allclose(ho_end, lpo_end)
#LPO Gaussian kernel
rls1 = RLS(X,
Y,
regparam=11.0,
kenerl="PolynomialKernel",
coef0=1,
degree=3)
lpo_start, lpo_end = rls1.leave_pair_out(start, end)
ho_start, ho_end = [], []
for i in range(len(start)):
P = rls1.holdout([start[i], end[i]])
ho_start.append(P[0])
ho_end.append(P[1])
ho_start = np.array(ho_start)
ho_end = np.array(ho_end)
assert_allclose(ho_start, lpo_start)
assert_allclose(ho_end, lpo_end)
def test_leave_pair_out(self):
#compares holdout and leave-pair-out
start = [0, 2, 3, 5]
end = [1, 3, 6, 8]
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
#LPO with linear kernel
rls1 = RLS(X, Y, regparam = 7.0, bias=3.0)
lpo_start, lpo_end = rls1.leave_pair_out(start, end)
ho_start, ho_end = [], []
for i in range(len(start)):
P = rls1.holdout([start[i], end[i]])
ho_start.append(P[0])
ho_end.append(P[1])
ho_start = np.array(ho_start)
ho_end = np.array(ho_end)
assert_allclose(ho_start, lpo_start)
assert_allclose(ho_end, lpo_end)
#LPO Gaussian kernel
rls1 = RLS(X, Y, regparam = 11.0, kenerl="PolynomialKernel", coef0=1, degree=3)
lpo_start, lpo_end = rls1.leave_pair_out(start, end)
ho_start, ho_end = [], []
for i in range(len(start)):
P = rls1.holdout([start[i], end[i]])
ho_start.append(P[0])
ho_end.append(P[1])
ho_start = np.array(ho_start)
ho_end = np.array(ho_end)
assert_allclose(ho_start, lpo_start)
assert_allclose(ho_end, lpo_end)
def lposcv_rls(coordinates, Xdata, Ydata, dzradii, regparam, visualization):
print("Starting lposcv-rls analysis...")
# Calculate sorted pairwise distance matrix and indexes
performanceTable = np.zeros([len(dzradii), 2])
data_distances, data_distance_indexes = distanceMatrix(coordinates)
negcount = len(np.where(Ydata == 0)[0])
folds = lpo_folds(Ydata, negcount)
for rind, dzradius in enumerate(dzradii):
print("Analysis on going, dead zone radius: " + str(dzradius) +
"m / " + str(dzradii[len(dzradii) - 1]) + "m")
# Calculate dead zone folds
dz_folds = dzfolds(dzradius, folds, data_distances,
data_distance_indexes)
learner = RLS(Xdata, Ydata, regparam=regparam)
perfs = []
for dz_fold in dz_folds:
preds = learner.holdout(dz_fold)
if preds[0] > preds[1]:
perfs.append(1.)
elif preds[0] == preds[1]:
perfs.append(0.5)
else:
perfs.append(0.)
if visualization: # Check for visualization
testcoords = coordinates[dz_fold[0:2], :]
dzcoords = coordinates[dz_fold, :]
visualize_lposcv(coordinates, testcoords, dzcoords, dzradius)
perf = np.mean(perfs)
performanceTable[rind, 0] = dzradius
performanceTable[rind, 1] = perf
plotRes_lposcv(performanceTable, rind, len(folds), 'rls')
print("Analysis done.")
return performanceTable
def skcv_rls(coordinates, Xdata, Ydata, number_of_folds, dzradii, regparam, visualization):
print("Starting skcv-rls analysis...")
# Calculate sorted pairwise distance matrix and indexes
performanceTable = np.zeros([len(dzradii),2])
data_distances, data_distance_indexes = distanceMatrix(coordinates)
folds = random_folds(len(Ydata), number_of_folds)
for rind, dzradius in enumerate(dzradii):
print("Analysis ongoing, dead zone radius: " + str(dzradius) + "m / " + str(dzradii[len(dzradii)-1]) + "m")
# Calculate dead zone folds
dz_folds = dzfolds(dzradius, folds, data_distances, data_distance_indexes)
learner = RLS(Xdata, Ydata, regparam=regparam)
P = np.zeros(Ydata.shape)
for fold_id, dz_fold in enumerate(dz_folds):
preds = learner.holdout(dz_fold)
if preds.ndim == 0:
P[folds[fold_id]] = preds
else:
P[folds[fold_id]] = preds[0:len(folds[fold_id])]
if visualization: # Check for visualization
testcoords = coordinates[folds[fold_id],:]
dzcoords = coordinates[dz_fold, :]
visualize_skcv(coordinates, testcoords, dzcoords, dzradius)
perf = cindex(Ydata, P)
performanceTable[rind,0] = dzradius
performanceTable[rind, 1] = perf
plotRes_skcv(performanceTable, rind, number_of_folds, "rls")
print("Analysis done.")
return performanceTable
def test_holdout(self):
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
m = X.shape[0]
hoindices = [3, 5, 8, 10, 17, 21]
hocompl = list(set(range(m)) - set(hoindices))
#Holdout with linear kernel
rls1 = RLS(X, Y)
rls2 = RLS(X[hocompl], Y[hocompl])
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
#Holdout with bias
rls1 = RLS(X, Y, bias=3.0)
rls2 = RLS(X[hocompl], Y[hocompl], bias=3.0)
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
#Fast regularization
for i in range(-15, 15):
rls1.solve(2**i)
rls2.solve(2**i)
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
#Kernel holdout
rls1 = RLS(X, Y, kernel="GaussianKernel", gamma=0.01)
rls2 = RLS(X[hocompl],
Y[hocompl],
kernel="GaussianKernel",
gamma=0.01)
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
for i in range(-15, 15):
rls1.solve(2**i)
rls2.solve(2**i)
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
#Incorrect indices
I = [0, 3, 100]
self.assertRaises(IndexError, rls1.holdout, I)
I = [-1, 0, 2]
self.assertRaises(IndexError, rls1.holdout, I)
I = [1, 1, 2]
self.assertRaises(IndexError, rls1.holdout, I)
def lgo_core(X, y, groups, regparam):
logo = LeaveOneGroupOut()
rls = RLS(X, y, regparam=regparam, kernel="GaussianKernel", gamma=0.01)
errors = []
for train, test in logo.split(X, y, groups=groups):
p = rls.holdout(test)
e = sqerror(y[test], p)
errors.append(e)
return np.mean(errors)
def lgo_core(X,y, groups, regparam):
logo = LeaveOneGroupOut()
rls = RLS(X,y, regparam=regparam, kernel="GaussianKernel", gamma=0.01)
errors = []
for train, test in logo.split(X, y, groups=groups):
p = rls.holdout(test)
e = sqerror(y[test], p)
errors.append(e)
return np.mean(errors)
def test_holdout(self):
for X in [self.Xtrain1, self.Xtrain2]:
for Y in [self.Ytrain1, self.Ytrain2]:
m = X.shape[0]
hoindices = [3, 5, 8, 10, 17, 21]
hocompl = list(set(range(m)) - set(hoindices))
#Holdout with linear kernel
rls1 = RLS(X, Y)
rls2 = RLS(X[hocompl], Y[hocompl])
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
#Holdout with bias
rls1 = RLS(X, Y, bias = 3.0)
rls2 = RLS(X[hocompl], Y[hocompl], bias = 3.0)
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
#Fast regularization
for i in range(-15, 15):
rls1.solve(2**i)
rls2.solve(2**i)
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
#Kernel holdout
rls1 = RLS(X, Y, kernel = "GaussianKernel", gamma = 0.01)
rls2 = RLS(X[hocompl], Y[hocompl], kernel = "GaussianKernel", gamma = 0.01)
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
for i in range(-15, 15):
rls1.solve(2**i)
rls2.solve(2**i)
P1 = rls1.holdout(hoindices)
P2 = rls2.predict(X[hoindices])
assert_allclose(P1, P2)
#Incorrect indices
I = [0, 3, 100]
self.assertRaises(IndexError, rls1.holdout, I)
I = [-1, 0, 2]
self.assertRaises(IndexError, rls1.holdout, I)
I = [1,1,2]
self.assertRaises(IndexError, rls1.holdout, I)
def plot_rls():
#Select regparam with k-fold cross-validation,
#where instances related to a single sentence form
#together a fold
X_train = read_sparse("train_2000_x.txt")
Y_train = np.loadtxt("train_2000_y.txt")
X_test = read_sparse("test_2000_x.txt", X_train.shape[1])
Y_test = np.loadtxt("test_2000_y.txt")
#list of sentence ids
ids = np.loadtxt("train_2000_qids.txt")
#mapped to a list of lists, where each list
#contains indices for one fold
folds = map_ids(ids)
learner = RLS(X_train, Y_train)
best_regparam = None
best_error = float("inf")
#exponential grid of possible regparam values
log_regparams = range(-15, 16)
kfold_errors = []
loo_errors = []
test_errors = []
for log_regparam in log_regparams:
regparam = 2.**log_regparam
#RLS is re-trained with the new regparam, this
#is very fast due to computational short-cut
learner.solve(regparam)
#K-fold cross-validation
perfs = []
for fold in folds:
#computes holdout predictions, where instances
#in fold are left out of training set
P = learner.holdout(fold)
perfs.append(sqerror(Y_train[fold], P))
e_kfold = np.mean(perfs)
kfold_errors.append(e_kfold)
P_loo = learner.leave_one_out()
e_loo = sqerror(Y_train, P_loo)
loo_errors.append(e_loo)
P_test = learner.predict(X_test)
e_test = sqerror(Y_test, P_test)
test_errors.append(e_test)
plt.semilogy(log_regparams, loo_errors, label="leave-one-out")
plt.semilogy(log_regparams, kfold_errors, label="leave-sentence-out")
plt.semilogy(log_regparams, test_errors, label="test error")
plt.xlabel("$log_2(\lambda)$")
plt.ylabel("mean squared error")
plt.legend(loc=3)
plt.show()
def plot_rls():
#Select regparam with k-fold cross-validation,
#where instances related to a single sentence form
#together a fold
X_train = read_sparse("train_2000_x.txt")
Y_train = np.loadtxt("train_2000_y.txt")
X_test = read_sparse("test_2000_x.txt", X_train.shape[1])
Y_test = np.loadtxt("test_2000_y.txt")
#list of sentence ids
ids = np.loadtxt("train_2000_qids.txt")
#mapped to a list of lists, where each list
#contains indices for one fold
folds = map_ids(ids)
learner = RLS(X_train, Y_train)
best_regparam = None
best_error = float("inf")
#exponential grid of possible regparam values
log_regparams = range(-15, 16)
kfold_errors = []
loo_errors = []
test_errors = []
for log_regparam in log_regparams:
regparam = 2.**log_regparam
#RLS is re-trained with the new regparam, this
#is very fast due to computational short-cut
learner.solve(regparam)
#K-fold cross-validation
perfs = []
for fold in folds:
#computes holdout predictions, where instances
#in fold are left out of training set
P = learner.holdout(fold)
perfs.append(sqerror(Y_train[fold], P))
e_kfold = np.mean(perfs)
kfold_errors.append(e_kfold)
P_loo = learner.leave_one_out()
e_loo = sqerror(Y_train, P_loo)
loo_errors.append(e_loo)
P_test = learner.predict(X_test)
e_test = sqerror(Y_test, P_test)
test_errors.append(e_test)
plt.semilogy(log_regparams, loo_errors, label = "leave-one-out")
plt.semilogy(log_regparams, kfold_errors, label = "leave-sentence-out")
plt.semilogy(log_regparams, test_errors, label = "test error")
plt.xlabel("$log_2(\lambda)$")
plt.ylabel("mean squared error")
plt.legend(loc=3)
plt.show()
def train_rls():
#Select regparam with k-fold cross-validation,
#where instances related to a single sentence form
#together a fold
X_train = read_sparse("train_2000_x.txt")
Y_train = np.loadtxt("train_2000_y.txt")
X_test = read_sparse("test_2000_x.txt", X_train.shape[1])
Y_test = np.loadtxt("test_2000_y.txt")
#list of sentence ids
ids = np.loadtxt("train_2000_qids.txt")
#mapped to a list of lists, where each list
#contains indices for one fold
folds = map_ids(ids)
learner = RLS(X_train, Y_train)
best_regparam = None
best_error = float("inf")
#exponential grid of possible regparam values
log_regparams = range(-15, 16)
for log_regparam in log_regparams:
regparam = 2.**log_regparam
#RLS is re-trained with the new regparam, this
#is very fast due to computational short-cut
learner.solve(regparam)
#K-fold cross-validation
P = np.zeros(Y_train.shape)
for fold in folds:
#computes holdout predictions, where instances
#in fold are left out of training set
P[fold] = learner.holdout(fold)
e = sqerror(Y_train, P)
print("regparam 2**%d, k-fold error %f" % (log_regparam, e))
if e < best_error:
best_error = e
best_regparam = regparam
learner.solve(best_regparam)
P_test = learner.predict(X_test)
print("best regparam %f k-fold error %f" % (best_regparam, best_error))
print("test error %f" % sqerror(Y_test, P_test))
def train_rls():
#Select regparam with k-fold cross-validation,
#where instances related to a single sentence form
#together a fold
X_train = read_sparse("train_2000_x.txt")
Y_train = np.loadtxt("train_2000_y.txt")
X_test = read_sparse("test_2000_x.txt", X_train.shape[1])
Y_test = np.loadtxt("test_2000_y.txt")
#list of sentence ids
ids = np.loadtxt("train_2000_qids.txt")
#mapped to a list of lists, where each list
#contains indices for one fold
folds = map_ids(ids)
learner = RLS(X_train, Y_train)
best_regparam = None
best_error = float("inf")
#exponential grid of possible regparam values
log_regparams = range(-15, 16)
for log_regparam in log_regparams:
regparam = 2.**log_regparam
#RLS is re-trained with the new regparam, this
#is very fast due to computational short-cut
learner.solve(regparam)
#K-fold cross-validation
P = np.zeros(Y_train.shape)
for fold in folds:
#computes holdout predictions, where instances
#in fold are left out of training set
P[fold] = learner.holdout(fold)
e = sqerror(Y_train, P)
print("regparam 2**%d, k-fold error %f" %(log_regparam, e))
if e < best_error:
best_error = e
best_regparam = regparam
learner.solve(best_regparam)
P_test = learner.predict(X_test)
print("best regparam %f k-fold error %f" %(best_regparam, best_error))
print("test error %f" %sqerror(Y_test, P_test))
def testRLS(self):
print
print
print
print
print("Testing the cross-validation routines of the RLS module.")
print
print
floattype = np.float64
m, n = 400, 100
Xtrain = np.random.rand(m, n)
K = np.dot(Xtrain, Xtrain.T)
ylen = 2
Y = np.zeros((m, ylen), dtype=floattype)
Y = np.random.rand(m, ylen)
hoindices = [45]
hoindices2 = [45, 50]
hoindices3 = [45, 50, 55]
hocompl = list(set(range(m)) - set(hoindices))
Kho = K[np.ix_(hocompl, hocompl)]
Yho = Y[hocompl]
kwargs = {}
kwargs['Y'] = Y
kwargs['X'] = K
kwargs['kernel'] = 'PrecomputedKernel'
dualrls = RLS(**kwargs)
kwargs = {}
kwargs["X"] = Xtrain
kwargs["Y"] = Y
kwargs["bias"] = 0.
primalrls = RLS(**kwargs)
kwargs = {}
kwargs['Y'] = Yho
kwargs['X'] = Kho
kwargs['kernel'] = 'PrecomputedKernel'
dualrls_naive = RLS(**kwargs)
testkm = K[np.ix_(hocompl, hoindices)]
trainX = Xtrain[hocompl]
testX = Xtrain[hoindices]
kwargs = {}
kwargs['Y'] = Yho
kwargs['X'] = trainX
kwargs["bias"] = 0.
primalrls_naive = RLS(**kwargs)
loglambdas = range(-5, 5)
for j in range(0, len(loglambdas)):
regparam = 2. ** loglambdas[j]
print
print("Regparam 2^%1d" % loglambdas[j])
dumbho = np.dot(testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho))
dumbho = np.squeeze(dumbho)
print(str(dumbho) + ' Dumb HO (dual)')
dualrls_naive.solve(regparam)
predho1 = dualrls_naive.predictor.predict(testkm.T)
print(str(predho1) + ' Naive HO (dual)')
dualrls.solve(regparam)
predho2 = dualrls.holdout(hoindices)
print(str(predho2) + ' Fast HO (dual)')
dualrls.solve(regparam)
predho = dualrls.leave_one_out()[hoindices[0]]
print(str(predho) + ' Fast LOO (dual)')
primalrls_naive.solve(regparam)
predho3 = primalrls_naive.predictor.predict(testX)
print(str(predho3) + ' Naive HO (primal)')
primalrls.solve(regparam)
predho4 = primalrls.holdout(hoindices)
print(str(predho4) + ' Fast HO (primal)')
for predho in [predho1, predho2, predho3, predho4]:
self.assertEqual(dumbho.shape, predho.shape)
assert_allclose(dumbho, predho)
#for row in range(predho.shape[0]):
# for col in range(predho.shape[1]):
# self.assertAlmostEqual(dumbho[row,col],predho[row,col])
primalrls.solve(regparam)
predho = primalrls.leave_one_out()[hoindices[0]]
print(str(predho) + ' Fast LOO (primal)')
print()
hoindices = range(100, 300)
hocompl = list(set(range(m)) - set(hoindices))
Kho = K[np.ix_(hocompl, hocompl)]
Yho = Y[hocompl]
testkm = K[np.ix_(hocompl, hoindices)]
dumbho = np.dot(testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho))
kwargs = {}
kwargs['Y'] = Y
kwargs['X'] = Xtrain
dualrls.solve(regparam)
predho2 = dualrls.holdout(hoindices2)
print(str(predho2) + ' Fast HO')
hopred = dualrls.leave_pair_out(np.array([hoindices2[0], 4, 6]), np.array([hoindices2[1], 5, 7]))
print(str(hopred[0][0]) + '\n' + str(hopred[1][0]) + ' Fast LPO')
def testRLS(self):
print
print
print
print
print("Testing the cross-validation routines of the RLS module.")
print
print
floattype = np.float64
m, n = 400, 100
Xtrain = np.random.rand(m, n)
K = np.dot(Xtrain, Xtrain.T)
ylen = 2
Y = np.zeros((m, ylen), dtype=floattype)
Y = np.random.rand(m, ylen)
hoindices = [45]
hoindices2 = [45, 50]
hoindices3 = [45, 50, 55]
hocompl = list(set(range(m)) - set(hoindices))
Kho = K[np.ix_(hocompl, hocompl)]
Yho = Y[hocompl]
kwargs = {}
kwargs['Y'] = Y
kwargs['X'] = K
kwargs['kernel'] = 'PrecomputedKernel'
dualrls = RLS(**kwargs)
kwargs = {}
kwargs["X"] = Xtrain
kwargs["Y"] = Y
kwargs["bias"] = 0.
primalrls = RLS(**kwargs)
kwargs = {}
kwargs['Y'] = Yho
kwargs['X'] = Kho
kwargs['kernel'] = 'PrecomputedKernel'
dualrls_naive = RLS(**kwargs)
testkm = K[np.ix_(hocompl, hoindices)]
trainX = Xtrain[hocompl]
testX = Xtrain[hoindices]
kwargs = {}
kwargs['Y'] = Yho
kwargs['X'] = trainX
kwargs["bias"] = 0.
primalrls_naive = RLS(**kwargs)
loglambdas = range(-5, 5)
for j in range(0, len(loglambdas)):
regparam = 2.**loglambdas[j]
print
print("Regparam 2^%1d" % loglambdas[j])
dumbho = np.dot(
testkm.T,
np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])), Yho))
dumbho = np.squeeze(dumbho)
print(str(dumbho) + ' Dumb HO (dual)')
dualrls_naive.solve(regparam)
predho1 = dualrls_naive.predictor.predict(testkm.T)
print(str(predho1) + ' Naive HO (dual)')
dualrls.solve(regparam)
predho2 = dualrls.holdout(hoindices)
print(str(predho2) + ' Fast HO (dual)')
dualrls.solve(regparam)
predho = dualrls.leave_one_out()[hoindices[0]]
print(str(predho) + ' Fast LOO (dual)')
primalrls_naive.solve(regparam)
predho3 = primalrls_naive.predictor.predict(testX)
print(str(predho3) + ' Naive HO (primal)')
primalrls.solve(regparam)
predho4 = primalrls.holdout(hoindices)
print(str(predho4) + ' Fast HO (primal)')
for predho in [predho1, predho2, predho3, predho4]:
self.assertEqual(dumbho.shape, predho.shape)
assert_allclose(dumbho, predho)
#for row in range(predho.shape[0]):
# for col in range(predho.shape[1]):
# self.assertAlmostEqual(dumbho[row,col],predho[row,col])
primalrls.solve(regparam)
predho = primalrls.leave_one_out()[hoindices[0]]
print(str(predho) + ' Fast LOO (primal)')
print()
hoindices = range(100, 300)
hocompl = list(set(range(m)) - set(hoindices))
Kho = K[np.ix_(hocompl, hocompl)]
Yho = Y[hocompl]
testkm = K[np.ix_(hocompl, hoindices)]
dumbho = np.dot(
testkm.T, np.dot(la.inv(Kho + regparam * np.eye(Kho.shape[0])),
Yho))
kwargs = {}
kwargs['Y'] = Y
kwargs['X'] = Xtrain
dualrls.solve(regparam)
predho2 = dualrls.holdout(hoindices2)
print(str(predho2) + ' Fast HO')
hopred = dualrls.leave_pair_out(np.array([hoindices2[0], 4, 6]),
np.array([hoindices2[1], 5, 7]))
print(str(hopred[0][0]) + '\n' + str(hopred[1][0]) + ' Fast LPO')
