def test_kernel_ridge_regression2():
'''
We test the linear kernel case and gaussian kernel case
'''
n_feat = 10
n_rff_feat = 1000
X_train = np.ones([2, n_feat])
X_train[0, :] *= 1
X_train[0, :] *= 2
Y_train = np.ones([2, 1])
kernel = GaussianKernel(sigma=2.0)
kernel = RFF(n_rff_feat, n_feat, kernel)
use_cuda = torch.cuda.is_available()
kernel.torch(cuda=use_cuda)
reg_lambda = 1.0
regressor = KernelRidgeRegression(kernel, reg_lambda=reg_lambda)
if use_cuda:
regressor.fit(torch.cuda.DoubleTensor(X_train),
torch.cuda.DoubleTensor(Y_train))
else:
regressor.fit(torch.DoubleTensor(X_train), torch.DoubleTensor(Y_train))
# compare the two ways of calculating feature weights as sanity check
# feature weight using the approach inside KernelRidgeRegression
if use_cuda:
kernel.get_kernel_matrix(torch.cuda.DoubleTensor(X_train),
torch.cuda.DoubleTensor(X_train))
else:
kernel.get_kernel_matrix(torch.DoubleTensor(X_train),
torch.DoubleTensor(X_train))
# print kernel.rff_x2.size(), regressor.alpha.size()
w1 = torch.mm(torch.transpose(kernel.rff_x2, 0, 1), regressor.alpha)
# print w1.size()
# feature weight using alternative way of calculation
if use_cuda:
val = torch.inverse( (regressor.reg_lambda * torch.eye(n_rff_feat).double().cuda() \
+ torch.mm(torch.transpose(kernel.rff_x1, 0, 1), kernel.rff_x1) ) )
else:
val = torch.inverse( (regressor.reg_lambda * torch.eye(n_rff_feat).double() \
+ torch.mm(torch.transpose(kernel.rff_x1, 0, 1), kernel.rff_x1) ) )
val = torch.mm(val, torch.transpose(kernel.rff_x2, 0, 1))
if use_cuda:
w2 = torch.mm(val, torch.cuda.DoubleTensor(Y_train))
else:
w2 = torch.mm(val, torch.DoubleTensor(Y_train))
np.testing.assert_array_almost_equal(w1.cpu().numpy(), w2.cpu().numpy())
# print(w1.cpu().numpy().ravel()[-10:-1], w2.cpu().numpy().ravel()[-10:-1] )
print("kernel ridge regression test2 passed!")
def test_kernel_ridge_regression1():
'''
We test the linear kernel case and gaussian kernel case
'''
n_feat = 10
n_rff_feat = 1000000
X_train = np.ones([2, n_feat])
X_train[0, :] *= 1
X_train[0, :] *= 2
Y_train = np.ones([2, 1])
kernel = GaussianKernel(sigma=2.0)
kernel = RFF(n_rff_feat, n_feat, kernel)
use_cuda = torch.cuda.is_available()
kernel.torch(cuda=torch.cuda.is_available())
reg_lambda = 1.0
regressor = KernelRidgeRegression(kernel, reg_lambda=reg_lambda)
#regressor.torch(use_cuda=torch.cuda.is_available() )
if use_cuda:
regressor.fit(
torch.DoubleTensor(X_train).cuda(),
torch.DoubleTensor(Y_train).cuda())
else:
regressor.fit(torch.DoubleTensor(X_train), torch.DoubleTensor(Y_train))
regressor.torch(use_cuda=torch.cuda.is_available())
# if test data equals traing data, it should the same L2 error
X_test = np.copy(X_train)
Y_test = np.copy(Y_train)
if use_cuda:
test_pred = regressor.predict(torch.DoubleTensor(X_test).cuda())
else:
test_pred = regressor.predict(torch.DoubleTensor(X_test))
train_error = regressor.get_train_error()
if use_cuda:
test_error = regressor.get_test_error(
torch.DoubleTensor(Y_test).cuda())
else:
test_error = regressor.get_test_error(torch.DoubleTensor(Y_test))
assert np.abs(train_error - test_error) < 1e-6
# if test data is different from traing data, L2 error for train and test should be different
X_test = np.copy(X_train) * 2
Y_test = np.copy(Y_train)
if use_cuda:
test_pred = regressor.predict(torch.cuda.DoubleTensor(X_test))
else:
test_pred = regressor.predict(torch.DoubleTensor(X_test))
train_error = regressor.get_train_error()
if use_cuda:
test_error = regressor.get_test_error(torch.cuda.DoubleTensor(Y_test))
else:
test_error = regressor.get_test_error(torch.DoubleTensor(Y_test))
assert np.abs(train_error - test_error) >= 1e-3
X_test = np.copy(X_train)
Y_test = np.copy(Y_train) * 2
if use_cuda:
test_pred = regressor.predict(torch.cuda.DoubleTensor(X_test))
else:
test_pred = regressor.predict(torch.DoubleTensor(X_test))
train_error = regressor.get_train_error()
if use_cuda:
test_error = regressor.get_test_error(torch.cuda.DoubleTensor(Y_test))
else:
test_error = regressor.get_test_error(torch.DoubleTensor(Y_test))
assert np.abs(train_error - test_error) >= 1e-3
print("kernel ridge regression test1 passed!")
use_cuda=use_cuda,
for_lm_halp=((args.opt == "lm_halp_svrg")
or (args.opt == "lm_halp_sgd")))
print("feature quantization scale, bit ", quantizer.scale,
quantizer.nbit)
elif args.do_fp_feat == True:
print("fp circulant rff feature mode")
kernel_approx = CirculantRFF(args.n_feat,
n_input_feat,
kernel,
rand_seed=args.random_seed)
quantizer = None
else:
raise Exception(
"kernel approximation type not specified or not supported!")
kernel.torch(cuda=use_cuda)
kernel_approx.torch(cuda=use_cuda)
if args.fixed_design or args.closed_form_sol:
# for fixed design experiments and closed form solution form real setting
if args.fixed_design_auto_l2_reg:
# get kernel matrix and get the decomposition
assert isinstance(X_train, torch.DoubleTensor)
print("fixed design lambda calculation using kernel ",
type(kernel_approx))
kernel_mat = kernel_approx.get_kernel_matrix(
X_train, X_train, quantizer, quantizer)
assert isinstance(kernel_mat, torch.DoubleTensor)
U, S, _ = np.linalg.svd(kernel_mat.cpu().numpy().astype(
np.float64))
# numerically figure out the best lambda in the fixed design setting