def test_ensemble_nystrom_full_prec_three_learner():
# test if keep all the dimensions is the nystrom kernel matrix equals to the exact kernel
n_sample = 150
n_feat = n_sample
input_val1 = torch.DoubleTensor(
np.random.normal(size=[n_sample, n_feat])).double()
input_val2 = input_val1
# input_val2 = torch.DoubleTensor(np.random.normal(size=[n_sample - 1, n_feat] ) ).double()
# get exact gaussian kernel
kernel = GaussianKernel(sigma=10.0)
kernel_mat = kernel.get_kernel_matrix(input_val1, input_val2)
# nystrom method
approx = Nystrom(n_feat, kernel=kernel)
approx.setup(input_val1)
feat = approx.get_feat(input_val1)
approx_kernel_mat = approx.get_kernel_matrix(input_val1, input_val2)
# ensembleed nystrom method
approx_ensemble = EnsembleNystrom(n_feat, n_learner=3, kernel=kernel)
approx_ensemble.setup(input_val1)
feat_ensemble = approx_ensemble.get_feat(input_val1)
assert feat_ensemble.size(0) == n_sample
assert feat_ensemble.size(1) == n_feat
approx_kernel_mat_ensemble = approx_ensemble.get_kernel_matrix(
input_val1, input_val2)
print("single learner ensembled nystrom test passed!")
def test_ensemble_nystrom_full_prec_one_learner():
# test if keep all the dimensions is the nystrom kernel matrix equals to the exact kernel
n_sample = 150
n_feat = n_sample
input_val1 = torch.DoubleTensor(
np.random.normal(size=[n_sample, n_feat])).double()
input_val2 = input_val1
# input_val2 = torch.DoubleTensor(np.random.normal(size=[n_sample - 1, n_feat] ) ).double()
# get exact gaussian kernel
kernel = GaussianKernel(sigma=10.0)
kernel_mat = kernel.get_kernel_matrix(input_val1, input_val2)
# nystrom method
approx = Nystrom(n_feat, kernel=kernel)
approx.setup(input_val1)
feat = approx.get_feat(input_val1)
approx_kernel_mat = approx.get_kernel_matrix(input_val1, input_val2)
# ensembleed nystrom method
approx_ensemble = EnsembleNystrom(n_feat, n_learner=1, kernel=kernel)
approx_ensemble.setup(input_val1)
feat_ensemble = approx_ensemble.get_feat(input_val1)
approx_kernel_mat_ensemble = approx_ensemble.get_kernel_matrix(
input_val1, input_val2)
np.testing.assert_array_almost_equal(
np.sum(feat.cpu().numpy()**2), np.sum(feat_ensemble.cpu().numpy()**2))
np.testing.assert_array_almost_equal(
np.sum(approx_kernel_mat.cpu().numpy()**2),
np.sum(approx_kernel_mat_ensemble.cpu().numpy()**2))
print("single learner ensembled nystrom test passed!")
def test_pytorch_gaussian_kernel():
n_feat = 10
input_val = np.ones([2, n_feat])
input_val[0, :] *= 1
input_val[0, :] *= 2
# get exact gaussian kernel
kernel = GaussianKernel(sigma=2.0)
kernel_mat = kernel.get_kernel_matrix(input_val, input_val)
kernel_mat_torch = kernel.get_kernel_matrix(torch.Tensor(input_val),
torch.Tensor(input_val))
np.testing.assert_array_almost_equal(kernel_mat.cpu().numpy(),
kernel_mat_torch.cpu().numpy())
print("gaussian kernel pytorch version test passed!")
def test_circulant_rff():
'''
test if the circulant structure is observed in the circulant RFF object
'''
n_feat = 1000
n_rff_feat = 1000
seed = 2
input_val = torch.DoubleTensor(np.ones([100, n_feat]))
kernel = GaussianKernel(sigma=5.0)
kernel_cir = CirculantRFF(n_rff_feat,
n_feat,
kernel=kernel,
rand_seed=seed)
kernel_basic = RFF(n_rff_feat, n_feat, kernel=kernel, rand_seed=seed)
kernel_cir.torch()
kernel_basic.torch()
print("should see column circulant structure", kernel_cir.w.cpu().numpy())
np.testing.assert_array_almost_equal(np.abs(kernel_cir.b.cpu().numpy()),
np.abs(kernel_basic.b.cpu().numpy()))
# np.testing.assert_array_almost_equal(np.abs(kernel_cir.w.cpu().numpy() ),
# np.abs(kernel_basic.w.cpu().numpy() ) )
print("should see similar row std between basic rff and circulant rff",
np.std(np.abs(kernel_cir.w.cpu().numpy()), axis=1)[:10],
np.std(np.abs(kernel_basic.w.cpu().numpy()), axis=1)[:10])
print("circulant rff test passed!")
def test_rff_generation():
n_feat = 10
n_rff_feat = 1000000
input_val = np.ones([2, n_feat])
input_val[0, :] *= 1
input_val[0, :] *= 2
# get exact gaussian kernel
kernel = GaussianKernel(sigma=2.0)
kernel_mat = kernel.get_kernel_matrix(input_val, input_val)
# get RFF approximate kernel matrix
rff = RFF(n_rff_feat, n_feat, kernel=kernel)
rff.get_gaussian_wb()
approx_kernel_mat = rff.get_kernel_matrix(input_val, input_val)
np.testing.assert_array_almost_equal(approx_kernel_mat.cpu().numpy(),
kernel_mat.cpu().numpy(),
decimal=3)
print("rff generation test passed!")
def test_ensemble_nystrom_low_prec():
# test if keep all the dimensions is the nystrom kernel matrix equals to the exact kernel
n_sample = 150
n_feat = n_sample
input_val1 = torch.DoubleTensor(
np.random.normal(size=[n_sample, n_feat])).double()
input_val2 = input_val1
# input_val2 = torch.DoubleTensor(np.random.normal(size=[n_sample - 1, n_feat] ) ).double()
# get exact gaussian kernel
kernel = GaussianKernel(sigma=10.0)
kernel_mat = kernel.get_kernel_matrix(input_val1, input_val2)
# setup quantizer
quantizer = Quantizer(4,
torch.min(input_val1),
torch.max(input_val1),
rand_seed=2,
use_cuda=False)
# nystrom method
approx = Nystrom(n_feat, kernel=kernel)
approx.setup(input_val1)
feat = approx.get_feat(input_val1)
approx_kernel_mat = approx.get_kernel_matrix(input_val1, input_val2,
quantizer, quantizer)
# ensembleed nystrom method
approx_ensemble = EnsembleNystrom(n_feat, n_learner=1, kernel=kernel)
approx_ensemble.setup(input_val1)
feat_ensemble = approx_ensemble.get_feat(input_val1)
approx_kernel_mat_ensemble = approx_ensemble.get_kernel_matrix(
input_val1,
input_val2,
quantizer,
quantizer,
consistent_quant_seed=True)
approx_kernel_mat_ensemble = approx_ensemble.get_kernel_matrix(
input_val1,
input_val2,
quantizer,
quantizer,
consistent_quant_seed=True)
print("single learner ensembled nystrom quantizerd version test passed!")
def test_nystrom_full():
# test if keep all the dimensions is the nystrom kernel matrix equals to the exact kernel
n_sample = 15
n_feat = n_sample
input_val1 = torch.Tensor(
np.random.normal(size=[n_sample, n_feat])).double()
input_val2 = torch.Tensor(np.random.normal(size=[n_sample -
1, n_feat])).double()
# get exact gaussian kernel
kernel = GaussianKernel(sigma=np.random.normal())
kernel_mat = kernel.get_kernel_matrix(input_val1, input_val2)
approx = Nystrom(n_feat, kernel=kernel)
approx.setup(input_val1)
approx_kernel_mat = approx.get_kernel_matrix(input_val1, input_val2)
np.testing.assert_array_almost_equal(kernel_mat.cpu().numpy(),
approx_kernel_mat.cpu().numpy())
print("nystrom full dimension test passed!")
def test_kernel_ridge_regression2():
'''
We test the linear kernel case and gaussian kernel case
'''
n_feat = 10
n_rff_feat = 1000
X_train = np.ones([2, n_feat])
X_train[0, :] *= 1
X_train[0, :] *= 2
Y_train = np.ones([2, 1])
kernel = GaussianKernel(sigma=2.0)
kernel = RFF(n_rff_feat, n_feat, kernel)
use_cuda = torch.cuda.is_available()
kernel.torch(cuda=use_cuda)
reg_lambda = 1.0
regressor = KernelRidgeRegression(kernel, reg_lambda=reg_lambda)
if use_cuda:
regressor.fit(torch.cuda.DoubleTensor(X_train),
torch.cuda.DoubleTensor(Y_train))
else:
regressor.fit(torch.DoubleTensor(X_train), torch.DoubleTensor(Y_train))
# compare the two ways of calculating feature weights as sanity check
# feature weight using the approach inside KernelRidgeRegression
if use_cuda:
kernel.get_kernel_matrix(torch.cuda.DoubleTensor(X_train),
torch.cuda.DoubleTensor(X_train))
else:
kernel.get_kernel_matrix(torch.DoubleTensor(X_train),
torch.DoubleTensor(X_train))
# print kernel.rff_x2.size(), regressor.alpha.size()
w1 = torch.mm(torch.transpose(kernel.rff_x2, 0, 1), regressor.alpha)
# print w1.size()
# feature weight using alternative way of calculation
if use_cuda:
val = torch.inverse( (regressor.reg_lambda * torch.eye(n_rff_feat).double().cuda() \
+ torch.mm(torch.transpose(kernel.rff_x1, 0, 1), kernel.rff_x1) ) )
else:
val = torch.inverse( (regressor.reg_lambda * torch.eye(n_rff_feat).double() \
+ torch.mm(torch.transpose(kernel.rff_x1, 0, 1), kernel.rff_x1) ) )
val = torch.mm(val, torch.transpose(kernel.rff_x2, 0, 1))
if use_cuda:
w2 = torch.mm(val, torch.cuda.DoubleTensor(Y_train))
else:
w2 = torch.mm(val, torch.DoubleTensor(Y_train))
np.testing.assert_array_almost_equal(w1.cpu().numpy(), w2.cpu().numpy())
# print(w1.cpu().numpy().ravel()[-10:-1], w2.cpu().numpy().ravel()[-10:-1] )
print("kernel ridge regression test2 passed!")
def test_rff_generation2():
n_feat = 10
n_rff_feat = 1000000
input_val = np.ones([2, n_feat])
input_val[0, :] *= 1
input_val[0, :] *= 2
# get exact gaussian kernel
kernel = GaussianKernel(sigma=2.0)
# kernel_mat = kernel.get_kernel_matrix(input_val, input_val)
# get RFF approximate kernel matrix
rff = RFF(n_rff_feat, n_feat, kernel=kernel)
rff.get_gaussian_wb()
approx_kernel_mat = rff.get_kernel_matrix(input_val, input_val)
rff.torch(cuda=False)
approx_kernel_mat2 = rff.get_kernel_matrix(torch.DoubleTensor(input_val),
torch.DoubleTensor(input_val))
np.testing.assert_array_almost_equal(approx_kernel_mat.cpu().numpy(),
approx_kernel_mat2.cpu().numpy(),
decimal=6)
print("rff generation test 2 passed!")
def test_kernel_ridge_regression1():
'''
We test the linear kernel case and gaussian kernel case
'''
n_feat = 10
n_rff_feat = 1000000
X_train = np.ones([2, n_feat])
X_train[0, :] *= 1
X_train[0, :] *= 2
Y_train = np.ones([2, 1])
kernel = GaussianKernel(sigma=2.0)
kernel = RFF(n_rff_feat, n_feat, kernel)
use_cuda = torch.cuda.is_available()
kernel.torch(cuda=torch.cuda.is_available())
reg_lambda = 1.0
regressor = KernelRidgeRegression(kernel, reg_lambda=reg_lambda)
#regressor.torch(use_cuda=torch.cuda.is_available() )
if use_cuda:
regressor.fit(
torch.DoubleTensor(X_train).cuda(),
torch.DoubleTensor(Y_train).cuda())
else:
regressor.fit(torch.DoubleTensor(X_train), torch.DoubleTensor(Y_train))
regressor.torch(use_cuda=torch.cuda.is_available())
# if test data equals traing data, it should the same L2 error
X_test = np.copy(X_train)
Y_test = np.copy(Y_train)
if use_cuda:
test_pred = regressor.predict(torch.DoubleTensor(X_test).cuda())
else:
test_pred = regressor.predict(torch.DoubleTensor(X_test))
train_error = regressor.get_train_error()
if use_cuda:
test_error = regressor.get_test_error(
torch.DoubleTensor(Y_test).cuda())
else:
test_error = regressor.get_test_error(torch.DoubleTensor(Y_test))
assert np.abs(train_error - test_error) < 1e-6
# if test data is different from traing data, L2 error for train and test should be different
X_test = np.copy(X_train) * 2
Y_test = np.copy(Y_train)
if use_cuda:
test_pred = regressor.predict(torch.cuda.DoubleTensor(X_test))
else:
test_pred = regressor.predict(torch.DoubleTensor(X_test))
train_error = regressor.get_train_error()
if use_cuda:
test_error = regressor.get_test_error(torch.cuda.DoubleTensor(Y_test))
else:
test_error = regressor.get_test_error(torch.DoubleTensor(Y_test))
assert np.abs(train_error - test_error) >= 1e-3
X_test = np.copy(X_train)
Y_test = np.copy(Y_train) * 2
if use_cuda:
test_pred = regressor.predict(torch.cuda.DoubleTensor(X_test))
else:
test_pred = regressor.predict(torch.DoubleTensor(X_test))
train_error = regressor.get_train_error()
if use_cuda:
test_error = regressor.get_test_error(torch.cuda.DoubleTensor(Y_test))
else:
test_error = regressor.get_test_error(torch.DoubleTensor(Y_test))
assert np.abs(train_error - test_error) >= 1e-3
print("kernel ridge regression test1 passed!")
# setup dataloader
train_data = \
torch.utils.data.TensorDataset(X_train, Y_train)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=args.minibatch,
shuffle=False)
val_data = \
torch.utils.data.TensorDataset(X_val, Y_val)
val_loader = torch.utils.data.DataLoader(val_data,
batch_size=args.minibatch,
shuffle=False)
# setup gaussian kernel
n_input_feat = X_train.shape[1]
kernel = GaussianKernel(sigma=args.kernel_sigma)
if args.approx_type == "exact":
print("exact kernel mode")
# raise Exception("SGD based exact kernel is not implemented yet!")
kernel_approx = kernel
quantizer = None
elif args.approx_type == "nystrom":
print("fp nystrom mode")
kernel_approx = Nystrom(args.n_feat,
kernel=kernel,
rand_seed=args.random_seed)
kernel_approx.setup(X_train)
quantizer = None
elif args.approx_type == "ensemble_nystrom":
print("ensembled nystrom mode with ", args.n_ensemble_nystrom,
"learner")