def test_ensemble_nystrom_full_prec_three_learner(): # test if keep all the dimensions is the nystrom kernel matrix equals to the exact kernel n_sample = 150 n_feat = n_sample input_val1 = torch.DoubleTensor( np.random.normal(size=[n_sample, n_feat])).double() input_val2 = input_val1 # input_val2 = torch.DoubleTensor(np.random.normal(size=[n_sample - 1, n_feat] ) ).double() # get exact gaussian kernel kernel = GaussianKernel(sigma=10.0) kernel_mat = kernel.get_kernel_matrix(input_val1, input_val2) # nystrom method approx = Nystrom(n_feat, kernel=kernel) approx.setup(input_val1) feat = approx.get_feat(input_val1) approx_kernel_mat = approx.get_kernel_matrix(input_val1, input_val2) # ensembleed nystrom method approx_ensemble = EnsembleNystrom(n_feat, n_learner=3, kernel=kernel) approx_ensemble.setup(input_val1) feat_ensemble = approx_ensemble.get_feat(input_val1) assert feat_ensemble.size(0) == n_sample assert feat_ensemble.size(1) == n_feat approx_kernel_mat_ensemble = approx_ensemble.get_kernel_matrix( input_val1, input_val2) print("single learner ensembled nystrom test passed!")
def test_ensemble_nystrom_full_prec_one_learner(): # test if keep all the dimensions is the nystrom kernel matrix equals to the exact kernel n_sample = 150 n_feat = n_sample input_val1 = torch.DoubleTensor( np.random.normal(size=[n_sample, n_feat])).double() input_val2 = input_val1 # input_val2 = torch.DoubleTensor(np.random.normal(size=[n_sample - 1, n_feat] ) ).double() # get exact gaussian kernel kernel = GaussianKernel(sigma=10.0) kernel_mat = kernel.get_kernel_matrix(input_val1, input_val2) # nystrom method approx = Nystrom(n_feat, kernel=kernel) approx.setup(input_val1) feat = approx.get_feat(input_val1) approx_kernel_mat = approx.get_kernel_matrix(input_val1, input_val2) # ensembleed nystrom method approx_ensemble = EnsembleNystrom(n_feat, n_learner=1, kernel=kernel) approx_ensemble.setup(input_val1) feat_ensemble = approx_ensemble.get_feat(input_val1) approx_kernel_mat_ensemble = approx_ensemble.get_kernel_matrix( input_val1, input_val2) np.testing.assert_array_almost_equal( np.sum(feat.cpu().numpy()**2), np.sum(feat_ensemble.cpu().numpy()**2)) np.testing.assert_array_almost_equal( np.sum(approx_kernel_mat.cpu().numpy()**2), np.sum(approx_kernel_mat_ensemble.cpu().numpy()**2)) print("single learner ensembled nystrom test passed!")
def test_pytorch_gaussian_kernel(): n_feat = 10 input_val = np.ones([2, n_feat]) input_val[0, :] *= 1 input_val[0, :] *= 2 # get exact gaussian kernel kernel = GaussianKernel(sigma=2.0) kernel_mat = kernel.get_kernel_matrix(input_val, input_val) kernel_mat_torch = kernel.get_kernel_matrix(torch.Tensor(input_val), torch.Tensor(input_val)) np.testing.assert_array_almost_equal(kernel_mat.cpu().numpy(), kernel_mat_torch.cpu().numpy()) print("gaussian kernel pytorch version test passed!")
def test_circulant_rff(): ''' test if the circulant structure is observed in the circulant RFF object ''' n_feat = 1000 n_rff_feat = 1000 seed = 2 input_val = torch.DoubleTensor(np.ones([100, n_feat])) kernel = GaussianKernel(sigma=5.0) kernel_cir = CirculantRFF(n_rff_feat, n_feat, kernel=kernel, rand_seed=seed) kernel_basic = RFF(n_rff_feat, n_feat, kernel=kernel, rand_seed=seed) kernel_cir.torch() kernel_basic.torch() print("should see column circulant structure", kernel_cir.w.cpu().numpy()) np.testing.assert_array_almost_equal(np.abs(kernel_cir.b.cpu().numpy()), np.abs(kernel_basic.b.cpu().numpy())) # np.testing.assert_array_almost_equal(np.abs(kernel_cir.w.cpu().numpy() ), # np.abs(kernel_basic.w.cpu().numpy() ) ) print("should see similar row std between basic rff and circulant rff", np.std(np.abs(kernel_cir.w.cpu().numpy()), axis=1)[:10], np.std(np.abs(kernel_basic.w.cpu().numpy()), axis=1)[:10]) print("circulant rff test passed!")
def test_rff_generation(): n_feat = 10 n_rff_feat = 1000000 input_val = np.ones([2, n_feat]) input_val[0, :] *= 1 input_val[0, :] *= 2 # get exact gaussian kernel kernel = GaussianKernel(sigma=2.0) kernel_mat = kernel.get_kernel_matrix(input_val, input_val) # get RFF approximate kernel matrix rff = RFF(n_rff_feat, n_feat, kernel=kernel) rff.get_gaussian_wb() approx_kernel_mat = rff.get_kernel_matrix(input_val, input_val) np.testing.assert_array_almost_equal(approx_kernel_mat.cpu().numpy(), kernel_mat.cpu().numpy(), decimal=3) print("rff generation test passed!")
def test_ensemble_nystrom_low_prec(): # test if keep all the dimensions is the nystrom kernel matrix equals to the exact kernel n_sample = 150 n_feat = n_sample input_val1 = torch.DoubleTensor( np.random.normal(size=[n_sample, n_feat])).double() input_val2 = input_val1 # input_val2 = torch.DoubleTensor(np.random.normal(size=[n_sample - 1, n_feat] ) ).double() # get exact gaussian kernel kernel = GaussianKernel(sigma=10.0) kernel_mat = kernel.get_kernel_matrix(input_val1, input_val2) # setup quantizer quantizer = Quantizer(4, torch.min(input_val1), torch.max(input_val1), rand_seed=2, use_cuda=False) # nystrom method approx = Nystrom(n_feat, kernel=kernel) approx.setup(input_val1) feat = approx.get_feat(input_val1) approx_kernel_mat = approx.get_kernel_matrix(input_val1, input_val2, quantizer, quantizer) # ensembleed nystrom method approx_ensemble = EnsembleNystrom(n_feat, n_learner=1, kernel=kernel) approx_ensemble.setup(input_val1) feat_ensemble = approx_ensemble.get_feat(input_val1) approx_kernel_mat_ensemble = approx_ensemble.get_kernel_matrix( input_val1, input_val2, quantizer, quantizer, consistent_quant_seed=True) approx_kernel_mat_ensemble = approx_ensemble.get_kernel_matrix( input_val1, input_val2, quantizer, quantizer, consistent_quant_seed=True) print("single learner ensembled nystrom quantizerd version test passed!")
def test_nystrom_full(): # test if keep all the dimensions is the nystrom kernel matrix equals to the exact kernel n_sample = 15 n_feat = n_sample input_val1 = torch.Tensor( np.random.normal(size=[n_sample, n_feat])).double() input_val2 = torch.Tensor(np.random.normal(size=[n_sample - 1, n_feat])).double() # get exact gaussian kernel kernel = GaussianKernel(sigma=np.random.normal()) kernel_mat = kernel.get_kernel_matrix(input_val1, input_val2) approx = Nystrom(n_feat, kernel=kernel) approx.setup(input_val1) approx_kernel_mat = approx.get_kernel_matrix(input_val1, input_val2) np.testing.assert_array_almost_equal(kernel_mat.cpu().numpy(), approx_kernel_mat.cpu().numpy()) print("nystrom full dimension test passed!")
def test_kernel_ridge_regression2(): ''' We test the linear kernel case and gaussian kernel case ''' n_feat = 10 n_rff_feat = 1000 X_train = np.ones([2, n_feat]) X_train[0, :] *= 1 X_train[0, :] *= 2 Y_train = np.ones([2, 1]) kernel = GaussianKernel(sigma=2.0) kernel = RFF(n_rff_feat, n_feat, kernel) use_cuda = torch.cuda.is_available() kernel.torch(cuda=use_cuda) reg_lambda = 1.0 regressor = KernelRidgeRegression(kernel, reg_lambda=reg_lambda) if use_cuda: regressor.fit(torch.cuda.DoubleTensor(X_train), torch.cuda.DoubleTensor(Y_train)) else: regressor.fit(torch.DoubleTensor(X_train), torch.DoubleTensor(Y_train)) # compare the two ways of calculating feature weights as sanity check # feature weight using the approach inside KernelRidgeRegression if use_cuda: kernel.get_kernel_matrix(torch.cuda.DoubleTensor(X_train), torch.cuda.DoubleTensor(X_train)) else: kernel.get_kernel_matrix(torch.DoubleTensor(X_train), torch.DoubleTensor(X_train)) # print kernel.rff_x2.size(), regressor.alpha.size() w1 = torch.mm(torch.transpose(kernel.rff_x2, 0, 1), regressor.alpha) # print w1.size() # feature weight using alternative way of calculation if use_cuda: val = torch.inverse( (regressor.reg_lambda * torch.eye(n_rff_feat).double().cuda() \ + torch.mm(torch.transpose(kernel.rff_x1, 0, 1), kernel.rff_x1) ) ) else: val = torch.inverse( (regressor.reg_lambda * torch.eye(n_rff_feat).double() \ + torch.mm(torch.transpose(kernel.rff_x1, 0, 1), kernel.rff_x1) ) ) val = torch.mm(val, torch.transpose(kernel.rff_x2, 0, 1)) if use_cuda: w2 = torch.mm(val, torch.cuda.DoubleTensor(Y_train)) else: w2 = torch.mm(val, torch.DoubleTensor(Y_train)) np.testing.assert_array_almost_equal(w1.cpu().numpy(), w2.cpu().numpy()) # print(w1.cpu().numpy().ravel()[-10:-1], w2.cpu().numpy().ravel()[-10:-1] ) print("kernel ridge regression test2 passed!")
def test_rff_generation2(): n_feat = 10 n_rff_feat = 1000000 input_val = np.ones([2, n_feat]) input_val[0, :] *= 1 input_val[0, :] *= 2 # get exact gaussian kernel kernel = GaussianKernel(sigma=2.0) # kernel_mat = kernel.get_kernel_matrix(input_val, input_val) # get RFF approximate kernel matrix rff = RFF(n_rff_feat, n_feat, kernel=kernel) rff.get_gaussian_wb() approx_kernel_mat = rff.get_kernel_matrix(input_val, input_val) rff.torch(cuda=False) approx_kernel_mat2 = rff.get_kernel_matrix(torch.DoubleTensor(input_val), torch.DoubleTensor(input_val)) np.testing.assert_array_almost_equal(approx_kernel_mat.cpu().numpy(), approx_kernel_mat2.cpu().numpy(), decimal=6) print("rff generation test 2 passed!")
def test_kernel_ridge_regression1(): ''' We test the linear kernel case and gaussian kernel case ''' n_feat = 10 n_rff_feat = 1000000 X_train = np.ones([2, n_feat]) X_train[0, :] *= 1 X_train[0, :] *= 2 Y_train = np.ones([2, 1]) kernel = GaussianKernel(sigma=2.0) kernel = RFF(n_rff_feat, n_feat, kernel) use_cuda = torch.cuda.is_available() kernel.torch(cuda=torch.cuda.is_available()) reg_lambda = 1.0 regressor = KernelRidgeRegression(kernel, reg_lambda=reg_lambda) #regressor.torch(use_cuda=torch.cuda.is_available() ) if use_cuda: regressor.fit( torch.DoubleTensor(X_train).cuda(), torch.DoubleTensor(Y_train).cuda()) else: regressor.fit(torch.DoubleTensor(X_train), torch.DoubleTensor(Y_train)) regressor.torch(use_cuda=torch.cuda.is_available()) # if test data equals traing data, it should the same L2 error X_test = np.copy(X_train) Y_test = np.copy(Y_train) if use_cuda: test_pred = regressor.predict(torch.DoubleTensor(X_test).cuda()) else: test_pred = regressor.predict(torch.DoubleTensor(X_test)) train_error = regressor.get_train_error() if use_cuda: test_error = regressor.get_test_error( torch.DoubleTensor(Y_test).cuda()) else: test_error = regressor.get_test_error(torch.DoubleTensor(Y_test)) assert np.abs(train_error - test_error) < 1e-6 # if test data is different from traing data, L2 error for train and test should be different X_test = np.copy(X_train) * 2 Y_test = np.copy(Y_train) if use_cuda: test_pred = regressor.predict(torch.cuda.DoubleTensor(X_test)) else: test_pred = regressor.predict(torch.DoubleTensor(X_test)) train_error = regressor.get_train_error() if use_cuda: test_error = regressor.get_test_error(torch.cuda.DoubleTensor(Y_test)) else: test_error = regressor.get_test_error(torch.DoubleTensor(Y_test)) assert np.abs(train_error - test_error) >= 1e-3 X_test = np.copy(X_train) Y_test = np.copy(Y_train) * 2 if use_cuda: test_pred = regressor.predict(torch.cuda.DoubleTensor(X_test)) else: test_pred = regressor.predict(torch.DoubleTensor(X_test)) train_error = regressor.get_train_error() if use_cuda: test_error = regressor.get_test_error(torch.cuda.DoubleTensor(Y_test)) else: test_error = regressor.get_test_error(torch.DoubleTensor(Y_test)) assert np.abs(train_error - test_error) >= 1e-3 print("kernel ridge regression test1 passed!")
# setup dataloader train_data = \ torch.utils.data.TensorDataset(X_train, Y_train) train_loader = torch.utils.data.DataLoader(train_data, batch_size=args.minibatch, shuffle=False) val_data = \ torch.utils.data.TensorDataset(X_val, Y_val) val_loader = torch.utils.data.DataLoader(val_data, batch_size=args.minibatch, shuffle=False) # setup gaussian kernel n_input_feat = X_train.shape[1] kernel = GaussianKernel(sigma=args.kernel_sigma) if args.approx_type == "exact": print("exact kernel mode") # raise Exception("SGD based exact kernel is not implemented yet!") kernel_approx = kernel quantizer = None elif args.approx_type == "nystrom": print("fp nystrom mode") kernel_approx = Nystrom(args.n_feat, kernel=kernel, rand_seed=args.random_seed) kernel_approx.setup(X_train) quantizer = None elif args.approx_type == "ensemble_nystrom": print("ensembled nystrom mode with ", args.n_ensemble_nystrom, "learner")