def __init__(self, kernel, gamma=1.0, nmax=1000): super(BaseKernelReg, self).__init__(nmax) if kernel == 'rbf': self.kernel = lambda X, Y: rbf_kernel(X, Y, gamma) else: raise NotImplementedError('Kernel not supported')
def validation(train, train_r, val, val_r, test, test_r, perm_matrices, T): hyperparameters_c = [ 0.00009, 0.0004, 0.0005, 0.0006, 0.00077, 0.0008, 0.0009, 0.001 ] hyperparameters_l = [ 0.005, 0.006, 0.007, 0.008, 0.01, 0.02, 0.03, 0.05, 0.06, 0.07, 0.08 ] shape_1 = train.shape best_AP = 0 for c in hyperparameters_c: start_time = time.time() for l in hyperparameters_l: train_reshape = np.reshape(train, (shape_1[0], shape_1[1] * shape_1[2])) K = kernel.rbf_kernel(train_reshape, train_reshape, c) K_inv, gamma_train = training_MAP(train, train_r, K, l) AP = test_proc(train, val, val_r, K_inv, gamma_train, perm_matrices, c, T) if AP > best_AP: best_AP = AP l_opt = l c_opt = c print(best_AP, 'best_AP_so far', c_opt, l_opt) end_time = time.time() print("total time taken this loop: ", end_time - start_time) print(l_opt, c_opt)
def __init__(self, kernel, gamma=1.0, nmax=1000, max_sv=0.5): super(BaseKernelReg, self).__init__(nmax, max_sv) if kernel == 'rbf': self.kernel = lambda X, Y: rbf_kernel(X, Y, gamma) else: raise NotImplementedError('Kernel not supported')
def Stein_sampler(model, n_particles, max_iter, step_size, seed = 44, adagrad = True, kernel_type = 'mixture'): np.random.seed(seed) d = model.dimension sig = 1.5 x_initial = sig * np.random.randn(n_particles, d) # set initial particles as standard normal x = x_initial EVOLVEX = np.zeros([max_iter+1, n_particles, d]) EVOLVEX[0,:,:] = x adag = np.zeros([n_particles, d]) # for adagrad for i in range(max_iter): grad_logp = model.grad_log_p(x) # gradient information for each particles: n*d Hs = model.Hessian_log_p(x) # Hessian information for each particles: n*d*d A = np.mean(Hs, axis = 0) # Average Hessian: d*d if kernel_type == 'newton': # SVN v = SVN(x, grad_logp, Hs) elif kernel_type == 'mixture': # matrix SVGD(mixture) v = mixture_hessian_SVGD(x, grad_logp, Hs) else: if kernel_type == 'gaussian': # matrix SVGD(average) kernel = gaussian_kernel(A) B = model.inv_avg_Hessian(A) else: # vanilla SVGD if i > 30: kernel = rbf_kernel(d, decay = True) else: kernel = rbf_kernel(d) B = np.eye(d) v = matrix_SVGD(x, grad_logp, kernel, B) adag += v ** 2 # update sum of gradient's square if adagrad: x = x + step_size * v / np.sqrt(adag + 1e-12) else: x = x + step_size * v EVOLVEX[i+1,:,:] = x return EVOLVEX
def __sparse_features(self, X, sigma, rbf_kernel_type='conv'): """ :param X: inputs of size (N,3) :return: hinged features with intercept of size (N, # of features + 1) """ if rbf_kernel_type == 'conv': rbf_features = rbf_kernel_conv(X, self.grid, gamma=self.gamma, sigma=sigma, device=self.device) elif rbf_kernel_type == 'wass': rbf_features = rbf_kernel_wasserstein(X, self.grid, gamma=self.gamma, sigma=sigma, device=self.device) else: rbf_features = rbf_kernel(X, self.grid, gamma=self.gamma) return rbf_features
def check_rbf_kernel(): ex_name = "RBF kernel" n, m, d = 3, 5, 7 gamma = 0.5 X = np.random.random((n, d)) Y = np.random.random((m, d)) try: K = kernel.rbf_kernel(X, Y, gamma) except NotImplementedError: log(red("FAIL"), ex_name, ": not implemented") return True for i in range(n): for j in range(m): exp = np.exp(-gamma * (np.linalg.norm(X[i] - Y[j])**2)) got = K[i][j] if (not equals(exp, got)): log( red("FAIL"), ex_name, ": values at ({}, {}) do not match. Expected {}, got {}". format(i, j, exp, got)) log(green("PASS"), ex_name, "")
def test_proc(train, test, test_r, K_inv, gamma_train, perm_matrices, c, T): """ Tests on all test data points for a bias c, and training input K_inv, gamma_train (trained with l) :param train_r: matrix of training rel labels, padded with zeros np_array (nr_queries x nr_documents x dim_feature) :param test: matrix test data, padded with zeros np_array (nr_queries x nr_documents x dim_feature) :param test_r: matrix test rel labels , padded with zeros np_array (nr_queries x nr_documents x dim_feature) :param K_inv: (K + nlI)^{-1} (nr_documents x nr_documents) :param gamma_train: array of all gamma(r^{i}) (nr_queries x nr_documents x nr_documents) :param perm_matrices: All possible permutation matrices with dim nr_documents :param c: bias for linear kernel :param T: Upper triangle matrix defined for the MAP :return: Returns the mean average precision. """ nr_test_queries = test_r.shape[0] shape_1 = train.shape AP = 0 j = 0 for i in range(nr_test_queries): j += 1 x = np.expand_dims(test[i, :, :], axis=0) shape_x = x.shape x = np.reshape(x, (shape_x[0], shape_x[1] * shape_x[2])) train_reshape = np.reshape(train, (shape_1[0], shape_1[1] * shape_1[2])) Kx = kernel.rbf_kernel(train_reshape, x, c) prediction = g_MAP(train, Kx, K_inv, gamma_train) (value, optimal_perm) = train_max(prediction, T, perm_matrices) r = gamma_MAP(test_r[i, :], k=10) r[r > 1] == 1 AP = AP * (j - 1) / j + objective_fun(r, T, optimal_perm) / j return AP
train_r = np.concatenate((train_r, val_r), axis=0) shape_train = train.shape #FOR TRAINING train = np.reshape(train, (shape_train[0] * shape_train[1], shape_train[2])) scaler = preprocessing.StandardScaler().fit(train) train = scaler.transform(train) train = np.reshape(train, (shape_train[0], shape_train[1], shape_train[2])) train_reshape = np.reshape(train, (shape_train[0], shape_train[1] * shape_train[2])) shape_test = test.shape # FOR TESTING test = np.reshape(test, (shape_test[0] * shape_test[1], shape_test[2])) test = scaler.transform(test) test = np.reshape(test, (shape_test[0], shape_test[1], shape_test[2])) K = kernel.rbf_kernel(train_reshape, gamma=c) K_inv, gamma_train = training_MAP(train, train_r, K, l) test_proc(train, test, test_r, K_inv, gamma_train, perm_matrices, c, T) def validation(train, train_r, val, val_r, test, test_r, perm_matrices, T): hyperparameters_c = [ 0.00009, 0.0004, 0.0005, 0.0006, 0.00077, 0.0008, 0.0009, 0.001 ] hyperparameters_l = [ 0.005, 0.006, 0.007, 0.008, 0.01, 0.02, 0.03, 0.05, 0.06, 0.07, 0.08 ] shape_1 = train.shape best_AP = 0 for c in hyperparameters_c:
temp_parameter = 1 theta = run_kernel_softmax("lineal_pca18", linear_kernel, train_pca, train_y, temp_parameter) test_error = compute_kernel_test_error(test_pca, test_y, linear_kernel, theta, train_pca, temp_parameter) print('\nsoftmax_kernel lineal_pca18 \t\ttest_error:') print('(t = {}) \t\t\t\t{:.3}'.format(temp_parameter, test_error)) c, p = 0.5, 2 kernel = lambda X, Y: polynomial_kernel(X, Y, c, p) theta = run_kernel_softmax("polinomial_{}_{}_pca18".format(c, p), kernel, train_pca, train_y, temp_parameter) test_error = compute_kernel_test_error(test_pca, test_y, kernel, theta, train_pca, temp_parameter) print('\nsoftmax_kernel polinomial_pca18 \ttest_error:') print('(t = {}, c = {}, p = {}) \t\t{:.3}'.format(temp_parameter, c, p, test_error)) gamma = 1 kernel = lambda X, Y: rbf_kernel(X, Y, gamma) theta = run_kernel_softmax("rbf_{}_pca18".format(gamma), kernel, train_pca, train_y, temp_parameter) test_error = compute_kernel_test_error(test_pca, test_y, kernel, theta, train_pca, temp_parameter) print('\nsoftmax_kernel rbf_pca18 \t\ttest_error:') print('(t = {}, gamma = {}) \t\t\t{:.3}'.format(temp_parameter, gamma, test_error))