def test_feature_map_derivative_d_2n(): X = np.array([[1.], [3.]]) u = np.array([2.]) omega = np.array([[2.]]) d = 0 phi_derivative = feature_map_derivative_d(X, omega, u, d) phi_derivative_manual = -np.sin(X * omega + u) * omega[:, d] * np.sqrt(2.) assert_close(phi_derivative, phi_derivative_manual)
def test_compute_C_1d2n(): X = np.array([[1.], [2.]]) u = np.array([2.]) omega = np.array([[2.]]) d = 0 C_manual = np.mean(feature_map_derivative_d(X, omega, u, d)**2) C = compute_C_memory(X, omega, u) assert_allclose(C_manual, C)
def test_compute_C_1d1n(): X = np.array([[1.]]) u = np.array([2.]) omega = np.array([[2.]]) d = 0 phi = feature_map_derivative_d(X, omega, u, d).flatten() C_manual = np.outer(phi, phi) C = compute_C_memory(X, omega, u) assert_allclose(C_manual, C)
def test_feature_map_derivatives_loop_equals_map_derivative_d(): N = 10 D = 20 m = 3 X = np.random.randn(N, D) omega = np.random.randn(D, m) u = np.random.uniform(0, 2 * np.pi, m) derivatives = feature_map_derivatives_loop(X, omega, u) for d in range(D): derivative = feature_map_derivative_d(X, omega, u, d) assert_allclose(derivatives[d], derivative)
def test_objective_sym_equals_completely_manual_manually(): N = 100 D = 3 m = 3 omega = np.random.randn(D, m) u = np.random.uniform(0, 2 * np.pi, m) X = np.random.randn(N, D) lmbda = 1. theta = np.random.randn(m) J_manual = 0. for n in range(N): b_manual = np.zeros(m) C_manual = np.zeros((m, m)) J_n_manual = 0. for d in range(D): b_term_manual = -np.sqrt(2. / m) * np.cos(np.dot(X[n], omega) + u) * (omega[d, :]**2) b_term = feature_map_derivative2_d(X[n], omega, u, d) assert_allclose(b_term_manual, b_term) b_manual -= b_term_manual J_manual += np.dot(b_term_manual, theta) J_n_manual += np.dot(b_term_manual, theta) c_vec_manual = -np.sqrt(2. / m) * np.sin(np.dot(X[n], omega) + u) * omega[d, :] c_vec = feature_map_derivative_d(X[n], omega, u, d) assert_allclose(c_vec_manual, c_vec) C_term = np.outer(c_vec_manual, c_vec_manual) C_manual += C_term # not regularised here, done afterwards J_manual += 0.5 * np.dot(theta, np.dot(C_term, theta)) J_n_manual += 0.5 * np.dot(theta, np.dot(C_term, theta)) b = compute_b_memory(X[n].reshape(1, m), omega, u) C = compute_C_memory(X[n].reshape(1, m), omega, u) assert_allclose(b_manual, b) assert_allclose(C_manual, C) # discard regularisation for these internal checks J_n = objective(X[n].reshape(1, m), theta, 0, omega, u) J_n_2 = 0.5 * np.dot(theta, np.dot(C, theta)) - np.dot(theta, b) assert_allclose(J_n_2, J_n, rtol=1e-4) assert_allclose(J_n_manual, J_n, rtol=1e-4) J_manual /= N J_manual += 0.5 * lmbda * np.dot(theta, theta) J = objective(X, theta, lmbda, omega, u) assert_close(J, J_manual, decimal=5)
def test_feature_map_grad_single_equals_feature_map_derivative_d(): D = 2 m = 3 omega = np.random.randn(D, m) u = np.random.uniform(0, 2 * np.pi, m) x = np.random.randn(D) grad = feature_map_grad_single(x, omega, u) grad_manual = np.zeros((D, m)) for d in range(D): grad_manual[d, :] = feature_map_derivative_d(x, omega, u, d) assert_allclose(grad_manual, grad)
# sample random feature np.random.seed(seed_offset + i) omega, u = sample_basis(D=D, m=1, gamma=gamma) phi_x = feature_map(x, omega, u) # sample data point and predict f = predict(x, alphas[:i]) * phi_x # gradient of f at x f_grad = feature_map_grad_single(x, omega, u) * f # gradient grad = 0 for d in range(D): phi_derivative_d = feature_map_derivative_d(x, omega, u, d) phi_derivative2_d = feature_map_derivative2_d(x, omega, u, d) grad += phi_derivative_d * f_grad[d] + phi_derivative2_d # take gradient step r = learning_rate(i + 1) alphas[i] = -r * grad * phi_x # down-weight past alphas[:i] *= (1 - r * lmbda) # visualise log pdf log_pdf = lambda x: predict(x, alphas) res = 20 Xs = np.linspace(-3, 3, res)