def _gresnet(mlp_type, mlp, params, inputs): ravel, unravel = _make_ravelers(inputs.shape) mlp_params, (W, b1, b2) = params if mlp_type == 'mean': mu_mlp, sigmasq_mlp = mlp(mlp_params, inputs) mu_res = unravel(np.dot(ravel(inputs), W) + b1) sigmasq_res = log1pexp(b2) return make_tuple(mu_mlp + mu_res, sigmasq_mlp + sigmasq_res) else: J_mlp, h_mlp = mlp(mlp_params, inputs) J_res = -1./2 * log1pexp(b2) h_res = unravel(np.dot(ravel(inputs), W) + b1) return make_tuple(J_mlp + J_res, h_mlp + h_res)
def _gresnet(mlp_type, mlp, params, inputs): ravel, unravel = _make_ravelers(inputs.shape) mlp_params, (W, b1, b2) = params if mlp_type == 'mean': mu_mlp, sigmasq_mlp = mlp(mlp_params, inputs) mu_res = unravel(np.dot(ravel(inputs), W) + b1) sigmasq_res = log1pexp(b2) return make_tuple(mu_mlp + mu_res, sigmasq_mlp + sigmasq_res) else: J_mlp, h_mlp = mlp(mlp_params, inputs) J_res = -1. / 2 * log1pexp(b2) h_res = unravel(np.dot(ravel(inputs), W) + b1) return make_tuple(J_mlp + J_res, h_mlp + h_res)
def linear_recognize(x, psi): C, d = psi sigmasq = d**2 mu = np.dot(x, C.T) J = np.tile(1. / sigmasq, (x.shape[0], 1)) h = J * mu return make_tuple(-1. / 2 * J, h, np.zeros(x.shape[0]))
def linear_recognize(x, psi): C, D = psi J, Jzx, Jxx, _ = pair_mean_to_natural(C, np.dot(D, D.T)) T = x.shape[0] J = np.tile(np.diag(J), (T, 1)) h = np.dot(x, Jzx.T) logZ = np.zeros(x.shape[0]) return make_tuple(J, h, logZ)
def expectedstats_standard(nu, S, M, K, fudge=1e-8): m = M.shape[0] E_Sigmainv = nu*symmetrize(np.linalg.inv(S)) + fudge*np.eye(S.shape[0]) E_Sigmainv_A = nu*np.linalg.solve(S, M) E_AT_Sigmainv_A = m*K + nu*symmetrize(np.dot(M.T, np.linalg.solve(S, M))) \ + fudge*np.eye(K.shape[0]) E_logdetSigmainv = digamma((nu-np.arange(m))/2.).sum() \ + m*np.log(2) - np.linalg.slogdet(S)[1] assert is_posdef(E_Sigmainv) assert is_posdef(E_AT_Sigmainv_A) return make_tuple( -1./2*E_AT_Sigmainv_A, E_Sigmainv_A.T, -1./2*E_Sigmainv, 1./2*E_logdetSigmainv)
def natural_lds_inference_general_autograd(natparam, node_params, num_samples=None): init_params, pair_params = natparam def lds_log_normalizer(all_natparams): init_params, pair_params, node_params = all_natparams forward_messages, lognorm = natural_filter_forward_general( init_params, pair_params, node_params) return lognorm, (lognorm, forward_messages) all_natparams = make_tuple(init_params, pair_params, node_params) expected_stats, (lognorm, forward_messages) = agrad(lds_log_normalizer)(all_natparams) samples = natural_sample_backward_general(forward_messages, pair_params, num_samples) return samples, expected_stats, lognorm
def mlp_recognize(x, psi, tanh_scale=10.): nnet_params, ((W_h, b_h), (W_J, b_J)) = psi[:-2], psi[-2:] T, p = x.shape[0], b_h.shape[0] shape = x.shape[:-1] + (-1,) nnet = compose(tanh_layer(W, b) for W, b in nnet_params) h = linear_layer(W_h, b_h) log_J = linear_layer(W_J, b_J) nnet_outputs = nnet(np.reshape(x, (-1, x.shape[-1]))) J = -1./2 * np.exp(tanh_scale * np.tanh(log_J(nnet_outputs) / tanh_scale)) h = h(nnet_outputs) logZ = np.zeros(shape[:-1]) return make_tuple(np.reshape(J, shape), np.reshape(h, shape), logZ)
def natural_lds_inference_general_autograd(natparam, node_params, num_samples=None): init_params, pair_params = natparam def lds_log_normalizer(all_natparams): init_params, pair_params, node_params = all_natparams forward_messages, lognorm = natural_filter_forward_general( init_params, pair_params, node_params) return lognorm, (lognorm, forward_messages) all_natparams = make_tuple(init_params, pair_params, node_params) expected_stats, ( lognorm, forward_messages) = agrad(lds_log_normalizer)(all_natparams) samples = natural_sample_backward_general(forward_messages, pair_params, num_samples) return samples, expected_stats, lognorm
def bind(result, step): next_smooth, stats = result J, h, (mu, ExxT, ExxnT) = step(next_smooth) return make_tuple(J, h, mu), [(mu, ExxT, ExxnT)] + stats
def unit(filtered_message): J, h = filtered_message mu, Sigma = natural_to_mean(filtered_message) ExxT = Sigma + np.outer(mu, mu) return make_tuple(J, h, mu), [(mu, ExxT, 0.)]
def gaussian_mean(inputs, sigmoid_mean=False): mu_input, sigmasq_input = np.split(inputs, 2, axis=-1) mu = sigmoid(mu_input) if sigmoid_mean else mu_input sigmasq = log1pexp(sigmasq_input) return make_tuple(mu, sigmasq)
def pack_nodeparams(node_params): return make_tuple(*map(np.array, zip(*node_params)))
def gaussian_info(inputs): J_input, h = np.split(inputs, 2, axis=-1) J = -1. / 2 * log1pexp(J_input) return make_tuple(J, h)
from __future__ import division import autograd.numpy as np from autograd.scipy.special import multigammaln from autograd import grad from autograd.util import make_tuple import mniw # special case of mniw al2d = np.atleast_2d add_dims = lambda A, b, c, d: (A, b[:,None], al2d(c), d) remove_dims = lambda A, B, C, d: make_tuple(A, B.ravel(), C[0,0], d) def standard_to_natural(nu, S, m, kappa): A, B, C, d = mniw.standard_to_natural(nu, S, m[:,None], al2d(kappa)) return C, B.ravel(), A[0,0], d def expectedstats(natparam): return remove_dims(*mniw.expectedstats(add_dims(*natparam))) def logZ(natparam): return mniw.logZ(add_dims(*natparam)) def natural_sample(natparam): A, Sigma = mniw.natural_sample(add_dims(*natparam)) return np.ravel(A), Sigma
def gaussian_info(inputs): J_input, h = np.split(inputs, 2, axis=-1) J = -1./2 * log1pexp(J_input) return make_tuple(J, h)
def get_hmm_vlb(lds_global_natparam, hmm_local_natparam, lds_expected_stats): init_params, pair_params, _ = hmm_local_natparam node_params = get_arhmm_local_nodeparams(lds_global_natparam, lds_expected_stats) local_natparam = make_tuple(init_params, pair_params, node_params) return hmm_logZ(local_natparam)