def log_abs_det_jacobian(self, x, y, intermediates=None): return sum_rightmost( np.broadcast_to(np.log(np.abs(self.scale)), np.shape(x)), self.event_dim)
def __init__(self, logits=None, validate_args=None): self.logits = logits super(BernoulliLogits, self).__init__(batch_shape=jnp.shape(self.logits), validate_args=validate_args)
def support(self): return constraints.integer_interval(0, jnp.shape(self.logits)[-1] - 1)
def shape_i(x): return jnp.shape(x)[i]
def _predictive( rng_key, model, posterior_samples, batch_shape, return_sites=None, infer_discrete=False, parallel=True, model_args=(), model_kwargs={}, ): masked_model = numpyro.handlers.mask(model, mask=False) if infer_discrete: # inspect the model to get some structure rng_key, subkey = random.split(rng_key) batch_ndim = len(batch_shape) prototype_sample = tree_map( lambda x: jnp.reshape(x, (-1, ) + jnp.shape(x)[batch_ndim:])[0], posterior_samples, ) prototype_trace = trace( seed(substitute(masked_model, prototype_sample), subkey)).get_trace(*model_args, **model_kwargs) first_available_dim = -_guess_max_plate_nesting(prototype_trace) - 1 def single_prediction(val): rng_key, samples = val if infer_discrete: from numpyro.contrib.funsor import config_enumerate from numpyro.contrib.funsor.discrete import _sample_posterior model_trace = prototype_trace temperature = 1 pred_samples = _sample_posterior( config_enumerate(condition(model, samples)), first_available_dim, temperature, rng_key, *model_args, **model_kwargs, ) else: model_trace = trace( seed(substitute(masked_model, samples), rng_key)).get_trace(*model_args, **model_kwargs) pred_samples = { name: site["value"] for name, site in model_trace.items() } if return_sites is not None: if return_sites == "": sites = { k for k, site in model_trace.items() if site["type"] != "plate" } else: sites = return_sites else: sites = { k for k, site in model_trace.items() if (site["type"] == "sample" and k not in samples) or ( site["type"] == "deterministic") } return { name: value for name, value in pred_samples.items() if name in sites } num_samples = int(np.prod(batch_shape)) if num_samples > 1: rng_key = random.split(rng_key, num_samples) rng_key = rng_key.reshape((*batch_shape, 2)) chunk_size = num_samples if parallel else 1 return soft_vmap(single_prediction, (rng_key, posterior_samples), len(batch_shape), chunk_size)
def __init__(self, lmbda): self.event_shape = () self.batch_shape = broadcast_batch_shape(np.shape(lmbda)) self.lmbda = lmbda
def init(chain_state: HMCState) -> MassMatrixAdaptationState: """Initialize the mass matrix adaptation algorithm.""" n_dims = jnp.shape(chain_state.position)[-1] mm_state = mm_init(n_dims) return mm_state
def log_abs_det_jacobian(self, x, y, intermediates=None): return jnp.broadcast_to(jnp.log(jnp.abs(self.scale)), jnp.shape(x))
def log_abs_det_jacobian(self, x, y, intermediates=None): # Ref: http://web.mit.edu/18.325/www/handouts/handout2.pdf page 13 n = jnp.shape(x)[-1] order = -jnp.arange(n, 0, -1) return -n * jnp.log(2) + jnp.sum(order * jnp.log(jnp.diagonal(y, axis1=-2, axis2=-1)), axis=-1)
def shape(x): size = numpy.shape(x) if len(size) == 0: return (1, ) else: return size
def __init__(self, p): self.event_shape = () self.batch_shape = broadcast_batch_shape(np.shape(p)) self.p = p * 1.0 # will fail if p is int
x = nn.sigmoid(z) x = jnp.reshape(x, (x.shape[0], ) + self.input_shape) return x # `ae` is a detached module, which has no variables. ae = AutoEncoder(encoder_widths=(32, 32, 32), decoder_widths=(32, 32, 32), input_shape=(28, 28, 1)) # `ae.initialized` returnes a materialized copy of `ae` by # running through an input to create submodules defined lazily. params = ae.init({'params': random.PRNGKey(42)}, jnp.ones((1, 28, 28, 1))) # Now you can use `ae` as a normal object, calling any methods defined on AutoEncoder print("reconstruct", jnp.shape(ae.apply(params, jnp.ones((1, 28, 28, 1))))) print("encoder", jnp.shape(ae.apply(params, jnp.ones((1, 28, 28, 1)), method=ae.encode))) # `ae.variables` is a frozen dict that looks like # {'params': {"decoder": {"Dense_0": {"bias": ..., "kernel": ...}, ...}} print("var shapes", jax.tree_map(jnp.shape, params)) # TODO(avital, levskaya): resurrect this example once interactive api is restored. # You can access submodules defined in setup(), they are just references on # the autoencoder instance # encoder = ae.encoder # print("encoder var shapes", jax.tree_map(jnp.shape, encoder.variables)) # # You can also acccess submodules that were defined in-line.
def log_abs_det_jacobian(self, x, y, intermediates=None): return np.full(np.shape(x)[:-1], 0.)
def log_abs_det_jacobian(self, x, y, intermediates=None): return np.full( np.shape(x) if self.event_dim == 0 else np.shape(x)[:-1], 0.)
def __init__(self, log_factor, validate_args=None): batch_shape = jnp.shape(log_factor) event_shape = (0,) # This satisfies .size == 0. self.log_factor = log_factor super(Unit, self).__init__(batch_shape, event_shape, validate_args=validate_args)
def log_abs_det_jacobian(self, x, y, intermediates=None): # NB: see derivation in LKJCholesky implementation n = jnp.shape(x)[-1] order = -jnp.arange(n - 1, -1, -1) return jnp.sum(order * jnp.log(jnp.diagonal(y, axis1=-2, axis2=-1)), axis=-1)
def log_prob(self, value): shape = lax.broadcast_shapes(self.batch_shape, jnp.shape(value)[:-1]) return jnp.broadcast_to(self.log_factor, shape)
def _inverse(self, y): y = y - self.loc original_shape = jnp.shape(y) yt = jnp.reshape(y, (-1, original_shape[-1])).T xt = solve_triangular(self.scale_tril, yt, lower=True) return jnp.reshape(xt.T, original_shape)
def __init__(self, p): self.event_shape = () self.batch_shape = jnp.shape(p) self.p = p * 1.0 # will fail if p is int
def log_abs_det_jacobian(self, x, y, intermediates=None): return jnp.broadcast_to(jnp.log(jnp.diagonal(self.scale_tril, axis1=-2, axis2=-1)).sum(-1), jnp.shape(x)[:-1])
def shape(x): return jnp.shape(x)
def log_abs_det_jacobian(self, x, y, intermediates=None): return jnp.zeros(jnp.shape(x)[:-1])
def body_fn(state): i, key, _, _ = state key, subkey = random.split(key) if radius is None or prototype_params is None: # XXX: we don't want to apply enum to draw latent samples model_ = model if enum: from numpyro.contrib.funsor import enum as enum_handler if isinstance(model, substitute) and isinstance( model.fn, enum_handler): model_ = substitute(model.fn.fn, data=model.data) elif isinstance(model, enum_handler): model_ = model.fn # Wrap model in a `substitute` handler to initialize from `init_loc_fn`. seeded_model = substitute(seed(model_, subkey), substitute_fn=init_strategy) model_trace = trace(seeded_model).get_trace( *model_args, **model_kwargs) constrained_values, inv_transforms = {}, {} for k, v in model_trace.items(): if (v["type"] == "sample" and not v["is_observed"] and not v["fn"].support.is_discrete): constrained_values[k] = v["value"] with helpful_support_errors(v): inv_transforms[k] = biject_to(v["fn"].support) params = transform_fn( inv_transforms, {k: v for k, v in constrained_values.items()}, invert=True, ) else: # this branch doesn't require tracing the model params = {} for k, v in prototype_params.items(): if k in init_values: params[k] = init_values[k] else: params[k] = random.uniform(subkey, jnp.shape(v), minval=-radius, maxval=radius) key, subkey = random.split(key) potential_fn = partial(potential_energy, model, model_args, model_kwargs, enum=enum) if validate_grad: if forward_mode_differentiation: pe = potential_fn(params) z_grad = jacfwd(potential_fn)(params) else: pe, z_grad = value_and_grad(potential_fn)(params) z_grad_flat = ravel_pytree(z_grad)[0] is_valid = jnp.isfinite(pe) & jnp.all(jnp.isfinite(z_grad_flat)) else: pe = potential_fn(params) is_valid = jnp.isfinite(pe) z_grad = None return i + 1, key, (params, pe, z_grad), is_valid
def log_prob(self, value): shape = lax.broadcast_shapes(self.batch_shape, jnp.shape(value)[:max(jnp.ndim(value) - self.event_dim, 0)]) log_prob = self.base_dist.log_prob(value) return jnp.broadcast_to(log_prob, shape)
def discrete_gibbs_fn(model, model_args=(), model_kwargs={}, *, random_walk=False, modified=False): """ [EXPERIMENTAL INTERFACE] Returns a gibbs_fn to be used in :class:`HMCGibbs`, which works for discrete latent sites with enumerate support. The site update order is randomly permuted at each step. Note that those discrete latent sites that are not specified in the constructor of :class:`HMCGibbs` will be marginalized out by default (if they have enumerate supports). :param callable model: a callable with NumPyro primitives. This should be the same model as the one used in the `inner_kernel` of :class:`HMCGibbs`. :param tuple model_args: Arguments provided to the model. :param dict model_kwargs: Keyword arguments provided to the model. :param bool random_walk: If False, Gibbs sampling will be used to draw a sample from the conditional `p(gibbs_site | remaining sites)`. Otherwise, a sample will be drawn uniformly from the domain of `gibbs_site`. :param bool modified: whether to use a modified proposal, as suggested in reference [1], which always proposes a new state for the current Gibbs site. The modified scheme appears in the literature under the name "modified Gibbs sampler" or "Metropolised Gibbs sampler". :return: a callable `gibbs_fn` to be used in :class:`HMCGibbs` **References:** 1. *Peskun's theorem and a modified discrete-state Gibbs sampler*, Liu, J. S. (1996) **Example** .. doctest:: >>> from jax import random >>> import jax.numpy as jnp >>> import numpyro >>> import numpyro.distributions as dist >>> from numpyro.infer import MCMC, NUTS, HMCGibbs, discrete_gibbs_fn ... >>> def model(probs, locs): ... c = numpyro.sample("c", dist.Categorical(probs)) ... numpyro.sample("x", dist.Normal(locs[c], 0.5)) ... >>> probs = jnp.array([0.15, 0.3, 0.3, 0.25]) >>> locs = jnp.array([-2, 0, 2, 4]) >>> gibbs_fn = discrete_gibbs_fn(model, (probs, locs)) >>> kernel = HMCGibbs(NUTS(model), gibbs_fn, gibbs_sites=["c"]) >>> mcmc = MCMC(kernel, 1000, 100000, progress_bar=False) >>> mcmc.run(random.PRNGKey(0), probs, locs) >>> mcmc.print_summary() # doctest: +SKIP """ # NB: all of the information such as `model`, `model_args`, `model_kwargs` # can be accessed from HMCGibbs.sample but we require them here to # simplify the api of `gibbs_fn` prototype_trace = trace(seed(model, rng_seed=0)).get_trace( *model_args, **model_kwargs) support_sizes = { name: jnp.broadcast_to(site["fn"].enumerate_support(False).shape[0], jnp.shape(site["value"])) for name, site in prototype_trace.items() if site["type"] == "sample" and site["fn"].has_enumerate_support and not site["is_observed"] } max_plate_nesting = _guess_max_plate_nesting(prototype_trace) if random_walk: if modified: proposal_fn = partial(_discrete_modified_rw_proposal, stay_prob=0.) else: proposal_fn = _discrete_rw_proposal else: if modified: proposal_fn = partial(_discrete_modified_gibbs_proposal, stay_prob=0.) else: proposal_fn = _discrete_gibbs_proposal def gibbs_fn(rng_key, gibbs_sites, hmc_sites): # convert to unconstrained values z_hmc = { k: biject_to(prototype_trace[k]["fn"].support).inv(v) for k, v in hmc_sites.items() if k in prototype_trace and prototype_trace[k]["type"] == "sample" } use_enum = len(set(support_sizes) - set(gibbs_sites)) > 0 wrapped_model = _wrap_model(model) if use_enum: from numpyro.contrib.funsor import config_enumerate, enum wrapped_model = enum(config_enumerate(wrapped_model), -max_plate_nesting - 1) def potential_fn(z_discrete): model_kwargs_ = model_kwargs.copy() model_kwargs_["_gibbs_sites"] = z_discrete return potential_energy(wrapped_model, model_args, model_kwargs_, z_hmc, enum=use_enum) # get support_sizes of gibbs_sites support_sizes_flat, _ = ravel_pytree( {k: support_sizes[k] for k in gibbs_sites}) num_discretes = support_sizes_flat.shape[0] rng_key, rng_permute = random.split(rng_key) idxs = random.permutation(rng_key, jnp.arange(num_discretes)) def body_fn(i, val): idx = idxs[i] support_size = support_sizes_flat[idx] rng_key, z, pe = val rng_key, z_new, pe_new, log_accept_ratio = proposal_fn( rng_key, z, pe, potential_fn=potential_fn, idx=idx, support_size=support_size) rng_key, rng_accept = random.split(rng_key) # u ~ Uniform(0, 1), u < accept_ratio => -log(u) > -log_accept_ratio # and -log(u) ~ exponential(1) z, pe = cond( random.exponential(rng_accept) > -log_accept_ratio, (z_new, pe_new), identity, (z, pe), identity) return rng_key, z, pe init_val = (rng_key, gibbs_sites, potential_fn(gibbs_sites)) _, gibbs_sites, _ = fori_loop(0, num_discretes, body_fn, init_val) return gibbs_sites return gibbs_fn
def log_prob(self, value): batch_shape = jnp.shape(value)[:jnp.ndim(value) - len(self.event_shape)] batch_shape = lax.broadcast_shapes(batch_shape, self.batch_shape) return jnp.zeros(batch_shape)
def __init__(self, logits, total_count=1, validate_args=None): self.logits, self.total_count = promote_shapes(logits, total_count) batch_shape = lax.broadcast_shapes(jnp.shape(logits), jnp.shape(total_count)) super(BinomialLogits, self).__init__(batch_shape=batch_shape, validate_args=validate_args)
def _validate_sample(self, value): mask = super(ImproperUniform, self)._validate_sample(value) batch_dim = jnp.ndim(value) - len(self.event_shape) if batch_dim < jnp.ndim(mask): mask = jnp.all(jnp.reshape(mask, jnp.shape(mask)[:batch_dim] + (-1,)), -1) return mask
def __init__(self, rate, validate_args=None): self.rate = rate super(Poisson, self).__init__(jnp.shape(rate), validate_args=validate_args)
def apply_fun(params, inputs, **kwargs): del kwargs pe = params symbol_size = np.shape(inputs)[1] return inputs + pe[:, :symbol_size]