def auto_guide(data): probs_a = pyro.param("guide_probs_a") probs_c = pyro.param("guide_probs_c") a = pyro.sample("a", dist.Categorical(probs_a), infer={"enumerate": "parallel"}) with pyro.plate("data", 2, dim=-1): pyro.sample("c", dist.Categorical(probs_c[a]))
def constrained_model(data): locs = pyro.param("locs", torch.randn(3), constraint=constraints.real) scales = pyro.param("scales", ops.exp(torch.randn(3)), constraint=constraints.positive) p = torch.tensor([0.5, 0.3, 0.2]) x = pyro.sample("x", dist.Categorical(p)) pyro.sample("obs", dist.Normal(locs[x], scales[x]), obs=data)
def hand_guide(data): probs_a = pyro.param("guide_probs_a") probs_c = pyro.param("guide_probs_c") a = pyro.sample("a", dist.Categorical(probs_a), infer={"enumerate": "parallel"}) for i in range(2): pyro.sample("c_{}".format(i), dist.Categorical(probs_c[a]))
def test_elbo_plate_plate(backend, outer_dim, inner_dim): with pyro_backend(backend): pyro.get_param_store().clear() num_particles = 1 q = pyro.param("q", torch.tensor([0.75, 0.25], requires_grad=True)) p = 0.2693204236205713 # for which kl(Categorical(q), Categorical(p)) = 0.5 p = torch.tensor([p, 1 - p]) def model(): d = dist.Categorical(p) context1 = pyro.plate("outer", outer_dim, dim=-1) context2 = pyro.plate("inner", inner_dim, dim=-2) pyro.sample("w", d) with context1: pyro.sample("x", d) with context2: pyro.sample("y", d) with context1, context2: pyro.sample("z", d) def guide(): d = dist.Categorical(pyro.param("q")) context1 = pyro.plate("outer", outer_dim, dim=-1) context2 = pyro.plate("inner", inner_dim, dim=-2) pyro.sample("w", d, infer={"enumerate": "parallel"}) with context1: pyro.sample("x", d, infer={"enumerate": "parallel"}) with context2: pyro.sample("y", d, infer={"enumerate": "parallel"}) with context1, context2: pyro.sample("z", d, infer={"enumerate": "parallel"}) kl_node = kl_divergence( torch.distributions.Categorical(funsor.to_data(q)), torch.distributions.Categorical(funsor.to_data(p))) kl = (1 + outer_dim + inner_dim + outer_dim * inner_dim) * kl_node expected_loss = kl expected_grad = grad(kl, [funsor.to_data(q)])[0] elbo = infer.TraceEnum_ELBO(num_particles=num_particles, vectorize_particles=True, strict_enumeration_warning=True) elbo = elbo.differentiable_loss if backend == "pyro" else elbo actual_loss = funsor.to_data(elbo(model, guide)) actual_loss.backward() actual_grad = funsor.to_data(pyro.param('q')).grad assert_close(actual_loss, expected_loss, atol=1e-5) assert_close(actual_grad, expected_grad, atol=1e-5)
def main(args): # Define a basic model with a single Normal latent random variable `loc` # and a batch of Normally distributed observations. def model(data): loc = pyro.sample("loc", dist.Normal(0., 1.)) with pyro.plate("data", len(data), dim=-1): pyro.sample("obs", dist.Normal(loc, 1.), obs=data) # Define a guide (i.e. variational distribution) with a Normal # distribution over the latent random variable `loc`. def guide(data): guide_loc = pyro.param("guide_loc", torch.tensor(0.)) guide_scale = pyro.param("guide_scale", torch.tensor(1.), constraint=constraints.positive) pyro.sample("loc", dist.Normal(guide_loc, guide_scale)) # Generate some data. torch.manual_seed(0) data = torch.randn(100) + 3.0 # Because the API in minipyro matches that of Pyro proper, # training code works with generic Pyro implementations. with pyro_backend(args.backend), interpretation(monte_carlo): # Construct an SVI object so we can do variational inference on our # model/guide pair. Elbo = infer.JitTrace_ELBO if args.jit else infer.Trace_ELBO elbo = Elbo() adam = optim.Adam({"lr": args.learning_rate}) svi = infer.SVI(model, guide, adam, elbo) # Basic training loop pyro.get_param_store().clear() for step in range(args.num_steps): loss = svi.step(data) if args.verbose and step % 100 == 0: print("step {} loss = {}".format(step, loss)) # Report the final values of the variational parameters # in the guide after training. if args.verbose: for name in pyro.get_param_store(): value = pyro.param(name).data print("{} = {}".format(name, value.detach().cpu().numpy())) # For this simple (conjugate) model we know the exact posterior. In # particular we know that the variational distribution should be # centered near 3.0. So let's check this explicitly. assert (pyro.param("guide_loc") - 3.0).abs() < 0.1
def auto_model(): probs_a = pyro.param("probs_a") probs_b = pyro.param("probs_b") probs_c = pyro.param("probs_c") probs_d = pyro.param("probs_d") with pyro.plate("a_axis", 2, dim=-1): a = pyro.sample("a", dist.Categorical(probs_a), infer={"enumerate": "parallel"}) pyro.sample("b", dist.Categorical(probs_b[a]), obs=b_data) with pyro.plate("c_axis", 3, dim=-1): c = pyro.sample("c", dist.Categorical(probs_c), infer={"enumerate": "parallel"}) pyro.sample("d", dist.Categorical(probs_d[c]), obs=d_data)
def auto_model(data): probs_a = pyro.param("model_probs_a") probs_b = pyro.param("model_probs_b") probs_c = pyro.param("model_probs_c") probs_d = pyro.param("model_probs_d") probs_e = pyro.param("model_probs_e") a = pyro.sample("a", dist.Categorical(probs_a)) b = pyro.sample("b", dist.Categorical(probs_b[a]), infer={"enumerate": "parallel"}) with pyro.plate("data", 2, dim=-1): c = pyro.sample("c", dist.Categorical(probs_c[a])) d = pyro.sample("d", dist.Categorical(Vindex(probs_d)[b, c]), infer={"enumerate": "parallel"}) pyro.sample("obs", dist.Categorical(probs_e[d]), obs=data)
def hand_model(data): probs_a = pyro.param("model_probs_a") probs_b = pyro.param("model_probs_b") probs_c = pyro.param("model_probs_c") probs_d = pyro.param("model_probs_d") probs_e = pyro.param("model_probs_e") a = pyro.sample("a", dist.Categorical(probs_a)) b = pyro.sample("b", dist.Categorical(probs_b[a]), infer={"enumerate": "parallel"}) for i in range(2): c = pyro.sample("c_{}".format(i), dist.Categorical(probs_c[a])) d = pyro.sample("d_{}".format(i), dist.Categorical(Vindex(probs_d)[b, c]), infer={"enumerate": "parallel"}) pyro.sample("obs_{}".format(i), dist.Categorical(probs_e[d]), obs=data[i])
def hand_model(): probs_a = pyro.param("probs_a") probs_b = pyro.param("probs_b") probs_c = pyro.param("probs_c") probs_d = pyro.param("probs_d") for i in range(2): a = pyro.sample("a_{}".format(i), dist.Categorical(probs_a), infer={"enumerate": "parallel"}) pyro.sample("b_{}".format(i), dist.Categorical(probs_b[a]), obs=b_data[i]) for j in range(3): c = pyro.sample("c_{}".format(j), dist.Categorical(probs_c), infer={"enumerate": "parallel"}) pyro.sample("d_{}".format(j), dist.Categorical(probs_d[c]), obs=d_data[j])
def guide(): d = dist.Categorical(pyro.param("q")) context1 = pyro.plate("outer", outer_dim, dim=-1) context2 = pyro.plate("inner", inner_dim, dim=-2) pyro.sample("w", d, infer={"enumerate": "parallel"}) with context1: pyro.sample("x", d, infer={"enumerate": "parallel"}) with context2: pyro.sample("y", d, infer={"enumerate": "parallel"}) with context1, context2: pyro.sample("z", d, infer={"enumerate": "parallel"})
def model(data): T, N, D = data.shape # time steps, individuals, features # Gaussian initial distribution. init_loc = pyro.param("init_loc", torch.zeros(D)) init_scale = pyro.param("init_scale", 1e-2 * torch.eye(D), constraint=constraints.lower_cholesky) # Linear dynamics with Gaussian noise. trans_const = pyro.param("trans_const", torch.zeros(D)) trans_coeff = pyro.param("trans_coeff", torch.eye(D)) noise = pyro.param("noise", 1e-2 * torch.eye(D), constraint=constraints.lower_cholesky) obs_plate = pyro.plate("channel", D, dim=-1) with pyro.plate("data", N, dim=-2): state = None for t in range(T): # Transition. if t == 0: loc = init_loc scale_tril = init_scale else: loc = trans_const + funsor.torch.torch_tensordot( trans_coeff, state, 1) scale_tril = noise state = pyro.sample("state_{}".format(t), dist.MultivariateNormal(loc, scale_tril), infer={"exact": exact}) # Factorial probit likelihood model. with obs_plate: pyro.sample("obs_{}".format(t), dist.Bernoulli(logits=state["channel"]), obs=data[t])
def _check_loss_and_grads(expected_loss, actual_loss, atol=1e-4, rtol=1e-4): # copied from pyro expected_loss, actual_loss = funsor.to_data(expected_loss), funsor.to_data( actual_loss) assert_close(actual_loss, expected_loss, atol=atol, rtol=rtol) names = pyro.get_param_store().keys() params = [] for name in names: params.append(funsor.to_data(pyro.param(name)).unconstrained()) actual_grads = grad(actual_loss, params, allow_unused=True, retain_graph=True) expected_grads = grad(expected_loss, params, allow_unused=True, retain_graph=True) for name, actual_grad, expected_grad in zip(names, actual_grads, expected_grads): if actual_grad is None or expected_grad is None: continue assert_close(actual_grad, expected_grad, atol=atol, rtol=rtol)
def test_local_param_ok(backend, jit): data = torch.randn(10) def model(): locs = pyro.param("locs", torch.tensor([-1.0, 0.0, 1.0])) with pyro.plate("plate", len(data), dim=-1): x = pyro.sample("x", dist.Categorical(torch.ones(3) / 3)) pyro.sample("obs", dist.Normal(locs[x], 1.0), obs=data) def guide(): with pyro.plate("plate", len(data), dim=-1): p = pyro.param("p", torch.ones(len(data), 3) / 3, event_dim=1) pyro.sample("x", dist.Categorical(p)) return p with pyro_backend(backend): Elbo = infer.JitTrace_ELBO if jit else infer.Trace_ELBO elbo = Elbo(ignore_jit_warnings=True) assert_ok(model, guide, elbo) # Check that pyro.param() can be called without init_value. expected = guide() actual = pyro.param("p") assert_close(actual, expected)
def guide(): with pyro.plate("plate", len(data), dim=-1): p = pyro.param("p", torch.ones(len(data), 3) / 3, event_dim=1) pyro.sample("x", dist.Categorical(p)) return p
def model(): locs = pyro.param("locs", torch.tensor([-1.0, 0.0, 1.0])) with pyro.plate("plate", len(data), dim=-1): x = pyro.sample("x", dist.Categorical(torch.ones(3) / 3)) pyro.sample("obs", dist.Normal(locs[x], 1.0), obs=data)
def guide(): p = pyro.param("p", torch.tensor([0.5, 0.3, 0.2])) with pyro.plate("plate", len(data), dim=-1): pyro.sample("x", dist.Categorical(p))
def guide(): loc = pyro.param("loc", torch.tensor(0.0)) scale = pyro.param("scale", torch.tensor(1.0)) with pyro.plate("plate_outer", data.size(-1), dim=-1): pyro.sample("x", dist.Normal(loc, scale))
def model(data): loc = pyro.param("loc", torch.tensor(0.0)) pyro.sample("x", dist.Normal(loc, 1.0), obs=data)
def test_elbo_enumerate_plates_1(backend): # +-----------------+ # | a ----> b M=2 | # +-----------------+ # +-----------------+ # | c ----> d N=3 | # +-----------------+ # This tests two unrelated plates. # Each should remain uncontracted. with pyro_backend(backend): pyro.param("probs_a", torch.tensor([0.45, 0.55]), constraint=constraints.simplex) pyro.param("probs_b", torch.tensor([[0.6, 0.4], [0.4, 0.6]]), constraint=constraints.simplex) pyro.param("probs_c", torch.tensor([0.75, 0.25]), constraint=constraints.simplex) pyro.param("probs_d", torch.tensor([[0.4, 0.6], [0.3, 0.7]]), constraint=constraints.simplex) b_data = torch.tensor([0, 1]) d_data = torch.tensor([0, 0, 1]) def auto_model(): probs_a = pyro.param("probs_a") probs_b = pyro.param("probs_b") probs_c = pyro.param("probs_c") probs_d = pyro.param("probs_d") with pyro.plate("a_axis", 2, dim=-1): a = pyro.sample("a", dist.Categorical(probs_a), infer={"enumerate": "parallel"}) pyro.sample("b", dist.Categorical(probs_b[a]), obs=b_data) with pyro.plate("c_axis", 3, dim=-1): c = pyro.sample("c", dist.Categorical(probs_c), infer={"enumerate": "parallel"}) pyro.sample("d", dist.Categorical(probs_d[c]), obs=d_data) def hand_model(): probs_a = pyro.param("probs_a") probs_b = pyro.param("probs_b") probs_c = pyro.param("probs_c") probs_d = pyro.param("probs_d") for i in range(2): a = pyro.sample("a_{}".format(i), dist.Categorical(probs_a), infer={"enumerate": "parallel"}) pyro.sample("b_{}".format(i), dist.Categorical(probs_b[a]), obs=b_data[i]) for j in range(3): c = pyro.sample("c_{}".format(j), dist.Categorical(probs_c), infer={"enumerate": "parallel"}) pyro.sample("d_{}".format(j), dist.Categorical(probs_d[c]), obs=d_data[j]) def guide(): pass elbo = infer.TraceEnum_ELBO(max_plate_nesting=1) elbo = elbo.differentiable_loss if backend == "pyro" else elbo auto_loss = elbo(auto_model, guide) elbo = infer.TraceEnum_ELBO(max_plate_nesting=0) elbo = elbo.differentiable_loss if backend == "pyro" else elbo hand_loss = elbo(hand_model, guide) _check_loss_and_grads(hand_loss, auto_loss)
def guide(): q = pyro.param("q", torch.randn(3).exp(), constraint=constraints.simplex) pyro.sample("x", dist.Categorical(q))
def guide(data): guide_loc = pyro.param("guide_loc", torch.tensor(0.)) guide_scale = pyro.param("guide_scale", torch.tensor(1.), constraint=constraints.positive) pyro.sample("loc", dist.Normal(guide_loc, guide_scale))
def model(data): p = pyro.param("p", torch.tensor(0.5)) pyro.sample("x", dist.Bernoulli(p), obs=data)
def model(data=None): loc = pyro.param("loc", torch.tensor(expected_mean)) scale = pyro.param("scale", torch.tensor(1.0)) with pyro.plate("data", 1000, dim=-1): x = pyro.sample("x", dist.Normal(loc, scale), obs=data) return x
def guide(): loc = pyro.param("loc", torch.tensor(0.)) y = pyro.sample("y", dist.Normal(loc, 1.)) pyro.sample("x", dist.Normal(y, 1.))
def guide_constrained_model(data): q = pyro.param("q", ops.exp(torch.randn(3)), constraint=constraints.simplex) pyro.sample("x", dist.Categorical(q))
def model(): locs = pyro.param("locs", torch.tensor([0.2, 0.3, 0.5])) p = torch.tensor([0.2, 0.3, 0.5]) with pyro.plate("plate", len(data), dim=-1): x = pyro.sample("x", dist.Categorical(p)) pyro.sample("obs", dist.Normal(locs[x], 1.0), obs=data)
def model(data=None): loc = pyro.param("loc", torch.tensor(2.0)) scale = pyro.param("scale", torch.tensor(1.0)) x = pyro.sample("x", dist.Normal(loc, scale), obs=data) return x
def test_elbo_enumerate_plate_7(backend): # Guide Model # a -----> b # | | # +-|--------|----------------+ # | V V | # | c -----> d -----> e N=2 | # +---------------------------+ # This tests a mixture of model and guide enumeration. with pyro_backend(backend): pyro.param("model_probs_a", torch.tensor([0.45, 0.55]), constraint=constraints.simplex) pyro.param("model_probs_b", torch.tensor([[0.6, 0.4], [0.4, 0.6]]), constraint=constraints.simplex) pyro.param("model_probs_c", torch.tensor([[0.75, 0.25], [0.55, 0.45]]), constraint=constraints.simplex) pyro.param("model_probs_d", torch.tensor([[[0.4, 0.6], [0.3, 0.7]], [[0.3, 0.7], [0.2, 0.8]]]), constraint=constraints.simplex) pyro.param("model_probs_e", torch.tensor([[0.75, 0.25], [0.55, 0.45]]), constraint=constraints.simplex) pyro.param("guide_probs_a", torch.tensor([0.35, 0.64]), constraint=constraints.simplex) pyro.param( "guide_probs_c", torch.tensor([[0., 1.], [1., 0.]]), # deterministic constraint=constraints.simplex) def auto_model(data): probs_a = pyro.param("model_probs_a") probs_b = pyro.param("model_probs_b") probs_c = pyro.param("model_probs_c") probs_d = pyro.param("model_probs_d") probs_e = pyro.param("model_probs_e") a = pyro.sample("a", dist.Categorical(probs_a)) b = pyro.sample("b", dist.Categorical(probs_b[a]), infer={"enumerate": "parallel"}) with pyro.plate("data", 2, dim=-1): c = pyro.sample("c", dist.Categorical(probs_c[a])) d = pyro.sample("d", dist.Categorical(Vindex(probs_d)[b, c]), infer={"enumerate": "parallel"}) pyro.sample("obs", dist.Categorical(probs_e[d]), obs=data) def auto_guide(data): probs_a = pyro.param("guide_probs_a") probs_c = pyro.param("guide_probs_c") a = pyro.sample("a", dist.Categorical(probs_a), infer={"enumerate": "parallel"}) with pyro.plate("data", 2, dim=-1): pyro.sample("c", dist.Categorical(probs_c[a])) def hand_model(data): probs_a = pyro.param("model_probs_a") probs_b = pyro.param("model_probs_b") probs_c = pyro.param("model_probs_c") probs_d = pyro.param("model_probs_d") probs_e = pyro.param("model_probs_e") a = pyro.sample("a", dist.Categorical(probs_a)) b = pyro.sample("b", dist.Categorical(probs_b[a]), infer={"enumerate": "parallel"}) for i in range(2): c = pyro.sample("c_{}".format(i), dist.Categorical(probs_c[a])) d = pyro.sample("d_{}".format(i), dist.Categorical(Vindex(probs_d)[b, c]), infer={"enumerate": "parallel"}) pyro.sample("obs_{}".format(i), dist.Categorical(probs_e[d]), obs=data[i]) def hand_guide(data): probs_a = pyro.param("guide_probs_a") probs_c = pyro.param("guide_probs_c") a = pyro.sample("a", dist.Categorical(probs_a), infer={"enumerate": "parallel"}) for i in range(2): pyro.sample("c_{}".format(i), dist.Categorical(probs_c[a])) data = torch.tensor([0, 0]) elbo = infer.TraceEnum_ELBO(max_plate_nesting=1) elbo = elbo.differentiable_loss if backend == "pyro" else elbo auto_loss = elbo(auto_model, auto_guide, data) elbo = infer.TraceEnum_ELBO(max_plate_nesting=0) elbo = elbo.differentiable_loss if backend == "pyro" else elbo hand_loss = elbo(hand_model, hand_guide, data) _check_loss_and_grads(hand_loss, auto_loss)
def guide(data): guide_loc = pyro.param("guide_loc", torch.tensor(0.)) guide_scale = ops.exp(pyro.param("guide_scale_log", torch.tensor(0.))) pyro.sample("loc", dist.Normal(guide_loc, guide_scale))
def guide(data): guide_loc = pyro.param("guide_loc", torch.tensor(0.)) guide_scale = pyro.param( "guide_scale_log", torch.tensor(0.), torch.distributions.constraints.positive).exp() pyro.sample("loc", dist.Normal(guide_loc, guide_scale))