def doADVI(n, xx, xy, yy, x):
# Optional setting for reproducibility
use_seed = False
d = xx.shape[0]
ns = 5000
if use_seed: # optional setting for reproducibility
seed = 42
# Disable printing
sys.stdout = open(os.devnull, 'w')
# Sufficient statistics
NXX = shared(xx)
NXY = shared(xy)
NYY = shared(yy)
# Define model and perform MCMC sampling
with Model() as model:
# Fixed hyperparameters for priors
b0 = Deterministic('b0', th.zeros((d), dtype='float64'))
ide = Deterministic('ide', th.eye(d, m=d, k=0, dtype='float64'))
# Priors for parameters
l0 = Gamma('l0', alpha=2.0, beta=2.0)
l = Gamma('l', alpha=2.0, beta=2.0)
b = MvNormal('b', mu=b0, tau=l0 * ide, shape=d)
# Custom log likelihood
def logp(xtx, xty, yty):
return (n / 2.0) * th.log(l / (2 * np.pi)) + (-l / 2.0) * (
th.dot(th.dot(b, xtx), b) - 2 * th.dot(b, xty) + yty)
# Likelihood
delta = DensityDist('delta',
logp,
observed={
'xtx': NXX,
'xty': NXY,
'yty': NYY
})
# Inference
if use_seed:
v_params = advi(n=ns, random_seed=seed)
trace = sample_vp(v_params, draws=ns, random_seed=seed)
else:
v_params = advi(n=ns)
trace = sample_vp(v_params, draws=ns)
# Enable printing
sys.stdout = sys.__stdout__
# Compute prediction over posterior
return np.mean([np.dot(x, trace['b'][i]) for i in range(ns)], 0)
def test_advi():
n = 1000
sd0 = 2.
mu0 = 4.
sd = 3.
mu = -5.
data = sd * np.random.RandomState(0).randn(n) + mu
d = n / sd**2 + 1 / sd0**2
mu_post = (n * np.mean(data) / sd**2 + mu0 / sd0**2) / d
with Model() as model:
mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0)
x = Normal('x', mu=mu_, sd=sd, observed=data)
advi_fit = advi(
model=model, n=1000, accurate_elbo=False, learning_rate=1e-1,
random_seed=1)
np.testing.assert_allclose(advi_fit.means['mu'], mu_post, rtol=0.1)
trace = sample_vp(advi_fit, 10000, model)
np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.4)
np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.4)
def test_advi():
n = 1000
sd0 = 2.
mu0 = 4.
sd = 3.
mu = -5.
data = sd * np.random.RandomState(0).randn(n) + mu
d = n / sd**2 + 1 / sd0**2
mu_post = (n * np.mean(data) / sd**2 + mu0 / sd0**2) / d
with Model() as model:
mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0)
x = Normal('x', mu=mu_, sd=sd, observed=data)
advi_fit = advi(model=model,
n=1000,
accurate_elbo=False,
learning_rate=1e-1,
random_seed=1)
np.testing.assert_allclose(advi_fit.means['mu'], mu_post, rtol=0.1)
trace = sample_vp(advi_fit, 10000, model)
np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.4)
np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.4)
def test_sample_vp():
n_samples = 100
rng = np.random.RandomState(0)
xs = rng.binomial(n=1, p=0.2, size=n_samples)
with pm.Model() as model:
p = pm.Beta('p', alpha=1, beta=1)
pm.Binomial('xs', n=1, p=p, observed=xs)
v_params = advi(n=1000)
with model:
trace = sample_vp(v_params, hide_transformed=True)
assert(set(trace.varnames) == set('p'))
with model:
trace = sample_vp(v_params, hide_transformed=False)
assert(set(trace.varnames) == set(('p', 'p_logodds_')))
def test_advi_minibatch():
n = 1000
sd0 = 2.
mu0 = 4.
sd = 3.
mu = -5.
data = sd * np.random.RandomState(0).randn(n) + mu
d = n / sd**2 + 1 / sd0**2
mu_post = (n * np.mean(data) / sd**2 + mu0 / sd0**2) / d
data_t = tt.vector()
data_t.tag.test_value = np.zeros(1, )
with Model() as model:
mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0)
x = Normal('x', mu=mu_, sd=sd, observed=data_t)
minibatch_RVs = [x]
minibatch_tensors = [data_t]
def create_minibatch(data):
while True:
data = np.roll(data, 100, axis=0)
yield data[:100]
minibatches = [create_minibatch(data)]
with model:
advi_fit = advi_minibatch(n=1000,
minibatch_tensors=minibatch_tensors,
minibatch_RVs=minibatch_RVs,
minibatches=minibatches,
total_size=n,
learning_rate=1e-1,
random_seed=1)
np.testing.assert_allclose(advi_fit.means['mu'], mu_post, rtol=0.1)
trace = sample_vp(advi_fit, 10000)
np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.4)
np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.4)
def test_advi_minibatch():
n = 1000
sd0 = 2.
mu0 = 4.
sd = 3.
mu = -5.
data = sd * np.random.RandomState(0).randn(n) + mu
d = n / sd**2 + 1 / sd0**2
mu_post = (n * np.mean(data) / sd**2 + mu0 / sd0**2) / d
data_t = tt.vector()
data_t.tag.test_value=np.zeros(1,)
with Model() as model:
mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0)
x = Normal('x', mu=mu_, sd=sd, observed=data_t)
minibatch_RVs = [x]
minibatch_tensors = [data_t]
def create_minibatch(data):
while True:
data = np.roll(data, 100, axis=0)
yield data[:100]
minibatches = [create_minibatch(data)]
with model:
advi_fit = advi_minibatch(
n=1000, minibatch_tensors=minibatch_tensors,
minibatch_RVs=minibatch_RVs, minibatches=minibatches,
total_size=n, learning_rate=1e-1, random_seed=1
)
np.testing.assert_allclose(advi_fit.means['mu'], mu_post, rtol=0.1)
trace = sample_vp(advi_fit, 10000)
np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.4)
np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.4)
