def fit_kwargs(inference, use_minibatch):
_select = {
(ADVI, "full"): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), n=5000),
(ADVI, "mini"): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000
),
(NFVI, "full"): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000
),
(NFVI, "mini"): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000
),
(FullRankADVI, "full"): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), n=6000
),
(FullRankADVI, "mini"): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), n=12000
),
(SVGD, "full"): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.075, n_win=7), n=300),
(SVGD, "mini"): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.075, n_win=7), n=300),
(ASVGD, "full"): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), n=500, obj_n_mc=300
),
(ASVGD, "mini"): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), n=500, obj_n_mc=300
),
}
if use_minibatch:
key = "mini"
# backward compat for PR#3071
inference.approx.scale_cost_to_minibatch = False
else:
key = "full"
return _select[(type(inference), key)]
def fit_kwargs(inference, using_minibatch):
_select = {
(ADVI, 'full'):
dict(obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50),
n=5000),
(ADVI, 'mini'):
dict(obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50),
n=12000),
(FullRankADVI, 'full'):
dict(obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50),
n=6000),
(FullRankADVI, 'mini'):
dict(obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50),
n=12000),
(SVGD, 'full'):
dict(obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7),
n=300),
(SVGD, 'mini'):
dict(obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7),
n=300),
(ASVGD, 'full'):
dict(obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10),
n=500,
obj_n_mc=300),
(ASVGD, 'mini'):
dict(obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10),
n=500,
obj_n_mc=300)
}
if using_minibatch:
key = 'mini'
else:
key = 'full'
return _select[(type(inference), key)]
def fit_kwargs(inference, use_minibatch):
_select = {
(ADVI, 'full'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50),
n=5000
),
(ADVI, 'mini'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50),
n=12000
),
(NFVI, 'full'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50),
n=12000
),
(NFVI, 'mini'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50),
n=12000
),
(FullRankADVI, 'full'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50),
n=6000
),
(FullRankADVI, 'mini'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50),
n=12000
),
(SVGD, 'full'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.075, n_win=7),
n=300
),
(SVGD, 'mini'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.075, n_win=7),
n=300
),
(ASVGD, 'full'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10),
n=500, obj_n_mc=300
),
(ASVGD, 'mini'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10),
n=500, obj_n_mc=300
)
}
if use_minibatch:
key = 'mini'
# backward compat for PR#3071
inference.approx.scale_cost_to_minibatch = False
else:
key = 'full'
return _select[(type(inference), key)]
def fit_kwargs(inference, use_minibatch):
_select = {
(ADVI, 'full'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50),
n=5000
),
(ADVI, 'mini'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50),
n=12000
),
(NFVI, 'full'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50),
n=12000
),
(NFVI, 'mini'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50),
n=12000
),
(FullRankADVI, 'full'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50),
n=6000
),
(FullRankADVI, 'mini'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50),
n=12000
),
(SVGD, 'full'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7),
n=300
),
(SVGD, 'mini'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7),
n=300
),
(ASVGD, 'full'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10),
n=500, obj_n_mc=300
),
(ASVGD, 'mini'): dict(
obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10),
n=500, obj_n_mc=300
)
}
if use_minibatch:
key = 'mini'
else:
key = 'full'
return _select[(type(inference), key)]
log_vn_ss = T.log(T.clip(vn_ss[:, v_inds], 1E-8, 1E8))
log_vn_ss = T.clip(log_vn_ss, -1.5, 1.5)
chi_clip = T.clip(chi_ss[:, x_inds], -1.5, 1.5)
chi_obs = pm.Normal('chi_obs',
mu=chi_clip,
sd=0.2,
observed=xn.clip(lower=-1.5, upper=1.5))
log_vn_obs = pm.Normal('vn_obs',
mu=log_vn_ss,
sd=0.1,
observed=np.log(vn).clip(lower=-1.5, upper=1.5))
if __name__ == "__main__":
with pymc_model:
approx = pm.ADVI()
hist = approx.fit(n=40000,
obj_optimizer=pm.adagrad_window(learning_rate=0.005),
total_grad_norm_constraint=100)
# trace = hist.sample(500)
# ppc = pm.sample_ppc(trace)
import gzip
import pickle
with gzip.open('data/hackett_advi.pgz', 'wb') as f:
pickle.dump(approx, f)
print("Saving the model...")
with open("data/mc_model_{}.pkl".format(args.tag), "wb") as buff:
pickle.dump(model, buff)
print("Saving the pca...")
with open("data/mc_model_{}_pca.pkl".format(args.tag), "wb") as buff:
pickle.dump(model.pca, buff)
print("Fitting...")
save = SaveCallback(file_base="data/mc_model_{}".format(args.tag),
note=vars(args))
save.note["gene_ids"] = list(model.counts.index)
save.note["sample_ids"] = list(model.counts.columns)
if args.learnrate:
if args.optimizer == "adam":
obj_optimizer = pm.adam(learning_rate=args.learnrate)
elif args.optimizer == "adagrad_window":
obj_optimizer = pm.adagrad_window(learning_rate=args.learnrate,
n_win=args.nwin)
elif args.optimizer == "nesterov_momentum":
obj_optimizer = pm.nesterov_momentum(learning_rate=args.learnrate)
elif args.optimizer == "adagrad":
obj_optimizer = pm.adagrad_window(learning_rate=args.learnrate)
elif args.optimizer == "momentum":
obj_optimizer = pm.momentum(learning_rate=args.learnrate)
else:
raise ValueError(
f'The given optimizer "{args.optimizer}" is unknown.')
else:
if args.optimizer == "adam":
obj_optimizer = pm.adam()
elif args.optimizer == "adagrad_window":
obj_optimizer = pm.adagrad_window(n_win=args.nwin)
elif args.optimizer == "nesterov_momentum":
class Base(SeededTest):
inference = None
NITER = 12000
optimizer = pm.adagrad_window(learning_rate=0.01, n_win=50)
conv_cb = property(lambda self: [
pm.callbacks.CheckParametersConvergence(
every=500, diff='relative', tolerance=0.001),
pm.callbacks.CheckParametersConvergence(
every=500, diff='absolute', tolerance=0.0001)
])
def test_vars_view(self):
_, model, _ = models.multidimensional_model()
with model:
app = self.inference().approx
posterior = app.random(10)
x_sampled = app.view(posterior, 'x').eval()
assert x_sampled.shape == (10, ) + model['x'].dshape
def test_vars_view_dynamic_size(self):
_, model, _ = models.multidimensional_model()
with model:
app = self.inference().approx
i = tt.iscalar('i')
i.tag.test_value = 1
posterior = app.random(i)
x_sampled = app.view(posterior, 'x').eval({i: 10})
assert x_sampled.shape == (10, ) + model['x'].dshape
x_sampled = app.view(posterior, 'x').eval({i: 1})
assert x_sampled.shape == (1, ) + model['x'].dshape
def test_vars_view_dynamic_size_numpy(self):
_, model, _ = models.multidimensional_model()
with model:
app = self.inference().approx
i = tt.iscalar('i')
i.tag.test_value = 1
x_sampled = app.view(app.random_fn(10), 'x')
assert x_sampled.shape == (10, ) + model['x'].dshape
x_sampled = app.view(app.random_fn(1), 'x')
assert x_sampled.shape == (1, ) + model['x'].dshape
x_sampled = app.view(app.random_fn(), 'x')
assert x_sampled.shape == () + model['x'].dshape
def test_sample(self):
n_samples = 100
xs = np.random.binomial(n=1, p=0.2, size=n_samples)
with pm.Model():
p = pm.Beta('p', alpha=1, beta=1)
pm.Binomial('xs', n=1, p=p, observed=xs)
app = self.inference().approx
trace = app.sample(draws=1, include_transformed=False)
assert trace.varnames == ['p']
assert len(trace) == 1
trace = app.sample(draws=10, include_transformed=True)
assert sorted(trace.varnames) == ['p', 'p_logodds__']
assert len(trace) == 10
def test_sample_node(self):
n_samples = 100
xs = np.random.binomial(n=1, p=0.2, size=n_samples)
with pm.Model():
p = pm.Beta('p', alpha=1, beta=1)
pm.Binomial('xs', n=1, p=p, observed=xs)
app = self.inference().approx
app.sample_node(p).eval() # should be evaluated without errors
def test_optimizer_with_full_data(self):
n = 1000
sd0 = 2.
mu0 = 4.
sd = 3.
mu = -5.
data = sd * np.random.randn(n) + mu
d = n / sd**2 + 1 / sd0**2
mu_post = (n * np.mean(data) / sd**2 + mu0 / sd0**2) / d
with Model():
mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0)
Normal('x', mu=mu_, sd=sd, observed=data)
inf = self.inference(start={})
approx = inf.fit(
self.NITER,
obj_optimizer=self.optimizer,
callbacks=self.conv_cb,
)
trace = approx.sample(10000)
np.testing.assert_allclose(np.mean(trace['mu']),
mu_post,
rtol=0.05)
np.testing.assert_allclose(np.std(trace['mu']),
np.sqrt(1. / d),
rtol=0.1)
def test_optimizer_minibatch_with_generator(self):
n = 1000
sd0 = 2.
mu0 = 4.
sd = 3.
mu = -5.
data = sd * np.random.randn(n) + mu
d = n / sd**2 + 1 / sd0**2
mu_post = (n * np.mean(data) / sd**2 + mu0 / sd0**2) / d
def create_minibatch(data):
while True:
data = np.roll(data, 100, axis=0)
yield data[:100]
minibatches = create_minibatch(data)
with Model():
mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0)
Normal('x', mu=mu_, sd=sd, observed=minibatches, total_size=n)
inf = self.inference()
approx = inf.fit(self.NITER * 3,
obj_optimizer=self.optimizer,
callbacks=self.conv_cb)
trace = approx.sample(10000)
np.testing.assert_allclose(np.mean(trace['mu']),
mu_post,
rtol=0.05)
np.testing.assert_allclose(np.std(trace['mu']),
np.sqrt(1. / d),
rtol=0.1)
def test_optimizer_minibatch_with_callback(self):
n = 1000
sd0 = 2.
mu0 = 4.
sd = 3.
mu = -5.
data = sd * np.random.randn(n) + mu
d = n / sd**2 + 1 / sd0**2
mu_post = (n * np.mean(data) / sd**2 + mu0 / sd0**2) / d
def create_minibatch(data):
while True:
data = np.roll(data, 100, axis=0)
yield pm.floatX(data[:100])
minibatches = create_minibatch(data)
with Model():
data_t = theano.shared(next(minibatches))
def cb(*_):
data_t.set_value(next(minibatches))
mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0)
Normal('x', mu=mu_, sd=sd, observed=data_t, total_size=n)
inf = self.inference(scale_cost_to_minibatch=True)
approx = inf.fit(self.NITER * 3,
callbacks=[cb] + self.conv_cb,
obj_optimizer=self.optimizer)
trace = approx.sample(10000)
np.testing.assert_allclose(np.mean(trace['mu']),
mu_post,
rtol=0.05)
np.testing.assert_allclose(np.std(trace['mu']),
np.sqrt(1. / d),
rtol=0.1)
def test_n_obj_mc(self):
n_samples = 100
xs = np.random.binomial(n=1, p=0.2, size=n_samples)
with pm.Model():
p = pm.Beta('p', alpha=1, beta=1)
pm.Binomial('xs', n=1, p=p, observed=xs)
inf = self.inference(scale_cost_to_minibatch=True)
# should just work
inf.fit(10, obj_n_mc=10, obj_optimizer=self.optimizer)
def test_pickling(self):
with models.multidimensional_model()[1]:
inference = self.inference()
inference = pickle.loads(pickle.dumps(inference))
inference.fit(20)
def test_profile(self):
with models.multidimensional_model()[1]:
self.inference().run_profiling(10)
def test_multiple_replacements(self):
_, model, _ = models.exponential_beta(n=2)
x = model.x
y = model.y
xy = x * y
xpy = x + y
with model:
mf = self.inference().approx
xy_, xpy_ = mf.apply_replacements([xy, xpy])
xy_s, xpy_s = mf.sample_node([xy, xpy])
xy_.eval()
xpy_.eval()
xy_s.eval()
xpy_s.eval()
class TestASVGD(TestApproximates.Base):
NITER = 15000
inference = ASVGD
test_aevb = _test_aevb
optimizer = pm.adagrad_window(learning_rate=0.002)
conv_cb = []
