def fit_kwargs(inference, use_minibatch): _select = { (ADVI, "full"): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), n=5000), (ADVI, "mini"): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000 ), (NFVI, "full"): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000 ), (NFVI, "mini"): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000 ), (FullRankADVI, "full"): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), n=6000 ), (FullRankADVI, "mini"): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), n=12000 ), (SVGD, "full"): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.075, n_win=7), n=300), (SVGD, "mini"): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.075, n_win=7), n=300), (ASVGD, "full"): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), n=500, obj_n_mc=300 ), (ASVGD, "mini"): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), n=500, obj_n_mc=300 ), } if use_minibatch: key = "mini" # backward compat for PR#3071 inference.approx.scale_cost_to_minibatch = False else: key = "full" return _select[(type(inference), key)]
def fit_kwargs(inference, using_minibatch): _select = { (ADVI, 'full'): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), n=5000), (ADVI, 'mini'): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000), (FullRankADVI, 'full'): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), n=6000), (FullRankADVI, 'mini'): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), n=12000), (SVGD, 'full'): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7), n=300), (SVGD, 'mini'): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7), n=300), (ASVGD, 'full'): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), n=500, obj_n_mc=300), (ASVGD, 'mini'): dict(obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), n=500, obj_n_mc=300) } if using_minibatch: key = 'mini' else: key = 'full' return _select[(type(inference), key)]
def fit_kwargs(inference, use_minibatch): _select = { (ADVI, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), n=5000 ), (ADVI, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000 ), (NFVI, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000 ), (NFVI, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000 ), (FullRankADVI, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), n=6000 ), (FullRankADVI, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), n=12000 ), (SVGD, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.075, n_win=7), n=300 ), (SVGD, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.075, n_win=7), n=300 ), (ASVGD, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), n=500, obj_n_mc=300 ), (ASVGD, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), n=500, obj_n_mc=300 ) } if use_minibatch: key = 'mini' # backward compat for PR#3071 inference.approx.scale_cost_to_minibatch = False else: key = 'full' return _select[(type(inference), key)]
def fit_kwargs(inference, use_minibatch): _select = { (ADVI, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), n=5000 ), (ADVI, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000 ), (NFVI, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000 ), (NFVI, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), n=12000 ), (FullRankADVI, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), n=6000 ), (FullRankADVI, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), n=12000 ), (SVGD, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7), n=300 ), (SVGD, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7), n=300 ), (ASVGD, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), n=500, obj_n_mc=300 ), (ASVGD, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), n=500, obj_n_mc=300 ) } if use_minibatch: key = 'mini' else: key = 'full' return _select[(type(inference), key)]
log_vn_ss = T.log(T.clip(vn_ss[:, v_inds], 1E-8, 1E8)) log_vn_ss = T.clip(log_vn_ss, -1.5, 1.5) chi_clip = T.clip(chi_ss[:, x_inds], -1.5, 1.5) chi_obs = pm.Normal('chi_obs', mu=chi_clip, sd=0.2, observed=xn.clip(lower=-1.5, upper=1.5)) log_vn_obs = pm.Normal('vn_obs', mu=log_vn_ss, sd=0.1, observed=np.log(vn).clip(lower=-1.5, upper=1.5)) if __name__ == "__main__": with pymc_model: approx = pm.ADVI() hist = approx.fit(n=40000, obj_optimizer=pm.adagrad_window(learning_rate=0.005), total_grad_norm_constraint=100) # trace = hist.sample(500) # ppc = pm.sample_ppc(trace) import gzip import pickle with gzip.open('data/hackett_advi.pgz', 'wb') as f: pickle.dump(approx, f)
print("Saving the model...") with open("data/mc_model_{}.pkl".format(args.tag), "wb") as buff: pickle.dump(model, buff) print("Saving the pca...") with open("data/mc_model_{}_pca.pkl".format(args.tag), "wb") as buff: pickle.dump(model.pca, buff) print("Fitting...") save = SaveCallback(file_base="data/mc_model_{}".format(args.tag), note=vars(args)) save.note["gene_ids"] = list(model.counts.index) save.note["sample_ids"] = list(model.counts.columns) if args.learnrate: if args.optimizer == "adam": obj_optimizer = pm.adam(learning_rate=args.learnrate) elif args.optimizer == "adagrad_window": obj_optimizer = pm.adagrad_window(learning_rate=args.learnrate, n_win=args.nwin) elif args.optimizer == "nesterov_momentum": obj_optimizer = pm.nesterov_momentum(learning_rate=args.learnrate) elif args.optimizer == "adagrad": obj_optimizer = pm.adagrad_window(learning_rate=args.learnrate) elif args.optimizer == "momentum": obj_optimizer = pm.momentum(learning_rate=args.learnrate) else: raise ValueError( f'The given optimizer "{args.optimizer}" is unknown.') else: if args.optimizer == "adam": obj_optimizer = pm.adam() elif args.optimizer == "adagrad_window": obj_optimizer = pm.adagrad_window(n_win=args.nwin) elif args.optimizer == "nesterov_momentum":
class Base(SeededTest): inference = None NITER = 12000 optimizer = pm.adagrad_window(learning_rate=0.01, n_win=50) conv_cb = property(lambda self: [ pm.callbacks.CheckParametersConvergence( every=500, diff='relative', tolerance=0.001), pm.callbacks.CheckParametersConvergence( every=500, diff='absolute', tolerance=0.0001) ]) def test_vars_view(self): _, model, _ = models.multidimensional_model() with model: app = self.inference().approx posterior = app.random(10) x_sampled = app.view(posterior, 'x').eval() assert x_sampled.shape == (10, ) + model['x'].dshape def test_vars_view_dynamic_size(self): _, model, _ = models.multidimensional_model() with model: app = self.inference().approx i = tt.iscalar('i') i.tag.test_value = 1 posterior = app.random(i) x_sampled = app.view(posterior, 'x').eval({i: 10}) assert x_sampled.shape == (10, ) + model['x'].dshape x_sampled = app.view(posterior, 'x').eval({i: 1}) assert x_sampled.shape == (1, ) + model['x'].dshape def test_vars_view_dynamic_size_numpy(self): _, model, _ = models.multidimensional_model() with model: app = self.inference().approx i = tt.iscalar('i') i.tag.test_value = 1 x_sampled = app.view(app.random_fn(10), 'x') assert x_sampled.shape == (10, ) + model['x'].dshape x_sampled = app.view(app.random_fn(1), 'x') assert x_sampled.shape == (1, ) + model['x'].dshape x_sampled = app.view(app.random_fn(), 'x') assert x_sampled.shape == () + model['x'].dshape def test_sample(self): n_samples = 100 xs = np.random.binomial(n=1, p=0.2, size=n_samples) with pm.Model(): p = pm.Beta('p', alpha=1, beta=1) pm.Binomial('xs', n=1, p=p, observed=xs) app = self.inference().approx trace = app.sample(draws=1, include_transformed=False) assert trace.varnames == ['p'] assert len(trace) == 1 trace = app.sample(draws=10, include_transformed=True) assert sorted(trace.varnames) == ['p', 'p_logodds__'] assert len(trace) == 10 def test_sample_node(self): n_samples = 100 xs = np.random.binomial(n=1, p=0.2, size=n_samples) with pm.Model(): p = pm.Beta('p', alpha=1, beta=1) pm.Binomial('xs', n=1, p=p, observed=xs) app = self.inference().approx app.sample_node(p).eval() # should be evaluated without errors def test_optimizer_with_full_data(self): n = 1000 sd0 = 2. mu0 = 4. sd = 3. mu = -5. data = sd * np.random.randn(n) + mu d = n / sd**2 + 1 / sd0**2 mu_post = (n * np.mean(data) / sd**2 + mu0 / sd0**2) / d with Model(): mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0) Normal('x', mu=mu_, sd=sd, observed=data) inf = self.inference(start={}) approx = inf.fit( self.NITER, obj_optimizer=self.optimizer, callbacks=self.conv_cb, ) trace = approx.sample(10000) np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.05) np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.1) def test_optimizer_minibatch_with_generator(self): n = 1000 sd0 = 2. mu0 = 4. sd = 3. mu = -5. data = sd * np.random.randn(n) + mu d = n / sd**2 + 1 / sd0**2 mu_post = (n * np.mean(data) / sd**2 + mu0 / sd0**2) / d def create_minibatch(data): while True: data = np.roll(data, 100, axis=0) yield data[:100] minibatches = create_minibatch(data) with Model(): mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0) Normal('x', mu=mu_, sd=sd, observed=minibatches, total_size=n) inf = self.inference() approx = inf.fit(self.NITER * 3, obj_optimizer=self.optimizer, callbacks=self.conv_cb) trace = approx.sample(10000) np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.05) np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.1) def test_optimizer_minibatch_with_callback(self): n = 1000 sd0 = 2. mu0 = 4. sd = 3. mu = -5. data = sd * np.random.randn(n) + mu d = n / sd**2 + 1 / sd0**2 mu_post = (n * np.mean(data) / sd**2 + mu0 / sd0**2) / d def create_minibatch(data): while True: data = np.roll(data, 100, axis=0) yield pm.floatX(data[:100]) minibatches = create_minibatch(data) with Model(): data_t = theano.shared(next(minibatches)) def cb(*_): data_t.set_value(next(minibatches)) mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0) Normal('x', mu=mu_, sd=sd, observed=data_t, total_size=n) inf = self.inference(scale_cost_to_minibatch=True) approx = inf.fit(self.NITER * 3, callbacks=[cb] + self.conv_cb, obj_optimizer=self.optimizer) trace = approx.sample(10000) np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.05) np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.1) def test_n_obj_mc(self): n_samples = 100 xs = np.random.binomial(n=1, p=0.2, size=n_samples) with pm.Model(): p = pm.Beta('p', alpha=1, beta=1) pm.Binomial('xs', n=1, p=p, observed=xs) inf = self.inference(scale_cost_to_minibatch=True) # should just work inf.fit(10, obj_n_mc=10, obj_optimizer=self.optimizer) def test_pickling(self): with models.multidimensional_model()[1]: inference = self.inference() inference = pickle.loads(pickle.dumps(inference)) inference.fit(20) def test_profile(self): with models.multidimensional_model()[1]: self.inference().run_profiling(10) def test_multiple_replacements(self): _, model, _ = models.exponential_beta(n=2) x = model.x y = model.y xy = x * y xpy = x + y with model: mf = self.inference().approx xy_, xpy_ = mf.apply_replacements([xy, xpy]) xy_s, xpy_s = mf.sample_node([xy, xpy]) xy_.eval() xpy_.eval() xy_s.eval() xpy_s.eval()
class TestASVGD(TestApproximates.Base): NITER = 15000 inference = ASVGD test_aevb = _test_aevb optimizer = pm.adagrad_window(learning_rate=0.002) conv_cb = []