def bayesian_random_effects(data, labels, group, n_samples=2000, n_burnin=500): import pymc as pm #preparing the data donors = data[group].unique() donors_lookup = dict(zip(donors, range(len(donors)))) data['donor_code'] = data[group].replace(donors_lookup).values n_donors = len(data[group].unique()) donor_idx = data['donor_code'].values #setting up the model with pm.Model() as hierarchical_model: # Hyperpriors for group nodes group_intercept_mean = pm.Normal('group intercept (mean)', mu=0., sd=100**2) group_intercept_variance = pm.Uniform('group intercept (variance)', lower=0, upper=100) group_slope_mean = pm.Normal('group slope (mean)', mu=0., sd=100**2) group_slope_variance = pm.Uniform('group slope (variance)', lower=0, upper=100) individual_intercepts = pm.Normal('individual intercepts', mu=group_intercept_mean, sd=group_intercept_variance, shape=n_donors) individual_slopes = pm.Normal('individual slopes', mu=group_slope_mean, sd=group_slope_variance, shape=n_donors) # Model error residuals = pm.Uniform('residuals', lower=0, upper=100) expression_est = individual_slopes[donor_idx] * data[labels[0]].values + individual_intercepts[donor_idx] # Data likelihood expression_like = pm.Normal('expression_like', mu=expression_est, sd=residuals, observed=data[labels[1]]) start = pm.find_MAP() step = pm.NUTS(scaling=start) hierarchical_trace = pm.sample(n_samples, step, start=start, progressbar=True) mean_slope = hierarchical_trace['group slope (mean)'][n_burnin:].mean() zero_percentile = percentileofscore(hierarchical_trace['group slope (mean)'][n_burnin:], 0) print "Mean group level slope was %g (zero was %g percentile of the posterior distribution)"%(mean_slope, zero_percentile) pm.summary(hierarchical_trace[n_burnin:], vars=['group slope (mean)']) pm.traceplot(hierarchical_trace[n_burnin:]) selection = donors fig, axis = plt.subplots(2, 3, figsize=(12, 6), sharey=True, sharex=True) axis = axis.ravel() xvals = np.linspace(data[labels[0]].min(), data[labels[0]].max()) for i, c in enumerate(selection): c_data = data.ix[data[group] == c] c_data = c_data.reset_index(drop = True) z = list(c_data['donor_code'])[0] for a_val, b_val in zip(hierarchical_trace['individual intercepts'][n_burnin::10][z], hierarchical_trace['individual slopes'][n_burnin::10][z]): axis[i].plot(xvals, a_val + b_val * xvals, 'g', alpha=.1) axis[i].plot(xvals, hierarchical_trace['individual intercepts'][n_burnin:][z].mean() + hierarchical_trace['individual slopes'][n_burnin:][z].mean() * xvals, 'g', alpha=1, lw=2.) axis[i].hexbin(c_data[labels[0]], c_data[labels[1]], mincnt=1, cmap=plt.cm.YlOrRd_r) axis[i].set_title(c) axis[i].set_xlabel(labels[0]) axis[i].set_ylabel(labels[1]) plt.show() return mean_slope, zero_percentile
def make_step(cls): args = {} if hasattr(cls, "step_args"): args.update(cls.step_args) if "scaling" not in args: _, step = pm.sampling.init_nuts(n_init=10000, **args) else: step = pm.NUTS(**args) return step
def bayesian_regression(): import warnings warnings.simplefilter("ignore") import pymc as pm import numpy as np np.random.seed(1000) import matplotlib.pyplot as plt %matplotlib inline #PYMC3 x = np.linspace(0, 10, 500) y = 4 + 2 * x + np.random.standard_normal(len(x)) * 2 reg = np.polyfit(x, y, 1) plt.figure(figsize=(8, 4)) plt.scatter(x, y, c=y, marker="v") plt.plot(x, reg[1] + reg[0] * x, lw=2.0) plt.colorbar() plt.grid(True) plt.xlabel("x") plt.ylabel("y") reg with pm.Model() as model: # model specifications in PyMC3 # are wrapped in a with statement # define priors alpha = pm.Normal("alpha", mu=0, sd=20) beta = pm.Normal("beta", mu=0, sd=20) sigma = pm.Uniform("sigma", lower=0, upper=10) # define linear regression y_est = alpha + beta * x # define likelihood likelihood = pm.Normal("y", mu=y_est, sd=sigma, observed=y) # inference start = pm.find_MAP() # find starting value by optimization step = pm.NUTS(state=start) # instantiate MCMC sampling algorithm trace = pm.sample(100, step, start=start, progressbar=False) # draw 100 posterior samples using NUTS sampling trace[0] fig = pm.traceplot(trace, lines={"alpha": 4, "beta": 2, "sigma": 2}) plt.figure(figsize=(8, 8)) plt.figure(figsize=(8, 4)) plt.scatter(x, y, c=y, marker="v") plt.colorbar() plt.grid(True) plt.xlabel("x") plt.ylabel("y") for i in range(len(trace)): plt.plot(x, trace["alpha"][i] + trace["beta"][i] * x) pass
def test_nuts_tuning(): with pymc.Model(): pymc.Normal("mu", mu=0, sigma=1) step = pymc.NUTS() idata = pymc.sample( 10, step=step, tune=5, discard_tuned_samples=False, progressbar=False, chains=1 ) assert not step.tune ss_tuned = idata.warmup_sample_stats["step_size"][0, -1] ss_posterior = idata.sample_stats["step_size"][0, :] np.testing.assert_array_equal(ss_posterior, ss_tuned)
def test_iterator(): with pm.Model() as model: a = pm.Normal("a", shape=1) b = pm.HalfNormal("b") step1 = pm.NUTS([model.rvs_to_values[a]]) step2 = pm.Metropolis([model.rvs_to_values[b]]) step = pm.CompoundStep([step1, step2]) start = {"a": floatX(np.array([1.0])), "b_log__": floatX(np.array(2.0))} sampler = ps.ParallelSampler(10, 10, 3, 2, [2, 3, 4], [start] * 3, step, 0, False) with sampler: for draw in sampler: pass
def test_user_potential(): model = pymc.Model() with model: pymc.Normal("a", mu=0, sigma=1) # Work around missing nonlocal in python2 called = [] class Potential(quadpotential.QuadPotentialDiag): def energy(self, x, velocity=None): called.append(1) return super().energy(x, velocity) pot = Potential(floatX([1])) with model: step = pymc.NUTS(potential=pot) pymc.sample(10, init=None, step=step, chains=1) assert called
def test_edge_case(self): # Edge case discovered in #2948 ndim = 3 with pm.Model() as m: pm.LogNormal("sigma", mu=np.zeros(ndim), tau=np.ones(ndim), shape=ndim) # variance for the correlation matrix pm.HalfCauchy("nu", beta=10) step = pm.NUTS() func = step._logp_dlogp_func func.set_extra_values(m.initial_point) q = func.dict_to_array(m.initial_point) logp, dlogp = func(q) assert logp.size == 1 assert dlogp.size == 4 npt.assert_allclose(dlogp, 0.0, atol=1e-5)
def test_bad_unpickle(): with pm.Model() as model: pm.Normal("x") with model: step = pm.NUTS() step.no_unpickle = NoUnpickle() with pytest.raises(Exception) as exc_info: pm.sample( tune=2, draws=2, mp_ctx="spawn", step=step, cores=2, chains=2, compute_convergence_checks=False, ) assert "could not be unpickled" in str(exc_info.getrepr(style="short"))
def __init__(self, sim, params, database=None, overwrite=False): raise ValueError('database not implemented yet') self.sim = sim self.params = params self.model = self.make_model(self.sim, dict(self.params)) with model: self.MAP = pymc.find_MAP() self.step = pymc.NUTS() self.dbname = database if self.dbname is None: pass else: raise ValueError('database not implemented yet') self.MCMCparams = {} self.number_of_bins = 200
def test_abort(mp_start_method): with pm.Model() as model: a = pm.Normal("a", shape=1) b = pm.HalfNormal("b") step1 = pm.NUTS([model.rvs_to_values[a]]) step2 = pm.Metropolis([model.rvs_to_values[b]]) step = pm.CompoundStep([step1, step2]) # on Windows we cannot fork if platform.system() == "Windows" and mp_start_method == "fork": return if mp_start_method == "spawn": step_method_pickled = cloudpickle.dumps(step, protocol=-1) else: step_method_pickled = None for abort in [False, True]: ctx = multiprocessing.get_context(mp_start_method) proc = ps.ProcessAdapter( 10, 10, step, chain=3, seed=1, mp_ctx=ctx, start={ "a": floatX(np.array([1.0])), "b_log__": floatX(np.array(2.0)) }, step_method_pickled=step_method_pickled, ) proc.start() while True: proc.write_next() out = ps.ProcessAdapter.recv_draw([proc]) if out[1]: break if abort: proc.abort() proc.join()
def test_full_adapt_sampling(seed=289586): np.random.seed(seed) L = np.random.randn(5, 5) L[np.diag_indices_from(L)] = np.exp(L[np.diag_indices_from(L)]) L[np.triu_indices_from(L, 1)] = 0.0 with pymc.Model() as model: pymc.MvNormal("a", mu=np.zeros(len(L)), chol=L, size=len(L)) initial_point = model.recompute_initial_point() initial_point_size = sum(initial_point[n.name].size for n in model.value_vars) pot = quadpotential.QuadPotentialFullAdapt( initial_point_size, np.zeros(initial_point_size)) step = pymc.NUTS(model=model, potential=pot) pymc.sample(draws=10, tune=1000, random_seed=seed, step=step, cores=1, chains=1)
def sample_lambda_posterior(self, num_steps): """ Sample values of lambda from the model posterior using the NUTS Hamiltonian MC algorithm. Performed with automatic tuning courtesy of pymc. Parameters ---------- num_steps : int The number of MCMC steps to take. Returns ------- trace : np.ndarray A shape-(num_steps, num_measurements) array with sequential samples of lambda corresponding to each experimental measurement. """ # right now regenerating the model object each time we want to sample # this will cost a little, but ensure we have the latest # -- so could be sped up if we check to see what expts are in a chached # model m = self._lambda_posterior_model() trace = m.sample(num_steps, pymc.NUTS(**self.nuts_params)) # stack the sampled lambdas for each experiment lambda_trace = np.concatenate([ trace.samples[('lambda-likelihood' + '-e' + str(i))].vals[0] for i in range(self.num_experiments) ], axis=1) lambda_trace = np.squeeze(lambda_trace) assert lambda_trace.shape == (num_steps, self.num_measurements) return trace
# MAP found: # x: 2.08679716543 # y: [[ 7.93662462] # [ 9.10291792]] # Compare with true values: # ytrue [[ 8.09247902] # [ 9.01023203]] # xtrue [ 2.03525495] # # We will use NUTS to sample from the posterior as implemented by the `NUTS` step method class. # In[6]: with model: step = pm.NUTS() # The `sample` function takes a number of steps to sample, a step method, a starting point. It returns a trace object which contains our samples. # In[7]: with model: trace = pm.sample(3000, step, start) # Out[7]: # To use more than one sampler, pass a list of step methods to `sample`. # # The trace object can be indexed by the variables in the model, returning an array with the first index being the sample index # and the other indexes the shape of the parameter. Thus for this example:
def foo(self, discrete): student_ids = [] timestep_ids = [] y = [] ids = collections.defaultdict(itertools.count().next) for t in range(0, len(self)): student_ids += [ids[o.id] for o in self[t]] timestep_ids += [t for o in self[t]] y += [o.value for o in self[t]] n_students = len(set(student_ids)) n_timesteps = len(self) print student_ids, "!", n_students with pm.Model() as hierarchical_model: # Hyperpriors for group nodes mu_student = pm.Normal('mu_student', mu=0., sd=100**2) sigma_student = pm.Uniform('sigma_student', lower=0, upper=100) #mu_timestep = pm.Normal('mu_beta', mu=0., sd=100**2) #sigma_timestep = pm.Uniform('sigma_beta', lower=0, upper=100) student = pm.Normal('student', mu=mu_student, sd=sigma_student, shape=n_students) #random effect timestep = pm.Normal('timestep', mu=0, sd=100**2, shape=n_timesteps) #fixed effect # Model error eps = pm.Uniform('eps', lower=0, upper=100) theta = student[student_ids] + timestep[timestep_ids] # Data likelihood if discrete: ll = pm.Bernoulli('theta', p=self.invlogit(theta), observed=y) else: ll = pm.Normal('theta', mu=theta, sd=eps, observed=y) with hierarchical_model: print "Find MAP..." start = pm.find_MAP() #if discrete: # step = pm.BinaryMetropolis(scaling=start) # else: print "NUTS..." step = pm.NUTS(scaling=start) print "Samples..." hierarchical_trace = pm.sample(2000, step, start=start, progressbar=False) print "done..." print "Plot..." pl.figure(figsize=(10, 10)) f = pm.traceplot(hierarchical_trace[500:]) f.savefig("a.png") return hierarchical_trace
# Bayesian Regression # with无法使用,pm.Model中没有__enter__与__exit__结构 with pm.Model() as model: # define priors alpha = pm.Normal('alpha', mu=0, sd=20) beta = pm.Normal('beta', mu=0, sd=20) sigma = pm.Uniform('sigma', lower=0, upper=10) # define linear regression y_est = alpha + beta * x # define likelihood likelihood = pm.Normal('y', mu=y_est, sd=sigma, observed=y) # inference start = pm.find_MAP() # find starting value by optimization step = pm.NUTS(state=start) # instantiate MCMC sampling algorithm trace = pm.sample(100, step, start=start, progressbar=False) # draw 100 posterior samples using NUTS sampling fig = pm.traceplot(trace, lines={'alpha': 4, 'beta': 2, 'sigma': 2}) plt.figure(figsize=(8, 8)) # GUI # 出现NotImplementedError,被调用却没被实现,需要用到pyqt4 # 使用Anaconda新版本会强制安装pyqt5,如不兼容则卸载重装pyqt4 import numpy as np import traits.api as trapi import traitsui.api as trui
# add scatter to points xdata = np.random.normal(xdata, 10) ydata = np.random.normal(ydata, 10) data = {'x': xdata, 'y': ydata} with pymc.Model() as model: alpha = pymc.Uniform('intercept', -100, 100) # Create custom densities beta = pymc.DensityDist('slope', lambda value: -1.5 * T.log(1 + value**2), testval=0) sigma = pymc.DensityDist('sigma', lambda value: -T.log(T.abs_(value)), testval=1) # Create likelihood like = pymc.Normal('y_est', mu=alpha + beta * xdata, sd=sigma, observed=ydata) start = pymc.find_MAP() step = pymc.NUTS(scaling=start) # Instantiate sampler trace = pymc.sample(10000, step, start=start) ################################################# # Create some convenience routines for plotting # All functions below written by Jake Vanderplas def compute_sigma_level(trace1, trace2, nbins=20): """From a set of traces, bin by number of standard deviations""" L, xbins, ybins = np.histogram2d(trace1, trace2, nbins) L[L == 0] = 1E-16 logL = np.log(L) shape = L.shape L = L.ravel()
]) condition = np.repeat([0, 1, 2, 3], nSubj) # Specify the model in PyMC with pm.Model() as model: kappa = pm.Gamma('kappa', 1, 0.1, shape=ncond) mu = pm.Beta('mu', 1, 1, shape=ncond) theta = pm.Beta('theta', mu[condition] * kappa[condition], (1 - mu[condition]) * kappa[condition], shape=len(z)) y = pm.Binomial('y', p=theta, n=N, observed=z) start = pm.find_MAP() step1 = pm.Metropolis([mu]) step2 = pm.Metropolis([theta]) step3 = pm.NUTS([kappa]) # samplers = [pm.Metropolis([rv]) for rv in model.unobserved_RVs] trace = pm.sample(10000, [step1, step2, step3], start=start, progressbar=False) ## Check the results. burnin = 5000 # posterior samples to discard thin = 10 # posterior samples to discard ## Print summary for each trace #pm.summary(trace[burnin::thin]) #pm.summary(trace) ## Check for mixing and autocorrelation #pm.autocorrplot(trace, vars =[mu, kappa])
# define hyperpriors muB = pm.Normal('muB', 0, .100) tauB = pm.Gamma('tauB', .01, .01) udfB = pm.Uniform('udfB', 0, 1) tdfB = 1 + tdfBgain * (-pm.log(1 - udfB)) # define the priors tau = pm.Gamma('tau', 0.01, 0.01) beta0 = pm.Normal('beta0', mu=0, tau=1.0E-12) beta1 = pm.T('beta1', mu=muB, lam=tauB, nu=tdfB, shape=n_predictors) mu = beta0 + pm.dot(beta1, x.values.T) # define the likelihood #mu = beta0 + beta1[0] * x.values[:,0] + beta1[1] * x.values[:,1] yl = pm.Normal('yl', mu=mu, tau=tau, observed=y) # Generate a MCMC chain start = pm.find_MAP() step1 = pm.NUTS([beta1]) step2 = pm.Metropolis([beta0, tau, muB, tauB, udfB]) trace = pm.sample(10000, [step1, step2], start, progressbar=False) # EXAMINE THE RESULTS burnin = 2000 thin = 1 # Print summary for each trace #pm.summary(trace[burnin::thin]) #pm.summary(trace) # Check for mixing and autocorrelation #pm.autocorrplot(trace[burnin::thin], vars =[mu, tau]) #pm.autocorrplot(trace, vars =[beta0])
def realdata(): import warnings warnings.simplefilter("ignore") import zipline import pytz import datetime as dt data = zipline.data.load_from_yahoo(stocks=["GLD", "GDX"], end=dt.datetime(2014, 3, 15, 0, 0, 0, 0, pytz.utc)).dropna() data.info() data.plot(figsize=(8, 4)) data.ix[-1] / data.ix[0] - 1 data.corr() data.index import matplotlib as mpl mpl_dates = mpl.dates.date2num(data.index) mpl_dates plt.figure(figsize=(8, 4)) plt.scatter(data["GDX"], data["GLD"], c=mpl_dates, marker="o") plt.grid(True) plt.xlabel("GDX") plt.ylabel("GLD") plt.colorbar(ticks=mpl.dates.DayLocator(interval=250), format=mpl.dates.DateFormatter("%d %b %y")) with pm.Model() as model: alpha = pm.Normal("alpha", mu=0, sd=20) beta = pm.Normal("beta", mu=0, sd=20) sigma = pm.Uniform("sigma", lower=0, upper=50) y_est = alpha + beta * data["GDX"].values likelihood = pm.Normal("GLD", mu=y_est, sd=sigma, observed=data["GLD"].values) start = pm.find_MAP() step = pm.NUTS(state=start) trace = pm.sample(100, step, start=start, progressbar=False) fig = pm.traceplot(trace) plt.figure(figsize=(8, 8)) plt.figure(figsize=(8, 4)) plt.scatter(data["GDX"], data["GLD"], c=mpl_dates, marker="o") plt.grid(True) plt.xlabel("GDX") plt.ylabel("GLD") for i in range(len(trace)): plt.plot(data["GDX"], trace["alpha"][i] + trace["beta"][i] * data ["GDX"]) plt.colorbar(ticks=mpl.dates.DayLocator(interval=250), format=mpl.dates.DateFormatter("%d %b %y")) model_randomwalk = pm.Model() with model_randomwalk: # std of random walk best sampled in log space sigma_alpha, log_sigma_alpha = \ model_randomwalk.TransformedVar("sigma_alpha", pm.Exponential.dist(1. / .02, testval=.1), pm.logtransform) sigma_beta, log_sigma_beta = \ model_randomwalk.TransformedVar("sigma_beta", pm.Exponential.dist(1. / .02, testval=.1), pm.logtransform) from pymc.distributions.timeseries import GaussianRandomWalk # to make the model simpler, we will apply the same coefficients # to 50 data points at a time subsample_alpha = 50 subsample_beta = 50 with model_randomwalk: alpha = GaussianRandomWalk("alpha", sigma_alpha**-2, shape=len(data) / subsample_alpha) beta = GaussianRandomWalk("beta", sigma_beta**-2, shape=len(data) / subsample_beta) # make coefficients have the same length as prices alpha_r = np.repeat(alpha, subsample_alpha) beta_r = np.repeat(beta, subsample_beta) len(data.dropna().GDX.values) with model_randomwalk: # define regression regression = alpha_r + beta_r * data.GDX.values[:1950] # assume prices are normally distributed # the mean comes from the regression sd = pm.Uniform("sd", 0, 20) likelihood = pm.Normal("GLD", mu=regression, sd=sd, observed=data.GLD.values[:1950]) import scipy.optimize as sco with model_randomwalk: # first optimize random walk start = pm.find_MAP(vars=[alpha, beta], fmin=sco.fmin_l_bfgs_b) # sampling step = pm.NUTS(scaling=start) trace_rw = pm.sample(100, step, start=start, progressbar=False) np.shape(trace_rw["alpha"]) part_dates = np.linspace(min(mpl_dates), max(mpl_dates), 39) fig, ax1 = plt.subplots(figsize=(10, 5)) plt.plot(part_dates, np.mean(trace_rw["alpha"], axis=0), "b", lw=2.5, label="alpha") for i in range(45, 55): plt.plot(part_dates, trace_rw["alpha"][i], "b-.", lw=0.75) plt.xlabel("date") plt.ylabel("alpha") plt.axis("tight") plt.grid(True) plt.legend(loc=2) ax1.xaxis.set_major_formatter(mpl.dates.DateFormatter("%d %b %y") ) ax2 = ax1.twinx() plt.plot(part_dates, np.mean(trace_rw["beta"], axis=0), "r", lw=2.5, label="beta") for i in range(45, 55): plt.plot(part_dates, trace_rw["beta"][i], "r-.", lw=0.75) plt.ylabel("beta") plt.legend(loc=4) fig.autofmt_xdate() plt.figure(figsize=(10, 5)) plt.scatter(data["GDX"], data["GLD"], c=mpl_dates, marker="o") plt.colorbar(ticks=mpl.dates.DayLocator(interval=250), format=mpl.dates.DateFormatter("%d %b %y")) plt.grid(True) plt.xlabel("GDX") plt.ylabel("GLD") x = np.linspace(min(data["GDX"]), max(data["GDX"])) for i in range(39): alpha_rw = np.mean(trace_rw["alpha"].T[i]) beta_rw = np.mean(trace_rw["beta"].T[i]) plt.plot(x, alpha_rw + beta_rw * x, color=plt.cm.jet(256 * i / 39)) pass
import pymc as pm from pymc.distributions.timeseries import GaussianRandomWalk from scipy.sparse import csc_matrix from scipy import optimize with pm.Model() as model: sigma, log_sigma = model.TransformedVar( 'sigma', pm.Exponential.dist(1. / .02, testval=.1), pm.logtransform) nu = pm.Exponential('nu', 1. / 10) s = GaussianRandomWalk('s', sigma**-2, shape=len(nreturns)) r = pm.T('r', nu, lam=pm.exp(-2 * s), observed=nreturns) with model: start = pm.find_MAP(vars=[s], fmin=optimize.fmin_l_bfgs_b) step = pm.NUTS(scaling=start) trace = pm.sample(2000, step, start, progressbar=False) plt.plot(trace[s][::10].T, 'b', alpha=.03) plt.title('log volatility') with model: pm.traceplot(trace, model.vars[:2]) exps = np.exp(trace[s][::10].T) plt.plot(returns[:600][::-1]) plt.plot(exps, 'r', alpha=.03) plt.plot(-exps, 'r', alpha=.03) plt.show()