def bayesian_random_effects(data, labels, group, n_samples=2000, n_burnin=500):
import pymc as pm
#preparing the data
donors = data[group].unique()
donors_lookup = dict(zip(donors, range(len(donors))))
data['donor_code'] = data[group].replace(donors_lookup).values
n_donors = len(data[group].unique())
donor_idx = data['donor_code'].values
#setting up the model
with pm.Model() as hierarchical_model:
# Hyperpriors for group nodes
group_intercept_mean = pm.Normal('group intercept (mean)', mu=0., sd=100**2)
group_intercept_variance = pm.Uniform('group intercept (variance)', lower=0, upper=100)
group_slope_mean = pm.Normal('group slope (mean)', mu=0., sd=100**2)
group_slope_variance = pm.Uniform('group slope (variance)', lower=0, upper=100)
individual_intercepts = pm.Normal('individual intercepts', mu=group_intercept_mean, sd=group_intercept_variance, shape=n_donors)
individual_slopes = pm.Normal('individual slopes', mu=group_slope_mean, sd=group_slope_variance, shape=n_donors)
# Model error
residuals = pm.Uniform('residuals', lower=0, upper=100)
expression_est = individual_slopes[donor_idx] * data[labels[0]].values + individual_intercepts[donor_idx]
# Data likelihood
expression_like = pm.Normal('expression_like', mu=expression_est, sd=residuals, observed=data[labels[1]])
start = pm.find_MAP()
step = pm.NUTS(scaling=start)
hierarchical_trace = pm.sample(n_samples, step, start=start, progressbar=True)
mean_slope = hierarchical_trace['group slope (mean)'][n_burnin:].mean()
zero_percentile = percentileofscore(hierarchical_trace['group slope (mean)'][n_burnin:], 0)
print "Mean group level slope was %g (zero was %g percentile of the posterior distribution)"%(mean_slope, zero_percentile)
pm.summary(hierarchical_trace[n_burnin:], vars=['group slope (mean)'])
pm.traceplot(hierarchical_trace[n_burnin:])
selection = donors
fig, axis = plt.subplots(2, 3, figsize=(12, 6), sharey=True, sharex=True)
axis = axis.ravel()
xvals = np.linspace(data[labels[0]].min(), data[labels[0]].max())
for i, c in enumerate(selection):
c_data = data.ix[data[group] == c]
c_data = c_data.reset_index(drop = True)
z = list(c_data['donor_code'])[0]
for a_val, b_val in zip(hierarchical_trace['individual intercepts'][n_burnin::10][z], hierarchical_trace['individual slopes'][n_burnin::10][z]):
axis[i].plot(xvals, a_val + b_val * xvals, 'g', alpha=.1)
axis[i].plot(xvals, hierarchical_trace['individual intercepts'][n_burnin:][z].mean() + hierarchical_trace['individual slopes'][n_burnin:][z].mean() * xvals,
'g', alpha=1, lw=2.)
axis[i].hexbin(c_data[labels[0]], c_data[labels[1]], mincnt=1, cmap=plt.cm.YlOrRd_r)
axis[i].set_title(c)
axis[i].set_xlabel(labels[0])
axis[i].set_ylabel(labels[1])
plt.show()
return mean_slope, zero_percentile
def make_step(cls):
args = {}
if hasattr(cls, "step_args"):
args.update(cls.step_args)
if "scaling" not in args:
_, step = pm.sampling.init_nuts(n_init=10000, **args)
else:
step = pm.NUTS(**args)
return step
def bayesian_regression():
import warnings
warnings.simplefilter("ignore")
import pymc as pm
import numpy as np
np.random.seed(1000)
import matplotlib.pyplot as plt
%matplotlib inline
#PYMC3
x = np.linspace(0, 10, 500)
y = 4 + 2 * x + np.random.standard_normal(len(x)) * 2
reg = np.polyfit(x, y, 1)
plt.figure(figsize=(8, 4))
plt.scatter(x, y, c=y, marker="v")
plt.plot(x, reg[1] + reg[0] * x, lw=2.0)
plt.colorbar()
plt.grid(True)
plt.xlabel("x")
plt.ylabel("y")
reg
with pm.Model() as model:
# model specifications in PyMC3
# are wrapped in a with statement
# define priors
alpha = pm.Normal("alpha", mu=0, sd=20)
beta = pm.Normal("beta", mu=0, sd=20)
sigma = pm.Uniform("sigma", lower=0, upper=10)
# define linear regression
y_est = alpha + beta * x
# define likelihood
likelihood = pm.Normal("y", mu=y_est, sd=sigma, observed=y)
# inference
start = pm.find_MAP()
# find starting value by optimization
step = pm.NUTS(state=start)
# instantiate MCMC sampling algorithm
trace = pm.sample(100, step, start=start, progressbar=False)
# draw 100 posterior samples using NUTS sampling
trace[0]
fig = pm.traceplot(trace, lines={"alpha": 4, "beta": 2, "sigma": 2})
plt.figure(figsize=(8, 8))
plt.figure(figsize=(8, 4))
plt.scatter(x, y, c=y, marker="v")
plt.colorbar()
plt.grid(True)
plt.xlabel("x")
plt.ylabel("y")
for i in range(len(trace)):
plt.plot(x, trace["alpha"][i] + trace["beta"][i] * x)
pass
def test_nuts_tuning():
with pymc.Model():
pymc.Normal("mu", mu=0, sigma=1)
step = pymc.NUTS()
idata = pymc.sample(
10, step=step, tune=5, discard_tuned_samples=False, progressbar=False, chains=1
)
assert not step.tune
ss_tuned = idata.warmup_sample_stats["step_size"][0, -1]
ss_posterior = idata.sample_stats["step_size"][0, :]
np.testing.assert_array_equal(ss_posterior, ss_tuned)
def test_iterator():
with pm.Model() as model:
a = pm.Normal("a", shape=1)
b = pm.HalfNormal("b")
step1 = pm.NUTS([model.rvs_to_values[a]])
step2 = pm.Metropolis([model.rvs_to_values[b]])
step = pm.CompoundStep([step1, step2])
start = {"a": floatX(np.array([1.0])), "b_log__": floatX(np.array(2.0))}
sampler = ps.ParallelSampler(10, 10, 3, 2, [2, 3, 4], [start] * 3, step, 0, False)
with sampler:
for draw in sampler:
pass
def test_user_potential():
model = pymc.Model()
with model:
pymc.Normal("a", mu=0, sigma=1)
# Work around missing nonlocal in python2
called = []
class Potential(quadpotential.QuadPotentialDiag):
def energy(self, x, velocity=None):
called.append(1)
return super().energy(x, velocity)
pot = Potential(floatX([1]))
with model:
step = pymc.NUTS(potential=pot)
pymc.sample(10, init=None, step=step, chains=1)
assert called
def test_edge_case(self):
# Edge case discovered in #2948
ndim = 3
with pm.Model() as m:
pm.LogNormal("sigma",
mu=np.zeros(ndim),
tau=np.ones(ndim),
shape=ndim) # variance for the correlation matrix
pm.HalfCauchy("nu", beta=10)
step = pm.NUTS()
func = step._logp_dlogp_func
func.set_extra_values(m.initial_point)
q = func.dict_to_array(m.initial_point)
logp, dlogp = func(q)
assert logp.size == 1
assert dlogp.size == 4
npt.assert_allclose(dlogp, 0.0, atol=1e-5)
def test_bad_unpickle():
with pm.Model() as model:
pm.Normal("x")
with model:
step = pm.NUTS()
step.no_unpickle = NoUnpickle()
with pytest.raises(Exception) as exc_info:
pm.sample(
tune=2,
draws=2,
mp_ctx="spawn",
step=step,
cores=2,
chains=2,
compute_convergence_checks=False,
)
assert "could not be unpickled" in str(exc_info.getrepr(style="short"))
def __init__(self, sim, params, database=None, overwrite=False):
raise ValueError('database not implemented yet')
self.sim = sim
self.params = params
self.model = self.make_model(self.sim, dict(self.params))
with model:
self.MAP = pymc.find_MAP()
self.step = pymc.NUTS()
self.dbname = database
if self.dbname is None:
pass
else:
raise ValueError('database not implemented yet')
self.MCMCparams = {}
self.number_of_bins = 200
def test_abort(mp_start_method):
with pm.Model() as model:
a = pm.Normal("a", shape=1)
b = pm.HalfNormal("b")
step1 = pm.NUTS([model.rvs_to_values[a]])
step2 = pm.Metropolis([model.rvs_to_values[b]])
step = pm.CompoundStep([step1, step2])
# on Windows we cannot fork
if platform.system() == "Windows" and mp_start_method == "fork":
return
if mp_start_method == "spawn":
step_method_pickled = cloudpickle.dumps(step, protocol=-1)
else:
step_method_pickled = None
for abort in [False, True]:
ctx = multiprocessing.get_context(mp_start_method)
proc = ps.ProcessAdapter(
10,
10,
step,
chain=3,
seed=1,
mp_ctx=ctx,
start={
"a": floatX(np.array([1.0])),
"b_log__": floatX(np.array(2.0))
},
step_method_pickled=step_method_pickled,
)
proc.start()
while True:
proc.write_next()
out = ps.ProcessAdapter.recv_draw([proc])
if out[1]:
break
if abort:
proc.abort()
proc.join()
def test_full_adapt_sampling(seed=289586):
np.random.seed(seed)
L = np.random.randn(5, 5)
L[np.diag_indices_from(L)] = np.exp(L[np.diag_indices_from(L)])
L[np.triu_indices_from(L, 1)] = 0.0
with pymc.Model() as model:
pymc.MvNormal("a", mu=np.zeros(len(L)), chol=L, size=len(L))
initial_point = model.recompute_initial_point()
initial_point_size = sum(initial_point[n.name].size
for n in model.value_vars)
pot = quadpotential.QuadPotentialFullAdapt(
initial_point_size, np.zeros(initial_point_size))
step = pymc.NUTS(model=model, potential=pot)
pymc.sample(draws=10,
tune=1000,
random_seed=seed,
step=step,
cores=1,
chains=1)
def sample_lambda_posterior(self, num_steps):
"""
Sample values of lambda from the model posterior using the NUTS
Hamiltonian MC algorithm. Performed with automatic tuning courtesy of
pymc.
Parameters
----------
num_steps : int
The number of MCMC steps to take.
Returns
-------
trace : np.ndarray
A shape-(num_steps, num_measurements) array with sequential samples
of lambda corresponding to each experimental measurement.
"""
# right now regenerating the model object each time we want to sample
# this will cost a little, but ensure we have the latest
# -- so could be sped up if we check to see what expts are in a chached
# model
m = self._lambda_posterior_model()
trace = m.sample(num_steps, pymc.NUTS(**self.nuts_params))
# stack the sampled lambdas for each experiment
lambda_trace = np.concatenate([
trace.samples[('lambda-likelihood' + '-e' + str(i))].vals[0]
for i in range(self.num_experiments)
],
axis=1)
lambda_trace = np.squeeze(lambda_trace)
assert lambda_trace.shape == (num_steps, self.num_measurements)
return trace
# MAP found:
# x: 2.08679716543
# y: [[ 7.93662462]
# [ 9.10291792]]
# Compare with true values:
# ytrue [[ 8.09247902]
# [ 9.01023203]]
# xtrue [ 2.03525495]
#
# We will use NUTS to sample from the posterior as implemented by the `NUTS` step method class.
# In[6]:
with model:
step = pm.NUTS()
# The `sample` function takes a number of steps to sample, a step method, a starting point. It returns a trace object which contains our samples.
# In[7]:
with model:
trace = pm.sample(3000, step, start)
# Out[7]:
# To use more than one sampler, pass a list of step methods to `sample`.
#
# The trace object can be indexed by the variables in the model, returning an array with the first index being the sample index
# and the other indexes the shape of the parameter. Thus for this example:
def foo(self, discrete):
student_ids = []
timestep_ids = []
y = []
ids = collections.defaultdict(itertools.count().next)
for t in range(0, len(self)):
student_ids += [ids[o.id] for o in self[t]]
timestep_ids += [t for o in self[t]]
y += [o.value for o in self[t]]
n_students = len(set(student_ids))
n_timesteps = len(self)
print student_ids, "!", n_students
with pm.Model() as hierarchical_model:
# Hyperpriors for group nodes
mu_student = pm.Normal('mu_student', mu=0., sd=100**2)
sigma_student = pm.Uniform('sigma_student', lower=0, upper=100)
#mu_timestep = pm.Normal('mu_beta', mu=0., sd=100**2)
#sigma_timestep = pm.Uniform('sigma_beta', lower=0, upper=100)
student = pm.Normal('student',
mu=mu_student,
sd=sigma_student,
shape=n_students) #random effect
timestep = pm.Normal('timestep',
mu=0,
sd=100**2,
shape=n_timesteps) #fixed effect
# Model error
eps = pm.Uniform('eps', lower=0, upper=100)
theta = student[student_ids] + timestep[timestep_ids]
# Data likelihood
if discrete:
ll = pm.Bernoulli('theta', p=self.invlogit(theta), observed=y)
else:
ll = pm.Normal('theta', mu=theta, sd=eps, observed=y)
with hierarchical_model:
print "Find MAP..."
start = pm.find_MAP()
#if discrete:
# step = pm.BinaryMetropolis(scaling=start)
# else:
print "NUTS..."
step = pm.NUTS(scaling=start)
print "Samples..."
hierarchical_trace = pm.sample(2000,
step,
start=start,
progressbar=False)
print "done..."
print "Plot..."
pl.figure(figsize=(10, 10))
f = pm.traceplot(hierarchical_trace[500:])
f.savefig("a.png")
return hierarchical_trace
# Bayesian Regression
# with无法使用，pm.Model中没有__enter__与__exit__结构
with pm.Model() as model:
# define priors
alpha = pm.Normal('alpha', mu=0, sd=20)
beta = pm.Normal('beta', mu=0, sd=20)
sigma = pm.Uniform('sigma', lower=0, upper=10)
# define linear regression
y_est = alpha + beta * x
# define likelihood
likelihood = pm.Normal('y', mu=y_est, sd=sigma, observed=y)
# inference
start = pm.find_MAP()
# find starting value by optimization
step = pm.NUTS(state=start)
# instantiate MCMC sampling algorithm
trace = pm.sample(100, step, start=start, progressbar=False)
# draw 100 posterior samples using NUTS sampling
fig = pm.traceplot(trace, lines={'alpha': 4, 'beta': 2, 'sigma': 2})
plt.figure(figsize=(8, 8))
# GUI
# 出现NotImplementedError，被调用却没被实现,需要用到pyqt4
# 使用Anaconda新版本会强制安装pyqt5，如不兼容则卸载重装pyqt4
import numpy as np
import traits.api as trapi
import traitsui.api as trui
# add scatter to points
xdata = np.random.normal(xdata, 10)
ydata = np.random.normal(ydata, 10)
data = {'x': xdata, 'y': ydata}
with pymc.Model() as model:
alpha = pymc.Uniform('intercept', -100, 100)
# Create custom densities
beta = pymc.DensityDist('slope', lambda value: -1.5 * T.log(1 + value**2), testval=0)
sigma = pymc.DensityDist('sigma', lambda value: -T.log(T.abs_(value)), testval=1)
# Create likelihood
like = pymc.Normal('y_est', mu=alpha + beta * xdata, sd=sigma, observed=ydata)
start = pymc.find_MAP()
step = pymc.NUTS(scaling=start) # Instantiate sampler
trace = pymc.sample(10000, step, start=start)
#################################################
# Create some convenience routines for plotting
# All functions below written by Jake Vanderplas
def compute_sigma_level(trace1, trace2, nbins=20):
"""From a set of traces, bin by number of standard deviations"""
L, xbins, ybins = np.histogram2d(trace1, trace2, nbins)
L[L == 0] = 1E-16
logL = np.log(L)
shape = L.shape
L = L.ravel()
])
condition = np.repeat([0, 1, 2, 3], nSubj)
# Specify the model in PyMC
with pm.Model() as model:
kappa = pm.Gamma('kappa', 1, 0.1, shape=ncond)
mu = pm.Beta('mu', 1, 1, shape=ncond)
theta = pm.Beta('theta',
mu[condition] * kappa[condition],
(1 - mu[condition]) * kappa[condition],
shape=len(z))
y = pm.Binomial('y', p=theta, n=N, observed=z)
start = pm.find_MAP()
step1 = pm.Metropolis([mu])
step2 = pm.Metropolis([theta])
step3 = pm.NUTS([kappa])
# samplers = [pm.Metropolis([rv]) for rv in model.unobserved_RVs]
trace = pm.sample(10000, [step1, step2, step3],
start=start,
progressbar=False)
## Check the results.
burnin = 5000 # posterior samples to discard
thin = 10 # posterior samples to discard
## Print summary for each trace
#pm.summary(trace[burnin::thin])
#pm.summary(trace)
## Check for mixing and autocorrelation
#pm.autocorrplot(trace, vars =[mu, kappa])
# define hyperpriors
muB = pm.Normal('muB', 0, .100)
tauB = pm.Gamma('tauB', .01, .01)
udfB = pm.Uniform('udfB', 0, 1)
tdfB = 1 + tdfBgain * (-pm.log(1 - udfB))
# define the priors
tau = pm.Gamma('tau', 0.01, 0.01)
beta0 = pm.Normal('beta0', mu=0, tau=1.0E-12)
beta1 = pm.T('beta1', mu=muB, lam=tauB, nu=tdfB, shape=n_predictors)
mu = beta0 + pm.dot(beta1, x.values.T)
# define the likelihood
#mu = beta0 + beta1[0] * x.values[:,0] + beta1[1] * x.values[:,1]
yl = pm.Normal('yl', mu=mu, tau=tau, observed=y)
# Generate a MCMC chain
start = pm.find_MAP()
step1 = pm.NUTS([beta1])
step2 = pm.Metropolis([beta0, tau, muB, tauB, udfB])
trace = pm.sample(10000, [step1, step2], start, progressbar=False)
# EXAMINE THE RESULTS
burnin = 2000
thin = 1
# Print summary for each trace
#pm.summary(trace[burnin::thin])
#pm.summary(trace)
# Check for mixing and autocorrelation
#pm.autocorrplot(trace[burnin::thin], vars =[mu, tau])
#pm.autocorrplot(trace, vars =[beta0])
def realdata():
import warnings
warnings.simplefilter("ignore")
import zipline
import pytz
import datetime as dt
data = zipline.data.load_from_yahoo(stocks=["GLD", "GDX"],
end=dt.datetime(2014, 3, 15, 0, 0, 0, 0, pytz.utc)).dropna()
data.info()
data.plot(figsize=(8, 4))
data.ix[-1] / data.ix[0] - 1
data.corr()
data.index
import matplotlib as mpl
mpl_dates = mpl.dates.date2num(data.index)
mpl_dates
plt.figure(figsize=(8, 4))
plt.scatter(data["GDX"], data["GLD"], c=mpl_dates, marker="o")
plt.grid(True)
plt.xlabel("GDX")
plt.ylabel("GLD")
plt.colorbar(ticks=mpl.dates.DayLocator(interval=250),
format=mpl.dates.DateFormatter("%d %b %y"))
with pm.Model() as model:
alpha = pm.Normal("alpha", mu=0, sd=20)
beta = pm.Normal("beta", mu=0, sd=20)
sigma = pm.Uniform("sigma", lower=0, upper=50)
y_est = alpha + beta * data["GDX"].values
likelihood = pm.Normal("GLD", mu=y_est, sd=sigma,
observed=data["GLD"].values)
start = pm.find_MAP()
step = pm.NUTS(state=start)
trace = pm.sample(100, step, start=start, progressbar=False)
fig = pm.traceplot(trace)
plt.figure(figsize=(8, 8))
plt.figure(figsize=(8, 4))
plt.scatter(data["GDX"], data["GLD"], c=mpl_dates, marker="o")
plt.grid(True)
plt.xlabel("GDX")
plt.ylabel("GLD")
for i in range(len(trace)):
plt.plot(data["GDX"], trace["alpha"][i] + trace["beta"][i] * data
["GDX"])
plt.colorbar(ticks=mpl.dates.DayLocator(interval=250),
format=mpl.dates.DateFormatter("%d %b %y"))
model_randomwalk = pm.Model()
with model_randomwalk:
# std of random walk best sampled in log space
sigma_alpha, log_sigma_alpha = \
model_randomwalk.TransformedVar("sigma_alpha",
pm.Exponential.dist(1. / .02, testval=.1),
pm.logtransform)
sigma_beta, log_sigma_beta = \
model_randomwalk.TransformedVar("sigma_beta",
pm.Exponential.dist(1. / .02, testval=.1),
pm.logtransform)
from pymc.distributions.timeseries import GaussianRandomWalk
# to make the model simpler, we will apply the same coefficients
# to 50 data points at a time
subsample_alpha = 50
subsample_beta = 50
with model_randomwalk:
alpha = GaussianRandomWalk("alpha", sigma_alpha**-2,
shape=len(data) / subsample_alpha)
beta = GaussianRandomWalk("beta", sigma_beta**-2,
shape=len(data) / subsample_beta)
# make coefficients have the same length as prices
alpha_r = np.repeat(alpha, subsample_alpha)
beta_r = np.repeat(beta, subsample_beta)
len(data.dropna().GDX.values)
with model_randomwalk:
# define regression
regression = alpha_r + beta_r * data.GDX.values[:1950]
# assume prices are normally distributed
# the mean comes from the regression
sd = pm.Uniform("sd", 0, 20)
likelihood = pm.Normal("GLD",
mu=regression,
sd=sd,
observed=data.GLD.values[:1950])
import scipy.optimize as sco
with model_randomwalk:
# first optimize random walk
start = pm.find_MAP(vars=[alpha, beta], fmin=sco.fmin_l_bfgs_b)
# sampling
step = pm.NUTS(scaling=start)
trace_rw = pm.sample(100, step, start=start, progressbar=False)
np.shape(trace_rw["alpha"])
part_dates = np.linspace(min(mpl_dates), max(mpl_dates), 39)
fig, ax1 = plt.subplots(figsize=(10, 5))
plt.plot(part_dates, np.mean(trace_rw["alpha"], axis=0), "b", lw=2.5, label="alpha")
for i in range(45, 55):
plt.plot(part_dates, trace_rw["alpha"][i], "b-.", lw=0.75)
plt.xlabel("date")
plt.ylabel("alpha")
plt.axis("tight")
plt.grid(True)
plt.legend(loc=2)
ax1.xaxis.set_major_formatter(mpl.dates.DateFormatter("%d %b %y") )
ax2 = ax1.twinx()
plt.plot(part_dates, np.mean(trace_rw["beta"], axis=0), "r", lw=2.5, label="beta")
for i in range(45, 55):
plt.plot(part_dates, trace_rw["beta"][i], "r-.", lw=0.75)
plt.ylabel("beta")
plt.legend(loc=4)
fig.autofmt_xdate()
plt.figure(figsize=(10, 5))
plt.scatter(data["GDX"], data["GLD"], c=mpl_dates, marker="o")
plt.colorbar(ticks=mpl.dates.DayLocator(interval=250),
format=mpl.dates.DateFormatter("%d %b %y"))
plt.grid(True)
plt.xlabel("GDX")
plt.ylabel("GLD")
x = np.linspace(min(data["GDX"]), max(data["GDX"]))
for i in range(39):
alpha_rw = np.mean(trace_rw["alpha"].T[i])
beta_rw = np.mean(trace_rw["beta"].T[i])
plt.plot(x, alpha_rw + beta_rw * x, color=plt.cm.jet(256 * i / 39))
pass
import pymc as pm
from pymc.distributions.timeseries import GaussianRandomWalk
from scipy.sparse import csc_matrix
from scipy import optimize
with pm.Model() as model:
sigma, log_sigma = model.TransformedVar(
'sigma', pm.Exponential.dist(1. / .02, testval=.1), pm.logtransform)
nu = pm.Exponential('nu', 1. / 10)
s = GaussianRandomWalk('s', sigma**-2, shape=len(nreturns))
r = pm.T('r', nu, lam=pm.exp(-2 * s), observed=nreturns)
with model:
start = pm.find_MAP(vars=[s], fmin=optimize.fmin_l_bfgs_b)
step = pm.NUTS(scaling=start)
trace = pm.sample(2000, step, start, progressbar=False)
plt.plot(trace[s][::10].T, 'b', alpha=.03)
plt.title('log volatility')
with model:
pm.traceplot(trace, model.vars[:2])
exps = np.exp(trace[s][::10].T)
plt.plot(returns[:600][::-1])
plt.plot(exps, 'r', alpha=.03)
plt.plot(-exps, 'r', alpha=.03)
plt.show()
