def test_shade_alpha_algorithm2(self):
consumer = ChainConsumer()
consumer.add_chain(self.data)
consumer.add_chain(self.data)
consumer.configure()
alpha0 = consumer.chains[0].config["shade_alpha"]
alpha1 = consumer.chains[0].config["shade_alpha"]
assert alpha0 == 1.0 / 2.0
assert alpha1 == 1.0 / 2.0
def test_covariance_1d(self):
data = np.random.normal(0, 2, size=2000000)
parameters = ["x"]
c = ChainConsumer()
c.add_chain(data, parameters=parameters)
p, cor = c.analysis.get_covariance()
assert p[0] == "x"
assert np.isclose(cor[0, 0], 4, atol=1e-2)
assert cor.shape == (1, 1)
def test_gelman_rubin_index_not_converged():
data = np.vstack((np.random.normal(loc=0.0, size=100000),
np.random.normal(loc=1.0, size=100000))).T
data[80000:, :] *= 2
data[80000:, :] += 1
consumer = ChainConsumer()
consumer.add_chain(data, walkers=4)
assert not consumer.diagnostic.gelman_rubin(chain=0)
def __init__(self, data, truth):
self.c = ChainConsumer()
parameters = [
r"$\zeta$", r"$R_{mfp}$", r"$Mh_{min} \times 10^8 M\odot$"
]
self.parameters = parameters
self.data = np.reshape(np.array([data[:, 1], data[:, 2], data[:, 0]]),
(len(data[:, 1]), len(parameters)))
self.truth = [truth[1], truth[2], truth[0]]
def test_geweke_default_failed():
data = np.vstack((np.random.normal(loc=0.0, size=100000),
np.random.normal(loc=1.0, size=100000))).T
consumer = ChainConsumer()
consumer.add_chain(data, walkers=20, name="c1")
data2 = data.copy()
data2[98000:, :] += 0.3
consumer.add_chain(data2, walkers=20, name="c2")
assert not consumer.diagnostic.geweke()
def test_summary1(self):
tolerance = 5e-2
consumer = ChainConsumer()
consumer.add_chain(self.data)
consumer.configure_general(bins=1.6)
summary = consumer.get_summary()
actual = np.array(list(summary.values())[0])
expected = np.array([3.5, 5.0, 6.5])
diff = np.abs(expected - actual)
assert np.all(diff < tolerance)
def test_convergence_failure(self):
data = np.concatenate(
(np.random.normal(loc=0.0,
size=10000), np.random.normal(loc=4.0,
size=10000)))
consumer = ChainConsumer()
consumer.add_chain(data)
summary = consumer.analysis.get_summary()
actual = np.array(list(summary.values())[0])
assert actual[0] is None and actual[2] is None
def test_stats_cum_normal(self):
tolerance = 5e-2
consumer = ChainConsumer()
consumer.add_chain(self.data)
consumer.configure(statistics="cumulative")
summary = consumer.analysis.get_summary()
actual = np.array(list(summary.values())[0])
expected = np.array([3.5, 5.0, 6.5])
diff = np.abs(expected - actual)
assert np.all(diff < tolerance)
def test_stats_cum_skew(self):
tolerance = 3e-2
consumer = ChainConsumer()
consumer.add_chain(self.data_skew)
consumer.configure(statistics="cumulative")
summary = consumer.analysis.get_summary()
actual = np.array(list(summary.values())[0])
expected = np.array([1.27, 2.01, 3.11])
diff = np.abs(expected - actual)
assert np.all(diff < tolerance)
def test_summary_no_smooth(self):
tolerance = 5e-2
consumer = ChainConsumer()
consumer.add_chain(self.data)
consumer.configure(smooth=0, bins=2.4)
summary = consumer.analysis.get_summary()
actual = np.array(list(summary.values())[0])
expected = np.array([3.5, 5.0, 6.5])
diff = np.abs(expected - actual)
assert np.all(diff < tolerance)
def test_summary(self):
tolerance = 4e-2
consumer = ChainConsumer()
consumer.add_chain(self.data[::10])
consumer.configure(kde=True)
summary = consumer.analysis.get_summary()
actual = np.array(list(summary.values())[0])
expected = np.array([3.5, 5.0, 6.5])
diff = np.abs(expected - actual)
assert np.all(diff < tolerance)
def test_summary_power(self):
tolerance = 4e-2
consumer = ChainConsumer()
data = np.random.normal(loc=0, scale=np.sqrt(2), size=1000000)
consumer.add_chain(data, power=2.0)
summary = consumer.analysis.get_summary()
actual = np.array(list(summary.values())[0])
expected = np.array([-1.0, 0.0, 1.0])
diff = np.abs(expected - actual)
assert np.all(diff < tolerance)
def test_summary_specific(self):
tolerance = 5e-2
consumer = ChainConsumer()
consumer.add_chain(self.data, name="A")
consumer.configure(bins=0.8)
summary = consumer.analysis.get_summary(chains="A")
actual = np.array(list(summary.values())[0])
expected = np.array([3.5, 5.0, 6.5])
diff = np.abs(expected - actual)
assert np.all(diff < tolerance)
def test_gelman_rubin_default_not_converge():
data = np.vstack((np.random.normal(loc=0.0, size=100000),
np.random.normal(loc=1.0, size=100000))).T
consumer = ChainConsumer()
consumer.add_chain(data, walkers=4, name="c1")
consumer.add_chain(data, walkers=4, name="c2")
data2 = data.copy()
data2[:, 0] += np.linspace(-5, 5, 100000)
consumer.add_chain(data2, walkers=4, name="c3")
assert not consumer.diagnostic.gelman_rubin()
def run_mcmc(img_xobs, img_yobs, fig_dir, d, lenses, pars, cov, nwalkers=100, nsteps=200, burn=20, fits_file=None, img_name=''):
names = img_xobs.keys()
# Define likelihood function
@pymc.observed
def logL(value=0., tmp=pars):
""" Calculate log-likelihood probability.
Minimise the variance in the source position from all images. """
for lens in lenses:
lens.setPars()
x_src, y_src = {}, {}
lnlike_dict = {}
for name in names:
x_src[name], y_src[name] = pylens.getDeflections(lenses, [img_xobs[name], img_yobs[name]], d[name])
lnlike_dict[name] = -0.5 * (x_src[name].var() + y_src[name].var())
print(sum(lnlike_dict.values()), float(lenses[0].x), float(lenses[0].y), float(lenses[0].b), float(lenses[0].q), float(lenses[0].pa))
return sum(lnlike_dict.values())
# Run MCMC
sampler = myEmcee.Emcee(pars+[logL], cov, nwalkers=nwalkers, nthreads=46)
sampler.sample(nsteps)
# Plot chains
result = sampler.result()
posterior, samples, _, best = result
print("best", best)
import pylab
for j in range(nwalkers):
pylab.plot(posterior[:, j])
for i in range(len(pars)):
pylab.figure()
for j in range(nwalkers):
pylab.plot(samples[:, j, i])
# Trim initial samples (ie the burn-in) and concatenate chains
samples = samples[burn:].reshape(((nsteps-burn) * nwalkers, len(pars)))
# Plot parameter contours and mcmc chains
param_names = ['$x_{lens}$', '$y_{lens}$', '$b_{lens}$', '$q_{lens}$', '$pa_{lens}$']
if len(pars) == 7:
param_names += ['$b_{shear}$', '$pa_{shear}$']
c = ChainConsumer()
c.add_chain(samples, parameters=param_names)
c.configure(summary=True, cloud=True)
fig = c.plotter.plot()
fig.savefig(os.path.join(fig_dir, 'parameter_contours.png'), transparent=False)
fig = c.plotter.plot_walks(convolve=100)
fig.savefig(os.path.join(fig_dir, 'mcmc_walks.png'), transparent=False)
if fits_file:
fig = plt.figure(figsize=(13, 13))
plot_image_and_contours(fits_file, samples, fig_dir, img_name, save=False, figname=fig)
plot_source_and_pred_lens_positions(best, img_xobs, img_yobs, d, fig_dir, plotimage=False)
def test_aic_data_dependence():
d = norm.rvs(size=1000)
p = norm.logpdf(d)
c = ChainConsumer()
c.add_chain(d, posterior=p, num_free_params=1, num_eff_data_points=1000)
c.add_chain(d, posterior=p, num_free_params=1, num_eff_data_points=500)
aics = c.comparison.aic()
assert len(aics) == 2
assert aics[0] == 0
expected = (2.0 * 1 * 2 / (500 - 1 - 1)) - (2.0 * 1 * 2 / (1000 - 1 - 1))
assert np.isclose(aics[1], expected, atol=1e-3)
def test_bic_data_dependence2():
d = norm.rvs(size=1000)
p = norm.logpdf(d)
c = ChainConsumer()
c.add_chain(d, posterior=p, num_free_params=2, num_eff_data_points=1000)
c.add_chain(d, posterior=p, num_free_params=3, num_eff_data_points=500)
bics = c.comparison.bic()
assert len(bics) == 2
assert bics[0] == 0
expected = 3 * np.log(500) - 2 * np.log(1000)
assert np.isclose(bics[1], expected, atol=1e-3)
def test_file_loading2(self):
data = self.data[:1000]
directory = tempfile._get_default_tempdir()
filename = next(tempfile._get_candidate_names())
filename = directory + os.sep + filename + ".npy"
np.save(filename, data)
consumer = ChainConsumer()
consumer.add_chain(filename)
summary = consumer.analysis.get_summary()
actual = np.array(list(summary.values())[0])
assert np.abs(actual[1] - 5.0) < 0.5
def test_summary_some_params(self):
consumer = ChainConsumer()
consumer.add_chain(self.data_combined,
parameters=["a", "b"],
name="chain1")
summary = consumer.analysis.get_summary(parameters=["a"],
squeeze=False)
k1 = list(summary[0].keys())
assert len(k1) == 1
assert "a" in k1
assert "b" not in k1
def test_weights(self):
tolerance = 3e-2
samples = np.linspace(-4, 4, 200000)
weights = norm.pdf(samples)
c = ChainConsumer()
c.add_chain(samples, weights=weights)
expected = np.array([-1.0, 0.0, 1.0])
summary = c.analysis.get_summary()
actual = np.array(list(summary.values())[0])
diff = np.abs(expected - actual)
assert np.all(diff < tolerance)
def plot(burnout=0):
chain = np.loadtxt('samples.txt')[burnout::1]
print(chain.shape)
mean = chain.mean(axis=0)
c = ChainConsumer()
c.add_chain(chain)
# c.configure(kde=[2.0])
c.plotter.plot(filename="example.jpg", figsize="column", truth=mean)
# pygtc.plotGTC(chains=[chain])
# plt.plot(chain[:,0],chain[:,1],'-o',markersize=1.0)
plt.show()
def test_shade_alpha_algorithm3(self):
consumer = ChainConsumer()
consumer.add_chain(self.data)
consumer.add_chain(self.data)
consumer.add_chain(self.data)
consumer.configure()
alphas = [c.config["shade_alpha"] for c in consumer.chains]
assert len(alphas) == 3
assert alphas[0] == 1.0 / np.sqrt(3.0)
assert alphas[1] == 1.0 / np.sqrt(3.0)
assert alphas[2] == 1.0 / np.sqrt(3.0)
def test_max_likelihood_1(self):
c = ChainConsumer()
data = np.random.multivariate_normal([0, 0], [[1, 0], [0, 1]],
size=10000)
posterior = norm.logpdf(data).sum(axis=1)
data[:, 1] += 1
c.add_chain(data, parameters=["x", "y"], posterior=posterior, name="A")
result = c.analysis.get_max_posteriors()
x, y = result["x"], result["y"]
assert np.isclose(x, 0, atol=0.05)
assert np.isclose(y, 1, atol=0.05)
def make_real_corner(i):
chain = pd.read_csv(chainout%i, dtype='float64', delim_whitespace=True)
chain = chain.as_matrix()
from chainconsumer import ChainConsumer
labs=[r"$d_0'$",r"$d_1'$",r"$e_1'$",r"$f_0'$",r"$f_1'$",r"$g_0'$",r"$g_1'$"]
c = ChainConsumer()
c.add_chain(chain, parameters=labs)
c.configure(kde=True, tick_font_size=10, label_font_size=24, max_ticks=3, sigmas=[0,1,2,3], usetex=True)#, statistics='max_symmetric')
fig = c.plotter.plot()#legend=True)
plt.subplots_adjust(hspace=0, wspace=0)
plt.savefig("fig_Rcorner.pdf", bbox_inches='tight')
plt.show()
def test_aic_posterior_dependence():
d = norm.rvs(size=1000)
p = norm.logpdf(d)
p2 = norm.logpdf(d, scale=2)
c = ChainConsumer()
c.add_chain(d, posterior=p, num_free_params=1, num_eff_data_points=1000)
c.add_chain(d, posterior=p2, num_free_params=1, num_eff_data_points=1000)
aics = c.comparison.aic()
assert len(aics) == 2
assert aics[0] == 0
expected = 2 * np.log(2)
assert np.isclose(aics[1], expected, atol=1e-3)
def view():
import matplotlib.pyplot as plt
pkl_file = open('c2mag.pkl', 'r')
data = pickle.load(pkl_file)
pkl_file.close()
mn=numpy.zeros((data['alpha'].shape[0], len(inda)))
for i in xrange(len(inda)):
for j in xrange(data['alpha'].shape[0]):
mn[j,i] = numpy.dot(data['alpha'][j,:]*alpha_scale[:-1],data['snparameters'][j,indapy[i],:-1])
mn_meas = numpy.mean(mn,axis=0)
mn_cov = numpy.cov(mn,rowvar=False)
dm = inputfit['Delta']-inputfit['Delta'][:,0][:,None]
dm = dm[:,1:]
dm = dm[:,indapy]
dm_meas = numpy.mean(dm,axis=0)
dm_cov = numpy.cov(dm,rowvar=False)
res = dm_meas-mn_meas
res_cov = dm_cov + mn_cov
ax = plt.errorbar(mass, res, marker='o',linestyle="None",xerr=[emass,emass],yerr = [numpy.sqrt(numpy.diag(res_cov)), numpy.sqrt(numpy.diag(res_cov))])
x=numpy.arange(6.8,13,0.05)
(step,stepm,stepp)= numpy.percentile(data['stephigh'],(50,50-34,50+34),axis=0)/2
zterm = numpy.mean(data['zeroterm'] + data['stephigh']/2*numpy.tanh(10*(data['mass0_0']-10)))
plt.plot(x,step*numpy.tanh(10*(x-10))-zterm,color='black')
plt.plot(x,stepm*numpy.tanh(10*(x-10))-zterm,linestyle='--',color='r')
plt.plot(x,stepp*numpy.tanh(10*(x-10))-zterm,linestyle='--',color='g')
plt.ylabel(r'Relative magnitude offset $\vec{\Delta}_{.0} - \mu[0:N]$ (mag)')
plt.xlabel(r'$\log{(M_{\mathrm{host}}/M_{\odot})}$')
plt.xlim((6.8,13))
plt.savefig("mass.pdf",bbox_inches='tight')
# the outlier
print "outlier ", nms[res > 0.5],names[inda[res>0.5]], zcmb[inda[res>0.5]-1],zerr[inda[res>0.5]-1]
plt.clf()
(fitmass,fitmassm,fitmassp) = numpy.percentile(data['mass_0'],(50,50-34,50+34),axis=0)
plt.errorbar(mass,fitmass,xerr=[emass,emass],yerr=[fitmass-fitmassm, fitmassp-fitmass],linestyle='None',alpha=0.5,color='b')
plt.plot(mass,fitmass,'o',linestyle='None',markersize=4,color='b')
plt.plot([7.1,12.5],[7.1,12.5])
plt.xlabel(r'$\log{(M_{\mathrm{host}}/M_{\odot})}$')
plt.ylabel(r'$\theta_{M_{\mathrm{host}}}$')
plt.savefig("masses.pdf",bbox_inches='tight')
from chainconsumer import ChainConsumer
c = ChainConsumer()
c.add_chain([data['steplow'],data['stephigh'],data['mass_mn'],2*numpy.tan(data['mass_unif'])], parameters= \
[r"$M_{\mathrm{low}}$", r"$\Delta_{\mathrm{high}}$", r"$\langle M_{\mathrm{host}} \rangle$", r"$\sigma_{M_{\mathrm{host}}}$"],name='Master')
fig = c.plotter.plot(figsize="page",truth=numpy.zeros(4))
fig.savefig("mass_fit.pdf",bbox_inches='tight')
def test_plotter_extents5(self):
x, y = np.linspace(-3, 3, 200), np.linspace(-5, 5, 200)
xx, yy = np.meshgrid(x, y, indexing='ij')
xs, ys = xx.flatten(), yy.flatten()
chain = np.vstack((xs, ys)).T
pdf = (1 / (2 * np.pi)) * np.exp(-0.5 * (xs * xs + ys * ys / 4))
c = ChainConsumer()
c.add_chain(chain, parameters=['x', 'y'], weights=pdf, grid=True)
c.configure()
minv, maxv = c.plotter._get_parameter_extents("x", c.chains)
assert np.isclose(minv, -3, atol=0.001)
assert np.isclose(maxv, 3, atol=0.001)
def test_remove_chain_by_name(self):
tolerance = 5e-2
consumer = ChainConsumer()
consumer.add_chain(self.data * 2, name="a")
consumer.add_chain(self.data, name="b")
consumer.remove_chain(chain="a")
consumer.configure()
summary = consumer.analysis.get_summary()
assert isinstance(summary, dict)
actual = np.array(list(summary.values())[0])
expected = np.array([3.5, 5.0, 6.5])
diff = np.abs(expected - actual)
assert np.all(diff < tolerance)
def plot_corner_ecosw(self):
c = ChainConsumer()
chain = self.chain_without_burnin()
cols = 'ecosw2 ecosw3'.split()
parameters = [key2tex(k) for k in cols]
c.add_chain(np.array(chain[cols]), parameters=parameters)
c.configure(plot_hists=False)
c.plotter.plot(figsize=(3, 3))
fig = gcf()
axL = fig.get_axes()
setp(axL, xlim=(-0.301, 0.3))
setp(axL, ylim=(-0.301, 0.3))
fig.set_tight_layout(True)
def get_instance():
np.random.seed(0)
c = ChainConsumer()
parameters = ["$x$", r"$\Omega_\epsilon$", "$r^2(x_0)$"]
for name in ["Ref. model", "Test A", "Test B", "Test C"]:
# Add some random data
mean = np.random.normal(loc=0, scale=3, size=3)
sigma = np.random.uniform(low=1, high=3, size=3)
data = np.random.multivariate_normal(mean=mean,
cov=np.diag(sigma**2),
size=100000)
c.add_chain(data, parameters=parameters, name=name)
return c