def print_samples_qa(samples):
print "Mean, standard devs, acor tau, acor mean, acor s ..."
for kk in range(len(samples[0])):
xs= numpy.array([s[kk] for s in samples])
#Auto-correlation time
tau, m, s= acor.acor(xs)
print numpy.mean(xs), numpy.std(xs), tau, m, s
def calculate_ess(self):
""" {{{ Docstrings
Calculates the effective sample size of data, given the name of the
BEAST output file as a string, utilizing "genfromtxt" provided by the
"numpy" python module and "acor" provided by the "acor" python module.
}}} """
# Read in data, ignoring comments, sample column, and header,
# respectively
data = genfromtxt(
self._BEAST_ID, comments='#', usecols=range(1, 17)
)[1:]
# Concatenate data by columns
data = zip(*data)
# Calculate autocorrelation times (and other statistics) for each
# column
stats = map(lambda x: acor(x), data)
# Extract autocorrelation times from statistics
auto_cor_times = zip(*stats)[0]
# Calculate MCMC chain length
chain_length = int(args.MCMC_BEAST * (1 - args.burnin_BEAST))
# Calculate effective sample size
eff_sample_size = map(lambda x: chain_length / x, auto_cor_times)
return eff_sample_size
def main():
data = LogData(args.logdir, args.output)
s0 = data.get_ice_configs()
model = IceModel()
model.set_ice(s0)
observs = []
data.set_end_index(100)
# you had better cal defect den as they do.
for i, loop in enumerate(data.loops):
model.apply_loop(loop)
st = model.get_ice()
##dd = inner_prod(st, s0)
dd = model.get_symm_vertex()
observs.append(dd)
obs = np.asarray(observs)
acorr = data.cal_autocorr(obs, "auto_corr")
#tau = integrated_autocorr_time(acorr)
#print ("Integrated autocorrelation time: {}".format(tau))
import acor
tau, mean, sigma = acor.acor(obs)
print("Acor: tau = {}, mean = {}, sigma = {}".format(tau, mean, sigma))
def stats(g):
acor_result = acor.acor(g)
#l = [numpy.mean(g), numpy.std(g), ACtime(g), error(g)]
l = [numpy.mean(g), numpy.std(g), acor_result[0], numpy.std(g)/math.sqrt(len(g))]
return l
def integrated_time(self):
"""
Estimate the integrated autocorrelation time of a time series.
Since it seems there is something wrong with the sampler.integrated_time(),
I have to calculated myself using acor package.
"""
sampler = self.__sampler
if sampler == "EnsembleSampler":
chain = self.sampler.chain
elif sampler == "PTSampler":
chain = self.sampler.chain[0, ...]
else:
raise ValueError("{0} is an unrecognised sampler!".format(sampler))
tauParList = []
for npar in range(self.__dim):
tauList = []
for nwal in range(self.__nwalkers):
pchain = chain[nwal, :, npar]
try:
tau, mean, sigma = acor.acor(pchain)
except:
tau = np.nan
tauList.append(tau)
tauParList.append(tauList)
return tauParList
def makePostPlots_show(chain, ndim, labels):
import acor
for ii in range(ndim):
xmajorLocator = matplotlib.ticker.MaxNLocator(nbins=6,prune='both')
ymajorLocator = matplotlib.ticker.MaxNLocator(nbins=6,prune='both')
fig = plt.figure(figsize=(10,4))
ax = fig.add_subplot(121)
acl = acor.acor(chain[:,ii])[0]
neff = len(chain[:,ii]) / acl * 10
ax.plot(chain[:,ii])
plt.title('Neff = {0}'.format(int(neff)))
plt.ylabel(labels[ii])
majorFormatter = FormatStrFormatter('%d')
ax.xaxis.set_major_formatter(majorFormatter)
ax = fig.add_subplot(122)
ax.hist(chain[:,ii], 50, lw=2, color='b', normed=True)
plt.xlabel(labels[ii])
ax.xaxis.set_major_locator(xmajorLocator)
ax.yaxis.set_major_locator(ymajorLocator)
def makePostPlots_show(chain, ndim, labels):
import acor
for ii in range(ndim):
xmajorLocator = matplotlib.ticker.MaxNLocator(nbins=6, prune='both')
ymajorLocator = matplotlib.ticker.MaxNLocator(nbins=6, prune='both')
fig = plt.figure(figsize=(10, 4))
ax = fig.add_subplot(121)
try:
acl = acor.acor(chain[:, ii])[0]
neff = len(chain[:, ii]) / acl * 10
ax.plot(chain[:, ii])
plt.title('Neff = {0}'.format(int(neff)))
except:
ax.plot(chain[:, ii])
plt.ylabel(labels[ii])
majorFormatter = FormatStrFormatter('%d')
ax.xaxis.set_major_formatter(majorFormatter)
ax = fig.add_subplot(122)
ax.hist(chain[:, ii], 50, lw=2, color='b', normed=True)
plt.xlabel(labels[ii])
ax.xaxis.set_major_locator(xmajorLocator)
ax.yaxis.set_major_locator(ymajorLocator)
def print_samples_qa(samples):
print "Mean, standard devs, acor tau, acor mean, acor s ..."
for kk in range(len(samples[0])):
xs = numpy.array([s[kk] for s in samples])
#Auto-correlation time
tau, m, s = acor.acor(xs)
print numpy.mean(xs), numpy.std(xs), tau, m, s
def acor(self):
"""
The autocorrelation time of each parameter in the chain (length:
``dim``) as estimated by the ``acor`` module.
"""
if acor is None:
raise ImportError("acor")
return acor.acor(self._chain.T)[0]
def acor(self):
"""
The autocorrelation time of each parameter in the chain (length:
`dim`) as estimated by the `acor` module.
"""
if acor is None:
raise ImportError("acor")
return acor.acor(self._chain.T)[0]
def acor(self):
"""
The autocorrelation time of each parameter in the chain (length:
``dim``) as estimated by the ``acor`` module.
"""
if acor is None:
raise ImportError("You need to install acor: "
"https://github.com/dfm/acor")
return acor.acor(self._chain.T)[0]
def acor(self):
"""
The autocorrelation time of each parameter in the chain (length:
``dim``) as estimated by the ``acor`` module.
"""
if acor is None:
raise ImportError("You need to install acor: "
"https://github.com/dfm/acor")
return acor.acor(self._chain.T)[0]
def runmore(pos,nruns):
start=time.time()
sampler.run_mcmc(pos, nruns)
x = sampler.flatchain[:,0]
tau, mean, sigma = acor.acor(x)
runsdone = len(x)/nwalkers
indsamples = runsdone/tau
end=time.time()
total = str("%.4f" % ((end-start)/(nruns*nwalkers)))
print "Time per run and per walker is " + total + " seconds (" + str(runsdone) + " runs / " + str(nwalkers) + " walkers)"
print "Autocorrelation time = " + str(tau) + " and independent samples = " + str(indsamples)
def autocorr_timescale(self, trace):
"""
Compute the autocorrelation time scale as estimated by the `acor` module.
:param trace: The parameter trace, a numpy array.
"""
acors = []
for i in range(trace.shape[1]):
tau, mean, sigma = acor.acor(trace[:, i].real) # Warning, does not work with numpy.complex
acors.append(tau)
return np.array(acors)
def AutoCorrelation(chain,burnin=None):
"""Calculate the autocorrelation time for a chain"""
c = reshape_chain(chain)
if burnin is not None:
c=c[:,burnin:,:]
nwalkers,nsteps,ndim = np.shape(c)
tau = np.zeros((nwalkers,ndim))
for i in np.arange(nwalkers):
for j in np.arange(ndim):
tau[i,j],mean,sigma = acor.acor(c[i,:,j]) #here we compute the autocorrelation time
return np.mean(tau,axis=0)
def autocorr_timescale(self, trace):
"""
Compute the autocorrelation time scale as estimated by the `acor` module.
:param trace: The parameter trace, a numpy array.
"""
acors = []
for i in range(trace.shape[1]):
tau, mean, sigma = acor.acor(
trace[:, i].real) # Warning, does not work with numpy.complex
acors.append(tau)
return np.array(acors)
def blockReduce(g):
acor_result = acor.acor(g)
blockSize = int(math.ceil(acor_result[0]))
Nlength = len(g)
reducedData = []
Nblock = int(Nlength / blockSize)
for i in range(Nblock):
reducedData.append(numpy.mean(g[i * blockSize : i * blockSize + blockSize]))
return reducedData
def odds_ratio(chain, models=[0, 1], uncertainty=True, thin=False):
if thin:
indep_samples = np.rint(chain.shape[0] / acor.acor(chain)[0])
samples = np.random.choice(chain.copy(), int(indep_samples))
else:
samples = chain.copy()
mask_top = np.rint(samples) == max(models)
mask_bot = np.rint(samples) == min(models)
top = float(np.sum(mask_top))
bot = float(np.sum(mask_bot))
if top == 0.0 and bot != 0.0:
bf = 1.0 / bot
elif bot == 0.0 and top != 0.0:
bf = top
else:
bf = top / bot
if uncertainty:
if bot == 0. or top == 0.:
sigma = 0.0
else:
# Counting transitions from model 1 model 2
ct_tb = 0
for ii in range(len(mask_top) - 1):
if mask_top[ii]:
if not mask_top[ii + 1]:
ct_tb += 1
# Counting transitions from model 2 to model 1
ct_bt = 0
for ii in range(len(mask_bot) - 1):
if mask_bot[ii]:
if not mask_bot[ii + 1]:
ct_bt += 1
try:
sigma = bf * np.sqrt((float(top) - float(ct_tb)) /
(float(top) * float(ct_tb)) +
(float(bot) - float(ct_bt)) /
(float(bot) * float(ct_bt)))
except ZeroDivisionError:
sigma = 0.0
return bf, sigma
elif not uncertainty:
return bf
def acor(self):
"""
The autocorrelation time of each parameter in the chain (length:
``dim``) as estimated by the ``acor`` module.
"""
if acor is None:
raise ImportError("acor")
s = self.dim
t = np.zeros(s)
for i in range(s):
t[i] = acor.acor(self.chain[:, :, i])[0]
return t
def acor(self):
"""
The autocorrelation time of each parameter in the chain (length:
``dim``) as estimated by the ``acor`` module.
"""
if acor is None:
raise ImportError("acor")
s = self.dim
t = np.zeros(s)
for i in range(s):
t[i] = acor.acor(self.chain[:, :, i])[0]
return t
def UL_uncert(chain, p=0.95):
corr = acor(chain)[0]
N = len(chain)
Neff = N / corr
hist = np.histogram(chain, bins=100)
pdf = ss.rv_histogram(hist).pdf
UL = np.percentile(chain, 100 * p) # 95 for 95% (not 0.95)
pUL = pdf(UL)
dUL = np.sqrt(p * (1 - p) / Neff) / pUL
return UL, dUL
def autocorrelation(chain,labels,plt_label):
npars = chain.shape[1]
maxlags = chain.shape[0]/5.
nlags = 100
lags = np.linspace(1,maxlags,nlags).astype(int)
tau = np.zeros(shape=(lags.shape[0],npars))
for l, lag in enumerate(lags):
#print('maxlag:{}').format(lag)
print l
for i in xrange(npars):
tau[l,i] = acor.acor(chain[:,i], maxlag=lag)[0]
#print('\t '+labels[i]+': {0}'.format(tau[l,i]))
### emcee version
from emcee import autocorr
c = 10
good = False
while good == False:
try:
emcee_tau = autocorr.integrated_time(chain, c=c)
good = True
except:
if c > 2:
c -= 0.5
else:
c = c ** 0.95
if (c-1) < 1e-3:
print 'FAILED TO CALCULATE AUTOCORRELATION TIMES'
emcee_tau = np.zeros(len(labels))
break
print 'AUTOCORRELATION LENGTHS'
for r, l in zip(emcee_tau,labels): print l+': '+"{:.2f}".format(r)
### plotting
fig, ax = plt.subplots(1,1, figsize=(8,8))
cmap = get_cmap(npars)
for i in xrange(npars):
ax.plot(lags,tau[:,i],label=labels[i]+'='+"{:.2f}".format(emcee_tau[i]),color=cmap(i),lw=2)
ax.set_xlabel('lag')
ax.set_ylabel('autocorrelation')
ax.legend(prop={'size':10},title='autocorrelation lengths',ncol=npars / 5,numpoints=1,markerscale=0.7)
fig.tight_layout()
plt.savefig('autocorrelation_time_'+plt_label+'.png',dpi=150)
plt.close()
def getIntegratedAutocorrelationTime_acor(self, paramIndex):
"""
Return the walkers average integrated autocorrelation time.
It uses acor.
"""
import acor
tauMean = 0
for walker in self.chains:
paramChain = [row[paramIndex] for row in walker]
tau, mean, sigma = acor.acor(paramChain)
tauMean += tau
return tauMean/len(self.chains)
def acor(self):
"""
Returns a matrix of autocorrelation lengths for each
parameter in each temperature of shape ``(Ntemps, Ndim)``.
"""
if acor is None:
raise ImportError('acor')
else:
acors = np.zeros((self.ntemps, self.dim))
for i in range(self.ntemps):
for j in range(self.dim):
acors[i, j] = acor.acor(self._chain[i, :, :, j])[0]
return acors
def acor(self):
"""
Returns a matrix of autocorrelation lengths for each
parameter in each temperature of shape ``(Ntemps, Ndim)``.
"""
if acor is None:
raise ImportError('acor')
else:
acors = np.zeros((self.ntemps, self.dim))
for i in range(self.ntemps):
for j in range(self.dim):
acors[i, j] = acor.acor(self._chain[i, :, :, j])[0]
return acors
def plot_chain(result, **kwargs):
""" Chain evolution
"""
efig, eaxes = pl.subplots(4, 4, sharex=True, figsize=(12, 8))
for i in range(result['chain'].shape[-1]):
ax = eaxes.flatten()[i]
tau = acor.acor(result['chain'][:, i])[0]
ax.plot(result['chain'][:, i])
ax.text(0.1,
0.85,
r'$\theta_{{{0}}}, \tau={1:4.1f}$'.format(i, tau),
bbox=dict(facecolor='white', edgecolor='black', pad=10.0),
transform=ax.transAxes,
fontsize=6)
efig.show()
pl.gcf().set_tight_layout(True)
return efig, eaxes
def thin_chain(chain, fburnin=0.5):
"""Takes a chain of shape ``(Niter, Nwalkers, Nparams)``, and discards
the first ``fburnin`` of the samples, thinning the remainder by
the autocorrelation length.
"""
istart = int(round(fburnin*chain.shape[0]))
chain = chain[istart:, :, :]
taumax = float('-inf')
for k in range(chain.shape[-1]):
tau = acor(np.mean(chain[:,:,k], axis=1))[0]
taumax = max(tau,taumax)
tau = int(np.ceil(taumax))
return chain[::tau, :,:]
def decorrelated_samples(pts):
"""Returns a decorrelated subset of ``pts``.
:param pts: The samples from a chain to be decorrelated. Shape
``(Nsamples, Nwalkers, Ndim)``."""
if acor is None:
raise ImportError('acor')
means=np.mean(pts, axis=1)
taumax=float('-inf')
for j in range(means.shape[1]):
tau,mu,sigma=acor.acor(means[:,j])
taumax=max(tau,taumax)
taumax=int(round(taumax))
return pts[::taumax, :, :]
def ul(chain, q=95.0):
"""
Computes upper limit and associated uncertainty.
:param chain: MCMC samples of GWB (or common red noise) amplitude
:param q: desired percentile of upper-limit value [out of 100, default=95]
:returns: (upper limit, uncertainty on upper limit)
"""
hist = np.histogram(10.0**chain, bins=100)
hist_dist = scistats.rv_histogram(hist)
A_ul = 10**np.percentile(chain, q=q)
p_ul = hist_dist.pdf(A_ul)
Aul_error = np.sqrt((q / 100.) * (1.0 - (q / 100.0)) /
(chain.shape[0] / acor.acor(chain)[0])) / p_ul
return A_ul, Aul_error
def postProcess(model, sampler, numberOfIterations, nrSteps, nrRep):
algo = sampler.algorithmName
algofile = algo
compTime = sampler.timeAv.getAverage() * numberOfIterations
totalNumberofSteps = numberOfIterations * nrSteps * nrRep
Cwlck = compTime / totalNumberofSteps
#save long traj
#np.save(algofile+'_'+model.potentialFlag+'_traj', sampler.sampled_trajectory)
tau, mean, sigma = acor.acor(
sampler.sampled_trajectory[:, 0] -
np.mean(sampler.sampled_trajectory[:, 0], axis=0))
print('\n' + algo)
print('Time per step ' + repr(Cwlck))
print('Stat. error in wallclock time ' + repr(sigma / np.sqrt(compTime)))
# file.write('Computational time is '+repr(compTime)+' s\n')
#
# file.write('Kinetic temperature is ' + repr(sampler.kineticTemperature.getAverage())+'\n' )
# file.write('<X> is ' + repr(sampler.averagePosition.getAverage()) +'\n')
#
#
#
# file.write ('Autocorrelation time is ' +repr(tau)+'\n')
# file.write ('Mean ' +repr(mean)+'\n')
# file.write ('Sigma ' +repr(sigma)+'\n')
# #statistical precision espilon=sigma^2/number_of_steps - in wallclock time esp=sigma^2/
# file.write ('Stat. error in wallclock time ' +repr(sigma/np.sqrt(compTime))+'\n')
#
# file.close()
return np.array([
tau, mean, sigma, compTime, totalNumberofSteps,
(sigma / np.sqrt(compTime))
])
def acor(self, k = 5):
"""
Autocorrelation time of the chain
return the autocorrelation time for each parameters
Inputs :
k :
parameter in self-consistent window
Outputs :
t :
autocorrelation time of the chain
"""
try:
import acor
except ImportError:
print "Can't import acor, please download."
return 0
n = np.shape(self._chain)[1]
t = np.zeros(n)
for i in xrange(n):
t[i] = acor.acor(self._chain[:,i],k)[0]
return t
def test_IAC():
"""
Test the un-metropolized XY model sampler and get the ACF and IAC.
Nsteps is the number of MCMC steps.
"""
L = 25 # Lattice size
beta = 0.1 # Inverse temperature
h = 0.05 # Step size
n = 5 # Number of velocity verlet steps.
Nsteps = int(1E4) # Number of MCMC steps
metropolize = False # Do metropolize or not.
if metropolize:
[xy, rej_rate, mags] = HybridMC(L, beta, h, n, Nsteps, True, True)
print("\nMetropolized Scheme")
print("\nRejection Rate = {:.2f}%".format(100 * rej_rate))
else:
print("\nUn-Metropolized Scheme")
[xy, mags] = HybridMC(L, beta, h, n, Nsteps, False, True)
acf = acor.function(mags)
# Time for the correlation to first reach 0 (within the tolerance).
cor_time = np.where(acf <= 1E-6)[0][0]
plt.figure(3)
plt.rc('text', usetex=True)
plt.rc('font', family='serif')
plt.xlabel('Lag')
plt.ylabel('Autocorrelation')
if metropolize:
plt.title('ACF of the Metropolized Scheme')
else:
plt.title('ACF of the Un-Metropolized Scheme')
plt.plot(np.arange(cor_time + 1), acf[:cor_time + 1], 'b-')
tau = acor.acor(mags, maxlag=cor_time)[0]
print("\nIAC = {:.1f}\n".format(tau))
def makePostPlots(chain, labels, outDir='./postplots'):
import acor
if not os.path.exists(outDir):
try:
os.makedirs(outDir)
except OSError:
pass
ndim = chain.shape[1]
for ii in range(ndim):
xmajorLocator = matplotlib.ticker.MaxNLocator(nbins=6,prune='both')
ymajorLocator = matplotlib.ticker.MaxNLocator(nbins=6,prune='both')
fig = plt.figure(figsize=(10,4))
ax = fig.add_subplot(121)
acl = acor.acor(chain[:,ii])[0]
neff = len(chain[:,ii]) / acl * 10
ax.plot(chain[:,ii])
plt.title('Neff = {0}'.format(int(neff)))
plt.ylabel(labels[ii])
ax = fig.add_subplot(122)
if 'equad' in labels[ii] or 'jitter' in labels[ii] or \
'Amplitude' in labels[ii]:
ax.hist(10**chain[:,ii], 50, lw=2, color='b', \
weights=10**chain[:,ii], normed=True)
else:
ax.hist(chain[:,ii], 50, lw=2, color='b', normed=True)
plt.xlabel(labels[ii])
ax.xaxis.set_major_locator(xmajorLocator)
ax.yaxis.set_major_locator(ymajorLocator)
plt.savefig(outDir + '/' + labels[ii] + '_post.png', bbox_inches='tight', \
dpi=200)
def postProcessTemporary(traj, model, sampler, numberOfIterations, nrSteps,
nrRep):
algo = sampler.algorithmName
algofile = algo
compTime = sampler.timeAv.getAverage() * numberOfIterations
totalNumberofSteps = traj.shape[0]
Cwlck = compTime / totalNumberofSteps
tau, mean, sigma = acor.acor(traj[:, 0] - np.mean(traj[:, 0], axis=0))
print('\n' + algo)
print('Time per step ' + repr(Cwlck))
print('Stat. error in wallclock time ' + repr(sigma / np.sqrt(compTime)))
return np.array([
tau, mean, sigma, compTime, totalNumberofSteps,
(sigma / np.sqrt(compTime))
])
def makePostPlots(chain, labels, outDir='./postplots'):
import acor
if not os.path.exists(outDir):
try:
os.makedirs(outDir)
except OSError:
pass
ndim = chain.shape[1]
for ii in range(ndim):
xmajorLocator = matplotlib.ticker.MaxNLocator(nbins=6,prune='both')
ymajorLocator = matplotlib.ticker.MaxNLocator(nbins=6,prune='both')
fig = plt.figure(figsize=(10,4))
ax = fig.add_subplot(121)
acl = acor.acor(chain[:,ii])[0]
neff = len(chain[:,ii]) / acl * 10
ax.plot(chain[:,ii])
plt.title('Neff = {0}'.format(int(neff)))
plt.ylabel(labels[ii])
ax = fig.add_subplot(122)
if 'equad' in labels[ii] or 'jitter' in labels[ii] or \
'Amplitude' in labels[ii]:
ax.hist(10**chain[:,ii], 50, lw=2, color='b', \
weights=10**chain[:,ii], normed=True)
else:
ax.hist(chain[:,ii], 50, lw=2, color='b', normed=True)
plt.xlabel(labels[ii])
ax.xaxis.set_major_locator(xmajorLocator)
ax.yaxis.set_major_locator(ymajorLocator)
plt.savefig(outDir + '/' + labels[ii] + '_post.png', bbox_inches='tight', \
dpi=200)
def ess(t):
'''returns the average sample size of a N,M ndarray (doesn't actually take a N,M ndarray'''
#extract data from dictoranry
data = []
for i in t:
dics = []
for j in i.keys():
dics.append(nu.ravel(i[j]))
data.append(nu.hstack(dics))
data = nu.asarray(data)
temp_ess,Len = [],float(len(data[:,0]))
#calculate ess for each parameter
for i in range(data.shape[1]):
try:
temp_ess.append(Len/acor.acor(data[:,i])[0])
except ZeroDivisionError:
pass
except RuntimeError:
pass
if len(temp_ess) == 0:
return 0.
#return max
return nu.nanmax(temp_ess)
def master_inference(path, all=False, outfile="combined_inference.hdf5"):
""" Create a master inference hdf5 file from output every 1000 steps """
# first see if relative
cache_path = os.path.join(streamspath, path)
print(cache_path)
if not os.path.exists(cache_path): raise IOError("Path doesn't exist!")
for filename in sorted(glob.glob(os.path.join(cache_path,"inference_*.hdf5"))):
print(filename)
with h5py.File(filename, "r") as f:
try:
chain = np.hstack((chain,f["chain"].value))
except NameError:
chain = f["chain"].value
accfrac = f["acceptance_fraction"].value
ix = accfrac > 0.02
print(sum(~ix), "<2% acceptance")
#tau,mm,xx = acor.acor(np.mean(chain[ix],axis=0).T)
tau = [acor.acor(np.mean(chain[ix],axis=0).T[i])[0] for i in range(chain.shape[-1])]
acor_time = int(np.max(tau))
print("Autocorrelation times: ", tau)
print("Max autocorrelation time: ", acor_time)
if not all:
_chain = chain[:,::acor_time].copy()
else:
_chain = chain.copy()
fn = os.path.join(cache_path, outfile)
with h5py.File(fn, "w") as f:
f["chain"] = _chain
f["acor"] = tau
print('Found new best likelihood of {0:.1f}.'.format(old_best_lnlike))
print('Resetting around parameters ')
best_params = lnposterior.to_params(best).squeeze()
for n in best_params.dtype.names:
print(n + ':', best_params[n])
print('')
sys.stdout.flush()
means.append(np.mean(p0[0, :, :], axis=0))
ameans = np.array(means)
ameans = ameans[int(round(0.2*ameans.shape[0])):, :]
taumax = float('-inf')
for j in range(ameans.shape[1]):
try:
tau = acor.acor(ameans[:,j])[0]
except:
tau = float('inf')
taumax = max(tau, taumax)
ndone = int(round(ameans.shape[0]/taumax))
print('Computed {0:d} effective ensembles (max correlation length is {1:g})'.format(ndone, taumax))
print('')
sys.stdout.flush()
if ndone > args.nensembles:
break
if len(f.shape) == 1:
return 1 + 2*np.sum(f[1:window])
# N-dimensional case.
m = [slice(None), ] * len(f.shape)
m[axis] = slice(1, window)
tau = 1 + 2*np.sum(f[m], axis=axis)
return tau
if __name__ == "__main__":
import time
import acor
N = 400000
a = 0.9
d = 3
x = np.empty((N, d))
x[0] = np.zeros(d)
for i in xrange(1, N):
x[i] = x[i-1] * a + np.random.rand(d)
strt = time.time()
print(integrated_time(x))
print(time.time() - strt)
strt = time.time()
print([acor.acor(x[:, i])[0] for i in range(d)])
print(time.time() - strt)
met0 = np.arange(-2.5, 0.5, 0.1)
sky0 = np.arange(0.0, 2.0, 0.1)
pos = [np.array([np.random.choice(teff0), np.random.choice(logg0), np.random.choice(vel0), np.random.choice(met0), np.random.choice(sky0)]) for i in range(nwalkers)]
# set up the emcee sampler, then run the MCMC for 100 steps from the stating position 'pos0', then reset the sampler
sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, args=(star, wave))
start=time.time()
pos, prob, state = sampler.run_mcmc(pos, nruns)
sampler.reset()
middle=time.time()
total = str("%.4f" % (middle-start))
print "Burn-in time is " + total + " seconds (for nwalkers=" + str(nwalkers) + ", nruns=" + str(nruns) + ")."
print "Starting main MCMC simulation now."
runsdone = 0
while (runsdone < 100 or indsamples < 10):
sampler.run_mcmc(pos, nruns)
x = sampler.flatchain[:,0]
tau, mean, sigma = acor.acor(x)
runsdone = len(x)/nwalkers
indsamples = runsdone/tau
print "Done " + str(runsdone) + " runs and so far have " + str(indsamples) + " independent samples."
end=time.time()
total = str("%.4f" % (end-start))
timeper = str("%.4f" % ((end-start)/(runsdone*nwalkers)))
print "Converged after " + str(runsdone) + " runs, which took " + total + " seconds"
print "Time per run and per walker is " + timeper + " seconds"
print "Autocorrelation time = " + str(tau) + " and independent samples = " + str(indsamples)
samps = np.array(samps).T
Likes = samps[-1]
samps = np.float64(samps[:-1])
TimingParams = open(root + "T2scaling.txt").readlines()
for i in range(len(TimingParams)):
TimingParams[i] = TimingParams[i].strip('\n').split()
NSamp = len(samps[0])
acors = np.zeros(len(samps))
means = np.zeros(len(samps))
stds = np.zeros(len(samps))
for i in range(len(samps)):
acors[i] = acor.acor(np.float64(samps[i]))[0]
means[i] = np.mean(samps[i])
stds[i] = np.std(samps[i])
for i in range(len(TimingParams)):
print i, TimingParams[i][0], means[i + 1] * np.float64(
TimingParams[i][-1]) + np.float64(
TimingParams[i][-2]), stds[i + 1] * np.float64(TimingParams[i][-1])
for p in range(len(samps)):
index = p
sum = 0
for i in range(len(samps[index]) - 1):
sum = sum + np.abs(samps[index][i + 1] - samps[index][i])
sum = sum / (NSamp - 1)
print p, sum, np.std(samps[index]), sum / np.std(samps[index])
# Burn in samples
##################
if args.intelburn:
loglike = chain[-3] # likelihood column
dim = chain.shape[1] - 4 # dimensionality of search
likemax = loglike.max() # max likelihood
burniter = np.where(loglike > (likemax - dim / 2.))[0]
burntdata = chain[burniter:, :]
# GWB amplitude index
amp_ind = int(param_list[param_list[:, 1] == 'Agwb', 0][0])
corrlength, mean, sigma = acor.acor(burntdata[:, amp_ind])
indsamples = burntdata[::int(corrlength)]
chain = indsamples
if not args.intelburn: chain = chain[args.manualburn:, :]
############################################################
# Doing a quick plot to manually cut-off the burn-in stage
############################################################
if 'gam4p33' in args.chaindir:
# checking Agwb
Agwb = chain[:, int(param_list[param_list[:, 1] == 'Agwb', 0][0])]
gwb_params = np.zeros((len(chain), 1))
gwb_params[:, 0] = Agwb
# now accept the step with probability min(1.0,gxtry/gx);
# if not accepted walker coordinate remains the same and is also added to the chain
#
if gxtry > gx:
x = xtry; y=ytry
naccept += 1
else:
aprob = gxtry/gx # acceptance probability
u = rnd.uniform(0,1)
if u < aprob:
x = xtry; y= ytry
naccept += 1
#
# whatever the outcome, add current walker coordinates to the chain
#
chain.append([x,y])
i += 1; ntry += 1
print "Generated n ",n," samples with a single walker"
print "with acceptance ratio", 1.0*naccept/ntry
xh = np.array(zip(*chain)[0]); yh = np.array(zip(*chain)[1])
xh = xh[nburn::nsel]; yh=yh[nburn::nsel]
print "computing autocorrelation time..."
t1 = time()
tacor = acor.acor(xh,100)
t2 = time()
print "done in",t2-t1," seconds."
print "tacor=",tacor
aprob = gxtry/gx # acceptance probability
u = rnd.uniform(0,1)
if u < aprob:
x[i] = xtry; y[i]= ytry
naccept += 1
if nd > nburn and (not nd%nsel) : # start the chain only after burn in
chain.append([x[i],y[i]])
ntry += 1
print "Generated n ",n*nwalkers," samples using", nwalkers," walkers"
print "with acceptance ratio", 1.0*naccept/ntry
xh = zip(*chain)[0]; yh = zip(*chain)[1]
print "computing autocorrelation time..."
tacorx = acor.acor(xh)[0]
tacory = acor.acor(yh)[0]
print "tacorx=",tacorx
print "tacory=",tacory
#
# plot results:
#
x = np.arange(-10.0,10.0,0.05); y = np.arange(-1.0,100,0.05)
X, Y = np.meshgrid(x,y)
Z = modelpdf(X,Y)
def __init__(self, infile, inject=False, **kwargs):
self.d = d = pf.getdata(infile,1)
m = isfinite(d.flux_1) & (~(d.mflags_1 & 2**3).astype(np.bool))
m &= ~binary_dilation((d.quality & 2**20) != 0)
self.Kp = pf.getval(infile,'kepmag')
self.Kp = self.Kp if not isinstance(self.Kp, Undefined) else nan
self.tm = MA(supersampling=12, nthr=1)
self.em = MA(supersampling=10, nldc=0, nthr=1)
self.epic = int(basename(infile).split('_')[1])
self.time = d.time[m]
self.flux = (d.flux_1[m]
- d.trend_t_1[m] + nanmedian(d.trend_t_1[m])
- d.trend_p_1[m] + nanmedian(d.trend_p_1[m]))
self.mflux = nanmedian(self.flux)
self.flux /= self.mflux
self.flux_e = d.error_1[m] / abs(self.mflux)
self.flux_r = d.flux_1[m] / self.mflux
self.trend_t = d.trend_t_1[m] / self.mflux
self.trend_p = d.trend_p_1[m] / self.mflux
self.period_range = kwargs.get('period_range', (0.7,0.98*(self.time.max()-self.time.min())))
self.nbin = kwargs.get('nbin',900)
self.qmin = kwargs.get('qmin',0.002)
self.qmax = kwargs.get('qmax',0.115)
self.nf = kwargs.get('nfreq',10000)
self.bls = BLS(self.time, self.flux, self.flux_e, period_range=self.period_range,
nbin=self.nbin, qmin=self.qmin, qmax=self.qmax, nf=self.nf, pmean='running_median')
def ndist(p=0.302):
return 1.-2*abs(((self.bls.period-p)%p)/p-0.5)
def cmask(s=0.05):
return 1.-np.exp(-ndist()/s)
self.bls.pmul = cmask()
try:
ar,ac,ap,at = acor(self.flux_r)[0], acor(self.flux)[0], acor(self.trend_p)[0], acor(self.trend_t)[0]
except RuntimeError:
ar,ac,ap,at = nan,nan,nan,nan
self.lcinfo = array((self.epic, self.mflux, self.flux.std(), nan, nan, ar, ac, ap, at), dtype=dt_lcinfo)
self._rbls = None
self._rtrf = None
self._rvar = None
self._rtoe = None
self._rpol = None
self._recl = None
## Transit fit pv [k u t0 p a i]
self._pv_bls = None
self._pv_trf = None
self.period = None
self.zero_epoch = None
self.duration = None
def MCMC(self,
niter=500,
nburn=200,
nwalkers=200,
threads=1,
fit_partial=False,
width=3,
savedir=None,
refit=False,
thin=10,
conf=0.95,
maxslope=MAXSLOPE,
debug=False,
p0=None):
"""
Fit transit signal to trapezoid model using MCMC
.. note:: As currently implemented, this method creates a
bunch of attributes relevant to the MCMC fit; I plan
to refactor this to define those attributes as properties
so as not to have their creation hidden away here. I plan
to refactor how this works.
"""
if fit_partial:
wok = np.where((np.absolute(self.ts - self.center) <
(width * self.dur)) & ~np.isnan(self.fs))
else:
wok = np.where(~np.isnan(self.fs))
if savedir is not None:
if not os.path.exists(savedir):
os.mkdir(savedir)
alreadydone = True
alreadydone &= savedir is not None
alreadydone &= os.path.exists('%s/ts.npy' % savedir)
alreadydone &= os.path.exists('%s/fs.npy' % savedir)
if savedir is not None and alreadydone:
ts_done = np.load('%s/ts.npy' % savedir)
fs_done = np.load('%s/fs.npy' % savedir)
alreadydone &= np.all(ts_done == self.ts[wok])
alreadydone &= np.all(fs_done == self.fs[wok])
if alreadydone and not refit:
logging.info('MCMC fit already done for %s. Loading chains.' %
self.name)
Ts = np.load('%s/duration_chain.npy' % savedir)
ds = np.load('%s/depth_chain.npy' % savedir)
slopes = np.load('%s/slope_chain.npy' % savedir)
tcs = np.load('%s/tc_chain.npy' % savedir)
else:
logging.info(
'Fitting data to trapezoid shape with MCMC for %s....' %
self.name)
if p0 is None:
p0 = self.trapfit.copy()
p0[0] = np.absolute(p0[0])
if p0[2] < 2:
p0[2] = 2.01
if p0[1] < 0:
p0[1] = 1e-5
logging.debug('p0 for MCMC = {}'.format(p0))
sampler = traptransit_MCMC(self.ts[wok],
self.fs[wok],
self.dfs[wok],
niter=niter,
nburn=nburn,
nwalkers=nwalkers,
threads=threads,
p0=p0,
return_sampler=True,
maxslope=maxslope)
Ts, ds, slopes, tcs = (sampler.flatchain[:,
0], sampler.flatchain[:,
1],
sampler.flatchain[:,
2], sampler.flatchain[:,
3])
self.sampler = sampler
if savedir is not None:
np.save('%s/duration_chain.npy' % savedir, Ts)
np.save('%s/depth_chain.npy' % savedir, ds)
np.save('%s/slope_chain.npy' % savedir, slopes)
np.save('%s/tc_chain.npy' % savedir, tcs)
np.save('%s/ts.npy' % savedir, self.ts[wok])
np.save('%s/fs.npy' % savedir, self.fs[wok])
if debug:
print(Ts)
print(ds)
print(slopes)
print(tcs)
N = len(Ts)
self.Ts_acor = acor.acor(Ts)[1]
self.ds_acor = acor.acor(ds)[1]
self.slopes_acor = acor.acor(slopes)[1]
self.tcs_acor = acor.acor(tcs)[1]
self.fit_converged = True
for t in [self.Ts_acor, self.ds_acor, self.slopes_acor, self.tcs_acor]:
if t > 0.1 * N:
self.fit_converged = False
ok = (Ts > 0) & (ds > 0) & (slopes > 0) & (slopes < self.maxslope)
logging.debug('trapezoidal fit has {} good sample points'.format(
ok.sum()))
if ok.sum() == 0:
if (Ts > 0).sum() == 0:
#logging.debug('{} points with Ts > 0'.format((Ts > 0).sum()))
logging.debug('{}'.format(Ts))
raise MCMCError('{}: 0 points with Ts > 0'.format(self.name))
if (ds > 0).sum() == 0:
#logging.debug('{} points with ds > 0'.format((ds > 0).sum()))
logging.debug('{}'.format(ds))
raise MCMCError('{}: 0 points with ds > 0'.format(self.name))
if (slopes > 0).sum() == 0:
#logging.debug('{} points with slopes > 0'.format((slopes > 0).sum()))
logging.debug('{}'.format(slopes))
raise MCMCError('{}: 0 points with slopes > 0'.format(
self.name))
if (slopes < self.maxslope).sum() == 0:
#logging.debug('{} points with slopes < maxslope ({})'.format((slopes < self.maxslope).sum(),self.maxslope))
logging.debug('{}'.format(slopes))
raise MCMCError('{} points with slopes < maxslope ({})'.format(
(slopes < self.maxslope).sum(), self.maxslope))
durs, deps, logdeps, slopes = (Ts[ok], ds[ok], np.log10(ds[ok]),
slopes[ok])
inds = (np.arange(len(durs) / thin) * thin).astype(int)
durs, deps, logdeps, slopes = (durs[inds], deps[inds], logdeps[inds],
slopes[inds])
self.durs, self.deps, self.logdeps, self.slopes = (durs, deps, logdeps,
slopes)
self._make_kde(conf=conf)
self.hasMCMC = True
logls = logls[::args.thin, :]
logps = logps[::args.thin, :]
chain = chain.reshape((-1, chain.shape[2]))
logls = logls.reshape((-1, 1))
logps = logps.reshape((-1, 1))
if args.decorrelated:
chain = chain.reshape((-1, args.nwalkers, chain.shape[1]))
logls = logls.reshape((-1, args.nwalkers))
logps = logps.reshape((-1, args.nwalkers))
maxtau = float('-inf')
for k in range(chain.shape[2]):
maxtau = max(maxtau, acor.acor(np.mean(chain[:,:,k], axis=1))[0])
maxtau = int(round(maxtau))
chain = chain[::maxtau, ...]
logls = logls[::maxtau, :]
logps = logps[::maxtau, :]
min_n = min(chain.shape[0], logls.shape[0], logps.shape[0])
chain = chain[:min_n,:]
logls = logls[:min_n]
logps = logps[:min_n]
out_base, ext = os.path.splitext(args.output)
if ext == '.gz':
o = gzip.open
else:
def sample(self, p0, Niter, ladder=None, Tmin=1, Tmax=None, Tskip=100, \
isave=1000, covUpdate=1000, KDEupdate=1000, SCAMweight=20, \
AMweight=20, DEweight=20, KDEweight=0, burn=10000, \
maxIter=None, thin=10, i0=0, neff=100000):
"""
Function to carry out PTMCMC sampling.
@param p0: Initial parameter vector
@param self.Niter: Number of iterations to use for T = 1 chain
@param ladder: User defined temperature ladder
@param Tmin: Minimum temperature in ladder (default=1)
@param Tmax: Maximum temperature in ladder (default=None)
@param Tskip: Number of steps between proposed temperature swaps (default=100)
@param isave: Number of iterations before writing to file (default=1000)
@param covUpdate: Number of iterations between AM covariance updates (default=1000)
@param KDEUpdate: Number of iterations between KDE updates (default=1000)
@param SCAMweight: Weight of SCAM jumps in overall jump cycle (default=20)
@param AMweight: Weight of AM jumps in overall jump cycle (default=20)
@param DEweight: Weight of DE jumps in overall jump cycle (default=20)
@param KDEweight: Weight of KDE jumps in overall jump cycle (default=100)
@param burn: Burn in time (DE jumps added after this iteration) (default=10000)
@param maxIter: Maximum number of iterations for high temperature chains
(default=2*self.Niter)
@param self.thin: Save every self.thin MCMC samples
@param i0: Iteration to start MCMC (if i0 !=0, do not re-initialize)
@param neff: Number of effective samples to collect before terminating
"""
# get maximum number of iterations
if maxIter is None and self.MPIrank > 0:
maxIter = 2*Niter
elif maxIter is None and self.MPIrank == 0:
maxIter = Niter
# set up arrays to store lnprob, lnlike and chain
N = int(maxIter/thin)
# if picking up from previous run, don't re-initialize
if i0 == 0:
self.initialize(Niter, ladder=ladder, Tmin=Tmin, Tmax=Tmax, Tskip=Tskip, \
isave=isave, covUpdate=covUpdate, KDEupdate=KDEupdate, SCAMweight=SCAMweight, \
AMweight=AMweight, DEweight=DEweight, KDEweight=KDEweight, burn=burn, \
maxIter=maxIter, thin=thin, i0=i0, neff=neff)
### compute lnprob for initial point in chain ###
# if resuming, just start with first point in chain
if self.resume and self.resumeLength > 0:
if self.newFileOrder:
p0, lnlike0, lnprob0 = self.resumechain[0,:-4], \
self.resumechain[0,-3], self.resumechain[0,-4]
else:
p0, lnlike0, lnprob0 = self.resumechain[0,3:], \
self.resumechain[0,1], self.resumechain[0,0]
else:
# compute prior
lp = self.logp(p0)
if lp == float(-np.inf):
lnprob0 = -np.inf
lnlike0 = -np.inf
else:
lnlike0 = self.logl(p0)
lnprob0 = 1/self.temp * lnlike0 + lp
# record first values
self.updateChains(p0, lnlike0, lnprob0, i0)
self.comm.barrier()
# start iterations
iter = i0
self.tstart = time.time()
runComplete = False
Neff = 0
while runComplete == False:
iter += 1
accepted = 0
# call PTMCMCOneStep
p0, lnlike0, lnprob0 = self.PTMCMCOneStep(p0, lnlike0, lnprob0, iter)
# compute effective number of samples
if iter % 1000 == 0 and iter > 2*self.burn and self.MPIrank == 0:
try:
Neff = int(iter/np.nanmax([acor.acor(self._AMbuffer[self.burn:(iter-1),ii])[0] \
for ii in range(self.ndim)]))
#print '\n {0} effective samples'.format(Neff)
except (NameError, OverflowError):
Neff = 0
pass
# stop if reached maximum number of iterations
if self.MPIrank == 0 and iter >= self.Niter-1:
if self.verbose:
print '\nRun Complete'
runComplete = True
# stop if reached effective number of samples
if self.MPIrank == 0 and Neff > self.neff:
if self.verbose:
print '\nRun Complete with {0} effective samples'.format(Neff)
runComplete = True
if self.MPIrank == 0 and runComplete:
for jj in range(1, self.nchain):
self.comm.send(runComplete, dest=jj, tag=55)
# check for other chains
if self.MPIrank > 0:
runComplete = self.comm.Iprobe(source=0, tag=55)
time.sleep(0.000001) # trick to get around
def MCMC(self, niter=500, nburn=200, nwalkers=200, threads=1,
fit_partial=False, width=3, savedir=None, refit=False,
thin=10, conf=0.95, maxslope=30, debug=False, p0=None):
"""
Fit transit signal to trapezoid model using MCMC
.. note:: As currently implemented, this method creates a
bunch of attributes relevant to the MCMC fit; I plan
to refactor this to define those attributes as properties
so as not to have their creation hidden away here. I plan
to refactor how this works.
"""
if fit_partial:
wok = np.where((np.absolute(self.ts-self.center) < (width*self.dur)) &
~np.isnan(self.fs))
else:
wok = np.where(~np.isnan(self.fs))
if savedir is not None:
if not os.path.exists(savedir):
os.mkdir(savedir)
alreadydone = True
alreadydone &= savedir is not None
alreadydone &= os.path.exists('%s/ts.npy' % savedir)
alreadydone &= os.path.exists('%s/fs.npy' % savedir)
if savedir is not None and alreadydone:
ts_done = np.load('%s/ts.npy' % savedir)
fs_done = np.load('%s/fs.npy' % savedir)
alreadydone &= np.all(ts_done == self.ts[wok])
alreadydone &= np.all(fs_done == self.fs[wok])
if alreadydone and not refit:
logging.info('MCMC fit already done for %s. Loading chains.' % self.name)
Ts = np.load('%s/duration_chain.npy' % savedir)
ds = np.load('%s/depth_chain.npy' % savedir)
slopes = np.load('%s/slope_chain.npy' % savedir)
tcs = np.load('%s/tc_chain.npy' % savedir)
else:
logging.info('Fitting data to trapezoid shape with MCMC for %s....' % self.name)
if p0 is None:
p0 = self.trapfit.copy()
p0[0] = np.absolute(p0[0])
if p0[2] < 2:
p0[2] = 2.01
if p0[1] < 0:
p0[1] = 1e-5
logging.debug('p0 for MCMC = {}'.format(p0))
sampler = traptransit_MCMC(self.ts[wok],self.fs[wok],self.dfs[wok],
niter=niter,nburn=nburn,nwalkers=nwalkers,
threads=threads,p0=p0,return_sampler=True,
maxslope=maxslope)
Ts,ds,slopes,tcs = (sampler.flatchain[:,0],sampler.flatchain[:,1],
sampler.flatchain[:,2],sampler.flatchain[:,3])
self.sampler = sampler
if savedir is not None:
np.save('%s/duration_chain.npy' % savedir,Ts)
np.save('%s/depth_chain.npy' % savedir,ds)
np.save('%s/slope_chain.npy' % savedir,slopes)
np.save('%s/tc_chain.npy' % savedir,tcs)
np.save('%s/ts.npy' % savedir,self.ts[wok])
np.save('%s/fs.npy' % savedir,self.fs[wok])
if debug:
print(Ts)
print(ds)
print(slopes)
print(tcs)
N = len(Ts)
self.Ts_acor = acor.acor(Ts)[1]
self.ds_acor = acor.acor(ds)[1]
self.slopes_acor = acor.acor(slopes)[1]
self.tcs_acor = acor.acor(tcs)[1]
self.fit_converged = True
for t in [self.Ts_acor,self.ds_acor,
self.slopes_acor,self.tcs_acor]:
if t > 0.1*N:
self.fit_converged = False
ok = (Ts > 0) & (ds > 0) & (slopes > 0) & (slopes < self.maxslope)
logging.debug('trapezoidal fit has {} good sample points'.format(ok.sum()))
if ok.sum()==0:
if (Ts > 0).sum()==0:
#logging.debug('{} points with Ts > 0'.format((Ts > 0).sum()))
logging.debug('{}'.format(Ts))
raise MCMCError('{}: 0 points with Ts > 0'.format(self.name))
if (ds > 0).sum()==0:
#logging.debug('{} points with ds > 0'.format((ds > 0).sum()))
logging.debug('{}'.format(ds))
raise MCMCError('{}: 0 points with ds > 0'.format(self.name))
if (slopes > 0).sum()==0:
#logging.debug('{} points with slopes > 0'.format((slopes > 0).sum()))
logging.debug('{}'.format(slopes))
raise MCMCError('{}: 0 points with slopes > 0'.format(self.name))
if (slopes < self.maxslope).sum()==0:
#logging.debug('{} points with slopes < maxslope ({})'.format((slopes < self.maxslope).sum(),self.maxslope))
logging.debug('{}'.format(slopes))
raise MCMCError('{} points with slopes < maxslope ({})'.format((slopes < self.maxslope).sum(),self.maxslope))
durs,deps,logdeps,slopes = (Ts[ok],ds[ok],np.log10(ds[ok]),
slopes[ok])
inds = (np.arange(len(durs)/thin)*thin).astype(int)
durs,deps,logdeps,slopes = (durs[inds],deps[inds],logdeps[inds],
slopes[inds])
self.durs,self.deps,self.logdeps,self.slopes = (durs,deps,logdeps,slopes)
self._make_kde(conf=conf)
self.hasMCMC = True
samps=np.array(samps).T
Likes=samps[-1]
samps=np.float64(samps[:-1])
TimingParams = open(root+"T2scaling.txt").readlines()
for i in range(len(TimingParams)):
TimingParams[i] = TimingParams[i].strip('\n').split()
NSamp = len(samps[0])
acors=np.zeros(len(samps))
means=np.zeros(len(samps))
stds=np.zeros(len(samps))
for i in range(len(samps)):
acors[i] = acor.acor(np.float64(samps[i]))[0]
means[i]=np.mean(samps[i])
stds[i] = np.std(samps[i])
for i in range(len(TimingParams)):
print i, TimingParams[i][0], means[i+1]*np.float64(TimingParams[i][-1])+np.float64(TimingParams[i][-2]), stds[i+1]*np.float64(TimingParams[i][-1])
for p in range(len(samps)):
index=p
sum=0
for i in range(len(samps[index])-1):
sum=sum+np.abs(samps[index][i+1]-samps[index][i])
sum=sum/(NSamp-1)
print p, sum, np.std(samps[index]), sum/np.std(samps[index])
def MCMC(self,
niter=500,
nburn=200,
nwalkers=200,
threads=1,
fit_partial=False,
width=3,
savedir=None,
refit=False,
thin=10,
conf=0.95,
maxslope=30,
debug=False,
p0=None):
if fit_partial:
wok = np.where((np.absolute(self.ts - self.center) <
(width * self.dur)) & ~np.isnan(self.fs))
else:
wok = np.where(~np.isnan(self.fs))
if not os.path.exists(savedir):
os.mkdir(savedir)
alreadydone = True
alreadydone &= savedir is not None
alreadydone &= os.path.exists('%s/ts.npy' % savedir)
alreadydone &= os.path.exists('%s/fs.npy' % savedir)
if savedir is not None and alreadydone:
ts_done = np.load('%s/ts.npy' % savedir)
fs_done = np.load('%s/fs.npy' % savedir)
alreadydone &= np.all(ts_done == self.ts[wok])
alreadydone &= np.all(fs_done == self.fs[wok])
if alreadydone and not refit:
logging.info('MCMC fit already done for %s. Loading chains.' %
self.name)
Ts = np.load('%s/duration_chain.npy' % savedir)
ds = np.load('%s/depth_chain.npy' % savedir)
slopes = np.load('%s/slope_chain.npy' % savedir)
tcs = np.load('%s/tc_chain.npy' % savedir)
else:
logging.info(
'Fitting data to trapezoid shape with MCMC for %s....' %
self.name)
if p0 is None:
p0 = self.trapfit.copy()
p0[0] = np.absolute(p0[0])
if p0[2] < 2:
p0[2] = 2.01
if p0[1] < 0:
p0[1] = 1e-5
logging.debug('p0 for MCMC = {}'.format(p0))
sampler = traptransit_MCMC(self.ts[wok],
self.fs[wok],
self.dfs[wok],
niter=niter,
nburn=nburn,
nwalkers=nwalkers,
threads=threads,
p0=p0,
return_sampler=True,
maxslope=maxslope)
Ts, ds, slopes, tcs = (sampler.flatchain[:,
0], sampler.flatchain[:,
1],
sampler.flatchain[:,
2], sampler.flatchain[:,
3])
self.sampler = sampler
if savedir is not None:
np.save('%s/duration_chain.npy' % savedir, Ts)
np.save('%s/depth_chain.npy' % savedir, ds)
np.save('%s/slope_chain.npy' % savedir, slopes)
np.save('%s/tc_chain.npy' % savedir, tcs)
np.save('%s/ts.npy' % savedir, self.ts[wok])
np.save('%s/fs.npy' % savedir, self.fs[wok])
if debug:
print(Ts)
print(ds)
print(slopes)
print(tcs)
N = len(Ts)
self.Ts_acor = acor.acor(Ts)[1]
self.ds_acor = acor.acor(ds)[1]
self.slopes_acor = acor.acor(slopes)[1]
self.tcs_acor = acor.acor(tcs)[1]
self.fit_converged = True
for t in [self.Ts_acor, self.ds_acor, self.slopes_acor, self.tcs_acor]:
if t > 0.1 * N:
self.fit_converged = False
ok = (Ts > 0) & (ds > 0) & (slopes > 0) & (slopes < self.maxslope)
logging.debug('trapezoidal fit has {} good sample points'.format(
ok.sum()))
if ok.sum() == 0:
if (Ts > 0).sum() == 0:
#logging.debug('{} points with Ts > 0'.format((Ts > 0).sum()))
logging.debug('{}'.format(Ts))
raise MCMCError('{}: 0 points with Ts > 0'.format(self.name))
if (ds > 0).sum() == 0:
#logging.debug('{} points with ds > 0'.format((ds > 0).sum()))
logging.debug('{}'.format(ds))
raise MCMCError('{}: 0 points with ds > 0'.format(self.name))
if (slopes > 0).sum() == 0:
#logging.debug('{} points with slopes > 0'.format((slopes > 0).sum()))
logging.debug('{}'.format(slopes))
raise MCMCError('{}: 0 points with slopes > 0'.format(
self.name))
if (slopes < self.maxslope).sum() == 0:
#logging.debug('{} points with slopes < maxslope ({})'.format((slopes < self.maxslope).sum(),self.maxslope))
logging.debug('{}'.format(slopes))
raise MCMCError('{} points with slopes < maxslope ({})'.format(
(slopes < self.maxslope).sum(), self.maxslope))
durs, deps, logdeps, slopes = (Ts[ok], ds[ok], np.log10(ds[ok]),
slopes[ok])
inds = (np.arange(len(durs) / thin) * thin).astype(int)
durs, deps, logdeps, slopes = (durs[inds], deps[inds], logdeps[inds],
slopes[inds])
self.durs, self.logdeps, self.slopes = (durs, logdeps, slopes)
self.durkde = gaussian_kde(durs)
self.depthkde = gaussian_kde(deps)
self.slopekde = gaussian_kde(slopes)
self.logdepthkde = gaussian_kde(logdeps)
if self.fit_converged:
try:
durconf = kdeconf(self.durkde, conf)
depconf = kdeconf(self.depthkde, conf)
logdepconf = kdeconf(self.logdepthkde, conf)
slopeconf = kdeconf(self.slopekde, conf)
except:
raise
raise MCMCError(
'Error generating confidence intervals...fit must not have worked.'
)
durmed = np.median(durs)
depmed = np.median(deps)
logdepmed = np.median(logdeps)
slopemed = np.median(slopes)
self.durfit = (durmed,
np.array([durmed - durconf[0],
durconf[1] - durmed]))
self.depthfit = (depmed,
np.array(
[depmed - depconf[0], depconf[1] - depmed]))
self.logdepthfit = (logdepmed,
np.array([
logdepmed - logdepconf[0],
logdepconf[1] - logdepmed
]))
self.slopefit = (slopemed,
np.array([
slopemed - slopeconf[0],
slopeconf[1] - slopemed
]))
else:
self.durfit = (np.nan, np.nan, np.nan)
self.depthfit = (np.nan, np.nan, np.nan)
self.logdepthfit = (np.nan, np.nan, np.nan)
self.slopefit = (np.nan, np.nan, np.nan)
points = np.array([durs, logdeps, slopes])
self.kde = gaussian_kde(points)
self.hasMCMC = True
# Burn in samples
##################
if args.intelburn:
loglike = chain[-3] # likelihood column
dim = chain.shape[1]-4 # dimensionality of search
likemax = loglike.max() # max likelihood
burniter = np.where(loglike > (likemax-dim/2.))[0]
burntdata = chain[burniter:,:]
# GWB amplitude index
amp_ind = int(param_list[param_list[:,1]=='Agwb',0][0])
corrlength, mean, sigma = acor.acor(burntdata[:,amp_ind])
indsamples = burntdata[::int(corrlength)]
chain = indsamples
if not args.intelburn: chain = chain[args.manualburn:,:]
############################################################
# Doing a quick plot to manually cut-off the burn-in stage
############################################################
if 'gam4p33' in args.chaindir:
# checking Agwb
Agwb = chain[:,int(param_list[param_list[:,1]=='Agwb',0][0])]
gwb_params = np.zeros((len(chain),1))
gwb_params[:,0] = Agwb
# In[302]:
plt.hist(gibbs_4_d, normed = 1)
plt.title('Histogram: 4x4 Magnetization - Deterministic')
plt.xlabel('Magnetization')
plt.ylabel('Probability')
plt.show()
# In[320]:
tau, mean, sigma = acor.acor(gibbs_4_d )
print("The gibbs sampling estimator (deterministic) is {}, with integrated autocorrelation time of {}".format(mean, tau))
# Now let's try gibbs sampling on the ising model with randomly selected points on the same 4x4 lattice:
# In[309]:
def gibbs_ising_magnetization_random(L, beta, n):
'''
Gibbs sampling using a random point on the lattice to resample.
params:
- n: number of samples
- L: nxn np matrix representing the periodic lattice. Values should ONLY be {-1, 1}
returns:
y = ytry
naccept += 1
else:
aprob = gxtry / gx # acceptance probability
u = rnd.uniform(0, 1)
if u < aprob:
x = xtry
y = ytry
naccept += 1
#
# whatever the outcome, add current walker coordinates to the chain
#
chain.append([x, y])
i += 1
ntry += 1
print "Generated n ", n, " samples with a single walker"
print "with acceptance ratio", 1.0 * naccept / ntry
xh = np.array(zip(*chain)[0])
yh = np.array(zip(*chain)[1])
xh = xh[nburn::nsel]
yh = yh[nburn::nsel]
print "computing autocorrelation time..."
t1 = time()
tacor = acor.acor(xh, 100)
t2 = time()
print "done in", t2 - t1, " seconds."
print "tacor=", tacor
def mcmc_sample(x,
nparams=2,
nwalkers=100,
nRval=100,
modelpdf=None,
ipar_active=None,
params=[],
nsteps=1000000000,
Rlim=1.001):
"""
MCMC sampler implementing the Goodman & Weare (2010) affine-invariant algorithm
inner loop is vectorized
run for nsteps or until R_GR=Rlim is reached, whichever comes first
"""
try:
import acor
except:
raise Exception("acor package is not installed! do: pip install acor")
# parameters used to draw random number with the GW10 proposal distribution
ap = 2.0
api = 1.0 / ap
asqri = 1.0 / np.sqrt(ap)
afact = (ap - 1.0)
# calculate effective number of parameters if some are specified to be fixed
ia = (ipar_active == 1)
npareff = np.size(ipar_active[ia])
print((
"starting sampling with %d active parameters of the total %d parameters"
% (npareff, nparams)))
# initialize some auxiliary arrays and variables
chain = []
Rval = []
naccept = 0
ntry = 0
nchain = 0
mw = np.zeros((nwalkers, npareff))
sw = np.zeros((nwalkers, npareff))
m = np.zeros(npareff)
Wgr = np.zeros(npareff)
Bgr = np.zeros(npareff)
Rgr = np.zeros(npareff)
mutx = []
taux = []
for i in range(npareff):
mutx.append([])
taux.append([])
Rval.append([])
gxo = np.zeros((2, nwalkers / 2))
gxo[0, :] = modelpdf(x[0, :, :], params)
gxo[1, :] = modelpdf(x[1, :, :], params)
converged = False
while not converged:
# for parallelization (not implemented here but the MPI version is available)
# the walkers are split into two complementary sub-groups (see GW10)
for kd in range(2):
k = abs(kd - 1)
# vectorized inner loop of walkers stretch move in the Goodman & Weare sampling algorithm
xchunk = x[k, :, :]
jcompl = np.random.randint(0, nwalkers / 2, nwalkers / 2)
xcompl = x[kd, jcompl, :]
gxold = gxo[k, :]
zf = np.random.rand(
nwalkers / 2
) # the next few steps implement Goodman & Weare sampling algorithm
zf = zf * afact
zr = (1.0 + zf) * (1.0 + zf) * api
zrtile = np.transpose(np.tile(
zr, (nparams, 1))) # duplicate zr for nparams
xtry = xcompl + zrtile * (xchunk - xcompl)
gxtry = modelpdf(xtry, params)
gx = gxold
gr = gxtry - gx
iacc = np.where(gr > 0.)
xchunk[iacc] = xtry[iacc]
gxold[iacc] = gxtry[iacc]
aprob = (npareff - 1) * np.log(zr) + (gxtry - gx)
u = np.random.uniform(0.0, 1.0, np.shape(xchunk)[0])
iprob = np.where(aprob > np.log(u))
xchunk[iprob] = xtry[iprob]
gxold[iprob] = gxtry[iprob]
naccept += len(iprob[0])
x[k, :, ia] = np.transpose(xchunk[:, ia])
gxo[k, :] = gxold
xdum = x[:, :, ia]
for i in range(nwalkers / 2):
chain.append(np.array(xdum[k, i, :]))
for i in range(nwalkers / 2):
mw[k * nwalkers / 2 + i, :] += xdum[k, i, :]
sw[k * nwalkers / 2 + i, :] += xdum[k, i, :]**2
ntry += 1
nchain += 1
# compute means for the auto-correlation time estimate
for i in range(npareff):
mutx[i].append(np.sum(xdum[:, :, i]) / (nwalkers))
# compute Gelman-Rubin indicator for all parameters
if (nchain % nRval == 0):
# calculate Gelman & Rubin convergence indicator
mwc = mw / (nchain - 1.0)
swc = sw / (nchain - 1.0) - np.power(mwc, 2)
for i in range(npareff):
# within chain variance
Wgr[i] = np.sum(swc[:, i]) / nwalkers
# mean of the means over Nwalkers
m[i] = np.sum(mwc[:, i]) / nwalkers
# between chain variance
Bgr[i] = nchain * np.sum(np.power(mwc[:, i] - m[i],
2)) / (nwalkers - 1.0)
# Gelman-Rubin R factor
Rgr[i] = (1.0 - 1.0 / nchain + Bgr[i] / Wgr[i] / nchain) * (
nwalkers + 1.0) / nwalkers - (nchain - 1.0) / (nchain *
nwalkers)
tacorx = acor.acor(np.abs(mutx[i]))[0]
taux[i].append(np.max(tacorx))
Rval[i].append(Rgr[i] - 1.0)
print(("nchain = %d; tcorr = %.2e" % (nchain, np.max(tacorx))))
print(("R_GR = ", Rgr))
if (np.max(np.abs(Rgr - 1.0)) < np.abs(Rlim - 1.0)) or (nchain >=
nsteps):
converged = True
print(("MCMC sampler generated %d samples using %d walkers" %
(ntry, nwalkers)))
print(("with step acceptance ratio of %.3f" % (1.0 * naccept / ntry)))
# record integer auto-correlation time at the final iteration
nthin = int(tacorx)
return chain, Rval, nthin
plt.rc('font', family='serif')
plt.hist(magsb005, bins=50, density=True)
plt.xlabel('Average Magnetization $f(\sigma)/L^2$')
plt.ylabel('Relative Frequency')
plt.title('Histogram of the Average Magnetization for $\\beta = 1$, $L=10$')
##
## Plot the autocorrelation function for beta = 0.05 with random and sweeping.
##
N = int(1E7) # Length of the chain.
mags = IsingSamplers.IsingSampler(N, L, beta, 'Gibbs', method='random')[1]
magSweep = IsingSamplers.IsingSampler(N, L, beta, 'Gibbs', method='sweep')[1]
tau = acor.acor(mags, maxlag=700)[0]
tauSweep = acor.acor(magSweep, maxlag=700)[0]
print("\n")
print("Estimated IAC of Gibbs Sampler (random): ", tau)
print("Estimated IAC of Gibbs Sampler (sweeping): ", tauSweep)
print("\n")
autocorr = acor.function(mags, maxt=800)
autocorrSweep = acor.function(magSweep, maxt=800)
plt.figure(3)
plt.rc('text', usetex=True)
plt.rc('font', family='serif')
plt.plot(autocorr, c='b', label='Gibbs')
plt.plot(autocorrSweep, c='r', label='Metroplis Hastings')
plt.xlabel('Lag')
