def test(challenge,lim=None):
meta,desi,times_f,resid_f,error_f = load(challenge,limit=int(lim))
alphaab = alphamat(meta)
print 'Working with {0} pulsars.'.format(len(meta))
with timing('GW covariance matrix [recurring]'):
cgw = Cgw_100ns(alphaab,times_f,-2.0/3.0,fL=1.0/500,approx_ksum=True)
cgw2 = cgw.copy()
with timing('Cgw interpolation'):
cgw3 = 0.2 * cgw + 0.8 * cgw2
with timing('PN covariance matrix'):
cpn = Cpn(error_f)
with timing('Design matrix'):
gmat = Gdesi2(desi,meta) # note this takes meta, not len(meta)
with timing('Reduced data'):
resid_f = N.dot(gmat.T,resid_f)
with timing('Reduced Cpn'):
cpn = blockmul(cpn,gmat,meta,blas=True)
with timing('Reduced Cgw [recurring]'):
cgw = blockmul(cgw,gmat,meta,blas=True)
cgw2 = cgw.copy()
with timing('Reduced-Cgw interpolation'):
cgw3 = 0.2 * cgw + 0.8 * cgw2
with timing('Likelihood [recurring]'):
logl = logL(resid_f,cgw,cpn)
def checklike(challenge,procs,yL=500.0,lim=None,gproj=True,inject=False,A=5e-14,alpha=-2.0/3.0,debug=1,prange=None):
"""Load challenge data and compute the likelihood for a range of background amplitudes.
Returns a two-column array of (A,logL)."""
global resid_f,cgw,alphaab,times_f,gmat,meta,cpn,error_f
meta,desi,times_f,resid_f,error_f = load(challenge,limit=lim)
alphaab = alphamat(meta)
with timing("Initial setup",1,debug):
cgw = Cgw_100ns(alphaab,times_f,alpha,fL=1.0/float(yL))
cpn = Cpn(error_f)
if inject:
resid_f = simulate(alphaab,times_f,cgw,cpn,A=A,n=1)
if gproj:
with timing("Timing for gmat setup",2,debug):
if desi is None:
gmat = Gproj(times_f,len(meta))
else:
print "Using tempo2 design matrix"
gmat = Gdesi2(desi,meta) # gmat = Gdesi(desi,len(meta))
resid_f = N.dot(gmat.T,resid_f)
cgw = blockmul(cgw,gmat,meta) # cgw = N.dot(gmat.T,N.dot(cgw,gmat))
cpn = blockmul(cpn,gmat,meta) # cpn = N.dot(gmat.T,N.dot(cpn,gmat))
pool = multiprocessing.Pool(int(procs))
if prange is None:
x = N.linspace(1e-14,9e-14,20) # range of A
else:
x = N.linspace(prange[0],prange[1],20) # assigned range of A
with timing("Total timing for {0} likelihoods".format(len(x)),1,debug):
# l = pool.map(lnprob,[[x0] for x0 in x])
l = pool.map(lnprob2,[[x0,alpha] for x0 in x])
pool.close()
pool.join()
if debug is True or debug >= 2:
print "Maximum %s found at par %s" % (N.max(l),x[N.argmax(l)])
return N.array([x,l]).T
def emceehammer(challenge,procs=10,suffix=None,ndim=None,nwalkers=200,iters=100,limit=None,inject=False,resume=False,checkpoint=None):
"""Load challenge data and perform a single-parameter (A) emcee Hammer run on them.
Save chain and probabilities to numpy arrays."""
global resid_f,cgw,alphaab,times_f,gmat,meta,cpn,error_f
meta,desi,times_f,resid_f,error_f = load(challenge,limit=limit)
alphaab = alphamat(meta)
if inject == 'raw':
print "Loading clean data from raw challenge files"
resid_f = loadraw(challenge,limit=None)
if ndim is None and challenge in ['closed1','closed2','closed3']:
ndim = 2*len(meta) + 2
with timing("Initial setup"):
cgw = Cgw_100ns(alphaab,times_f,alpha=-2.0/3.0,fL=1.0/500)
cpn = Cpn(error_f)
if challenge == 'open3':
cpn = cpn + Cred_100ns(alphaab,times_f,A=5.77e-22,alpha=1.7,fL=1.0/500)
if inject == 'inject':
print "Injecting synthetic signals at dataset times"
resid_f = simulate(alphaab,times_f,cgw,cpn,A=5e-14,n=1)
if desi is None:
gmat = Gproj(times_f,len(meta))
else:
print "Using tempo2 design matrix"
gmat = Gdesi2(desi,meta) # gmat = Gdesi(desi,len(meta))
resid_f = N.dot(gmat.T,resid_f)
if ndim == 1: # otherwise the multiplication is done in logL
cgw = blockmul(cgw,gmat,meta) # cgw = N.dot(gmat.T,N.dot(cgw,gmat))
cpn = blockmul(cpn,gmat,meta) # cpn = N.dot(gmat.T,N.dot(cpn,gmat))
if N.any(N.isnan(cgw.flatten())) or N.any(N.isinf(cgw.flatten())):
raise ArithmeticError
# multiprocessing seems to work better if nwalkers >> procs
# also keep in mind that the ensemble is split in two...
trueA, truealpha = 5e-14, -2.0/3.0
trueAred, truealphared = 5.77e-22, 1.7
if ndim == 1:
# initial walker positions - a list of numpy arrays
p0 = [random.uniform(trueA*0.5,trueA*1.5) for i in range(nwalkers)]
sampler = emcee.EnsembleSampler(nwalkers,ndim,lnprob,args=[],threads=int(procs))
elif ndim == 2:
p0 = [[random.uniform(trueA*0.5,trueA*1.5),
random.uniform(alpha_min,alpha_max)] for i in range(nwalkers)]
sampler = emcee.EnsembleSampler(nwalkers,ndim,lnprob2,args=[],threads=int(procs))
elif ndim == 4:
p0 = [[random.uniform(trueA*0.5,trueA*1.5),
random.uniform(alpha_min,alpha_max),
random.uniform(trueAred*0.1,trueAred*10),
random.uniform(alphared_min,alphared_max)] for i in range(nwalkers)]
sampler = emcee.EnsembleSampler(nwalkers,ndim,lnprob4,args=[],threads=int(procs))
elif ndim == 2*len(meta) + 2:
p0 = [[random.uniform(trueA*0.5,trueA*1.5),random.uniform(alpha_min,alpha_max)] +
[value for pulsar in meta
for value in [random.uniform(math.log10(trueAred*0.1),math.log10(trueAred*10)),random.uniform(alphared_min,alphared_max)]]
# for value in [random.uniform(trueAred*0.1,trueAred*10),random.uniform(alphared_min,alphared_max)]]
for i in range(nwalkers)]
sampler = emcee.EnsembleSampler(nwalkers,ndim,lnprob22Nlog,args=[],threads=int(procs))
# sampler = emcee.EnsembleSampler(nwalkers,ndim,lnprob22N,args=[],threads=int(procs))
elif ndim == 3*len(meta) + 2:
p0 = [[random.uniform(trueA*0.5,trueA*1.5),random.uniform(alpha_min,alpha_max)] +
[value for pulsar in meta
for value in [random.uniform(trueAred*0.1,trueAred*10),
random.uniform(alphared_min,alphared_max),
random.uniform(log10_efac_min,log10_efac_max)]]
for i in range(nwalkers)]
sampler = emcee.EnsembleSampler(nwalkers,ndim,lnprob23N,args=[],threads=int(procs))
suffix = (suffix + '-' + str(ndim)) if suffix else str(ndim)
resumefile = '../runs/resume-{0}-{1}.npy'.format(challenge,suffix)
chainfile = '../runs/chain-{0}-{1}.npy'.format(challenge,suffix)
lnprobfile = '../runs/lnprob-{0}-{1}.npy'.format(challenge,suffix)
if resume:
p0 = N.load(resumefile)
print "Resuming run from file", resumefile
if checkpoint:
for subrun in range(int(iters/checkpoint)):
with timing("{0} x {1} samples (subrun {2})".format(checkpoint,nwalkers,subrun)):
sampler.run_mcmc(p0,checkpoint)
p0 = sampler.chain[:,-1,:]
N.save(resumefile,p0)
N.save(chainfile, sampler.chain)
N.save(lnprobfile,sampler.lnprobability)
else:
with timing("{0} x {1} samples".format(iters,nwalkers)):
sampler.run_mcmc(p0,iters)
N.save(resumefile,sampler.chain[:,-1,:])
N.save(chainfile, sampler.chain)
N.save(lnprobfile,sampler.lnprobability)
print "Done! Mean acceptance fraction:", N.mean(sampler.acceptance_fraction)