def getPlanckstepmat(fparams,
fbase,
paramlist,
outfname,
steprescale=2.4,
nchains=8):
"""
return a step matrix for the input chain using mcmcutils.
"""
## copying from readPlanckchain, need a hack to make readable to mcmcutils.
names = ['weight', 'lnlike']
ifpp = open(fparams, 'r')
for line in ifpp:
nn = line.split('\t')[0].strip(' ').strip('*')
names.append(nn)
## write to a file
ofp = open('planckcolstmp.dat', 'w')
for i, nn in zip(range(len(names)), names):
if i == 0:
ofp.write('%s' % (nn))
else:
ofp.write(',%s' % (nn))
ofp.close()
mystep = []
for i in range(1, nchains + 1):
fchain = fbase + '_%d.txt' % i
cc = mcmcutils.chain(fchain, 'planckcolstmp.dat')
## hack -- don't vary anything except the parameters we care about.
cc.mcmcfixed[:] = 1 ## set everything to fixed.
for pp in paramlist:
cc.mcmcfixed[cc.mcmcp[pp]] = 0
cc.fillstepmatrix(steprescale=steprescale)
if i == 1:
mystep = cc.step_mat.copy()
else:
mystep = mystep + cc.step_mat
mystep = mystep / float(nchains)
## hack!!
## take the average and then print.
cc.step_mat = mystep
# cc.printstepmatrix(outfname)
ofp = open(outfname, 'w')
## print a list o the parameters.
xx = np.where(cc.mcmcfixed == 0)[0]
# print cc.mcmcpreverse[xx]
orderednames = [cc.mcmcpreverse[xx[ii]] for ii in range(len(xx))]
assert len(orderednames) == len(paramlist)
for i in range(len(paramlist)):
if i == 0:
ofp.write('# %s' % (orderednames[i]))
else:
ofp.write(',%s' % (orderednames[i]))
ofp.write('\n')
for i in range(len(paramlist)):
for j in range(len(paramlist)):
ofp.write('%e ' % (mystep[i, j]))
ofp.write('\n')
ofp.close()
def subsamplechainslow(chainfname, colfname, nsubsample, pbase, use5002or5003=1):
ccx = mcmcutils.chain(chainfname, colfname)
wgtsum = int(ccx.chain["weight"][:].sum())
cumwgt = ccx.chain["weight"][:].cumsum()
xx = np.random.randint(0, wgtsum, nsubsample)
xxi = np.zeros(len(xx))
for i in range(len(xxi)):
xxi[i] = np.where(cumwgt < xx[i])[0][-1]
assert cumwgt[xxi[i]] < xx[i]
if xxi[i] < nsubsample - 1:
assert cumwgt[xxi[i] + 1] >= xx[i]
assert ((xxi >= 0) & (xxi < len(ccx.chain["weight"][:]))).all()
print "generated this many subsamples", len(xxi)
for i in range(len(xxi)):
ftag = chainfname.split("/chains/")[1].split(".chain")[0] + "_precp0_%06d_" % xxi[i]
runchainmodel(
ccx.chain[xxi[i]],
pbase=pbase,
maskedopt=1,
whichfit=[0, 1, 1],
runslow=1,
ftag=ftag,
use5002or5003=use5002or5003,
)
def subsamplechainslow(chainfname,
colfname,
nsubsample,
pbase,
use5002or5003=1):
ccx = mcmcutils.chain(chainfname, colfname)
wgtsum = int(ccx.chain['weight'][:].sum())
cumwgt = ccx.chain['weight'][:].cumsum()
xx = np.random.randint(0, wgtsum, nsubsample)
xxi = np.zeros(len(xx))
for i in range(len(xxi)):
xxi[i] = np.where(cumwgt < xx[i])[0][-1]
assert cumwgt[xxi[i]] < xx[i]
if xxi[i] < nsubsample - 1:
assert cumwgt[xxi[i] + 1] >= xx[i]
assert ((xxi >= 0) & (xxi < len(ccx.chain['weight'][:]))).all()
print 'generated this many subsamples', len(xxi)
for i in range(len(xxi)):
ftag = chainfname.split('/chains/')[1].split(
'.chain')[0] + '_precp0_%06d_' % xxi[i]
runchainmodel(ccx.chain[xxi[i]],
pbase=pbase,
maskedopt=1,
whichfit=[0, 1, 1],
runslow=1,
ftag=ftag,
use5002or5003=use5002or5003)
def getPlanckstepmat(fparams, fbase, paramlist, outfname, steprescale=2.4, nchains=8):
"""
return a step matrix for the input chain using mcmcutils.
"""
## copying from readPlanckchain, need a hack to make readable to mcmcutils.
names = ["weight", "lnlike"]
ifpp = open(fparams, "r")
for line in ifpp:
nn = line.split("\t")[0].strip(" ").strip("*")
names.append(nn)
## write to a file
ofp = open("planckcolstmp.dat", "w")
for i, nn in zip(range(len(names)), names):
if i == 0:
ofp.write("%s" % (nn))
else:
ofp.write(",%s" % (nn))
ofp.close()
mystep = []
for i in range(1, nchains + 1):
fchain = fbase + "_%d.txt" % i
cc = mcmcutils.chain(fchain, "planckcolstmp.dat")
## hack -- don't vary anything except the parameters we care about.
cc.mcmcfixed[:] = 1 ## set everything to fixed.
for pp in paramlist:
cc.mcmcfixed[cc.mcmcp[pp]] = 0
cc.fillstepmatrix(steprescale=steprescale)
if i == 1:
mystep = cc.step_mat.copy()
else:
mystep = mystep + cc.step_mat
mystep = mystep / float(nchains)
## hack!!
## take the average and then print.
cc.step_mat = mystep
# cc.printstepmatrix(outfname)
ofp = open(outfname, "w")
## print a list o the parameters.
xx = np.where(cc.mcmcfixed == 0)[0]
# print cc.mcmcpreverse[xx]
orderednames = [cc.mcmcpreverse[xx[ii]] for ii in range(len(xx))]
assert len(orderednames) == len(paramlist)
for i in range(len(paramlist)):
if i == 0:
ofp.write("# %s" % (orderednames[i]))
else:
ofp.write(",%s" % (orderednames[i]))
ofp.write("\n")
for i in range(len(paramlist)):
for j in range(len(paramlist)):
ofp.write("%e " % (mystep[i, j]))
ofp.write("\n")
ofp.close()
def runBFfromchainbat(chainbatfname, writecat=None, PBold=None, PBnew=None):
"""
Scrape the .bat file for the chain, keep settings the same.
Open chain and find the best fit.
writecat is the filename to write the catalog to.
If you want to swap PBold in the bat file for PBnew
(example PBold = PB00, PBnew = PB01), set them both to not None.
"""
doswap = 0
if PBold is not None and PBnew is not None:
doswap = 1
assert writecat is not None
## note could set this up to do other tasks, but need to also edit FASTP_FBASE and FASTP_FBASE
batfp = open(chainbatfname, "r")
if doswap == 0:
outfname = chainbatfname.split(".bat")[0] + ".BF.bat"
else:
outfname = chainbatfname.split(".bat")[0] + ".BF.Run%s.bat" % (PBnew)
ofp = open(outfname, "w")
for line in batfp:
if writecat is not None:
if re.search("^USE_FASTP_COUNTS", line):
ofp.write("USE_FASTP_COUNTS 0\n")
continue
if re.search("^WRITE_CAT", line):
ofp.write("WRITE_CAT 1\n")
continue
if re.search("^CATFNAME", line):
ofp.write("CATFNAME %s\n" % (writecat))
continue
if re.search("^PARAMFNAME", line):
continue
# ofp.write('PARAMFNAME blah\n')
elif re.search("^OUTFILETAG", line):
outfiletag = line.split("OUTFILETAG")[1]
## remove spaces.
outfiletag = outfiletag.strip(" ").strip("\n")
elif re.search("^MCMCOPT", line):
ofp.write("MCMCOPT 0\n")
elif re.search("^CHAINNUM", line):
chainnum = int(line.split("CHAINNUM")[1].split("%")[0].strip(" ").strip("\n"))
ofp.write("%s" % (line))
elif re.search("^HVSCALE", line) or re.search("^IHVSCALE", line) or re.search("CENVFRAC", line):
continue
elif doswap == 1 and (re.search("^HaloFileName", line) or re.search("HaloDmFileName", line)):
newline = PBnew.join(line.split(PBold))
print "new line"
print newline
ofp.write(newline)
else:
ofp.write(line)
chainfname = "chains/" + outfiletag + "_" + str(chainnum) + ".chain"
## read chain.
## get best fit params from chain file.
cc = mcmcutils.chain(chainfname, colfname="chains/stdcols.dat")
xbest = np.where(cc.chain["chi2_tot"] == cc.chain["chi2_tot"].min())[0]
celt = cc.chain[xbest]
print celt
logopt = 0
if celt["M_min"] < 1e10:
logopt = 1
if logopt == 1:
Mmin = 10 ** (celt["M_min"])
Mcut = 10 ** (celt["M_cut"])
M1 = 10 ** (celt["M1"])
else:
Mmin = celt["M_min"]
Mcut = celt["M_cut"]
M1 = celt["M1"]
myihv = celt["ihvscale"]
mycenv = celt["cenvfrac"]
myhv = celt["hvscale"]
outnew = outfiletag + "_" + str(chainnum) + ".chain.BF"
paramfnameout = "hodparams/" + outnew + ".hod"
paramfnamein = outnew + ".hod"
## write param file.
alpha = celt["alpha"]
sigmalogM = celt["sigma_logM"]
ofpp = open(paramfnameout, "w")
for pp in [Mmin, sigmalogM, M1, alpha, Mcut]:
ofpp.write("%e\n" % (pp))
ofpp.close()
if doswap == 1:
ofp.write("OUTFILETAG %s\n" % (outnew + PBnew))
else:
ofp.write("OUTFILETAG %s\n" % (outnew))
ofp.write("PARAMFNAME %s\n" % (paramfnamein))
ofp.write("IHVSCALE %f\n" % (myihv))
ofp.write("HVSCALE %f\n" % (myhv))
ofp.write("CENVFRAC %f\n" % (mycenv))
ofp.close()
mystr = "./runall %s" % (outfname)
print mystr
os.system(mystr)
def runBFfromchainbat(chainbatfname, writecat=None, PBold=None, PBnew=None):
"""
Scrape the .bat file for the chain, keep settings the same.
Open chain and find the best fit.
writecat is the filename to write the catalog to.
If you want to swap PBold in the bat file for PBnew
(example PBold = PB00, PBnew = PB01), set them both to not None.
"""
doswap = 0
if PBold is not None and PBnew is not None:
doswap = 1
assert writecat is not None
## note could set this up to do other tasks, but need to also edit FASTP_FBASE and FASTP_FBASE
batfp = open(chainbatfname, 'r')
if doswap == 0:
outfname = chainbatfname.split('.bat')[0] + '.BF.bat'
else:
outfname = chainbatfname.split('.bat')[0] + '.BF.Run%s.bat' % (PBnew)
ofp = open(outfname, 'w')
for line in batfp:
if writecat is not None:
if re.search("^USE_FASTP_COUNTS", line):
ofp.write('USE_FASTP_COUNTS 0\n')
continue
if re.search("^WRITE_CAT", line):
ofp.write('WRITE_CAT 1\n')
continue
if re.search("^CATFNAME", line):
ofp.write('CATFNAME %s\n' % (writecat))
continue
if re.search('^PARAMFNAME', line):
continue
#ofp.write('PARAMFNAME blah\n')
elif re.search("^OUTFILETAG", line):
outfiletag = line.split('OUTFILETAG')[1]
## remove spaces.
outfiletag = outfiletag.strip(' ').strip('\n')
elif re.search("^MCMCOPT", line):
ofp.write('MCMCOPT 0\n')
elif re.search("^CHAINNUM", line):
chainnum = int(
line.split('CHAINNUM')[1].split('%')[0].strip(' ').strip('\n'))
ofp.write('%s' % (line))
elif re.search('^HVSCALE', line) or re.search(
'^IHVSCALE', line) or re.search('CENVFRAC', line):
continue
elif doswap == 1 and (re.search('^HaloFileName', line)
or re.search('HaloDmFileName', line)):
newline = PBnew.join(line.split(PBold))
print 'new line'
print newline
ofp.write(newline)
else:
ofp.write(line)
chainfname = 'chains/' + outfiletag + '_' + str(chainnum) + '.chain'
## read chain.
## get best fit params from chain file.
cc = mcmcutils.chain(chainfname, colfname='chains/stdcols.dat')
xbest = np.where(cc.chain['chi2_tot'] == cc.chain['chi2_tot'].min())[0]
celt = cc.chain[xbest]
print celt
logopt = 0
if celt['M_min'] < 1e10:
logopt = 1
if logopt == 1:
Mmin = 10**(celt['M_min'])
Mcut = 10**(celt['M_cut'])
M1 = 10**(celt['M1'])
else:
Mmin = (celt['M_min'])
Mcut = (celt['M_cut'])
M1 = (celt['M1'])
myihv = celt['ihvscale']
mycenv = celt['cenvfrac']
myhv = celt['hvscale']
outnew = outfiletag + '_' + str(chainnum) + '.chain.BF'
paramfnameout = 'hodparams/' + outnew + '.hod'
paramfnamein = outnew + '.hod'
## write param file.
alpha = celt['alpha']
sigmalogM = celt['sigma_logM']
ofpp = open(paramfnameout, 'w')
for pp in [Mmin, sigmalogM, M1, alpha, Mcut]:
ofpp.write('%e\n' % (pp))
ofpp.close()
if doswap == 1:
ofp.write('OUTFILETAG %s\n' % (outnew + PBnew))
else:
ofp.write('OUTFILETAG %s\n' % (outnew))
ofp.write('PARAMFNAME %s\n' % (paramfnamein))
ofp.write('IHVSCALE %f\n' % (myihv))
ofp.write('HVSCALE %f\n' % (myhv))
ofp.write('CENVFRAC %f\n' % (mycenv))
ofp.close()
mystr = './runall %s' % (outfname)
print mystr
os.system(mystr)
