def preloadNFWFit(configname):
config = simutils.readConfiguration(configname)
simreader = config['simreader']
nfwutils.global_cosmology.set_cosmology(simreader.getCosmology())
return config, simreader
def run(simtype, chaindir, outfile, delta, massbin=0):
config = simutils.readConfiguration('%s/config.py' % chaindir)
simreader = config['simreader']
nfwutils.global_cosmology.set_cosmology(simreader.getCosmology())
model = config['model']
if massbin == -1:
selector = rundln.takeAllMasses(simtype, delta)[0]
else:
selectors = rundln.defineMassEdges(simtype, delta)
selector = selectors[massbin]
isPDF = True
halos = dln.loadPosteriors(chaindir,
simtype,
simreader,
delta,
selector,
reader=dln.PDFReader,
model=model)
if len(halos) < 10:
sys.exit(0)
maxlikes = np.zeros(len(halos))
truths = np.zeros_like(maxlikes)
for i in range(len(halos)):
curhalo = halos[i]
maxlikes[i] = curhalo['masses'][curhalo['pdf'] == np.max(
curhalo['pdf'])][0]
truths[i] = curhalo['true_mass']
ratios = maxlikes / truths
with open('{}.maxlike'.format(outfile), 'w') as output:
output.write('mean stddev meanerr\n')
output.write('{} {} {}\n'.format(
np.mean(ratios), np.std(ratios, ddof=1),
np.std(ratios, ddof=1) / np.sqrt(len(halos))))
def run(simtype,
chaindir,
outfile,
delta,
pdftype,
massbin=0,
sigmapriorfile=None):
config = simutils.readConfiguration('%s/config.py' % chaindir)
simreader = config['simreader']
nfwutils.global_cosmology.set_cosmology(simreader.getCosmology())
model = config['model']
if massbin == -1:
selector = takeAllMasses(simtype, delta)[0]
else:
selectors = defineMassEdges(simtype, delta)
selector = selectors[massbin]
isPDF = False
if pdftype == 'pdf':
isPDF = True
halos = dln.loadPosteriors(chaindir,
simtype,
simreader,
delta,
selector,
reader=dln.PDFReader,
model=model)
elif pdftype == 'mcmc':
halos = dln.loadPosteriors(chaindir,
simtype,
simreader,
delta,
selector,
reader=dln.MCMCReader,
cprior=100.)
if len(halos) < 10:
sys.exit(0)
if len(halos) > 1000:
print 'Down sampling'
halos = random.sample(halos, 1000)
sigmapriors = None
if sigmapriorfile is not None:
print 'Using Sigma Priors!'
sigmapriors = cPickle.load(open(sigmaprior, 'rb'))[massbin]
success = False
for i in range(20):
# Occasionally, the pymc initialization is so far off that the
# probability of the model is essentially zero. Typically,
# just trying again will allow pymc to find a reasonable set
# of initial parameters to start with. If it doesn't find it
# within 20, something else is wrong in the model.
try:
if isPDF == True:
parts = dln.buildPDFModel(halos, sigmapriors=sigmapriors)
else:
parts = dln.buildMCMCModel(halos)
model = pymc.Model(parts)
assert (np.isfinite(model.logp))
success = True
break
except (AssertionError, pymc.ZeroProbability) as e:
continue
assert (success is True)
with open('%s.massrange' % outfile, 'w') as output:
output.write('%f\n%f\n' % (selector.masslow, selector.masshigh))
# Note: This output corresponds to one data point/errorbar
# point on the bias plot.
dln.memsample(model, 10000, outputFile=outfile)
def run(simtype, chaindir, outfile, delta, modelname, massbin=0, sigmapriorfile = None):
'''
simtype: something like mxxl41,
chaindir: output from nfwfit
outfile: output of the map step,
delta: overdensity (e.g. 200)
modelname: function name of model, e.g. buildMCMCModel
massbin: the index of the massbin that we are running (bins are defined above), depends on mass range, simtype, and overdensity
Note - pdftype: either 'pdf' or 'mcmc' --> format of output of map step determined in config file
'''
config = simutils.readConfiguration('%s/config.py' % chaindir)
simreader = config['simreader']
nfwutils.global_cosmology.set_cosmology(simreader.getCosmology())
model = config['model']
pdftype = config['fitter'].output_type
if massbin == -1:
selector = takeAllMasses(simtype, delta)[0]
else:
selectors = defineMassEdges(simtype, delta)
selector = selectors[massbin]
isPDF = False
if pdftype == 'pdf':
isPDF = True
halos = dln.loadPosteriors(chaindir, simtype, simreader, delta, selector,
reader = dln.PDFReader, model = model)
elif pdftype == 'mcmc':
halos = dln.loadPosteriors(chaindir, simtype, simreader, delta, selector,
reader = dln.MCMCReader,
cprior = 100.)
if len(halos) < 10:
sys.exit(0)
if len(halos) > 1000:
# Should be sufficient - some mass bins have ~7000.
print 'Down sampling'
halos = random.sample(halos, 1000)
sigmapriors = None
if sigmapriorfile is not None:
print 'Using Sigma Priors!'
sigmapriors = cPickle.load(open(sigmaprior, 'rb'))[massbin]
success = False
for i in range(20):
# Occasionally, the pymc initialization is so far off that the
# probability of the model is essentially zero. Typically,
# just trying again will allow pymc to find a reasonable set
# of initial parameters to start with. If it doesn't find it
# within 20, something else is wrong in the model.
try:
buildmodel = getattr(dln, modelname)
parts = buildmodel(halos, sigmapriors = sigmapriors)
model = pymc.Model(parts)
assert(np.isfinite(model.logp))
success=True
break
except (AssertionError, pymc.ZeroProbability) as e:
continue
assert(success is True)
with open('%s.massrange' % outfile, 'w') as output:
output.write('%f\n%f\n' % (selector.masslow, selector.masshigh))
# Note: This output corresponds to one data point/errorbar
# point on the bias plot.
# dln.memsample(model, num_samples, out)
#dln.memsample(model, 10000, outputFile = outfile)
dln.sample(model, outfile+'.debug', 1000, tempoutputdir='./debug/', singlecore=True)
def runMultiConfigs(jobparams, jobname=''):
inputfiles = jobparams['inputfiles']
outputExt = jobparams['outputExt']
workbase = jobparams['workbase']
doTransfer = workbase is not None
inputname = jobparams['catname']
outbasename = jobparams['outbasename']
if doTransfer:
inputbase = os.path.basename(inputname)
if jobname != '':
jobname = '-' + jobname
workdir = '{0}/{1}{2}'.format(workbase, inputbase, jobname)
print 'WORKDIR: ' + workdir
if not os.path.exists(workdir):
os.mkdir(workdir)
for inputfile in inputfiles:
shutil.copy(inputfile, workdir)
inputname = '{0}/{1}'.format(workdir, inputbase)
simreader = None
for configfile in jobparams['configurations']:
outdir = os.path.dirname(configfile)
outputname = '{0}/{1}{2}'.format(outdir, outbasename, outputExt)
print configfile, outputname
if simreader is None:
config, simreader = nfwfit.preloadNFWFit(configfile)
else:
config = simutils.readConfiguration(configfile)
nfwfit.runNFWFit_Preloaded(simreader, inputname, config, outputname)
if doTransfer:
shutil.rmtree(workdir)