def createFakeChains(config,
nclusters,
zcluster,
r_mpc_edges,
beta_s,
galdensity,
shapenoise,
nsamples=1000,
mass=10**15.2):
mtrues = mass * np.ones(nclusters)
r_mpcs, shearprofiles, shearerrs = createClusterSet(
config, mtrues, zcluster, r_mpc_edges, beta_s, galdensity, shapenoise)
fitter = nfwfit.buildFitter(config)
chains = []
for i in range(nclusters):
mcmc_model = None
for j in range(10):
try:
mcmc_model = fitter.model.makeMCMCModel(
r_mpcs[i], shearprofiles[i], shearerrs[i], beta_s,
beta_s**2, zcluster)
break
except pymc.ZeroProbability:
pass
if mcmc_model is None:
raise pymc.ZeroProbability
manager = varcontainer.VarContainer()
options = varcontainer.VarContainer()
manager.options = options
options.singlecore = True
options.adapt_every = 100
options.adapt_after = 100
options.nsamples = nsamples
manager.model = mcmc_model
runner = pma.MyMCMemRunner()
runner.run(manager)
runner.finalize(manager)
chains.append(manager.chain['m200'][200:])
return mtrues, chains
def sample(parts,
outputfile,
samples,
adaptevery=100,
adaptafter=100,
singlecore=False,
tempoutputdir=None,
init_with_MAP=True):
'''This function saves to disk the samples from the chain as they're drawn. '''
del_tempoutdir = False
if tempoutputdir is None:
tempoutputdir = tempfile.mkdtemp()
del_tempoutdir = True
outputdir, outputbase = os.path.split(outputfile)
options = varcontainer.VarContainer()
options.outputFile = '%s/%s' % (tempoutputdir, outputbase)
options.singlecore = singlecore
options.nsamples = samples
options.adapt_every = adaptevery
options.adapt_after = adaptafter
options.restore = True
manager = varcontainer.VarContainer()
manager.options = options
manager.model = pymc.Model(parts)
if init_with_MAP:
model_map = pymc.MAP(parts)
model_map.fit()
print "initialized at: ", model_map.logmu.value, model_map.logmu_c.value
assert (np.isfinite(manager.model.logp))
runner = pma.MyMCRunner()
try:
runner.run(manager)
runner.finalize(manager)
finally:
outputfiles = glob.glob('%s/*' % tempoutputdir)
for curoutputfile in outputfiles:
curoutputbase = os.path.basename(curoutputfile)
destination_output = '%s/%s' % (outputdir, curoutputbase)
if os.path.exists(destination_output):
os.remove(destination_output)
shutil.copyfile(curoutputfile, destination_output)
if del_tempoutdir:
shutil.rmtree(tempoutputdir)
def createOptions(self,
deltaz95low=-1,
deltaz95high=2.5,
zbhigh=1.25,
zcut=0.1,
masslow=1e13,
masshigh=1e16,
ztypecut=False,
radlow=0.75,
radhigh=3.0,
concentration=None,
logprior=False,
options=None,
args=None):
if options is None:
options = varcontainer.VarContainer()
options.deltaz95low = deltaz95low
options.deltaz95high = deltaz95high
options.zbhigh = zbhigh
options.zcut = zcut
options.masslow = masslow
options.masshigh = masshigh
options.ztypecut = ztypecut
options.radlow = radlow
options.radhigh = radhigh
options.concentration = concentration
options.logprior = logprior
return options, None
def parseCL(self):
parser = optparse.OptionParser(usage=usage)
def addGroup(strategy, printedname):
if hasattr(strategy, 'addCLOps'):
optgroup = optparse.OptionGroup(parser,
'%s Options' % printedname, '')
strategy.addCLOps(optgroup)
parser.add_option_group(optgroup)
addGroup(self, 'Controller')
addGroup(self.modelbuilder, 'Model')
addGroup(self.filehandler, 'File Handler')
addGroup(self.runmethod, 'Run Method')
varoptions = varcontainer.VarContainer()
options, args = parser.parse_args()
for key, val in vars(options).iteritems():
varoptions[key] = val
self.replace('options', varoptions)
self.replace('args', args)
return options, args
def memsample(parts, samples, adaptevery = 100, adaptafter = 100):
options = varcontainer.VarContainer()
options.singlecore = True
options.nsamples = samples
options.adapt_every = adaptevery
options.adapt_after = adaptafter
manager = varcontainer.VarContainer()
manager.options = options
manager.model = pymc.Model(parts)
runner = pma.MyMCMemRunner()
runner.run(manager)
runner.finalize(manager)
return manager.chain
def sample(parts, outputfile, samples, adaptevery = 100, adaptafter = 100, singlecore = False):
options = varcontainer.VarContainer()
options.outputFile = outputfile
options.singlecore = singlecore
options.nsamples = samples
options.adapt_every = adaptevery
options.adapt_after = adaptafter
manager = varcontainer.VarContainer()
manager.options = options
manager.model = pymc.Model(parts)
runner = pma.MyMCRunner()
runner.run(manager)
runner.finalize(manager)
def __call__(self, profile):
chains = {}
for delta in self.deltas:
mcmc_model = None
for i in range(20):
try:
mcmc_model = self.model.makeMCMCModel(profile, delta = delta)
break
except pymc.ZeroProbability:
pass
if mcmc_model is None:
raise pymc.ZeroProbability
# This sets up Adam Mantz's version of an MCMC sampler for production code calculations.
# This is stored in mymcmc_adapter.py (converts to talk with other MCMC code)
# Imported as pma above.
manager = varcontainer.VarContainer()
options = varcontainer.VarContainer()
manager.options = options
options.singlecore = True
options.adapt_every = 100
options.adapt_after = 100
options.nsamples = self.nsamples
manager.model = mcmc_model
runner = pma.MyMCMemRunner()
runner.run(manager)
runner.finalize(manager)
reducedchain = dict(cdelta = np.hstack(manager.chain['cdelta'][5000::2]).astype(np.float32),
mdelta = np.hstack(manager.chain['mdelta'][5000::2]).astype(np.float32),
likelihood = np.hstack(manager.chain['likelihood'][5000::2]).astype(np.float32))
chains[delta] = reducedchain
return chains
def __call__(self, profile):
chains = {}
for delta in self.deltas:
mcmc_model = None
for i in range(20):
try:
mcmc_model = self.model.makeMCMCModel(profile, delta=delta)
break
except pymc.ZeroProbability:
pass
if mcmc_model is None:
raise pymc.ZeroProbability
manager = varcontainer.VarContainer()
options = varcontainer.VarContainer()
manager.options = options
options.singlecore = True
options.adapt_every = 100
options.adapt_after = 100
options.nsamples = self.nsamples
manager.model = mcmc_model
runner = pma.MyMCMemRunner()
runner.run(manager)
runner.finalize(manager)
reducedchain = dict(
cdelta=np.hstack(manager.chain['cdelta'][5000::2]).astype(
np.float32),
mdelta=np.hstack(manager.chain['mdelta'][5000::2]).astype(
np.float32),
likelihood=np.hstack(
manager.chain['likelihood'][5000::2]).astype(np.float32))
chains[delta] = reducedchain
return chains
def sample(parts,
outputfile,
samples,
adaptevery=100,
adaptafter=100,
singlecore=False):
tempoutputdir = tempfile.mkdtemp()
outputdir, outputbase = os.path.split(outputfile)
options = varcontainer.VarContainer()
options.outputFile = '%s/%s' % (tempoutputdir, outputbase)
options.singlecore = singlecore
options.nsamples = samples
options.adapt_every = adaptevery
options.adapt_after = adaptafter
options.restore = True
manager = varcontainer.VarContainer()
manager.options = options
manager.model = pymc.Model(parts)
assert (np.isfinite(manager.model.logp))
runner = pma.MyMCRunner()
try:
runner.run(manager)
runner.finalize(manager)
finally:
outputfiles = glob.glob('%s/*' % tempoutputdir)
for curoutputfile in outputfiles:
curoutputbase = os.path.basename(curoutputfile)
destination_output = '%s/%s' % (outputdir, curoutputbase)
if os.path.exists(destination_output):
os.remove(destination_output)
shutil.copyfile(curoutputfile, destination_output)
shutil.rmtree(tempoutputdir)
def readData(self, manager):
options = manager.options
newoptions = varcontainer.VarContainer()
newoptions.update(options)
self.buildFileNames(manager, newoptions)
manager.open('lensingcat', newoptions.lensingcat, ldac.openObjectFile)
if options.nearestneighbor is True:
nncat = ldac.openObjectFile(newoptions.neighborcat)
manager.nearestneighbors = nncat.matchById(manager.lensingcat)
newoptions.psfsize = readPSFSize(options.workdir, options.cluster, options.filter, options.image)
newoptions.zcluster = parseZCluster(options.cluster)
newoptions.r500 = readR500(options.cluster)
newoptions.centerx, newoptions.centery = readClusterCenters(options.cluster)
newoptions.pixscale = 0.2
newoptions.xcol = 'Xpos'
newoptions.ycol = 'Ypos'
newoptions.g1col = 'gs1'
newoptions.g2col = 'gs2'
newoptions.sizecol = 'rh'
newoptions.centerx = 5000
newoptions.centery = 5000
newoptions.snratio = 'snratio_scaled1'
newoptions.zbcol = 'BPZ_Z_B'
manager.replace('options', newoptions)
maxlike_general_filehandler.readData(manager)
def memsample(model, samples, adaptevery=100, adaptafter=100, outputFile=None):
options = varcontainer.VarContainer()
options.singlecore = True
options.nsamples = samples
options.adapt_every = adaptevery
options.adapt_after = adaptafter
if outputFile:
options.outputFile = outputFile
manager = varcontainer.VarContainer()
manager.options = options
manager.model = model
runner = pma.MyMCMemRunner()
runner.run(manager)
if outputFile:
runner.dump(manager)
runner.finalize(manager)
return manager.chain
def createOptions(self,
inputCatFile,
inputBPZ,
inputPDZ,
options=None,
args=None):
if options is None:
options = varcontainer.VarContainer()
options.inputCatFile = inputCatFile
options.inputBPZ = inputBPZ
options.inputPDZ = inputPDZ
return options, args
def createOptions(self, cluster, filter, image, ldaclensing='../ldaclensing',
snlow = 3, snhigh = 9999, oddscut = 0., minmag = 25., rhcut = 1.15,
nearestneighbor = True, redseqcat = None, shapecut = False,
options = None, args = None):
if options is None:
options = varcontainer.VarContainer()
options.cluster = cluster
options.filter = filter
options.image = image
options.ldaclensing = ldaclensing
options.snlow = snlow
options.snhigh = snhigh
options.oddscut = oddscut
options.minmag = minmag
options.rhcut = rhcut
options.nearestneighbor = nearestneighbor
options.redseqcat = redseqcat
options.shapecut = shapecut
return options, None
def makeModelParts(self, datamanager, parts=None):
if parts is None:
parts = varcontainer.VarContainer()
parts.zcluster = datamanager.zcluster
for i in range(10):
try:
self.makeShapePrior(datamanager, parts)
self.makeModelPrior(datamanager, parts)
self.makeLikelihood(datamanager, parts)
return parts
except pymc.ZeroProbability:
pass
raise ModelInitException
def calcMaxlikeDistro(outputfile='maxlikedistro_paperplot_lowsn.pkl',
ngalsperarcmin=5):
'''show that using maximum likelihood estimators does not recover truth'''
figs = []
#megacamstyle
ntrials = 2000
samplesize = 25
r_mpc_edges = np.linspace(0.75, 3., 13)
r_mpc = (r_mpc_edges[1:] + r_mpc_edges[:-1]) / 2.
zcluster = 0.3
duffy = basicMassCon.Duffy(None)
c200 = 4.0
beta_s = nfwutils.global_cosmology.beta_s([1.0], zcluster)[0]
r_arcmin_edges = (r_mpc_edges /
nfwutils.global_cosmology.angulardist(zcluster)) * (
180. / np.pi) * 60
bin_areas = np.pi * (r_arcmin_edges[1:]**2 - r_arcmin_edges[:-1]**2)
ngals_per_bin = ngalsperarcmin * bin_areas
g_err = 0.25 / np.sqrt(ngals_per_bin)
m200s = np.arange(1e12, 5e15, 1e13)
config = varcontainer.VarContainer()
config.massconmodule = 'basicMassCon'
config.massconrelation = 'constant'
config.concentration = c200
model = nfwfit.buildModel(config)
likelihood = nfwnoise.Likelihood(config)
unweightedbootstraps_mean = np.zeros(ntrials)
weightedaverages = np.zeros(ntrials)
wlssaverage = np.zeros(ntrials)
linear_fit = np.zeros(ntrials)
smithaverages = np.zeros(ntrials)
pdfaverages = np.zeros(ntrials)
for curtrial in range(ntrials):
true_masses = 10**np.random.uniform(np.log10(5e14), np.log10(2e15),
samplesize)
lens_masses = true_masses * np.exp(
0.2 * np.random.standard_normal(size=samplesize))
#compute individual cluster likelihoods
maxlike_masses = np.zeros(samplesize)
maxlike_errs = np.zeros((2, samplesize))
pdfs = np.zeros((samplesize, len(m200s)))
logpdf = np.zeros(len(m200s))
for curcluster in range(samplesize):
g_obs = nfwnoise.createPerfectProfile(lens_masses[curcluster], c200,
zcluster, r_mpc, beta_s) + \
g_err*np.random.standard_normal(size=len(g_err))
fitter = nfwfit.NFWFitter(None, model, config)
maxlikemass, maxlikeerr = fitter.minChisqMethod(
r_mpc,
g_obs,
g_err,
beta_s,
beta_s**2,
zcluster,
guess=[true_masses[curcluster] / model.massScale])
maxlike_masses[curcluster] = maxlikemass['m200']
try:
maxlike_errs[:, curcluster] = maxlikeerr['m200']
except TypeError:
maxlike_errs[:, curcluster] = (maxlikeerr['m200'],
maxlikeerr['m200'])
likelihood.bindData(r_mpc, g_obs, g_err, beta_s, zcluster)
for i, m200 in enumerate(m200s):
logpdf[i] = likelihood(m200)
pdf = np.exp(logpdf - np.max(logpdf))
pdf /= np.trapz(pdf, m200s)
pdfs[curcluster, :] = pdf
maxlike_masses *= model.massScale
maxlike_errs = np.abs(maxlike_errs * model.massScale)
maxlike_logratio = np.log(maxlike_masses / true_masses)
#combine using different formulisms
bootstraps = np.zeros(1000)
for curboot in range(1000):
booted = np.random.randint(0, samplesize, samplesize)
bootstraps[curboot] = np.mean(maxlike_logratio[booted])
unweightedbootstraps_mean[curtrial] = np.mean(bootstraps)
maxlike_simpleerr = np.sum(maxlike_errs,
axis=0) / (2. * maxlike_masses)
weight = 1. / maxlike_simpleerr**2
weightedaverages[curtrial] = np.sum(
weight * maxlike_logratio) / np.sum(weight)
wlssaverage_weights = 1. / (maxlike_simpleerr**2 +
(0.2 * maxlike_masses)**2)
wlssaverage[curtrial] = np.sum(
wlssaverage_weights *
maxlike_logratio) / np.sum(wlssaverage_weights)
smithweight = 1. / (np.sum(maxlike_errs, axis=0) / 2.)**2
smithaverages[curtrial] = np.sum(
smithweight * maxlike_logratio) / np.sum(smithweight)
halos = [
dict(id=curcluster,
true_mass=true_masses[curcluster],
masses=m200s,
pdf=pdfs[curcluster, :]) for curcluster in range(samplesize)
]
pdfmodel = deconvolvedlognorm.buildPDFModel(halos)
pdfmap = pymc.MAP(pdfmodel)
pdfmap.fit()
pdfaverages[curtrial] = pdfmap.logmu.value
linefitter = fitmodel.FitModel(true_masses / 1e14,
maxlike_masses / 1e14,
np.sum(maxlike_errs / 1e14, axis=0) /
2.,
fitmodel.RatioModel,
guess=[1.])
linefitter.fit()
linear_fit[curtrial] = linefitter.par_vals['a0']
results = dict(unweightedbootstrapsmean=np.exp(unweightedbootstraps_mean),
weightedaverages=np.exp(weightedaverages),
wlssaverage=np.exp(wlssaverage),
smithaverages=np.exp(smithaverages),
pdfaverages=np.exp(pdfaverages),
linearfit=linear_fit)
with open(outputfile, 'wb') as output:
cPickle.dump(results, output, -1)
