def createFakeChains(config, nclusters, zcluster, r_mpc_edges, beta_s, galdensity, shapenoise, nsamples=1000, mass=10**15.2): mtrues = mass * np.ones(nclusters) r_mpcs, shearprofiles, shearerrs = createClusterSet( config, mtrues, zcluster, r_mpc_edges, beta_s, galdensity, shapenoise) fitter = nfwfit.buildFitter(config) chains = [] for i in range(nclusters): mcmc_model = None for j in range(10): try: mcmc_model = fitter.model.makeMCMCModel( r_mpcs[i], shearprofiles[i], shearerrs[i], beta_s, beta_s**2, zcluster) break except pymc.ZeroProbability: pass if mcmc_model is None: raise pymc.ZeroProbability manager = varcontainer.VarContainer() options = varcontainer.VarContainer() manager.options = options options.singlecore = True options.adapt_every = 100 options.adapt_after = 100 options.nsamples = nsamples manager.model = mcmc_model runner = pma.MyMCMemRunner() runner.run(manager) runner.finalize(manager) chains.append(manager.chain['m200'][200:]) return mtrues, chains
def sample(parts, outputfile, samples, adaptevery=100, adaptafter=100, singlecore=False, tempoutputdir=None, init_with_MAP=True): '''This function saves to disk the samples from the chain as they're drawn. ''' del_tempoutdir = False if tempoutputdir is None: tempoutputdir = tempfile.mkdtemp() del_tempoutdir = True outputdir, outputbase = os.path.split(outputfile) options = varcontainer.VarContainer() options.outputFile = '%s/%s' % (tempoutputdir, outputbase) options.singlecore = singlecore options.nsamples = samples options.adapt_every = adaptevery options.adapt_after = adaptafter options.restore = True manager = varcontainer.VarContainer() manager.options = options manager.model = pymc.Model(parts) if init_with_MAP: model_map = pymc.MAP(parts) model_map.fit() print "initialized at: ", model_map.logmu.value, model_map.logmu_c.value assert (np.isfinite(manager.model.logp)) runner = pma.MyMCRunner() try: runner.run(manager) runner.finalize(manager) finally: outputfiles = glob.glob('%s/*' % tempoutputdir) for curoutputfile in outputfiles: curoutputbase = os.path.basename(curoutputfile) destination_output = '%s/%s' % (outputdir, curoutputbase) if os.path.exists(destination_output): os.remove(destination_output) shutil.copyfile(curoutputfile, destination_output) if del_tempoutdir: shutil.rmtree(tempoutputdir)
def createOptions(self, deltaz95low=-1, deltaz95high=2.5, zbhigh=1.25, zcut=0.1, masslow=1e13, masshigh=1e16, ztypecut=False, radlow=0.75, radhigh=3.0, concentration=None, logprior=False, options=None, args=None): if options is None: options = varcontainer.VarContainer() options.deltaz95low = deltaz95low options.deltaz95high = deltaz95high options.zbhigh = zbhigh options.zcut = zcut options.masslow = masslow options.masshigh = masshigh options.ztypecut = ztypecut options.radlow = radlow options.radhigh = radhigh options.concentration = concentration options.logprior = logprior return options, None
def parseCL(self): parser = optparse.OptionParser(usage=usage) def addGroup(strategy, printedname): if hasattr(strategy, 'addCLOps'): optgroup = optparse.OptionGroup(parser, '%s Options' % printedname, '') strategy.addCLOps(optgroup) parser.add_option_group(optgroup) addGroup(self, 'Controller') addGroup(self.modelbuilder, 'Model') addGroup(self.filehandler, 'File Handler') addGroup(self.runmethod, 'Run Method') varoptions = varcontainer.VarContainer() options, args = parser.parse_args() for key, val in vars(options).iteritems(): varoptions[key] = val self.replace('options', varoptions) self.replace('args', args) return options, args
def memsample(parts, samples, adaptevery = 100, adaptafter = 100): options = varcontainer.VarContainer() options.singlecore = True options.nsamples = samples options.adapt_every = adaptevery options.adapt_after = adaptafter manager = varcontainer.VarContainer() manager.options = options manager.model = pymc.Model(parts) runner = pma.MyMCMemRunner() runner.run(manager) runner.finalize(manager) return manager.chain
def sample(parts, outputfile, samples, adaptevery = 100, adaptafter = 100, singlecore = False): options = varcontainer.VarContainer() options.outputFile = outputfile options.singlecore = singlecore options.nsamples = samples options.adapt_every = adaptevery options.adapt_after = adaptafter manager = varcontainer.VarContainer() manager.options = options manager.model = pymc.Model(parts) runner = pma.MyMCRunner() runner.run(manager) runner.finalize(manager)
def __call__(self, profile): chains = {} for delta in self.deltas: mcmc_model = None for i in range(20): try: mcmc_model = self.model.makeMCMCModel(profile, delta = delta) break except pymc.ZeroProbability: pass if mcmc_model is None: raise pymc.ZeroProbability # This sets up Adam Mantz's version of an MCMC sampler for production code calculations. # This is stored in mymcmc_adapter.py (converts to talk with other MCMC code) # Imported as pma above. manager = varcontainer.VarContainer() options = varcontainer.VarContainer() manager.options = options options.singlecore = True options.adapt_every = 100 options.adapt_after = 100 options.nsamples = self.nsamples manager.model = mcmc_model runner = pma.MyMCMemRunner() runner.run(manager) runner.finalize(manager) reducedchain = dict(cdelta = np.hstack(manager.chain['cdelta'][5000::2]).astype(np.float32), mdelta = np.hstack(manager.chain['mdelta'][5000::2]).astype(np.float32), likelihood = np.hstack(manager.chain['likelihood'][5000::2]).astype(np.float32)) chains[delta] = reducedchain return chains
def __call__(self, profile): chains = {} for delta in self.deltas: mcmc_model = None for i in range(20): try: mcmc_model = self.model.makeMCMCModel(profile, delta=delta) break except pymc.ZeroProbability: pass if mcmc_model is None: raise pymc.ZeroProbability manager = varcontainer.VarContainer() options = varcontainer.VarContainer() manager.options = options options.singlecore = True options.adapt_every = 100 options.adapt_after = 100 options.nsamples = self.nsamples manager.model = mcmc_model runner = pma.MyMCMemRunner() runner.run(manager) runner.finalize(manager) reducedchain = dict( cdelta=np.hstack(manager.chain['cdelta'][5000::2]).astype( np.float32), mdelta=np.hstack(manager.chain['mdelta'][5000::2]).astype( np.float32), likelihood=np.hstack( manager.chain['likelihood'][5000::2]).astype(np.float32)) chains[delta] = reducedchain return chains
def sample(parts, outputfile, samples, adaptevery=100, adaptafter=100, singlecore=False): tempoutputdir = tempfile.mkdtemp() outputdir, outputbase = os.path.split(outputfile) options = varcontainer.VarContainer() options.outputFile = '%s/%s' % (tempoutputdir, outputbase) options.singlecore = singlecore options.nsamples = samples options.adapt_every = adaptevery options.adapt_after = adaptafter options.restore = True manager = varcontainer.VarContainer() manager.options = options manager.model = pymc.Model(parts) assert (np.isfinite(manager.model.logp)) runner = pma.MyMCRunner() try: runner.run(manager) runner.finalize(manager) finally: outputfiles = glob.glob('%s/*' % tempoutputdir) for curoutputfile in outputfiles: curoutputbase = os.path.basename(curoutputfile) destination_output = '%s/%s' % (outputdir, curoutputbase) if os.path.exists(destination_output): os.remove(destination_output) shutil.copyfile(curoutputfile, destination_output) shutil.rmtree(tempoutputdir)
def readData(self, manager): options = manager.options newoptions = varcontainer.VarContainer() newoptions.update(options) self.buildFileNames(manager, newoptions) manager.open('lensingcat', newoptions.lensingcat, ldac.openObjectFile) if options.nearestneighbor is True: nncat = ldac.openObjectFile(newoptions.neighborcat) manager.nearestneighbors = nncat.matchById(manager.lensingcat) newoptions.psfsize = readPSFSize(options.workdir, options.cluster, options.filter, options.image) newoptions.zcluster = parseZCluster(options.cluster) newoptions.r500 = readR500(options.cluster) newoptions.centerx, newoptions.centery = readClusterCenters(options.cluster) newoptions.pixscale = 0.2 newoptions.xcol = 'Xpos' newoptions.ycol = 'Ypos' newoptions.g1col = 'gs1' newoptions.g2col = 'gs2' newoptions.sizecol = 'rh' newoptions.centerx = 5000 newoptions.centery = 5000 newoptions.snratio = 'snratio_scaled1' newoptions.zbcol = 'BPZ_Z_B' manager.replace('options', newoptions) maxlike_general_filehandler.readData(manager)
def memsample(model, samples, adaptevery=100, adaptafter=100, outputFile=None): options = varcontainer.VarContainer() options.singlecore = True options.nsamples = samples options.adapt_every = adaptevery options.adapt_after = adaptafter if outputFile: options.outputFile = outputFile manager = varcontainer.VarContainer() manager.options = options manager.model = model runner = pma.MyMCMemRunner() runner.run(manager) if outputFile: runner.dump(manager) runner.finalize(manager) return manager.chain
def createOptions(self, inputCatFile, inputBPZ, inputPDZ, options=None, args=None): if options is None: options = varcontainer.VarContainer() options.inputCatFile = inputCatFile options.inputBPZ = inputBPZ options.inputPDZ = inputPDZ return options, args
def createOptions(self, cluster, filter, image, ldaclensing='../ldaclensing', snlow = 3, snhigh = 9999, oddscut = 0., minmag = 25., rhcut = 1.15, nearestneighbor = True, redseqcat = None, shapecut = False, options = None, args = None): if options is None: options = varcontainer.VarContainer() options.cluster = cluster options.filter = filter options.image = image options.ldaclensing = ldaclensing options.snlow = snlow options.snhigh = snhigh options.oddscut = oddscut options.minmag = minmag options.rhcut = rhcut options.nearestneighbor = nearestneighbor options.redseqcat = redseqcat options.shapecut = shapecut return options, None
def makeModelParts(self, datamanager, parts=None): if parts is None: parts = varcontainer.VarContainer() parts.zcluster = datamanager.zcluster for i in range(10): try: self.makeShapePrior(datamanager, parts) self.makeModelPrior(datamanager, parts) self.makeLikelihood(datamanager, parts) return parts except pymc.ZeroProbability: pass raise ModelInitException
def calcMaxlikeDistro(outputfile='maxlikedistro_paperplot_lowsn.pkl', ngalsperarcmin=5): '''show that using maximum likelihood estimators does not recover truth''' figs = [] #megacamstyle ntrials = 2000 samplesize = 25 r_mpc_edges = np.linspace(0.75, 3., 13) r_mpc = (r_mpc_edges[1:] + r_mpc_edges[:-1]) / 2. zcluster = 0.3 duffy = basicMassCon.Duffy(None) c200 = 4.0 beta_s = nfwutils.global_cosmology.beta_s([1.0], zcluster)[0] r_arcmin_edges = (r_mpc_edges / nfwutils.global_cosmology.angulardist(zcluster)) * ( 180. / np.pi) * 60 bin_areas = np.pi * (r_arcmin_edges[1:]**2 - r_arcmin_edges[:-1]**2) ngals_per_bin = ngalsperarcmin * bin_areas g_err = 0.25 / np.sqrt(ngals_per_bin) m200s = np.arange(1e12, 5e15, 1e13) config = varcontainer.VarContainer() config.massconmodule = 'basicMassCon' config.massconrelation = 'constant' config.concentration = c200 model = nfwfit.buildModel(config) likelihood = nfwnoise.Likelihood(config) unweightedbootstraps_mean = np.zeros(ntrials) weightedaverages = np.zeros(ntrials) wlssaverage = np.zeros(ntrials) linear_fit = np.zeros(ntrials) smithaverages = np.zeros(ntrials) pdfaverages = np.zeros(ntrials) for curtrial in range(ntrials): true_masses = 10**np.random.uniform(np.log10(5e14), np.log10(2e15), samplesize) lens_masses = true_masses * np.exp( 0.2 * np.random.standard_normal(size=samplesize)) #compute individual cluster likelihoods maxlike_masses = np.zeros(samplesize) maxlike_errs = np.zeros((2, samplesize)) pdfs = np.zeros((samplesize, len(m200s))) logpdf = np.zeros(len(m200s)) for curcluster in range(samplesize): g_obs = nfwnoise.createPerfectProfile(lens_masses[curcluster], c200, zcluster, r_mpc, beta_s) + \ g_err*np.random.standard_normal(size=len(g_err)) fitter = nfwfit.NFWFitter(None, model, config) maxlikemass, maxlikeerr = fitter.minChisqMethod( r_mpc, g_obs, g_err, beta_s, beta_s**2, zcluster, guess=[true_masses[curcluster] / model.massScale]) maxlike_masses[curcluster] = maxlikemass['m200'] try: maxlike_errs[:, curcluster] = maxlikeerr['m200'] except TypeError: maxlike_errs[:, curcluster] = (maxlikeerr['m200'], maxlikeerr['m200']) likelihood.bindData(r_mpc, g_obs, g_err, beta_s, zcluster) for i, m200 in enumerate(m200s): logpdf[i] = likelihood(m200) pdf = np.exp(logpdf - np.max(logpdf)) pdf /= np.trapz(pdf, m200s) pdfs[curcluster, :] = pdf maxlike_masses *= model.massScale maxlike_errs = np.abs(maxlike_errs * model.massScale) maxlike_logratio = np.log(maxlike_masses / true_masses) #combine using different formulisms bootstraps = np.zeros(1000) for curboot in range(1000): booted = np.random.randint(0, samplesize, samplesize) bootstraps[curboot] = np.mean(maxlike_logratio[booted]) unweightedbootstraps_mean[curtrial] = np.mean(bootstraps) maxlike_simpleerr = np.sum(maxlike_errs, axis=0) / (2. * maxlike_masses) weight = 1. / maxlike_simpleerr**2 weightedaverages[curtrial] = np.sum( weight * maxlike_logratio) / np.sum(weight) wlssaverage_weights = 1. / (maxlike_simpleerr**2 + (0.2 * maxlike_masses)**2) wlssaverage[curtrial] = np.sum( wlssaverage_weights * maxlike_logratio) / np.sum(wlssaverage_weights) smithweight = 1. / (np.sum(maxlike_errs, axis=0) / 2.)**2 smithaverages[curtrial] = np.sum( smithweight * maxlike_logratio) / np.sum(smithweight) halos = [ dict(id=curcluster, true_mass=true_masses[curcluster], masses=m200s, pdf=pdfs[curcluster, :]) for curcluster in range(samplesize) ] pdfmodel = deconvolvedlognorm.buildPDFModel(halos) pdfmap = pymc.MAP(pdfmodel) pdfmap.fit() pdfaverages[curtrial] = pdfmap.logmu.value linefitter = fitmodel.FitModel(true_masses / 1e14, maxlike_masses / 1e14, np.sum(maxlike_errs / 1e14, axis=0) / 2., fitmodel.RatioModel, guess=[1.]) linefitter.fit() linear_fit[curtrial] = linefitter.par_vals['a0'] results = dict(unweightedbootstrapsmean=np.exp(unweightedbootstraps_mean), weightedaverages=np.exp(weightedaverages), wlssaverage=np.exp(wlssaverage), smithaverages=np.exp(smithaverages), pdfaverages=np.exp(pdfaverages), linearfit=linear_fit) with open(outputfile, 'wb') as output: cPickle.dump(results, output, -1)