def readData(manager): options = manager.options manager.zcluster = options.zcluster manager.r500 = options.r500 manager.psfsize = options.psfsize manager.pixscale = options.pixscale if 'lensingcat' not in manager: manager.open('lensingcat', options.lensingcat, ldac.openObjectFile) r_arc, E, B = sp.calcTangentialShear(cat=manager.lensingcat, center=(options.centerx, options.centery), pixscale=options.pixscale, xcol=options.xcol, ycol=options.ycol, g1col=options.g1col, g2col=options.g2col) r_pix = r_arc / options.pixscale r_mpc = r_arc * (1. / 3600.) * (np.pi / 180. ) * sp.angulardist(options.zcluster) size = manager.lensingcat[options.sizecol] / options.psfsize snratio = manager.lensingcat[options.snratio] manager.open('bpzcat', options.bpzfile, ldac.openObjectFile, 'STDTAB') manager.store('matched_bpzcat', manager.bpzcat.matchById, manager.lensingcat) z_b = manager.matched_bpzcat['BPZ_Z_B'] odds = manager.matched_bpzcat['BPZ_ODDS'] nfilt = manager.matched_bpzcat['NFILT'] galtype = manager.matched_bpzcat['BPZ_T_B'] cols = [pyfits.Column(name='SeqNr', format='J', array=manager.lensingcat['SeqNr']), pyfits.Column(name='r_mpc', format='E', array=r_mpc), pyfits.Column(name='size', format='E', array=size), pyfits.Column(name='snratio', format='E', array=snratio), pyfits.Column(name='z_b', format='E', array=z_b), pyfits.Column(name='z_t', format='E', array=galtype), pyfits.Column(name='odds', format='E', array=odds), pyfits.Column(name='nfilt', format='I', array=nfilt), pyfits.Column(name='ghats', format='E', array=E), pyfits.Column(name='B', format='E', array=B)] manager.store('inputcat', ldac.LDACCat(pyfits.BinTableHDU.from_columns(pyfits.ColDefs(cols)))) manager.open('pdzmanager', options.inputPDZ, pdzfile_utils.PDZManager.open) manager.store('pdzrange pdz'.split(), manager.pdzmanager.associatePDZ(manager.lensingcat['SeqNr'])) manager.replace('pdzmanager', None) manager.replace('matched_bpz', None) manager.replace('bpzcat', None)
def ML_NFW_Model(r_mpc, ghats, betas, pdz, dz, concentration, zcluster, shape_sigma): ####### # Data Prep ####### r_mpc = np.ascontiguousarray(r_mpc) pdz = np.ascontiguousarray(pdz) D_lens = sp.angulardist(zcluster) nzbins = len(betas) ghat_matrix = np.ascontiguousarray(np.column_stack(nzbins * [ghats])) ngals = len(ghats) beta_matrix = np.ascontiguousarray(np.vstack(ngals * [betas])) assert (ghat_matrix.shape == beta_matrix.shape == pdz.shape) ####### # Model ####### ## r_scale is log-uniform log_r_scale = pymc.Uniform('log_r_scale', np.log(.01), np.log(1.)) ##contamination contam_fraction = pymc.Uniform('contam_fraction', 0.0, 0.3) contam_mu = pymc.Uniform('contam_mu', -1.0, 1.0) contam_sig = pymc.Uniform('contam_sig', shape_sigma, 10 * shape_sigma) #assuming dz is constant over range!!! #assuming sigma is the same for all galaxies! @pymc.stochastic(observed=True) def data(value=ghats, log_r_scale=log_r_scale, contam_fraction=contam_fraction, contam_mu=contam_mu, contam_sig=contam_sig): raise Exception( "Out of date function; discovered & abandoned 2012-04-01; no joke") return tools.likelihood(log_r_scale, r_mpc, ghat_matrix, beta_matrix, pdz, concentration, zcluster, D_lens, shape_sigma, dz, contam_fraction, contam_mu, contam_sig) ######## return locals()
def beta_method(r_pix, r_mpc, ghat, beta_s, beta_s2, concentration, zcluster, pixscale=__DEFAULT_PIXSCALE__, fitrange=(0.3, 5.)): D_lens = sp.angulardist(zcluster) def NFWModel(r, r_scale): nfw_shear = beta_s * tools.NFWShear(r, concentration, r_scale, zcluster, D_lens) nfw_kappa = beta_s * tools.NFWKappa(r, concentration, r_scale, zcluster, D_lens) g = (1 + (beta_s2 / beta_s**2 - 1) * nfw_kappa) * nfw_shear / (1 - nfw_kappa) return g minPix = fitrange[0] * 3600. * (180. / np.pi) / (pixscale * D_lens) maxPix = fitrange[1] * 3600. * (180. / np.pi) / (pixscale * D_lens) profile = sp.shearprofile(np.zeros_like(r_pix), r_pix, ghat, np.zeros_like(ghat), sigma2=None, range=(minPix, maxPix), bins=15, center=(0, 0), logbin=True) profile_r_mpc = profile.r * pixscale * (1. / 3600.) * (np.pi / 180.) * D_lens rs, isCon = leastsq.leastsq(NFWModel, [0.4], profile_r_mpc, profile.E, profile.Eerr, fullOutput=False) if not isCon: return None return rs
########################## inputCatFile = sys.argv[1] inputPDZ = sys.argv[2] shapedistro_module = sys.argv[3] outputFile = sys.argv[4] ########################## inputcat = ldac.openObjectFile(inputCatFile) concentration = inputcat.hdu.header['CONCEN'] zcluster = inputcat.hdu.header['Z'] D_lens = sp.angulardist(zcluster) pixscale = 0.2 minPix = 0.3 * 3600. * (180. / np.pi) / (pixscale * D_lens) maxPix = 5. * 3600. * (180. / np.pi) / (pixscale * D_lens) goodObjs = np.logical_and( np.logical_and(inputcat['r_pix'] > minPix, inputcat['r_pix'] < maxPix), np.logical_and(inputcat['z_b'] > 0, inputcat['z_b'] < 1.25)) shapedistro = __import__(shapedistro_module) bin_selectors = [np.logical_and(goodObjs, selector) \ for selector in shapedistro.bin_selectors(inputcat)] pdzfile = open(inputPDZ, 'rb') pdzrange, pdz = cPickle.load(pdzfile)
def ML_NFW_Model(r_mpc, ghats, betas, pdz, concentration, zcluster, likelihood_func, shapedistro_samples): #r_mpc, ghats, pdz, and shapedistro_samples may be either arrays or lists of arrays # if lists of arrays, then each entry is associated across the lists and will be passed # to likelihood_func seperately, with the logProbs summed together ####### # Data Prep ####### if not isinstance(r_mpc, list): r_mpc = [r_mpc] ghats = [ghats] pdz = [pdz] shapedistro_samples = [shapedistro_samples] r_mpc = [np.ascontiguousarray(x) for x in r_mpc] pdz = [np.ascontiguousarray(x) for x in pdz] shapedistro_samples = [ np.ascontiguousarray(x) for x in shapedistro_samples ] D_lens = sp.angulardist(zcluster) nzbins = len(betas) ghats = [np.ascontiguousarray(x) for x in ghats] betas = np.ascontiguousarray(betas) nshapebins = len(shapedistro_samples) ####### # Model ####### ## shape parameter priors shape_sample_index = [ pymc.DiscreteUniform('shape_index_%d' % i, 0, len(x)) for i, x in enumerate(shapedistro_samples) ] shape_params = np.empty(nshapebins, dtype=object) for i, index, samples in zip(range(len(shapedistro_samples)), shape_sample_index, shapedistro_samples): @pymc.deterministic(name='shape_params_%d' % i) def shape_param_func(index=index, samples=samples): return samples[:, index] shape_params[i] = shape_param_func ## r_scale is log-uniform log_r_scale = pymc.Uniform('log_r_scale', np.log(.01), np.log(1.)) @pymc.stochastic(observed=True) def data(value=ghats, log_r_scale=log_r_scale, shape_params=shape_params): logprobs = np.array([likelihood_func(log_r_scale, cur_r_mpc, cur_ghats, betas, cur_pdz, cur_shapedistro_samples, concentration, zcluster, D_lens) \ for (cur_r_mpc, cur_ghats, cur_pdz, cur_shapedistro_samples) \ in zip(r_mpc, ghats, pdz, shape_params)]) return np.sum(logprobs) ######## return locals()
def beta_contamination(x, y, g1, g2, mag, useInContam, areamap, zdist, zcluster, norm_radii, concentration_prior=4., density_prior=[0, 30], contam_prior=[0, 10], fitrange=(0.4, 4.), bins=15, logbin=False, pixscale=__DEFAULT_PIXSCALE__, center=__DEFAULT_CENTER__): ####### # save config setting ####### momento = { 'x': x, 'y,': y, 'g1': g1, 'g2': g2, 'mag': mag, 'useInContam': useInContam, 'areamap': areamap, 'zdist': zdist, 'zcluster': zcluster, 'norm_radii': norm_radii, 'concentration_prior': concentration_prior, 'density_prior': density_prior, 'contam_prior': contam_prior, 'fitrange': fitrange, 'bins': bins, 'logbin': logbin, 'pixscale': pixscale, 'center': center } ####### # Data Prep ####### D_lens = sp.angulardist(zcluster) cat = {'Xpos': x, 'Ypos': y, 'gs1': g1, 'gs2': g2} r_arcsec, E, B = sp.calcTangentialShear(cat, center, pixscale) r_mpc = r_arcsec * (1. / 3600.) * (np.pi / 180.) * D_lens if logbin: dr = log(dr) rmin = log(rmin) rmax = log(rmax) leftedges, binwidth = linspace(rmin, rmax, bins + 1, endpoint=True, retstep=True) rbin = exp(leftedges) rbin = (rbin[1:] + rbin[:-1]) / 2. else: leftedges, binwidth = linspace(rmin, rmax, bins, endpoint=False, retstep=True) rbin = leftedges + binwidth / 2. # Set up binning grid: Ebin = zeros(bins, dtype=float64) Ebinerr = zeros(bins, dtype=float64) Bbin = zeros(bins) Bbinerr = zeros(bins) nbin = zeros(bins) index = ((dr - rmin) / binwidth).astype(int) goodentry = logical_and(index >= 0, index < bins) ####### # Model ####### ## r_scale is log-uniform log_r_scale = pymc.Uniform('log_r_scale', np.log(.01), np.log(1.)) normgal_density = pymc.Uniform('n_normgals', density_prior[0], density_prior[1]) contam_amp = pymc.Uniform('A_contam', contam_prior[0], contam_prior[1]) @pymc.stochastic(observed=True) def shear(value=ghats, log_r_scale=log_r_scale, normgal_density=normgal_density, contam_amp=contam_amp): return tools.likelihood(log_r_scale, r_mpc, ghat_matrix, beta_matrix, pdz, concentration, zcluster, D_lens, shape_sigma) @pymc.stochastic(observed=True) def numcounts(value=ngals, normgal_density=normgal_density, contam_amp=contam_amp): pass ######## return locals()
def addContamination(sourcebpz, source_snratio, source_size, simcat, simpdz, r500, zcluster, f500=0.04, pixscale=pixscale, refcat='/u/ki/dapple/nfs12/cosmos/cosmos2.cat', shape_distro=__DEFAULT_SHAPE_DISTRO__, shape_distro_args=[], shape_distro_kw={}): refcat = ldac.openObjectFile(refcat).matchById(sourcebpz, selfid='id') r_max = max(simcat['r_pix']) area = np.pi * (r_max * pixscale / 60.)**2 n_back = float(len(simcat)) / area x, y = nfwsim.stdcontamination((0, 0), pixscale, f500, n_back, r500, zcluster) r_pix = np.sqrt(x**2 + y**2) r_mpc = r_pix * pixscale * (1. / 3600.) * (np.pi / 180.) * sp.angulardist(zcluster) z = zcluster * np.ones(len(r_pix)) toKeepAvailable = {} for id in sourcebpz['SeqNr']: toKeepAvailable[id] = True for id in simcat['z_id']: toKeepAvailable[id] = False zkey = 'zp_best' if zkey not in sourcebpz: zkey = 'BPZ_Z_S' toKeep = np.logical_and( np.logical_and( np.array([toKeepAvailable[id] for id in sourcebpz['SeqNr']]), refcat['mod_gal'] > 8), np.logical_and(sourcebpz[zkey] > (zcluster - 0.05), sourcebpz[zkey] < (zcluster + 0.05))) #toKeep = np.logical_and(np.array([toKeepAvailable[id] for id in sourcebpz['SeqNr']]), # np.logical_and(sourcebpz[zkey] > (zcluster - 0.05), # sourcebpz[zkey] < (zcluster + 0.05))) # # availablebpz = sourcebpz.filter(toKeep) available_snratio = source_snratio[toKeep] available_size = source_size[toKeep] selected = np.random.randint(0, len(availablebpz), len(r_pix)) selectedbpz = availablebpz.filter(selected) selected_snratio = available_snratio[selected] selected_size = available_size[selected] col_collection = { 'Seqnr': -selectedbpz['SeqNr'], 'r_pix': r_pix, 'r_mpc': r_mpc, 'z': z, 'z_id': selectedbpz['SeqNr'], 'ghats': shape_distro(np.zeros(len(selectedbpz)), np.column_stack([selected_size, selected_snratio]), *shape_distro_args, **shape_distro_kw), 'true_shear': np.zeros_like(r_pix), 'true_z': zcluster * np.ones_like(r_pix), 'true_beta': np.zeros_like(r_pix), 'true_gamma': np.zeros_like(r_pix), 'true_kappa': np.zeros_like(r_pix) } cols = [] for name, arr in col_collection.iteritems(): if name == 'SeqNr': col = pyfits.Column(name=name, format='J', array=arr) else: col = pyfits.Column(name=name, format='E', array=arr) cols.append(col) contamcat = ldac.LDACCat( pyfits.BinTableHDU.from_columns(pyfits.ColDefs(cols))) finalcat = simcat.append(contamcat) return finalcat
def createCatalog(bpz, bpz_sizes, bpz_snratios, concentration, scale_radius, zcluster, ngals, shape_distro=__DEFAULT_SHAPE_DISTRO__, shape_distro_args=[], shape_distro_kw={}, maxpix=5000, radii_pix=None, contam=None, contam_args=[], contam_kw={}, idcol='ID'): # ngals == None -> no bootstrapping # bpz is ldac bpz output momento = { 'concentration': concentration, 'scale_radius': scale_radius, 'zcluster': zcluster, 'ngals': ngals, 'shape_distro': shape_distro.func_name, 'shape_distro_args': shape_distro_args, 'shape_distro_kw': shape_distro_kw, 'maxpix': maxpix, 'radii_pix': radii_pix } bootstrap = True if ngals == None: bootstrap = False ngals = len(bpz) if radii_pix is None: x_pix = np.random.uniform(0, maxpix, ngals) y_pix = np.random.uniform(0, maxpix, ngals) radii_pix = np.sqrt(x_pix**2 + y_pix**2) radii_mpc = radii_pix * pixscale * (1. / 3600.) * ( np.pi / 180.) * sp.angulardist(zcluster) chosenZs = bpz chosenSizes = bpz_sizes chosenSNratios = bpz_snratios if bootstrap: indices = np.random.randint(0, len(bpz), ngals) chosenZs = bpz.filter(indices) chosenSizes = bpz_sizes[indices] chosenSNratios = bpz_snratios[indices] print "adam-look: running createCatalog in cosmos_sim.py" z_id = chosenZs[idcol] zbins = numpy.arange(0, 6.01, .01) z_drawn = -1 * np.ones(len(chosenZs)) print '!!!', len(z_drawn) for id_indx, id in enumerate(z_id): try: cdf = id2pz_cdf[id] except: if id in gone_ids: print "adam-look: gone id ", id continue else: raise x = numpy.random.rand() try: zval = (zbins[cdf <= x])[-1] except: if id in bad_ids: print "adam-look: bad id ", id continue else: raise z_drawn[id_indx] = zval good_draws = z_drawn > -1 z_drawn = z_drawn[good_draws] radii_pix = radii_pix[good_draws] radii_mpc = radii_mpc[good_draws] chosenZs = chosenZs.filter(good_draws) chosenSizes = chosenSizes[good_draws] chosenSNratios = chosenSNratios[good_draws] ngals = len(z_drawn) true_shears, true_gamma, true_kappa = nfwsim.create_nfwmodel_shapedata( concentration, scale_radius, zcluster, ngals, z_drawn, radii_mpc) true_beta = nfwutils.beta_s(z_drawn, zcluster) ghats = shape_distro(true_shears, np.column_stack([chosenSizes, chosenSNratios]), *shape_distro_args, **shape_distro_kw) cols = [ pyfits.Column(name='Seqnr', format='J', array=np.arange(ngals)), pyfits.Column(name='r_pix', format='E', array=radii_pix), pyfits.Column(name='r_mpc', format='E', array=radii_mpc), pyfits.Column(name='z', format='E', array=z_drawn), pyfits.Column(name='z_id', format='J', array=chosenZs[idcol]), pyfits.Column(name='ghats', format='E', array=ghats), pyfits.Column(name='true_shear', format='E', array=true_shears), pyfits.Column(name='true_z', format='E', array=z_drawn), pyfits.Column(name='true_beta', format='E', array=true_beta), pyfits.Column(name='true_gamma', format='E', array=true_gamma), pyfits.Column(name='true_kappa', format='E', array=true_kappa) ] simcat = pyfits.new_table(pyfits.ColDefs(cols)) simcat.header.update('EXTNAME', 'OBJECTS') simcat.header.update('concen', concentration) simcat.header.update('r_s', scale_radius) simcat.header.update('z', zcluster) return ldac.LDACCat(simcat), momento
def createBootstrapCats(cluster, filter, image, outdir, nbootstraps = 100, startnum = 0): clusterdir = '%s/%s' % (subarudir, cluster) lensingdir = '%s/LENSING_%s_%s_aper/%s' % (clusterdir, filter, filter, image) lensingcat = '%s/coadd_photo2.cat' % lensingdir cutsfile = '%s/cc_cuts3.dat' % lensingdir outdir = '%s/%s' % (outdir, cluster) if not os.path.exists(outdir): os.mkdir(outdir) cat = ldac.openObjectFile(lensingcat) zcluster = msf.parseZCluster(cluster) center = msf.readClusterCenters(cluster) r_arc, E, B = sp.calcTangentialShear(cat = cat, center = center, pixscale = 0.2) r_mpc = r_arc * (1./3600.) * (np.pi / 180. ) * sp.angulardist(zcluster) cat = cat.filter(np.logical_and(r_mpc > 0.750, r_mpc < 3.0)) cat.hdu.header.update('EXTNAME','STDTAB') try: tmpspace = tempfile.mkdtemp(dir = progs.tempdir) cat.saveas('%s/base.cat' % tmpspace) subprocess.check_call("./prefilter_mlcc_bootstrap.sh %s %s %s %s %s %s" \ % (cluster, filter, image, tmpspace, '%s/base.cat' % tmpspace, '%s/prefiltered.cat' % tmpspace), shell=True) cat = ldac.openObjectFile('%s/prefiltered.cat' % tmpspace, 'STDTAB') cat.hdu.header.update('EXTNAME', 'OBJECTS') for i in range(startnum, startnum + nbootstraps): bootstrap = np.random.randint(0, len(cat), len(cat)) bootcat = cat.filter(bootstrap) bootcat.saveas('%s/bootstrap_%d.ml.cat' % (outdir, i), clobber=True) bootcat.hdu.header['EXTNAME'] = 'STDTAB' curcat = '%s/filter.cat' % tmpspace bootcat.saveas(curcat, clobber=True) with open(cutsfile) as cuts: for j, cut in enumerate(cuts.readlines()): cut = cut.strip() nextcat = '%s/filter_%d.cat' % (tmpspace, j) print "ldacfilter -i %s -o %s -t STDTAB -c '(%s);'" % (curcat, nextcat, cut) subprocess.check_call("ldacfilter -i %s -o %s -t STDTAB -c '(%s);'" % (curcat, nextcat, cut), shell=True) curcat = nextcat shutil.copyfile(curcat, '%s/bootstrap_%d.cc.cat' % (outdir, i)) for filterfile in glob.glob('%s/filter*' % tmpspace): os.remove(filterfile) finally: shutil.rmtree(tmpspace)
def readData(manager): options = manager.options manager.zcluster = options.zcluster manager.r500 = options.r500 manager.psfsize = options.psfsize manager.pixscale = options.pixscale if 'lensingcat' not in manager: manager.open('lensingcat', options.lensingcat, ldac.openObjectFile) r_arc, E, B = sp.calcTangentialShear(cat=manager.lensingcat, center=(options.centerx, options.centery), pixscale=options.pixscale, xcol=options.xcol, ycol=options.ycol, g1col=options.g1col, g2col=options.g2col) r_pix = r_arc / options.pixscale r_mpc = r_arc * (1. / 3600.) * (np.pi / 180.) * sp.angulardist( options.zcluster) size = manager.lensingcat[options.sizecol] / options.psfsize snratio = manager.lensingcat[options.snratio] manager.open('bpzcat', options.bpzfile, ldac.openObjectFile, 'STDTAB') manager.store('matched_bpzcat', manager.bpzcat.matchById, manager.lensingcat) z_b = manager.matched_bpzcat['BPZ_Z_B'] odds = manager.matched_bpzcat['BPZ_ODDS'] nfilt = manager.matched_bpzcat['NFILT'] galtype = manager.matched_bpzcat['BPZ_T_B'] cols = [ pyfits.Column(name='SeqNr', format='J', array=manager.lensingcat['SeqNr']), pyfits.Column(name='r_mpc', format='E', array=r_mpc), pyfits.Column(name='size', format='E', array=size), pyfits.Column(name='snratio', format='E', array=snratio), pyfits.Column(name='z_b', format='E', array=z_b), pyfits.Column(name='z_t', format='E', array=galtype), pyfits.Column(name='odds', format='E', array=odds), pyfits.Column(name='nfilt', format='I', array=nfilt), pyfits.Column(name='ghats', format='E', array=E), pyfits.Column(name='B', format='E', array=B) ] manager.store( 'inputcat', ldac.LDACCat(pyfits.BinTableHDU.from_columns( pyfits.ColDefs(cols)))) manager.open('pdzmanager', options.inputPDZ, pdzfile_utils.PDZManager.open) manager.store( 'pdzrange pdz'.split(), manager.pdzmanager.associatePDZ(manager.lensingcat['SeqNr'])) manager.replace('pdzmanager', None) manager.replace('matched_bpz', None) manager.replace('bpzcat', None)