def readData(manager):
options = manager.options
manager.zcluster = options.zcluster
manager.r500 = options.r500
manager.psfsize = options.psfsize
manager.pixscale = options.pixscale
if 'lensingcat' not in manager:
manager.open('lensingcat', options.lensingcat, ldac.openObjectFile)
r_arc, E, B = sp.calcTangentialShear(cat=manager.lensingcat,
center=(options.centerx, options.centery),
pixscale=options.pixscale,
xcol=options.xcol,
ycol=options.ycol,
g1col=options.g1col,
g2col=options.g2col)
r_pix = r_arc / options.pixscale
r_mpc = r_arc * (1. / 3600.) * (np.pi / 180. ) * sp.angulardist(options.zcluster)
size = manager.lensingcat[options.sizecol] / options.psfsize
snratio = manager.lensingcat[options.snratio]
manager.open('bpzcat', options.bpzfile, ldac.openObjectFile, 'STDTAB')
manager.store('matched_bpzcat', manager.bpzcat.matchById, manager.lensingcat)
z_b = manager.matched_bpzcat['BPZ_Z_B']
odds = manager.matched_bpzcat['BPZ_ODDS']
nfilt = manager.matched_bpzcat['NFILT']
galtype = manager.matched_bpzcat['BPZ_T_B']
cols = [pyfits.Column(name='SeqNr', format='J', array=manager.lensingcat['SeqNr']),
pyfits.Column(name='r_mpc', format='E', array=r_mpc),
pyfits.Column(name='size', format='E', array=size),
pyfits.Column(name='snratio', format='E', array=snratio),
pyfits.Column(name='z_b', format='E', array=z_b),
pyfits.Column(name='z_t', format='E', array=galtype),
pyfits.Column(name='odds', format='E', array=odds),
pyfits.Column(name='nfilt', format='I', array=nfilt),
pyfits.Column(name='ghats', format='E', array=E),
pyfits.Column(name='B', format='E', array=B)]
manager.store('inputcat', ldac.LDACCat(pyfits.BinTableHDU.from_columns(pyfits.ColDefs(cols))))
manager.open('pdzmanager', options.inputPDZ, pdzfile_utils.PDZManager.open)
manager.store('pdzrange pdz'.split(),
manager.pdzmanager.associatePDZ(manager.lensingcat['SeqNr']))
manager.replace('pdzmanager', None)
manager.replace('matched_bpz', None)
manager.replace('bpzcat', None)
def ML_NFW_Model(r_mpc, ghats, betas, pdz, dz, concentration, zcluster,
shape_sigma):
#######
# Data Prep
#######
r_mpc = np.ascontiguousarray(r_mpc)
pdz = np.ascontiguousarray(pdz)
D_lens = sp.angulardist(zcluster)
nzbins = len(betas)
ghat_matrix = np.ascontiguousarray(np.column_stack(nzbins * [ghats]))
ngals = len(ghats)
beta_matrix = np.ascontiguousarray(np.vstack(ngals * [betas]))
assert (ghat_matrix.shape == beta_matrix.shape == pdz.shape)
#######
# Model
#######
## r_scale is log-uniform
log_r_scale = pymc.Uniform('log_r_scale', np.log(.01), np.log(1.))
##contamination
contam_fraction = pymc.Uniform('contam_fraction', 0.0, 0.3)
contam_mu = pymc.Uniform('contam_mu', -1.0, 1.0)
contam_sig = pymc.Uniform('contam_sig', shape_sigma, 10 * shape_sigma)
#assuming dz is constant over range!!!
#assuming sigma is the same for all galaxies!
@pymc.stochastic(observed=True)
def data(value=ghats,
log_r_scale=log_r_scale,
contam_fraction=contam_fraction,
contam_mu=contam_mu,
contam_sig=contam_sig):
raise Exception(
"Out of date function; discovered & abandoned 2012-04-01; no joke")
return tools.likelihood(log_r_scale, r_mpc, ghat_matrix, beta_matrix,
pdz, concentration, zcluster, D_lens,
shape_sigma, dz, contam_fraction, contam_mu,
contam_sig)
########
return locals()
def beta_method(r_pix,
r_mpc,
ghat,
beta_s,
beta_s2,
concentration,
zcluster,
pixscale=__DEFAULT_PIXSCALE__,
fitrange=(0.3, 5.)):
D_lens = sp.angulardist(zcluster)
def NFWModel(r, r_scale):
nfw_shear = beta_s * tools.NFWShear(r, concentration, r_scale,
zcluster, D_lens)
nfw_kappa = beta_s * tools.NFWKappa(r, concentration, r_scale,
zcluster, D_lens)
g = (1 +
(beta_s2 / beta_s**2 - 1) * nfw_kappa) * nfw_shear / (1 -
nfw_kappa)
return g
minPix = fitrange[0] * 3600. * (180. / np.pi) / (pixscale * D_lens)
maxPix = fitrange[1] * 3600. * (180. / np.pi) / (pixscale * D_lens)
profile = sp.shearprofile(np.zeros_like(r_pix),
r_pix,
ghat,
np.zeros_like(ghat),
sigma2=None,
range=(minPix, maxPix),
bins=15,
center=(0, 0),
logbin=True)
profile_r_mpc = profile.r * pixscale * (1. / 3600.) * (np.pi /
180.) * D_lens
rs, isCon = leastsq.leastsq(NFWModel, [0.4],
profile_r_mpc,
profile.E,
profile.Eerr,
fullOutput=False)
if not isCon:
return None
return rs
##########################
inputCatFile = sys.argv[1]
inputPDZ = sys.argv[2]
shapedistro_module = sys.argv[3]
outputFile = sys.argv[4]
##########################
inputcat = ldac.openObjectFile(inputCatFile)
concentration = inputcat.hdu.header['CONCEN']
zcluster = inputcat.hdu.header['Z']
D_lens = sp.angulardist(zcluster)
pixscale = 0.2
minPix = 0.3 * 3600. * (180. / np.pi) / (pixscale * D_lens)
maxPix = 5. * 3600. * (180. / np.pi) / (pixscale * D_lens)
goodObjs = np.logical_and(
np.logical_and(inputcat['r_pix'] > minPix, inputcat['r_pix'] < maxPix),
np.logical_and(inputcat['z_b'] > 0, inputcat['z_b'] < 1.25))
shapedistro = __import__(shapedistro_module)
bin_selectors = [np.logical_and(goodObjs, selector) \
for selector in shapedistro.bin_selectors(inputcat)]
pdzfile = open(inputPDZ, 'rb')
pdzrange, pdz = cPickle.load(pdzfile)
def ML_NFW_Model(r_mpc, ghats, betas, pdz, concentration, zcluster,
likelihood_func, shapedistro_samples):
#r_mpc, ghats, pdz, and shapedistro_samples may be either arrays or lists of arrays
# if lists of arrays, then each entry is associated across the lists and will be passed
# to likelihood_func seperately, with the logProbs summed together
#######
# Data Prep
#######
if not isinstance(r_mpc, list):
r_mpc = [r_mpc]
ghats = [ghats]
pdz = [pdz]
shapedistro_samples = [shapedistro_samples]
r_mpc = [np.ascontiguousarray(x) for x in r_mpc]
pdz = [np.ascontiguousarray(x) for x in pdz]
shapedistro_samples = [
np.ascontiguousarray(x) for x in shapedistro_samples
]
D_lens = sp.angulardist(zcluster)
nzbins = len(betas)
ghats = [np.ascontiguousarray(x) for x in ghats]
betas = np.ascontiguousarray(betas)
nshapebins = len(shapedistro_samples)
#######
# Model
#######
## shape parameter priors
shape_sample_index = [
pymc.DiscreteUniform('shape_index_%d' % i, 0, len(x))
for i, x in enumerate(shapedistro_samples)
]
shape_params = np.empty(nshapebins, dtype=object)
for i, index, samples in zip(range(len(shapedistro_samples)),
shape_sample_index, shapedistro_samples):
@pymc.deterministic(name='shape_params_%d' % i)
def shape_param_func(index=index, samples=samples):
return samples[:, index]
shape_params[i] = shape_param_func
## r_scale is log-uniform
log_r_scale = pymc.Uniform('log_r_scale', np.log(.01), np.log(1.))
@pymc.stochastic(observed=True)
def data(value=ghats, log_r_scale=log_r_scale, shape_params=shape_params):
logprobs = np.array([likelihood_func(log_r_scale, cur_r_mpc,
cur_ghats, betas,
cur_pdz, cur_shapedistro_samples,
concentration, zcluster, D_lens) \
for (cur_r_mpc, cur_ghats, cur_pdz, cur_shapedistro_samples) \
in zip(r_mpc, ghats, pdz, shape_params)])
return np.sum(logprobs)
########
return locals()
def beta_contamination(x,
y,
g1,
g2,
mag,
useInContam,
areamap,
zdist,
zcluster,
norm_radii,
concentration_prior=4.,
density_prior=[0, 30],
contam_prior=[0, 10],
fitrange=(0.4, 4.),
bins=15,
logbin=False,
pixscale=__DEFAULT_PIXSCALE__,
center=__DEFAULT_CENTER__):
#######
# save config setting
#######
momento = {
'x': x,
'y,': y,
'g1': g1,
'g2': g2,
'mag': mag,
'useInContam': useInContam,
'areamap': areamap,
'zdist': zdist,
'zcluster': zcluster,
'norm_radii': norm_radii,
'concentration_prior': concentration_prior,
'density_prior': density_prior,
'contam_prior': contam_prior,
'fitrange': fitrange,
'bins': bins,
'logbin': logbin,
'pixscale': pixscale,
'center': center
}
#######
# Data Prep
#######
D_lens = sp.angulardist(zcluster)
cat = {'Xpos': x, 'Ypos': y, 'gs1': g1, 'gs2': g2}
r_arcsec, E, B = sp.calcTangentialShear(cat, center, pixscale)
r_mpc = r_arcsec * (1. / 3600.) * (np.pi / 180.) * D_lens
if logbin:
dr = log(dr)
rmin = log(rmin)
rmax = log(rmax)
leftedges, binwidth = linspace(rmin,
rmax,
bins + 1,
endpoint=True,
retstep=True)
rbin = exp(leftedges)
rbin = (rbin[1:] + rbin[:-1]) / 2.
else:
leftedges, binwidth = linspace(rmin,
rmax,
bins,
endpoint=False,
retstep=True)
rbin = leftedges + binwidth / 2.
# Set up binning grid:
Ebin = zeros(bins, dtype=float64)
Ebinerr = zeros(bins, dtype=float64)
Bbin = zeros(bins)
Bbinerr = zeros(bins)
nbin = zeros(bins)
index = ((dr - rmin) / binwidth).astype(int)
goodentry = logical_and(index >= 0, index < bins)
#######
# Model
#######
## r_scale is log-uniform
log_r_scale = pymc.Uniform('log_r_scale', np.log(.01), np.log(1.))
normgal_density = pymc.Uniform('n_normgals', density_prior[0],
density_prior[1])
contam_amp = pymc.Uniform('A_contam', contam_prior[0], contam_prior[1])
@pymc.stochastic(observed=True)
def shear(value=ghats,
log_r_scale=log_r_scale,
normgal_density=normgal_density,
contam_amp=contam_amp):
return tools.likelihood(log_r_scale, r_mpc, ghat_matrix, beta_matrix,
pdz, concentration, zcluster, D_lens,
shape_sigma)
@pymc.stochastic(observed=True)
def numcounts(value=ngals,
normgal_density=normgal_density,
contam_amp=contam_amp):
pass
########
return locals()
def addContamination(sourcebpz,
source_snratio,
source_size,
simcat,
simpdz,
r500,
zcluster,
f500=0.04,
pixscale=pixscale,
refcat='/u/ki/dapple/nfs12/cosmos/cosmos2.cat',
shape_distro=__DEFAULT_SHAPE_DISTRO__,
shape_distro_args=[],
shape_distro_kw={}):
refcat = ldac.openObjectFile(refcat).matchById(sourcebpz, selfid='id')
r_max = max(simcat['r_pix'])
area = np.pi * (r_max * pixscale / 60.)**2
n_back = float(len(simcat)) / area
x, y = nfwsim.stdcontamination((0, 0), pixscale, f500, n_back, r500,
zcluster)
r_pix = np.sqrt(x**2 + y**2)
r_mpc = r_pix * pixscale * (1. / 3600.) * (np.pi /
180.) * sp.angulardist(zcluster)
z = zcluster * np.ones(len(r_pix))
toKeepAvailable = {}
for id in sourcebpz['SeqNr']:
toKeepAvailable[id] = True
for id in simcat['z_id']:
toKeepAvailable[id] = False
zkey = 'zp_best'
if zkey not in sourcebpz:
zkey = 'BPZ_Z_S'
toKeep = np.logical_and(
np.logical_and(
np.array([toKeepAvailable[id] for id in sourcebpz['SeqNr']]),
refcat['mod_gal'] > 8),
np.logical_and(sourcebpz[zkey] > (zcluster - 0.05), sourcebpz[zkey] <
(zcluster + 0.05)))
#toKeep = np.logical_and(np.array([toKeepAvailable[id] for id in sourcebpz['SeqNr']]),
# np.logical_and(sourcebpz[zkey] > (zcluster - 0.05),
# sourcebpz[zkey] < (zcluster + 0.05)))
#
#
availablebpz = sourcebpz.filter(toKeep)
available_snratio = source_snratio[toKeep]
available_size = source_size[toKeep]
selected = np.random.randint(0, len(availablebpz), len(r_pix))
selectedbpz = availablebpz.filter(selected)
selected_snratio = available_snratio[selected]
selected_size = available_size[selected]
col_collection = {
'Seqnr':
-selectedbpz['SeqNr'],
'r_pix':
r_pix,
'r_mpc':
r_mpc,
'z':
z,
'z_id':
selectedbpz['SeqNr'],
'ghats':
shape_distro(np.zeros(len(selectedbpz)),
np.column_stack([selected_size, selected_snratio]),
*shape_distro_args, **shape_distro_kw),
'true_shear':
np.zeros_like(r_pix),
'true_z':
zcluster * np.ones_like(r_pix),
'true_beta':
np.zeros_like(r_pix),
'true_gamma':
np.zeros_like(r_pix),
'true_kappa':
np.zeros_like(r_pix)
}
cols = []
for name, arr in col_collection.iteritems():
if name == 'SeqNr':
col = pyfits.Column(name=name, format='J', array=arr)
else:
col = pyfits.Column(name=name, format='E', array=arr)
cols.append(col)
contamcat = ldac.LDACCat(
pyfits.BinTableHDU.from_columns(pyfits.ColDefs(cols)))
finalcat = simcat.append(contamcat)
return finalcat
def createCatalog(bpz,
bpz_sizes,
bpz_snratios,
concentration,
scale_radius,
zcluster,
ngals,
shape_distro=__DEFAULT_SHAPE_DISTRO__,
shape_distro_args=[],
shape_distro_kw={},
maxpix=5000,
radii_pix=None,
contam=None,
contam_args=[],
contam_kw={},
idcol='ID'):
# ngals == None -> no bootstrapping
# bpz is ldac bpz output
momento = {
'concentration': concentration,
'scale_radius': scale_radius,
'zcluster': zcluster,
'ngals': ngals,
'shape_distro': shape_distro.func_name,
'shape_distro_args': shape_distro_args,
'shape_distro_kw': shape_distro_kw,
'maxpix': maxpix,
'radii_pix': radii_pix
}
bootstrap = True
if ngals == None:
bootstrap = False
ngals = len(bpz)
if radii_pix is None:
x_pix = np.random.uniform(0, maxpix, ngals)
y_pix = np.random.uniform(0, maxpix, ngals)
radii_pix = np.sqrt(x_pix**2 + y_pix**2)
radii_mpc = radii_pix * pixscale * (1. / 3600.) * (
np.pi / 180.) * sp.angulardist(zcluster)
chosenZs = bpz
chosenSizes = bpz_sizes
chosenSNratios = bpz_snratios
if bootstrap:
indices = np.random.randint(0, len(bpz), ngals)
chosenZs = bpz.filter(indices)
chosenSizes = bpz_sizes[indices]
chosenSNratios = bpz_snratios[indices]
print "adam-look: running createCatalog in cosmos_sim.py"
z_id = chosenZs[idcol]
zbins = numpy.arange(0, 6.01, .01)
z_drawn = -1 * np.ones(len(chosenZs))
print '!!!', len(z_drawn)
for id_indx, id in enumerate(z_id):
try:
cdf = id2pz_cdf[id]
except:
if id in gone_ids:
print "adam-look: gone id ", id
continue
else:
raise
x = numpy.random.rand()
try:
zval = (zbins[cdf <= x])[-1]
except:
if id in bad_ids:
print "adam-look: bad id ", id
continue
else:
raise
z_drawn[id_indx] = zval
good_draws = z_drawn > -1
z_drawn = z_drawn[good_draws]
radii_pix = radii_pix[good_draws]
radii_mpc = radii_mpc[good_draws]
chosenZs = chosenZs.filter(good_draws)
chosenSizes = chosenSizes[good_draws]
chosenSNratios = chosenSNratios[good_draws]
ngals = len(z_drawn)
true_shears, true_gamma, true_kappa = nfwsim.create_nfwmodel_shapedata(
concentration, scale_radius, zcluster, ngals, z_drawn, radii_mpc)
true_beta = nfwutils.beta_s(z_drawn, zcluster)
ghats = shape_distro(true_shears,
np.column_stack([chosenSizes, chosenSNratios]),
*shape_distro_args, **shape_distro_kw)
cols = [
pyfits.Column(name='Seqnr', format='J', array=np.arange(ngals)),
pyfits.Column(name='r_pix', format='E', array=radii_pix),
pyfits.Column(name='r_mpc', format='E', array=radii_mpc),
pyfits.Column(name='z', format='E', array=z_drawn),
pyfits.Column(name='z_id', format='J', array=chosenZs[idcol]),
pyfits.Column(name='ghats', format='E', array=ghats),
pyfits.Column(name='true_shear', format='E', array=true_shears),
pyfits.Column(name='true_z', format='E', array=z_drawn),
pyfits.Column(name='true_beta', format='E', array=true_beta),
pyfits.Column(name='true_gamma', format='E', array=true_gamma),
pyfits.Column(name='true_kappa', format='E', array=true_kappa)
]
simcat = pyfits.new_table(pyfits.ColDefs(cols))
simcat.header.update('EXTNAME', 'OBJECTS')
simcat.header.update('concen', concentration)
simcat.header.update('r_s', scale_radius)
simcat.header.update('z', zcluster)
return ldac.LDACCat(simcat), momento
def createBootstrapCats(cluster, filter, image, outdir, nbootstraps = 100, startnum = 0):
clusterdir = '%s/%s' % (subarudir, cluster)
lensingdir = '%s/LENSING_%s_%s_aper/%s' % (clusterdir, filter, filter, image)
lensingcat = '%s/coadd_photo2.cat' % lensingdir
cutsfile = '%s/cc_cuts3.dat' % lensingdir
outdir = '%s/%s' % (outdir, cluster)
if not os.path.exists(outdir):
os.mkdir(outdir)
cat = ldac.openObjectFile(lensingcat)
zcluster = msf.parseZCluster(cluster)
center = msf.readClusterCenters(cluster)
r_arc, E, B = sp.calcTangentialShear(cat = cat,
center = center,
pixscale = 0.2)
r_mpc = r_arc * (1./3600.) * (np.pi / 180. ) * sp.angulardist(zcluster)
cat = cat.filter(np.logical_and(r_mpc > 0.750, r_mpc < 3.0))
cat.hdu.header.update('EXTNAME','STDTAB')
try:
tmpspace = tempfile.mkdtemp(dir = progs.tempdir)
cat.saveas('%s/base.cat' % tmpspace)
subprocess.check_call("./prefilter_mlcc_bootstrap.sh %s %s %s %s %s %s" \
% (cluster, filter, image,
tmpspace, '%s/base.cat' % tmpspace, '%s/prefiltered.cat' % tmpspace), shell=True)
cat = ldac.openObjectFile('%s/prefiltered.cat' % tmpspace, 'STDTAB')
cat.hdu.header.update('EXTNAME', 'OBJECTS')
for i in range(startnum, startnum + nbootstraps):
bootstrap = np.random.randint(0, len(cat), len(cat))
bootcat = cat.filter(bootstrap)
bootcat.saveas('%s/bootstrap_%d.ml.cat' % (outdir, i), clobber=True)
bootcat.hdu.header['EXTNAME'] = 'STDTAB'
curcat = '%s/filter.cat' % tmpspace
bootcat.saveas(curcat, clobber=True)
with open(cutsfile) as cuts:
for j, cut in enumerate(cuts.readlines()):
cut = cut.strip()
nextcat = '%s/filter_%d.cat' % (tmpspace, j)
print "ldacfilter -i %s -o %s -t STDTAB -c '(%s);'" % (curcat, nextcat, cut)
subprocess.check_call("ldacfilter -i %s -o %s -t STDTAB -c '(%s);'" % (curcat, nextcat, cut), shell=True)
curcat = nextcat
shutil.copyfile(curcat, '%s/bootstrap_%d.cc.cat' % (outdir, i))
for filterfile in glob.glob('%s/filter*' % tmpspace):
os.remove(filterfile)
finally:
shutil.rmtree(tmpspace)
def readData(manager):
options = manager.options
manager.zcluster = options.zcluster
manager.r500 = options.r500
manager.psfsize = options.psfsize
manager.pixscale = options.pixscale
if 'lensingcat' not in manager:
manager.open('lensingcat', options.lensingcat, ldac.openObjectFile)
r_arc, E, B = sp.calcTangentialShear(cat=manager.lensingcat,
center=(options.centerx,
options.centery),
pixscale=options.pixscale,
xcol=options.xcol,
ycol=options.ycol,
g1col=options.g1col,
g2col=options.g2col)
r_pix = r_arc / options.pixscale
r_mpc = r_arc * (1. / 3600.) * (np.pi / 180.) * sp.angulardist(
options.zcluster)
size = manager.lensingcat[options.sizecol] / options.psfsize
snratio = manager.lensingcat[options.snratio]
manager.open('bpzcat', options.bpzfile, ldac.openObjectFile, 'STDTAB')
manager.store('matched_bpzcat', manager.bpzcat.matchById,
manager.lensingcat)
z_b = manager.matched_bpzcat['BPZ_Z_B']
odds = manager.matched_bpzcat['BPZ_ODDS']
nfilt = manager.matched_bpzcat['NFILT']
galtype = manager.matched_bpzcat['BPZ_T_B']
cols = [
pyfits.Column(name='SeqNr',
format='J',
array=manager.lensingcat['SeqNr']),
pyfits.Column(name='r_mpc', format='E', array=r_mpc),
pyfits.Column(name='size', format='E', array=size),
pyfits.Column(name='snratio', format='E', array=snratio),
pyfits.Column(name='z_b', format='E', array=z_b),
pyfits.Column(name='z_t', format='E', array=galtype),
pyfits.Column(name='odds', format='E', array=odds),
pyfits.Column(name='nfilt', format='I', array=nfilt),
pyfits.Column(name='ghats', format='E', array=E),
pyfits.Column(name='B', format='E', array=B)
]
manager.store(
'inputcat',
ldac.LDACCat(pyfits.BinTableHDU.from_columns(
pyfits.ColDefs(cols))))
manager.open('pdzmanager', options.inputPDZ,
pdzfile_utils.PDZManager.open)
manager.store(
'pdzrange pdz'.split(),
manager.pdzmanager.associatePDZ(manager.lensingcat['SeqNr']))
manager.replace('pdzmanager', None)
manager.replace('matched_bpz', None)
manager.replace('bpzcat', None)