def clusterplot(data=None):
if data is None:
data = {}
if 'items' not in data:
data['items'] = readtxtfile('worklist')
items = data['items']
if 'clusters' not in data:
clusters = {}
for cluster, filter, image in items:
matchedfile = '/u/ki/dapple/subaru/%s/PHOTOMETRY_%s_aper/%s.matched.tab' % (
cluster, filter, cluster)
if not os.path.exists(matchedfile):
continue
bpz = ldac.openObjectFile(
'/u/ki/dapple/ki06/catalog_backup_2012-05-17/%s.%s.bpz.tab' %
(cluster, filter), 'STDTAB')
zcat = ldac.openObjectFile(matchedfile, 'STDTAB')
bpz = bpz.matchById(zcat, 'SeqNr_data')
bpz = bpz.filter(bpz['BPZ_ODDS'] > 0.5)
zcat = zcat.matchById(bpz, 'SeqNr', 'SeqNr_data')
clusters[cluster] = np.array([bpz['BPZ_Z_B'], zcat['z_spec']])
data['clusters'] = clusters
clusters = data['clusters']
if 'redshifts' not in data:
data['redshifts'] = np.column_stack([x for x in clusters.itervalues()])
redshifts = data['redshifts']
fig = makeHexbin(redshifts[1, :],
redshifts[0, :],
xlabel='Cluster Spectroscopic $z$',
ylabel='5 Filter Photo-$z$',
gridsize=45,
bins='log')
fig.axes[0].text(1.1, 0.05, '\emph{Log Color Scale}')
fig.savefig('publication/clustercomp.pdf')
return fig, data
def readCCSummary(outdir, clusters):
masses = {}
for cluster in clusters:
masses[cluster] = np.hstack(
readtxtfile('%s/%s.anja.mass.dat' % (outdir, cluster)))
print cluster, len(masses[cluster])
return masses, np.array([masses[x] for x in clusters])
def convertTxt2LDAC(file, keys, type):
rawfile = readtxtfile(file)
cols = []
for i, key, type in zip(xrange(len(keys)), keys, type):
cols.append(pyfits.Column(name=key, format=type, array=rawfile[:, i]))
hdu = pyfits.new_table(pyfits.ColDefs(cols))
hdu.header.update('EXTNAME', 'OBJECTS')
return ldac.LDACCat(hdu)
def publicationMLPullScript(data=None):
items = readtxtfile('worklist')
clusters = [x[0] for x in items]
if data is None:
MLbootmasses, MLmask = readMLBootstraps(
'/u/ki/dapple/ki06/bootstrap_2011-12-14', items)
MLdat = readDougMasses(
'/u/ki/dapple/subaru/doug/publication/baseline_2011-12-14')
pulls = np.hstack([
makePull(MLdat[tuple(x)][0], MLdat[tuple(x)][1],
MLbootmasses[x[0]], MLmask[x[0]]) for x in items
])
data = pulls
else:
pulls = data
fig = pylab.figure()
ax = fig.add_axes([0.14, 0.14, 0.95 - 0.14, 0.95 - 0.14])
ax.hist(pulls, bins=50, normed=True)
ax.axvline(0.0, c='k', linewidth=2)
gaussx = np.arange(-6, 6, 0.0002)
gauss = np.exp(-0.5 * gaussx**2) / np.sqrt(2 * np.pi)
ax.plot(gaussx, gauss, c='r', marker='None', linestyle='--', linewidth=1.5)
pullMean = np.mean(pulls)
pullStd = np.std(pulls)
ax.text(-5.5, 0.4, 'P(z)', fontsize=16)
ax.text(2.5, 0.4, '$\mu = %1.3f$' % pullMean, fontsize=14)
ax.text(2.5, 0.38, '$\sigma = %1.3f$' % pullStd, fontsize=14)
ax.set_xlabel('Stacked $\Delta/\sigma$ for %d Clusters' % len(clusters))
ax.set_ylabel('Probability ($\Delta/\sigma$)')
fig.savefig('publication/mlmass_pull.eps')
return fig, data
def galdensity(data = None):
if data is None:
data = {}
if 'ngals' not in data:
ngals = cPickle.load(open('galaxy_counts_pzmethod.pkl', 'rb'))
data['ngals'] = ngals
items = [tuple(x) for x in readtxtfile('worklist')]
clusters = [x[0] for x in items]
redshifts = cm.readClusterRedshifts()
properz = np.array([redshifts[x] for x in clusters])
data['properz'] = properz
Dl = np.array([nfwutils.angulardist(z) for z in properz])
data['Dl'] = Dl
inner_rad = np.arctan2(0.75, Dl) * (180./np.pi) * 60
outer_rad = np.arctan2(3., Dl) * (180 / np.pi) * 60
area = np.pi*(outer_rad**2 - inner_rad**2)
data['area'] = area
propercounts = np.array([ngals[x] for x in items])
density = propercounts / area
data['density'] = density
else:
properz = data['properz']
density = data['density']
fig = pylab.figure()
ax = fig.add_axes([0.12, 0.12, 0.95 - 0.12, 0.95 - 0.12])
ax.plot(properz, density, 'bo')
ax.set_xlabel('Cluster Redshift')
ax.set_ylabel('Input Galaxy Density')
return fig, data
def CCSimBiasPlotScript(data=None):
if data is None:
data = readtxtfile(
'/nfs/slac/g/ki/ki06/anja/SUBARU/cosmos_cats/simulations/publication/highsn/cluster2/cc_masses/masses_concat_anja.dat'
)
# ${cluster} ${zcluster} ${truemass} ${meanmass} ${emean} ${beta} ${beta2} ${nshells} ${nobjects}
clusters = [x[0] for x in data]
redshifts = np.array([float(x[1]) for x in data])
truemass = np.array([float(x[2]) for x in data])
meanmass = np.array([float(x[3]) for x in data])
errmass = np.array([float(x[4]) for x in data])
fig = pylab.figure()
ax = fig.add_axes([0.14, 0.14, 0.95 - 0.14, 0.95 - 0.14])
fracbias = (meanmass - truemass) / truemass
fracerr = errmass / truemass
ax.axhline(0.0, c='k', linewidth=1.5)
mean = 0.007
err = 0.0022
fillx = np.array([0.16, 0.72])
ax.fill_between(fillx, mean - err, mean + err, facecolor='r', alpha=0.1)
ax.axhline(mean, c='k', linestyle='--', alpha=0.5)
ax.errorbar(redshifts, fracbias, fracerr, fmt='bo')
ax.set_xlim(0.16, 0.72)
ax.set_ylim(-0.08, 0.08)
ax.set_xlabel('Cluster Redshift')
ax.set_ylabel(r'Fractional Mass Bias within 1.5 Mpc')
ax.text(0.2, 0.06, r'Color Cuts', fontsize=16)
fig.savefig('publication/clustersims_cc_compare.eps')
return fig, data
def ML5v6filterCompareScript(data=None):
if data is None:
worklist = readtxtfile('simclusterlist')
data = load5v6simdata(worklist)
Bfracbias = data[0]
Afracbias = data[1]
properredshifts = data[2]
fig = pylab.figure()
ax = fig.add_axes([0.14, 0.12, 0.95 - 0.14, 0.95 - 0.12])
Bmean, Berr = ss.bootstrapMean(Bfracbias)
Amean, Aerr = ss.bootstrapMean(Afracbias)
ax.errorbar(properredshifts - 0.0025,
Bmean,
Berr,
fmt='bo',
label=r'$BVr^+i^+z^+$')
ax.errorbar(properredshifts + 0.0025,
Amean,
Aerr,
fmt='rs',
label=r'$uBVr^+i^+z^+$')
ax.axhline(0.0, c='k')
ax.set_xlim(0.16, 0.72)
ax.set_ylim(-0.15, 0.08)
ax.set_xlabel('Cluster Redshift')
ax.set_ylabel(r'Fractional Mass Bias within 1.5 Mpc')
ax.legend(loc='lower left', numpoints=1, ncol=1)
ax.text(0.2, 0.06, r'10\% Contamination', fontsize=16)
fig.savefig('publication/clustersims_bvriz_aper_compare.eps')
return fig, data
def prepData(cluster, filter):
cat = ldac.openObjectFile('%s/%s/LENSING/%s_fbg.filtered.cat' %
(subarudir, cluster, cluster))
cat = cat.filter(
numpy.logical_and(
numpy.logical_and(cat['BPZ_Z_B_MAX-MIN_Z'] < 0.8,
cat['BPZ_ODDS'] > 0.95),
cat['MAG_AUTO'] < 23.85))
# cat = cat.filter(numpy.logical_and(cat['BPZ_Z_B_MAX-MIN_Z'] < 0.8,
# cat['BPZ_ODDS'] > 0.95))
coords = numpy.column_stack([cat['Xpos'], cat['Ypos']])
dX = coords - numpy.array([5000, 5000])
dR = numpy.sqrt(dX[:, 0]**2 + dX[:, 1]**2)
area = du.readtxtfile('%s/%s/%s/SCIENCE/coadd_%s_good/area.dat' %
(subarudir, cluster, filter, cluster))
return dR, area, cat['Z_BEST'], cat['BPZ_Z_B_MIN'], cat[
'BPZ_Z_B_MAX'], cat['MAG_AUTO']
def ML5filterContamCompareScript(data=None):
if data is None:
data = {}
if 'worklist' not in data:
data['worklist'] = readtxtfile('simclusterlist')
data['clusters'] = [x[0] for x in data['worklist']]
worklist = data['worklist']
clusters = data['clusters']
if 'redshifts' not in data:
data['redshifts'] = cm.readClusterRedshifts()
redshifts = data['redshifts']
data['properredshifts'] = np.array([redshifts[x] for x in clusters])
redshifts = data['redshifts']
properredshifts = data['properredshifts']
if 'fracbiases' not in data:
subdirs = ['%sBVRIZ' % x for x in ['', 'contam0p10/', 'contam0p20/']]
data['fracbiases'] = ss.processFracBiasData(
'/u/ki/dapple/nfs12/cosmos/simulations/clusters_2012-05-17-highdensity',
subdirs, clusters, redshifts)
fracbiases = data['fracbiases']
fig = publicationContamComparePlot(fracbiases[0], fracbiases[1],
fracbiases[2], properredshifts)
ax = fig.axes[0]
ax.text(0.2, 0.06, r'$B_{\mathrm J}V_{\mathrm J}r^+i^+z^+$', fontsize=16)
fig.savefig('publication/clustersims_bvriz_contam_compare.eps')
return fig, data
def applyFilter(filterfile):
filter = readtxtfile(filterfile)[:, :2]
step = filter[1, 0] - filter[0, 0]
#convert photon response filters to flux response filters
filterSpline = interp.interp1d(filter[:, 0],
filter[:, 1],
bounds_error=False,
fill_value=0.)
spec_mags = []
for spec in spectra:
specStep = spec[1, 0] - spec[0, 0]
resampFilter = filterSpline(spec[:, 0])
logEff = log10(sum(specStep * resampFilter * spec[:, 0] * spec[:, 1]))
logNorm = log10(
sum(resampFilter * c * specStep /
spec[:, 0])) #pivot wavelength, with cancelation of norm
mag = 2.5 * (logNorm - logEff)
spec_mags.append(mag)
return spec_mags
def calcBootstrapCovar(bootstrap):
sourcedir = '/u/ki/dapple/ki06/catalog_backup_2012-02-08'
bootdir = '/u/ki/dapple/ki06/bootstrap_2012-02-08'
data = {}
if 'items' not in data:
data['items'] = readtxtfile('worklist')
items = data['items']
if 'zbins' not in data:
data['zbins'] = np.unique(np.hstack([np.linspace(0., 1., 3.), np.linspace(1., 5., 10.), np.linspace(5., 10., 5.)]))
zbins = data['zbins']
bincenters = (zbins[1:] + zbins[:-1])/2.
if 'clusters' not in data:
clusters = {}
for cluster, filter, image in items:
key = (cluster, filter, image)
clusters[key] = {}
clusterdir='%s/%s' % (bootdir, cluster)
i = bootstrap
controller = driver.makeController()
options, args = controller.modelbuilder.createOptions()
options, args = controller.filehandler.createOptions(options = options, args = args,
workdir = sourcedir,
incatalog = '%s/bootstrap_%d.ml.cat' % (clusterdir, i),
cluster = cluster, filter = filter, image = image)
controller.load(options, args)
stats = cPickle.load(open('%s/bootstrap_%d.ml.out.mass15mpc.mass.summary.pkl' % (clusterdir, i)))
mass = stats['quantiles'][50]
rs = nfwutils.RsMassInsideR(mass, 4.0, controller.zcluster, 1.5)
scaledZ, estimators = sr.scaleShear(controller, rs, 4.0)
bins, weights, aveshear = sr.calcZBinWeights(scaledZ, controller.pdz, estimators, zbins)
clusters[key]['weights'] = weights
clusters[key]['shears'] = aveshear
data['clusters'] = clusters
else:
clusters = data['clusters']
if 'medians' not in data:
clusterboot = np.random.randint(0, len(items), len(items))
allweights = np.vstack([clusters[tuple(items[j])]['weights'] for j in clusterboot])
allshears = np.vstack([clusters[tuple(items[j])]['shears'] for j in clusterboot])
median, sig1, sig2 = sr.calcBinDistro(zbins, allweights, allshears)
data['median'] = median
data['sig1'] = sig1
data['sig2'] = sig2
return data
def MLUbandRatioScript(data=None):
simlist = readtxtfile('simclusterlist')
noUlist = readtxtfile('noUlist')
worklist = [x for x in simlist if x in noUlist]
del worklist[-1]
clusters = [x[0] for x in worklist]
if data is None:
redshifts = cm.readClusterRedshifts()
properredshifts = np.array([redshifts[x] for x in clusters])
noUsim_masses, noUsim_errs, noUsim_grid, scale_radii = ss.readMLMasses(
'/u/ki/dapple/nfs12/cosmos/simulations/publication/highsn/cluster3',
'contam0p10/BVRIZ', clusters)
Usim_masses, Usim_errs, Usim_grid, scale_radii = ss.readMLMasses(
'/u/ki/dapple/nfs12/cosmos/simulations/publication/highsn/cluster3',
'contam0p10/APER', clusters)
noUmasses, noUmask = cm.readMLBootstraps(
'/u/ki/dapple/ki06/bootstrap_2011-12-14', worklist, np.arange(100))
Umasses, Umask = cm.readMLBootstraps(
'/u/ki/dapple/ki06/bootstrap_U_2012-02-03', worklist,
np.arange(100))
data = [
noUsim_grid, Usim_grid, properredshifts, noUmasses, noUmask,
Umasses, Umask
]
else:
noUsim_grid = data[0]
Usim_grid = data[1]
properredshifts = data[2]
noUmasses = data[3]
noUmask = data[4]
Umasses = data[5]
Umask = data[6]
simRatio = noUsim_grid / Usim_grid
simMean, simErr = ss.bootstrapMean(simRatio.T)
dataRatios = cm.makeRatios(Umasses, Umask, noUmasses, noUmask)
dataMeans, dataErrs = cm.bootstrapMeans(dataRatios, clusters)
fig = pylab.figure()
ax = fig.add_axes([0.14, 0.12, 0.95 - 0.14, 0.95 - 0.12])
ax.errorbar(properredshifts + 0.0025,
dataMeans,
dataErrs,
fmt='rs',
label='Data')
ax.errorbar(properredshifts - 0.0025,
simMean,
simErr,
fmt='bo',
label='Simulations')
ax.axhline(1.0, c='k', linewidth=1.5)
ax.set_xlim(0.16, 0.72)
ax.set_ylim(0.85, 1.35)
ax.set_xlabel('Cluster Redshift')
ax.set_ylabel('Mass without U-band / Mass with U-band')
ax.legend(loc='upper left', numpoints=1)
fig.savefig('publication/dropU_comp.eps')
return fig, data
def publicationMLCCBootCompScript(worklist='worklist',
cosmology=None,
data=None):
items = readtxtfile(worklist)
clusters = [x[0] for x in items]
redshifts = readClusterRedshifts()
properredshifts = np.array([redshifts[x] for x in clusters])
if data is None:
MLmasses, MLmask = readMLBootstraps(
'/u/ki/dapple/ki06/bootstrap_2012-05-17', items, np.arange(0, 200))
CCmasses, CCmask = readCCSummary(
'/u/ki/dapple/ki06/bootstrap_2012-05-17', clusters,
np.arange(00, 200))
ratios = makeRatios(MLmasses, MLmask, CCmasses, CCmask)
medians, errs = bootstrapMedians(ratios, clusters)
# fit_ratio = np.hstack([pymc.database.pickle.load('ml_cc_rlog.out.%d' % i).trace('m', -1)[25000:] for i in range(1,6)])
data = [medians, errs, properredshifts]
else:
medians = data[0]
errs = data[1]
properredshifts = data[2]
# fit_ratio = data[3]
if cosmology is not None:
cosmoitems = readtxtfile(cosmology)
cosmoclusters = [x[0] for x in cosmoitems]
cosmomask = []
for x in clusters:
if x in cosmoclusters:
cosmomask.append(1)
else:
cosmomask.append(0)
cosmomask = np.array(cosmomask) == 1
print cosmomask
fig = pylab.figure()
try:
ax = fig.add_axes([0.12, 0.12, 0.95 - 0.12, 0.95 - 0.12])
ax.axhline(1.0, c='k', linewidth=1.25)
# ratio, ratioerr = sp.ConfidenceRegion(fit_ratio)
ratio = 0.998
ratioerr = [0.042, 0.044] #m, p
ratioerr2 = [0.081, 0.085] #m,p
fillx = [0.16, 0.72]
ax.fill_between(fillx,
ratio - ratioerr2[0],
ratio + ratioerr2[1],
facecolor=(1, .753, 0))
ax.fill_between(fillx,
ratio - ratioerr[0],
ratio + ratioerr[1],
facecolor=(1, 0.4, 0))
ax.axhline(ratio, c='k', linestyle='--')
if cosmology:
ratio = 0.965
ratioerr = [0.05, 0.06] #m, p
fillx = [0.17, 0.46]
ax.axhline(ratio, c='k', linestyle=':')
ax.fill_between(fillx,
ratio - ratioerr[0],
ratio + ratioerr[1],
facecolor='c',
alpha=0.3)
ax.plot(properredshifts[cosmomask],
medians[cosmomask],
'ko',
markersize=10,
markerfacecolor='None',
markeredgecolor='k')
ax.errorbar(properredshifts, medians, errs, fmt='ko')
ax.set_yscale('log')
ax.set_xlim(0.16, 0.72)
ax.set_ylim(0.45, 3.5)
ax.set_yticks([0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 2.0, 3.0])
ax.set_yticklabels(
['%2.1f' % x for x in [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 2.0, 3.0]])
ax.set_yticks([1.25, 1.5, 1.75, 2.25, 2.5, 2.75], minor=True)
ax.set_xlabel('Cluster Redshift')
ax.set_ylabel('CC Mass / P(z) Mass')
fig.savefig('publication/%s.eps' % worklist)
finally:
return fig, data
def measureMLCcBootIS(sample='worklist', data=None, diffR=None):
if data is None:
data = {}
if 'items' not in data:
items = readtxtfile(sample)
clusters = [x[0] for x in items]
redshifts = readClusterRedshifts()
properredshifts = np.array([redshifts[x] for x in clusters])
data['items'] = items
data['clusters'] = clusters
data['redshifts'] = redshifts
data['properredshifts'] = properredshifts
else:
items = data['items']
clusters = data['clusters']
redshifts = data['redshifts']
properredshifts = data['properredshifts']
if 'MLmasses' not in data or 'CCmasses' not in data:
if diffR is None:
if 'MLmasses' not in data:
MLmasses, MLmask = readMLBootstraps(
'/u/ki/dapple/ki06/bootstrap_2012-05-17', items,
np.arange(0, 200))
data['MLmasses'] = MLmasses
data['MLmask'] = MLmask
else:
MLmasses, MLmask = data['MLmasses'], data['MLmask']
if 'CCmasses' not in data:
CCmasses, CCmask = readCCSummary(
'/u/ki/dapple/ki06/bootstrap_2012-05-17', clusters,
np.arange(0, 200))
data['CCmasses'] = CCmasses
data['CCmask'] = CCmask
else:
CCmasses, CCmask = data['CCmasses'], data['CCmask']
else:
print 'Using Alternative diffR'
if 'MLmasses' not in data:
MLmasses, MLmask = readMLBootstraps_diffR(
'/u/ki/dapple/ki06/bootstrap_2012-05-17', items,
np.arange(0, 200), diffR, redshifts)
data['MLmasses'] = MLmasses
data['MLmask'] = MLmask
else:
MLmasses, MLmask = data['MLmasses'], data['MLmask']
if 'CCmasses' not in data:
CCmasses, CCmask = readCCSummary_diffR(
'/u/ki/dapple/ki06/bootstrap_2012-05-17', clusters,
np.arange(0, 200), diffR, redshifts)
data['CCmasses'] = CCmasses
data['CCmask'] = CCmask
else:
CCmasses, CCmask = data['CCmasses'], data['CCmask']
MLreduced = {}
CCreduced = {}
for key in MLmasses.keys():
totalmask = np.logical_and(MLmask[key], CCmask[key])
MLreduced[key] = MLmasses[key][totalmask]
CCreduced[key] = CCmasses[key][totalmask]
data['MLreduced'] = MLreduced
data['CCreduced'] = CCreduced
else:
MLreduced, CCreduced = data['MLreduced'], data['CCreduced']
grid, means, scatters = isg.intrinsicScatter(
MLreduced,
CCreduced,
means=np.arange(0.5, 1.5, 0.002),
scatters=np.arange(0.02, 0.2, 0.005))
data['grid'] = grid
data['means'] = means
data['scatters'] = scatters
figs = []
print
print
print '-----'
print 'var\tmode\t68%% +\t-\t95%% +\t-'
print '-----'
if 'meandist' not in data:
means = data['means']
scatters = data['scatters']
mode, (r68, r95) = isg.getdist_1d_hist(means[0],
means[1],
levels=[0.68, 0.95])
data['meandist'] = (mode, r68, r95)
mode, (r68, r95) = isg.getdist_1d_hist(scatters[0],
scatters[1],
levels=[0.68, 0.95])
data['scatterdist'] = (mode, r68, r95)
for varname in 'mean scatter'.split():
mode, r68, r95 = data['%sdist' % varname]
print mode, r68, r95
print '%s\t%2.4f\t+%2.4f\t-%2.4f\t+%2.4f\t-%2.4f' % (
varname, mode, r68[0][1] - mode, mode - r68[0][0],
r95[0][1] - mode, mode - r95[0][0])
x, prob = data['%ss' % varname]
fig = isgp.plotdist_1d_hist(x, prob, mode, [r68[0], r95[0]])
ax = fig.axes[0]
ax.set_title(varname)
figs.append(fig)
fig.show()
return figs, data
#!/usr/bin/env python
import ldac, dappleutils, sys, os, pyfits
filteredIDs = dappleutils.readtxtfile(sys.argv[1])
filteredIDcat = ldac.LDACCat(
pyfits.new_table(
pyfits.ColDefs(
[pyfits.Column(name='SeqNr', format='J', array=filteredIDs[:,
0])])))
fullCat = ldac.openObjectFile(sys.argv[2])
filteredCat = dappleutils.matchById(filteredIDcat, fullCat)
base, ext = os.path.splitext(sys.argv[2])
outputname = '%s.filtered%s' % (base, ext)
filteredCat.saveas(outputname, clobber=True)
#!/usr/bin/env python
import sys, glob, pyfits, os.path
from numpy import *
import scipy.interpolate.interpolate as interp
from dappleutils import readtxtfile
from optparse import OptionParser
c = 2.99792458e18 #Angstroms/s
spectrafiles = glob.glob('pickles/*.dat')
spectra = [readtxtfile(s)[:, :2] for s in spectrafiles]
nspectra = len(spectra)
def applyFilter(filterfile):
filter = readtxtfile(filterfile)[:, :2]
step = filter[1, 0] - filter[0, 0]
#convert photon response filters to flux response filters
filterSpline = interp.interp1d(filter[:, 0],
filter[:, 1],
bounds_error=False,
fill_value=0.)
spec_mags = []
for spec in spectra:
specStep = spec[1, 0] - spec[0, 0]
resampFilter = filterSpline(spec[:, 0])
def MLPointEstScript(data=None):
if data is None:
worklist = readtxtfile('simclusterlist')
clusters = [x[0] for x in worklist]
redshifts = cm.readClusterRedshifts()
properredshifts = np.array([redshifts[x] for x in clusters])
subdirs = ['contam0p10/BVRIZ']
# MLfracbias = ss.processFracBiasData('/u/ki/dapple/nfs12/cosmos/simulations/publication/highsn/cluster3',
# subdirs, clusters, redshifts)[0]
#
#
# Apointmass, Apointgrid, scale_radii = ss.readPointMasses('/u/ki/dapple/nfs12/cosmos/simulations/publication/highsn/cluster3', 'contam0p10/newman/APER', clusters)
Bpointmass, Bpointgrid, scale_radii = ss.readPointMasses(
'/u/ki/dapple/nfs12/cosmos/simulations/clusters_2012-05-17',
'contam0p10/newman/BVRIZ', clusters)
truemasses = [
nfwutils.massInsideR(scale_radii[x], 4., redshifts[x], 1.5)
for x in clusters
]
# Apointfracbias = ss.calcFracBias(Apointgrid, truemasses)
Bpointfracbias = ss.calcFracBias(Bpointgrid, truemasses)
data = [None, Bpointfracbias, properredshifts]
else:
# Apointfracbias = data[0]
Bpointfracbias = data[1]
properredshifts = data[2]
fig = pylab.figure()
try:
ax = fig.add_axes([0.15, 0.12, 0.95 - 0.15, 0.95 - 0.12])
ax.axhline(0.0, c='k', linewidth=1.25)
# Apointmean, Apointerr = ss.bootstrapMean(Apointfracbias)
Bpointmean, Bpointerr = ss.bootstrapMean(Bpointfracbias)
ax.errorbar(properredshifts,
Bpointmean,
Bpointerr,
fmt='bo',
label=r'$BVr^+i^+z^+$')
# ax.errorbar(properredshifts+0.0025, Apointmean, Apointerr, fmt='rs', label=r'$uBVr^+i^+z^+$')
ax.text(0.166, 0.135, r'$BVr^+i^+z^+$ Photo-Z Point Est', fontsize=16)
ax.set_xlim(0.16, 0.72)
ax.set_ylim(-0.05, 0.15)
ax.set_xlabel('Cluster Redshift', fontsize=16)
ax.set_ylabel(r'Fractional Mass Bias within 1.5 Mpc')
# ax.legend(loc='lower right', numpoints = 1, ncol=2)
fig.savefig('publication/clustersims_pointest_compare.eps')
finally:
return fig, data
def precisionZ(data = None):
#mass precision as a function of redshift
if data is None:
data = {}
if 'fracerrs' not in data:
items = [tuple(x) for x in readtxtfile('worklist')]
allmasses = cm.readDougMasses('/u/ki/dapple/subaru/doug/publication/baseline_2012-05-17')
redshifts = cm.readClusterRedshifts()
clusters = [x[0] for x in items]
properz = np.array([redshifts[x] for x in clusters])
masses, errs = cm.constructMassArray(allmasses, items)
fracerrs = errs / masses
aveerrs = np.mean(fracerrs, axis=0)
data['aveerrs'] = aveerrs
data['properz'] = properz
ccitems = [tuple(x) for x in readtxtfile('referenceset')]
ccmasses = cm.readAnjaMasses()
clusters = [x[0] for x in ccitems]
ccproperz = np.array([redshifts[x] for x in clusters])
data['ccproperz'] = ccproperz
masses, errs = cm.constructMassArray(ccmasses, ccitems)
fracerrs = errs/masses
ccaveerrs = np.mean(fracerrs, axis=0)
data['ccaveerrs'] = ccaveerrs
else:
aveerrs = data['aveaerrs']
properz = data['properz']
ccaveerrs = data['ccaveerrs']
ccproperz = data['ccproperz']
fig = pylab.figure()
ax = fig.add_axes([0.12, 0.12, 0.95-0.12, 0.95-0.12])
ax.plot(ccproperz, ccaveerrs, 'bo', label = 'Color-Cut', mfc = 'None', mew = 1.0, mec='b')
ax.plot(properz, aveerrs, 'rD', label = 'P($z$)')
ax.set_xlabel('Cluster Redshift')
ax.set_ylabel('Fractional Statistical Uncertainty M(r$<$1.5Mpc)')
ax.set_xlim([0.14, 0.72])
ax.legend(loc='upper left', numpoints = 1)
fig.savefig('publication/aveerr_redshift.eps')
return fig, data
def lostgals(data = None):
if data is None:
data = {}
items = readtxtfile('worklist')
del items[-1]
clusters = [x[0] for x in items]
if 'properz' not in data:
redshifts = cm.readClusterRedshifts()
properz = np.array([redshifts[x] for x in clusters])
data['properz'] = properz
else:
properz = data['properz']
if 'properbase' not in data:
basecuts = {}
for cluster, filter, image in items:
controller = driver.makeController()
options, args = controller.modelbuilder.createOptions()
options, args = controller.filehandler.createOptions(options = options, args = args,
workdir = '/u/ki/dapple/ki06/catalog_backup_2012-02-08',
incatalog = '/u/ki/dapple/ki06/catalog_backup_2012-02-08/%s.%s.%s.lensingbase.cat' % (cluster, filter, image),
cluster = cluster, filter = filter, image = image,
redseqcat = '/u/ki/dapple/ki06/catalog_backup_2012-02-08/%s.%s.%s.redsequence.cat' % (cluster, filter, image), shapecut = True)
controller.load(options, args)
basecuts[cluster] = controller.ngalaxies
data['basecuts'] = basecuts
properbase = np.array([basecuts[x[0]] for x in items])
data['properbase'] = properbase
else:
properbase = data['properbase']
if 'properloose' not in data:
loosecuts = {}
for cluster, filter, image in items:
controller = driver.makeController()
options, args = controller.modelbuilder.createOptions(deltaz95high = 9999, zbhigh = 9999)
options, args = controller.filehandler.createOptions(options = options, args = args,
workdir = '/u/ki/dapple/ki06/catalog_backup_2012-02-08',
incatalog = '/u/ki/dapple/ki06/catalog_backup_2012-02-08/%s.%s.%s.lensingbase.cat' % (cluster, filter, image),
cluster = cluster, filter = filter, image = image,
redseqcat = '/u/ki/dapple/ki06/catalog_backup_2012-02-08/%s.%s.%s.redsequence.cat' % (cluster, filter, image), shapecut = True)
controller.load(options, args)
loosecuts[cluster] = controller.ngalaxies
data['loosecuts'] = loosecuts
properloose = np.array([loosecuts[x[0]] for x in items])
data['properloose'] = properloose
else:
properloose = data['properloose']
if 'ratio' not in data:
ratio = 1 - (properbase.astype('float64') / properloose)
data['ratio'] = ratio
else:
ratio = data['ratio']
fig = pylab.figure()
ax = fig.add_axes([0.12, 0.12, 0.95 - 0.12, 0.95 - 0.12])
ax.plot(properz, ratio, 'bo')
ax.set_xlim([0.16, 0.72])
ax.set_xlabel('Cluster Redshift')
ax.set_ylabel('Fraction of Catalog Discarded')
return fig, data
def stackClusters(data = None, cosmology = nfwutils.std_cosmology, outdir = '/u/ki/dapple/subaru/doug/publication/baseline_2012-05-17'):
workdir = '/u/ki/dapple/ki06/catalog_backup_2012-05-17'
if data is None:
data = {}
if 'items' not in data:
data['items'] = readtxtfile('worklist')
items = data['items']
if 'clusters' not in data:
clusters = {}
for cluster, filter, image in items:
key = (cluster, filter, image)
clusters[key] = {}
controller = driver.makeController()
options, args = controller.modelbuilder.createOptions(zcut= None)
options, args = controller.filehandler.createOptions(options = options, args = args,
workdir = workdir,
incatalog = '%s/%s.%s.%s.lensingbase.cat' % (workdir, cluster, filter, image),
cluster = cluster, filter = filter, image = image,
shapecut = True,
redseqcat = '%s/%s.%s.%s.redsequence.cat' % (workdir, cluster, filter, image))
controller.load(options, args)
stats = cPickle.load(open('%s/%s.%s.%s.out.mass15mpc.mass.summary.pkl' % (outdir, cluster, filter, image)))
mass = stats['quantiles'][50]
rs = nfwutils.RsMassInsideR(mass, 4.0, controller.zcluster, 1.5)
scaledZ, estimators = sr.scaleShear(controller, rs, 4.0, cosmology = cosmology)
clusters[key]['scaledZ'] = scaledZ
clusters[key]['estimators'] = estimators
clusters[key]['pdz'] = controller.pdz
data['clusters'] = clusters
else:
clusters = data['clusters']
maxScaledZ = -1
for key in clusters.keys():
localMax = np.max(clusters[key]['scaledZ'])
maxScaledZ = max(localMax, maxScaledZ)
if 'zbins' not in data:
zbins = np.unique(np.hstack([np.linspace(0., 1., 3.), np.logspace(0.1, np.log10(maxScaledZ), 12.)]))
data['zbins'] = np.hstack([zbins[:-3], zbins[-1]])
zbins = data['zbins']
bincenters = (zbins[1:] + zbins[:-1])/2.
for key in clusters.keys():
bins, weights, aveshear = sr.calcZBinWeights(clusters[key]['scaledZ'], clusters[key]['estimators'], zbins)
clusters[key]['weights'] = weights
clusters[key]['shears'] = aveshear
if 'maxlike' not in data:
allweights = np.vstack([clusters[tuple(item)]['weights'] for item in items])
allshears = np.vstack([clusters[tuple(item)]['shears'] for item in items])
pointest = sr.calcBinDistro(zbins, allweights, allshears)
data['pointest'] = pointest
maxlike, sig1, sig2, maxlike_ests = sr.bootstrapBinDistro(zbins, allweights, allshears)
data['maxlike'] = maxlike
data['sig1'] = sig1
data['sig2'] = sig2
data['maxlike_ests'] = maxlike_ests
else:
maxlike = data['maxlike']
sig1 = data['sig1']
sig2 = data['sig2']
fig = pylab.figure()
ax = fig.add_axes([0.12, 0.12, 0.95 - 0.12, 0.95 - 0.12])
ax.errorbar(bincenters, maxlike, sig2, fmt='bo')
ax.errorbar(bincenters, maxlike, sig1, fmt='ro')
xplot = np.arange(0.01, np.max(zbins), 0.01)
ax.plot(xplot, sr.shearScaling(xplot), 'k-', linewidth=1.5)
ax.set_xlabel('$x = \omega_s/\omega_l$')
ax.set_ylabel('$\gamma(z)/\gamma(\infty)$')
ax.set_title('All Cluster Stack -- Maxlike Point Est')
fig.savefig('notes/shearratio/stack_maxlike.pdf')
return fig, data
#!/usr/bin/env python
####################
import sys
import numpy as np
import compare_masses as cm, intrinsicscatter as isc
import scatter_sims as ss, nfwutils
from dappleutils import readtxtfile
workdir = sys.argv[1]
subdir = sys.argv[2]
outfile = sys.argv[3]
nsamples = int(sys.argv[4])
items = readtxtfile('simclusterlist')
clusters = clusters = [ x[0] for x in items]
redshifts = cm.readClusterRedshifts()
properredshifts = np.array([redshifts[x] for x in clusters])
masses, errs, massgrid, scale_radii = ss.readMLMasses(workdir, subdir, clusters)
truemasses = {}
for cluster in clusters:
truemasses[cluster] = nfwutils.massInsideR(scale_radii[cluster], 4., redshifts[cluster], 1.5)
x = np.hstack([len(masses[c])*[truemasses[c]] for c in clusters])
y = np.hstack([masses[c] for c in clusters])
yerr = np.hstack([errs[c] for c in clusters])
def magSplit_script(category, data = None):
figs = []
if data is None:
data = {}
if 'items' not in data:
data['items'] = readtxtfile('worklist')
# try:
loc1 = '%s/%s/low' % (trialsdir, category)
loc2 = '%s/%s/high' % (trialsdir, category)
if category not in data:
masses = loadData(loc1, loc2,
data['items'])
data[category] = {}
data[category]['masses'] = masses
curdata = data[category]
if 'grid' not in curdata:
grid, means, scatters = isg.intrinsicScatter(curdata['masses'][0], curdata['masses'][1], means = np.arange(0.6, 1.4, 0.002), scatters = np.arange(0.02, 0.2, 0.01))
curdata['grid'] = grid
curdata['means'] = means
curdata['scatters'] = scatters
else:
grid = curdata['grid']
means = curdata['means']
scatters = curdata['scatters']
print
print
print category
print '-----'
print 'var\tmode\t68%% +\t-\t95%% +\t-'
print '-----'
if 'meandist' not in curdata:
mode, (r68, r95) = isg.getdist_1d_hist(means[0], means[1], levels = [0.68, 0.95])
curdata['meandist'] = (mode, r68, r95)
mode, (r68, r95) = isg.getdist_1d_hist(scatters[0], scatters[1], levels = [0.68, 0.95])
curdata['scatterdist'] = (mode, r68, r95)
for varname in 'mean scatter'.split():
mode, r68, r95 = curdata['%sdist' % varname]
print mode, r68, r95
print '%s\t%2.4f\t+%2.4f\t-%2.4f\t+%2.4f\t-%2.4f' % (varname, mode,
r68[0][1] - mode, mode - r68[0][0],
r95[0][1] - mode, mode - r95[0][0])
x, prob = curdata['%ss' % varname]
fig = isgp.plotdist_1d_hist(x, prob, mode, [r68[0], r95[0]])
ax = fig.axes[0]
ax.set_title(varname)
figs.append(fig)
fig.show()
xlabel, ylabel = label_conv[category]
fig, xmasses, ymasses = cm.publicationPlotMassMass(loc1, loc2, xlabel, ylabel)
ax = fig.axes[0]
mode, r68, r95 = curdata['meandist']
# ax.axhline(mode, c='r', ls='--', linewidth=1.5)
ax.fill_between([0.16, 0.72], r95[0][0], r95[0][1], facecolor=(1, 0.642, 0.610), zorder=-1)
ax.fill_between([0.16, 0.72], r68[0][0], r68[0][1], facecolor='#CC0000', zorder=-1)
ax.set_xlim(0.16, 0.72)
ax.set_ylim(0.1, 10)
fig.show()
fig.savefig(filename_conv[category])
figs.append(fig)
# finally:
return figs, data
def PointEstPzScript(data=None):
if data is None:
worklist = readtxtfile('simclusterlist')
clusters = [x[0] for x in worklist]
redshifts = cm.readClusterRedshifts()
properredshifts = np.array([redshifts[x] for x in clusters])
subdirs = ['contam0p10/BVRIZ']
MLfracbias = ss.processFracBiasData(
'/u/ki/dapple/nfs12/cosmos/simulations/clusters_2012-05-17-highdensity',
subdirs, clusters, redshifts)[0]
Bpointmass, Bpointgrid, scale_radii = ss.readPointMasses(
'/u/ki/dapple/nfs12/cosmos/simulations/clusters_2012-05-17',
'contam0p10/newman/BVRIZ', clusters)
truemasses = [
nfwutils.massInsideR(scale_radii[x], 4., redshifts[x], 1.5)
for x in clusters
]
Bpointfracbias = ss.calcFracBias(Bpointgrid, truemasses)
data = [MLfracbias, Bpointfracbias, properredshifts]
else:
MLfracbias = data[0]
Bpointfracbias = data[1]
properredshifts = data[2]
fig = pylab.figure()
try:
ax = fig.add_axes([0.15, 0.12, 0.96 - 0.15, 0.95 - 0.12])
ax.axhline(0.0, c='k', linewidth=1.25)
Apointmean, Apointerr = ss.bootstrapMean(MLfracbias)
Bpointmean, Bpointerr = ss.bootstrapMean(Bpointfracbias)
ax.errorbar(properredshifts - 0.0025,
Bpointmean,
Bpointerr,
fmt='cs',
label=r'Point Estimators',
color='#BFBFD4')
ax.errorbar(properredshifts + 0.0025,
Apointmean,
Apointerr,
fmt='ro',
label=r'P(z) Method')
ax.set_xlim(0.16, 0.72)
ax.set_ylim(-0.08, 0.19)
ax.set_xlabel('Cluster Redshift')
ax.set_ylabel(r'Fractional Mass Bias within 1.5 Mpc')
# ax.text(0.2, 0.12, r'$BVr^{+}i^{+}z^{+}$ Photo-$z$ Point Est', fontsize=16)
ax.legend(loc='upper left', numpoints=1, ncol=1)
fig.savefig('publication/clustersims_pointest_pz_compare.eps')
finally:
return fig, data
def stackClusters(data = None, doPlot = True, cosmology = nfwutils.std_cosmology):
workdir = '/u/ki/dapple/ki06/catalog_backup_2012-02-08'
outdir = '/u/ki/dapple/subaru/doug/publication/baseline_2012-02-08'
if data is None:
data = {}
if 'items' not in data:
data['items'] = readtxtfile('worklist')
items = data['items']
if 'zbins' not in data:
data['zbins'] = np.unique(np.hstack([np.linspace(0., 1., 3.), np.linspace(1., 5., 10.), np.linspace(5., 10., 5.)]))
zbins = data['zbins']
bincenters = (zbins[1:] + zbins[:-1])/2.
if 'clusters' not in data:
clusters = {}
for cluster, filter, image in items:
key = (cluster, filter, image)
clusters[key] = {}
controller = driver.makeController()
options, args = controller.modelbuilder.createOptions()
options, args = controller.filehandler.createOptions(options = options, args = args,
workdir = workdir,
incatalog = '%s/%s.%s.%s.lensingbase.cat' % (workdir, cluster, filter, image),
cluster = cluster, filter = filter, image = image,
shapecut = True,
redseqcat = '%s/%s.%s.%s.redsequence.cat' % (workdir, cluster, filter, image))
controller.load(options, args)
stats = cPickle.load(open('%s/%s.%s.%s.out.mass15mpc.mass.summary.pkl' % (outdir, cluster, filter, image)))
mass = stats['quantiles'][50]
rs = nfwutils.RsMassInsideR(mass, 4.0, controller.zcluster, 1.5)
scaledZ, estimators = sr.scaleShear(controller, rs, 4.0, cosmology = cosmology)
bins, weights, aveshear = sr.calcZBinWeights(scaledZ, controller.pdz, estimators, zbins)
clusters[key]['weights'] = weights
clusters[key]['shears'] = aveshear
data['clusters'] = clusters
else:
clusters = data['clusters']
if 'median' not in data:
allweights = np.vstack([clusters[tuple(item)]['weights'] for item in items])
allshears = np.vstack([clusters[tuple(item)]['shears'] for item in items])
median, sig1, sig2 = sr.calcBinDistro(zbins, allweights, allshears)
data['median'] = median
data['sig1'] = sig1
data['sig2'] = sig2
else:
median = data['median']
sig1 = data['sig1']
sig2 = data['sig2']
if doPlot:
fig = pylab.figure()
ax = fig.add_axes([0.12, 0.12, 0.95 - 0.12, 0.95 - 0.12])
ax.errorbar(bincenters, median, sig2, fmt='bo')
ax.errorbar(bincenters, median, sig1, fmt='ro')
xplot = np.arange(0., np.max(zbins), 0.01)
ax.plot(xplot, sr.shearScaling(xplot), 'k-', linewidth=1.5)
ax.set_xlabel('Scaled Redshift')
ax.set_ylabel('Lensing Power')
ax.set_title('All Cluster Stack -- Restricted to Fit Data')
fig.savefig('notes/shearratio/stack_restricted_manypoints.pdf')
return fig, data
return data
def fitAltOffsetScript(data=None):
if data is None:
data = {}
worklist = readtxtfile('worklist')
clusters = [x[0] for x in worklist]
workdir = '/u/ki/dapple/nfs12/cosmos/simulations/clusters_2012-05-17-highdensity/'
subdirs = ['%sBVRIZ' % x for x in ['', 'contam0p10/', 'contam0p20/']]
concentration = 4.
mradius = 1.5
redshifts = cm.readClusterRedshifts()
figs = []
for subdir in subdirs:
if subdir not in data:
data[subdir] = {}
curdata = data[subdir]
if 'masses' not in curdata:
curdata['masses'], errs, massgrid, curdata[
'scale_radii'] = ss.readMLMasses(workdir, subdir, clusters)
masses = curdata['masses']
scale_radii = curdata['scale_radii']
if 'grid' not in curdata:
refmasses = {}
for cluster in clusters:
refmasses[cluster] = nfwutils.massInsideR(
scale_radii[cluster], concentration, redshifts[cluster],
mradius) * np.ones_like(masses[cluster])
curdata['grid'], curdata['means'], curdata[
'scatters'] = isg.intrinsicScatter(
refmasses,
masses,
means=1. + np.arange(-0.08, 0.08, 0.0001),
scatters=np.arange(0.005, 0.05, 0.0025))
means = curdata['means']
scatters = curdata['scatters']
mode, (r68, r95) = isg.getdist_1d_hist(means[0],
means[1],
levels=[0.68, 0.95])
curdata['meandist'] = (mode, r68, r95)
mode, (r68, r95) = isg.getdist_1d_hist(scatters[0],
scatters[1],
levels=[0.68, 0.95])
curdata['scatterdist'] = (mode, r68, r95)
for varname in 'mean scatter'.split():
mode, r68, r95 = curdata['%sdist' % varname]
print mode, r68, r95
print '%s\t%2.4f\t+%2.4f\t-%2.4f\t+%2.4f\t-%2.4f' % (
varname, mode, r68[0][1] - mode, mode - r68[0][0],
r95[0][1] - mode, mode - r95[0][0])
x, prob = curdata['%ss' % varname]
fig = isgp.plotdist_1d_hist(x, prob, mode, [r68[0], r95[0]])
ax = fig.axes[0]
ax.set_title('%s %s' % (subdir, varname))
figs.append(fig)
fig.show()
return figs, data
#!/usr/bin/env python
####################
import sys
import numpy as np
import compare_masses as cm, intrinsicscatter2 as isc2
from dappleutils import readtxtfile
import pymc
outfile = sys.argv[1]
nsamples = int(sys.argv[2])
items = readtxtfile('worklist')
clusters = clusters = [ x[0] for x in items]
mlbootstraps, mlmasks = cm.readMLBootstraps('/u/ki/dapple/ki06/bootstrap_2011-12-14/', items)
ccbootstraps, ccbootmask = cm.readCCSummary('/u/ki/dapple/ki06/bootstrap_2011-12-14/', clusters, 100)
reducedML = {}
reducedCC = {}
for cluster in clusters:
totalmask = np.logical_and(mlmasks[cluster] == 1, ccbootmask[cluster] == 1)
reducedML[cluster] = mlbootstraps[cluster][totalmask]
reducedCC[cluster] = ccbootstraps[cluster][totalmask]
calibmodel = isc2.IntrinsicScatter2(reducedML, reducedCC)
calibMCMC = calibmodel.buildMCMC(outfile)
#calibMCMC.use_step_method(pymc.Metropolis, calibMCMC.m_angle, proposal_sd = 0.1)
#calibMCMC.use_step_method(pymc.Metropolis, calibMCMC.log10_intrinsic_scatter, proposal_sd = 0.8)
