def __init__(self, M0, M1, dM, logR0, logR1, dlogR, dmz0, dmz1, ddmz, drop,
z0, z1):
# dmz: the bin width of the differential K-correction
self.M0 = M0
self.M1 = M1
self.dM = dM
self.logR0 = logR0
self.logR1 = logR1
self.dlogR = dlogR
self.dmz0 = dmz0 # lower bound of differential K-correction
self.dmz1 = dmz1 # higher bound of differential K-correction
self.ddmz = ddmz # step size in differential K-correction, should be the same as model
# pixel width in the magnitude direction
self.dmzarray = N.arange(dmz0, dmz1 + ddmz, ddmz) # the bin boundaries
self.z0 = z0
self.z1 = z1
if drop == 'b':
zcent = 4.0
mc = mconvert('M1500_to_i.txt')
elif drop == 'v':
zcent = 5.0 # the fiducial central redshift of the given dropout sample
mc = mconvert('M1500_to_z.txt')
zarray = N.zeros(len(self.dmzarray))
# the redshifts corresponding to the magnitude of the differential K-corr dmz
for i in range(len(self.dmzarray)):
zarray[i] = mc.revert2z(self.dmzarray[i] + mc(zcent))
self.zarray = zarray # zarray could be larger than the [z0, z1] interval... be careful
self.Mlolims = N.arange(M0, M1, dM)
self.Mhilims = self.Mlolims + dM
self.logRlolims = N.arange(logR0, logR1, dlogR)
self.logRhilims = self.logRlolims + dlogR
if self.Mhilims[-1] > M1: self.Mhilims[-1] = M1
self.dkcorr = {}
if drop == 'b':
self.drop = 'b'
self.dropout = "B-dropout"
if drop == 'v':
self.drop = 'v'
self.dropout = "V-dropout"
def plot_RLdist_vdrops(logR0,
beta,
cat1=catdir + '/vdrops/vdrops_gf_v3.fits',
cat2=catdir + '/vdrops/vdrops_udf_gf_v3.fits',
marker='o',
ax=None,
ms=8**2,
color=pms.Bright_Blue,
bootstrap=False,
nsamp=10000):
c1 = Ftable(cat1)
c2 = Ftable(cat2)
mcz = bl.mconvert('kcorr/M1500_to_f850lp_omega_m_0.3.txt')
mean_kcorr = mcz(5.0)
gfcrit1 = (c1.f850lp_gfflag == True) & (c1.magout <= 26.5)
gfcrit2 = (c2.f850lp_gfflag == True) & (c2.magout <= 28.5)
mag_array1 = c1.magout[gfcrit1]
re_array1 = c1.reout[gfcrit1]
n_array1 = c1.nout[gfcrit1]
mag_array2 = c2.magout[gfcrit2]
re_array2 = c2.reout[gfcrit2]
n_array2 = c2.nout[gfcrit2]
mag_array = np.concatenate([mag_array1, mag_array2])
re_array = np.concatenate([re_array1, re_array2])
n_array = np.concatenate([n_array1, n_array2])
plot_RLdist(mag_array,
re_array,
'V-dropouts',
logR0,
beta,
mean_kcorr,
color=color,
marker=marker,
ax=ax,
ms=ms,
gfband='F850LP')
if bootstrap == True:
p_array = bootstrap_error(mag_array, np.log10(re_array), nsamp=nsamp)
return p_array
else:
return 0
def mc_goodness_fit(par, drop, niter, zlo=-1.):
# Determine goodness of fit given the parameters.
# Draw the same number of points from the model as observed, then calculate the
# loglikelihood of the drawn points. Calculate the probability that a simulated
# observation has lower loglikelihood than the observed points.
if drop == 'b':
cat1 = 'bdrops_gf_v2.cat'
cat2 = 'bdrops_udf_gf_v2.cat'
kgrid1 = mlutil.readkgrid('kernel_I.p')
kgrid2 = mlutil.readkgrid('kernel_I_udf.p')
zdf1 = 'zdgrid_bdrops.p'
zdf2 = 'zdgrid_bdrops_udf.p'
mcfile = 'M1500_to_i.txt'
mc = bl.mconvert(mcfile)
mk = mc(4.0)
chisqnulim = [0.4, 5.0]
elif drop == 'v':
cat1 = 'vdrops_gf_v2.cat'
cat2 = 'vdrops_udf_gf_v2.cat'
kgrid1 = mlutil.readkgrid('kernel_Z.p')
kgrid2 = mlutil.readkgrid('kernel_Z_udf.p')
zdf1 = 'zdgrid_vdrops.p'
zdf2 = 'zdgrid_vdrops_udf.p'
mcfile = 'M1500_to_z.txt'
mc = bl.mconvert(mcfile)
mk = mc(5.0)
chisqnulim = [0.5, 5.0]
cullflags = [0, 1, 2, 3, 4, 12, 13, 14, 18]
limits1 = array([[21.0, 26.5], [-2.0, 3.0]])
limits2 = array([[23.0, 28.5], [-2.0, 3.0]])
#limits1 = bl.limits1
#limits2 = bl.limits2
pixdx = array([0.02, 0.02])
mag1, re1, crit1 = fl.cleandata(cat1,
chisqnulim=chisqnulim[0],
magautolim=26.5,
cullflags=cullflags,
limits=limits1,
zlo=zlo,
drop=drop)
mag2, re2, crit2 = fl.cleandata(cat2,
chisqnulim=chisqnulim[1],
magautolim=28.5,
cullflags=cullflags,
limits=limits2,
zlo=zlo,
drop=drop)
data1 = array([mag1, log10(re1)])
data2 = array([mag2, log10(re2)])
N1 = len(mag1)
N2 = len(mag2)
print N1, N2, N1 + N2
model1 = bl.bivariate_lf(par, limits1, pixdx, drop, 'goods', kgrid=kgrid1, zdgridfile=zdf1,\
mcfile=mcfile, meankcorr=mk, add_interloper=True, norm=-1.)
model2 = bl.bivariate_lf(par, limits2, pixdx, drop, 'udf', kgrid=kgrid2, zdgridfile=zdf2,\
mcfile=mcfile, meankcorr=mk, add_interloper=True, norm=-1.)
sum1 = sum(model1.model.ravel()) * pixdx[0] * pixdx[1]
sum2 = sum(model2.model.ravel()) * pixdx[0] * pixdx[1]
phistar_mod = float(N1 + N2) / (sum1 + sum2)
print phistar_mod
model1.model = phistar_mod * model1.model
model2.model = phistar_mod * model2.model
logl_ref = bf.loglikelihood(data1, model1, floor=0.) + bf.loglikelihood(
data2, model2, floor=0.)
#logl_ref = bf.mlfunc(par, data1, data2, limits1, limits2, pixdx, kgrid1, kgrid2,
# 1.0, 1.0, -21.0, zdf1, zdf2, mcfile, 1, mk, 0, -1., 'phistar', drop)
print "logl_ref", logl_ref
simlogl_arr = zeros(niter) # actually -1*logL...
print "Start drawing simulated observations..."
t1 = time.time()
for i in range(niter):
if i % 1000 == 0: print i
simdata1 = mlutil.draw_from_pdf(N1, model1, model1.limits)
simdata2 = mlutil.draw_from_pdf(N2, model2, model2.limits)
simlogl1 = bf.loglikelihood(simdata1, model1)
simlogl2 = bf.loglikelihood(simdata2, model2)
simlogl = simlogl1 + simlogl2
simlogl_arr[i] = simlogl
t2 = time.time()
dt = t2 - t1
dtmin = int(floor(dt)) / 60
dtsec = dt % 60
n_worse = sum(simlogl_arr > logl_ref)
print "%d iterations took %d min %.1f sec" % (niter, dtmin, dtsec)
print "Percentage of simulated observations with lower log-likelihood: %.2f %%" % (
100. * float(n_worse) / float(niter))
return logl_ref, simlogl_arr
def reduced_chi2(par,
drop,
mbins1,
rbins1,
mbins2,
rbins2,
chisqnulim=(0.4, 5.0),
zlo=-1):
"""
Calculate the reduced chi2 of the best-fit model
mbins, rbins should include the upper limits.
"""
if drop == 'b':
cat1 = 'bdrops_gf_v2.cat'
cat2 = 'bdrops_udf_gf_v2.cat'
kgrid1 = mlutil.readkgrid('kernel_I.p')
kgrid2 = mlutil.readkgrid('kernel_I_udf.p')
zdf1 = 'zdgrid_bdrops.p'
zdf2 = 'zdgrid_bdrops_udf.p'
mcfile = 'M1500_to_i.txt'
mc = bl.mconvert(mcfile)
mk = mc(4.0)
elif drop == 'v':
cat1 = 'vdrops_gf_v2.cat'
cat2 = 'vdrops_udf_gf_v2.cat'
kgrid1 = mlutil.readkgrid('kernel_Z.p')
kgrid2 = mlutil.readkgrid('kernel_Z_udf.p')
zdf1 = 'zdgrid_vdrops.p'
zdf2 = 'zdgrid_vdrops_udf.p'
mcfile = 'M1500_to_z.txt'
mc = bl.mconvert(mcfile)
mk = mc(5.0)
cullflags = [0, 1, 2, 3, 4, 12, 13, 14, 18, 19]
limits1 = array([[21.0, 26.5], [-2.0, 3.0]])
limits2 = array([[23.0, 28.5], [-2.0, 3.0]])
pixdx = array([0.02, 0.02])
modshape1 = (limits1[:, 1] - limits1[:, 0]) / pixdx
modshape1 = around(modshape1).astype('int')
modshape2 = (limits2[:, 1] - limits2[:, 0]) / pixdx
modshape2 = around(modshape2).astype('int')
# bin the points & tally the counts
mag1, re1, crit1 = cleandata(cat1,
chisqnulim=chisqnulim[0],
magautolim=26.5,
cullflags=cullflags,
limits=limits1,
zlo=zlo)
mag2, re2, crit2 = cleandata(cat2,
chisqnulim=chisqnulim[1],
magautolim=28.5,
cullflags=cullflags,
limits=limits2,
zlo=zlo)
bincounts1 = histogram2d(mag1, log10(re1),
bins=[mbins1, rbins1])[0].astype('float')
bincounts2 = histogram2d(mag2, log10(re2),
bins=[mbins2, rbins2])[0].astype('float')
#print bincounts1, bincounts2
# calculate the best-fit models
model1 = bl.bivariate_lf(par, limits1, pixdx, kgrid=kgrid1, zdgridfile=zdf1, \
mcfile=mcfile, drop=drop, field='goods', meankcorr=mk, add_interloper=True)
model2 = bl.bivariate_lf(par, limits2, pixdx, kgrid=kgrid2, zdgridfile=zdf2, \
mcfile=mcfile, drop=drop, field='udf', meankcorr=mk, add_interloper=True)
phistar_mod = phistar(par, drop, zlo=zlo)
model1.model = phistar_mod * model1.model
model2.model = phistar_mod * model2.model
#model1.model = ones(modshape1)/(modshape1[0]*modshape1[1]*pixdx[0]*pixdx[1])*len(mag1)
#model2.model = ones(modshape2)/(modshape2[0]*modshape2[1]*pixdx[0]*pixdx[1])*len(mag2)
print sum(model1.model.ravel()) * pixdx[0] * pixdx[1]
chi2tot = 0.
nbins = 0
mindex1 = (mbins1 - 21.0) / 0.02
mindex1 = around(mindex1).astype('int')
rindex1 = (rbins1 - (-2.0)) / 0.02
rindex1 = around(rindex1).astype('int')
mindex2 = (mbins2 - 23.0) / 0.02
mindex2 = around(mindex2).astype('int')
rindex2 = (rbins2 - (-2.0)) / 0.02
rindex2 = around(rindex2).astype('int')
num_exp1 = [] # number of expected
num_exp2 = []
num_obs1 = bincounts1.ravel()[bincounts1.ravel() >= 5]
num_obs2 = bincounts2.ravel()[bincounts2.ravel() >= 5]
# iterate through all bins and calculate the chi2
for i in range(len(mbins1) - 1):
for j in range(len(rbins1) - 1):
if bincounts1[i, j] >= 5:
num_mod = sum(model1.model[mindex1[i]:mindex1[i + 1],
rindex1[j]:rindex1[j + 1]].ravel())
num_mod = num_mod * pixdx[0] * pixdx[1]
num_exp1 += [num_mod]
#chi2 = (bincounts1[i,j] - num_mod)**2 / num_mod
#print bincounts1[i,j], num_mod
#chi2tot += chi2
#nbins += 1
#if bincounts1[i,j] < nmin: nmin = bincounts1[i,j]
for i in range(len(mbins2) - 1):
for j in range(len(rbins2) - 1):
if bincounts2[i, j] >= 5:
num_mod = sum(model2.model[mindex2[i]:mindex2[i + 1],
rindex2[j]:rindex2[j + 1]].ravel())
num_mod = num_mod * pixdx[0] * pixdx[1]
num_exp2 += [num_mod]
#chi2 = (bincounts2[i,j] - num_mod)**2 / num_mod
#chi2tot += chi2
#nbins += 1
print "nbins", nbins
# Run chi-square test
num_exp = concatenate([num_exp1, num_exp2])
num_obs = concatenate([num_obs1, num_obs2])
ndeg = len(
num_exp) - 1 # degree of freedom = num. of contributing bins - 1
print "ndeg", ndeg
chi2, pval = stats.mstats.chisquare(num_obs, f_exp=num_exp)
print chi2, pval
#print num_exp, num_obs
#chi2nu = chi2tot / float(ndeg)
return chi2, pval, num_exp, num_obs
def __init__(self, parfile, reerr_lim=0.6, udf_maglim=30.0, expand=False):
"""
Defines the data sets, model limits, parameter boundaries.
udf_maglim: the magnitude limit in UDF to which we extend the fitting
to; beyond self.limits['udf'][0][1] I only fit 1D magnitude distribution
using SExtractor corrections.
"""
fl.FitBivariateRL.__init__(self, parfile)
self.catalog = self.DATAFILE
c = Ftable(self.catalog)
# i-dropout catalog with GALFIT results
self.fields = ['udf', 'deep', 'ers', 'wide']
#self.fields = ['deep', 'ers', 'wide']
self.delta_z = 1.5
self.expand = expand
# Create bl.bivariate_RL_class instances
self.models = {}
for f in self.fields:
self.models[f] = bl.bivariate_RL_class()
# Define limits
self.limits = {}
self.limits['udf'] = array(self.LIMITS1).reshape((2, 2))
self.limits['deep'] = array(self.LIMITS2).reshape((2, 2))
self.limits['ers'] = array(self.LIMITS3).reshape((2, 2))
self.limits['wide'] = array(self.LIMITS4).reshape((2, 2))
# Define mconvert instances
self.mc = {}
self.mc['udf'] = bl.mconvert(self.MCFILE1)
self.mc['deep'] = bl.mconvert(self.MCFILE2)
self.mc['ers'] = bl.mconvert(self.MCFILE3)
self.mc['wide'] = bl.mconvert(self.MCFILE4)
#print self.limits['deep']
# create self.dataset
self.datasets = {}
self.ra = {}
self.dec = {}
self.objid = {}
# UDF criteria
# udfcrit = (c.udf==True)
udfcrit = (c.wfc3_f160w_logwht >= 5.0)
# udfcrit = udfcrit & fl.between(c.f105w_magout_gf,
# self.limits['udf'][0])
# udfcrit = udfcrit & fl.between(log10(c.f105w_reout_gf),
# self.limits['udf'][1])
udfcrit = udfcrit & fl.between(c.f125w_magout_gf,
self.limits['udf'][0])
udfcrit = udfcrit & fl.between(log10(c.f125w_reout_gf),
self.limits['udf'][1])
udfcrit = udfcrit & (c.interloper_flag_udf == False)
# ERS criteria
# erscrit = (c.ers==True)
erscrit = (c.wfc3_f098m_weight > 0)
# erscrit = erscrit & fl.between(c.f098m_magout_gf,
# self.limits['ers'][0])
# erscrit = erscrit & fl.between(log10(c.f098m_reout_gf),
# self.limits['ers'][1])
erscrit = erscrit & fl.between(c.f125w_magout_gf,
self.limits['ers'][0])
erscrit = erscrit & fl.between(log10(c.f125w_reout_gf),
self.limits['ers'][1])
erscrit = erscrit & (c.interloper_flag_ers == False)
# Deep criteria
# deepcrit = (c.deep==True)
deepcrit = (c.wfc3_f098m_weight == 0) & ((c.wfc3_f160w_logwht < 5.0))
deepcrit = deepcrit & (c.wfc3_f160w_logwht>=4.2) & \
(c.wfc3_f160w_logwht<5.0)
# deepcrit = deepcrit & fl.between(c.f105w_magout_gf,
# self.limits['deep'][0])
# deepcrit = deepcrit & fl.between(log10(c.f105w_reout_gf),
# self.limits['deep'][1])
deepcrit = deepcrit & fl.between(c.f125w_magout_gf,
self.limits['deep'][0])
deepcrit = deepcrit & fl.between(log10(c.f125w_reout_gf),
self.limits['deep'][1])
deepcrit = deepcrit & (c.interloper_flag_deep == False)
# Wide criteria
# widecrit = (c.wide==True)
widecrit = (c.wfc3_f160w_logwht < 4.2) & (c.wfc3_f098m_weight == 0)
# widecrit = widecrit & fl.between(c.f105w_magout_gf,
# self.limits['wide'][0])
# widecrit = widecrit & fl.between(log10(c.f105w_reout_gf),
# self.limits['wide'][1])
widecrit = widecrit & fl.between(c.f125w_magout_gf,
self.limits['wide'][0])
widecrit = widecrit & fl.between(log10(c.f125w_reout_gf),
self.limits['wide'][1])
widecrit = widecrit & (c.interloper_flag_wide == False)
# Below is probably redundant, but just to be safe...
vfcrit = (c.vflag != 10) & (c.vflag != 4)
# Visual classification criteria
gfcrit = ((c.f125w_magflag == 0) & (c.f125w_reflag == 0))
gfcrit = (gfcrit & (c.f125w_nflag == 0))
gfcrit = (gfcrit &
(c.f125w_reout_err_gf / c.f125w_reout_gf <= self.REERR_LIM))
gfcrit = (gfcrit & (c.f125w_chi2nu_gf <= self.CHI2NU_LIM))
# Data set for UDF
# mag_udf = c.f105w_magout_gf[udfcrit]
# logre_udf = log10(c.f105w_reout_gf[udfcrit])
mag_udf = c.f125w_magout_gf[udfcrit & gfcrit & vfcrit]
logre_udf = log10(c.f125w_reout_gf[udfcrit & gfcrit & vfcrit])
self.datasets['udf'] = array([mag_udf, logre_udf])
self.ra['udf'] = c.ra[udfcrit & vfcrit & gfcrit]
self.dec['udf'] = c.dec[udfcrit & vfcrit & gfcrit]
self.objid['udf'] = c.number[udfcrit & vfcrit & gfcrit]
# Data set for GOODS-S Deep
# mag_deep = c.f105w_magout_gf[deepcrit]
# logre_deep = log10(c.f105w_reout_gf[deepcrit])
mag_deep = c.f125w_magout_gf[deepcrit & gfcrit & vfcrit]
logre_deep = log10(c.f125w_reout_gf[deepcrit & gfcrit & vfcrit])
self.datasets['deep'] = array([mag_deep, logre_deep])
self.ra['deep'] = c.ra[deepcrit & vfcrit & gfcrit]
self.dec['deep'] = c.dec[deepcrit & vfcrit & gfcrit]
self.objid['deep'] = c.number[deepcrit & vfcrit & gfcrit]
# Data set for ERS
# mag_ers = c.f098m_magout_gf[erscrit]
# logre_ers = log10(c.f098m_reout_gf[erscrit])
mag_ers = c.f125w_magout_gf[erscrit & gfcrit & vfcrit]
logre_ers = log10(c.f125w_reout_gf[erscrit & gfcrit & vfcrit])
self.datasets['ers'] = array([mag_ers, logre_ers])
self.ra['ers'] = c.ra[erscrit & vfcrit & gfcrit]
self.dec['ers'] = c.dec[erscrit & vfcrit & gfcrit]
self.objid['ers'] = c.number[erscrit & vfcrit & gfcrit]
# Data set for wide
# mag_wide = c.f105w_magout_gf[widecrit]
# logre_wide = log10(c.f105w_reout_gf[widecrit])
mag_wide = c.f125w_magout_gf[widecrit & gfcrit & vfcrit]
logre_wide = log10(c.f125w_reout_gf[widecrit & gfcrit & vfcrit])
self.datasets['wide'] = array([mag_wide, logre_wide])
self.ra['wide'] = c.ra[widecrit & vfcrit & gfcrit]
self.dec['wide'] = c.dec[widecrit & vfcrit & gfcrit]
self.objid['wide'] = c.number[widecrit & vfcrit & gfcrit]
print "Total numbers used in fitting:"
for f in self.fields:
print "%12s %6d" % (f.upper(), len(self.datasets[f][0]))
# Define & read GALFIT transfer function kernel grids
self.tfkgrids = {}
self.tfkgrids['udf'] = read_kgrid(self.KGRIDFILE1)
self.tfkgrids['deep'] = read_kgrid(self.KGRIDFILE2)
self.tfkgrids['ers'] = read_kgrid(self.KGRIDFILE3)
self.tfkgrids['wide'] = read_kgrid(self.KGRIDFILE4)
for f in self.fields:
self.tfkgrids[f].interpolate_completeness(self.limits[f],
self.pixdx)
# Define & read P(z) kernels
# Also calculate zdgrid.Pk
self.zdgrids = {}
self.zdgrids['udf'] = read_zdgrid(self.ZDGRIDFILE1)
self.zdgrids['deep'] = read_zdgrid(self.ZDGRIDFILE2)
self.zdgrids['ers'] = read_zdgrid(self.ZDGRIDFILE3)
self.zdgrids['wide'] = read_zdgrid(self.ZDGRIDFILE4)
# Calculate Pk
for f in self.fields:
bl.Pijk_dVdz(self.zdgrids[f], self.mc[f], self.limits[f],
self.pixdx)
# Define parameter boundaries
self.boundary = [self.BOUND_ALPHA, self.BOUND_MSTAR, self.BOUND_R0, \
self.BOUND_SIGMA, self.BOUND_BETA]
for i in range(5):
self.boundary[i] = fl.replaceNone(self.boundary[i])
if self.add_interloper == True:
self.sd['udf'] = cPickle.load(open(self.SDFILE1))
self.sd['deep'] = cPickle.load(open(self.SDFILE2))
self.sd['ers'] = cPickle.load(open(self.SDFILE3))
self.sd['wide'] = cPickle.load(open(self.SDFILE4))
self.intfrac['udf'] = Ftable(self.INTFRACFILE1)
self.intfrac['deep'] = Ftable(self.INTFRACFILE2)
self.intfrac['ers'] = Ftable(self.INTFRACFILE3)
self.intfrac['wide'] = Ftable(self.INTFRACFILE4)
self.mag_lolims['udf'] = self.MAG_LOLIMS[0]
self.mag_lolims['deep'] = self.MAG_LOLIMS[1]
self.mag_lolims['ers'] = self.MAG_LOLIMS[2]
self.mag_lolims['wide'] = self.MAG_LOLIMS[3]
# Specifically to extend UDF magnitude limit to constrain faint-end
# slope for i-dropouts
self.zdist_mag_udf = cPickle.load(open(self.ZDIST_MAG_UDF))
limits_ext = [self.limits['udf'][0][1] - 0.5, self.MAGLIM_UDF]
self.zdist_mag_udf.calc_Pk(limits_ext, self.pixdx[0], self.mc['udf'])
self.kgrid1d_udf = cPickle.load(open(self.KGRID1D_UDF))
mag_array_udf = c.wfc3_f125w_mag[(c.udf==True) & \
(c.wfc3_f125w_mag>=self.limits['udf'][0][1]) & \
(c.wfc3_f125w_mag<self.MAGLIM_UDF) & \
((c.wfc3_f125w_flux/c.wfc3_f125w_fluxerr)>=3.0)]
self.mag_array_udf = mag_array_udf
print "In addition, extend UDF down to %.1f magnitude using SExtractor transfer function only; %d more objects." % \
(self.MAGLIM_UDF, len(mag_array_udf))
# Define mlfunc_args
self.mlfunc_args = [
self.datasets, self.limits, self.pixdx, self.tfkgrids,
self.zdgrids, self.M0, self.mc, self.z_mean, self.drop,
self.add_interloper, self.sd, self.intfrac, self.mag_lolims,
self.nproc_model, self.zdist_mag_udf, self.kgrid1d_udf,
self.mag_array_udf, self.MAGLIM_UDF
]
def plot_RLdist_idrops(logR0,
beta,
cat=catdir +
'/idrops/idrops_goodss_130623_vflags_galfit.fits',
marker='o',
ax=None,
ms=8**2,
color=pms.Bright_Blue,
bootstrap=False,
nsamp=10000):
c = Ftable(cat)
mcy = bl.mconvert('kcorr/M1500_to_f105w_omega_m_0.3.txt')
mean_kcorr = mcy(6.0)
mag_array1 = c.f105w_magout_gf[(c.udf_fit == True)
& (c.f105w_magout_gf <= 27.5)]
re_array1 = c.f105w_reout_gf[(c.udf_fit == True)
& (c.f105w_magout_gf <= 27.5)]
n_array1 = c.f105w_nout_gf[(c.udf_fit == True)
& (c.f105w_magout_gf <= 27.5)]
mag_array2 = c.f105w_magout_gf[(c.deep_fit == True)
& (c.f105w_magout_gf <= 26.5)]
re_array2 = c.f105w_reout_gf[(c.deep_fit == True)
& (c.f105w_magout_gf <= 26.5)]
n_array2 = c.f105w_nout_gf[(c.deep_fit == True)
& (c.f105w_magout_gf <= 26.5)]
mag_array3 = c.f098m_magout_gf[(c.ers_fit == True)
& (c.f098m_magout_gf <= 26.5)]
re_array3 = c.f098m_reout_gf[(c.ers_fit == True)
& (c.f098m_magout_gf <= 26.5)]
n_array3 = c.f098m_nout_gf[(c.ers_fit == True)
& (c.f098m_magout_gf <= 26.5)]
mag_array4 = c.f105w_magout_gf[(c.wide_fit == True)
& (c.f105w_magout_gf <= 26.5)]
re_array4 = c.f105w_reout_gf[(c.wide_fit == True)
& (c.f105w_magout_gf <= 26.5)]
n_array4 = c.f105w_nout_gf[(c.wide_fit == True)
& (c.f105w_magout_gf <= 26.5)]
mag_array = np.concatenate(
[mag_array1, mag_array2, mag_array3, mag_array4])
re_array = np.concatenate([re_array1, re_array2, re_array3, re_array4])
n_array = np.concatenate([n_array1, n_array2, n_array3, n_array4])
plot_RLdist(mag_array,
re_array,
'i-dropouts',
logR0,
beta,
mean_kcorr,
color=color,
marker=marker,
ax=ax,
ms=ms,
gfband='F105W/F098M')
if bootstrap == True:
p_array = bootstrap_error(mag_array, np.log10(re_array), nsamp=nsamp)
return p_array
else:
return 0
#!/usr/bin/env python
from numpy import *
import bivRL as bl
import matplotlib as mpl
import matplotlib.pyplot as plt
from my_mplfonts import Helvetica
p1 = array([-1.7, -21.0, 0.7, 0.7, 0.3])
limits = array([[25.5, 27.0], [0.2, 1.0]])
pixdx = array([0.02, 0.02])
mci = bl.mconvert(
'/Users/khuang/Dropbox/Research/bivariate/bivariate_fit/kcorr/M1500_to_f775w.txt'
)
def bivmodel_pix_bin(param=p1, limits=limits, pixdx=pixdx):
"""
Plot a schematic diagram demonstrating the concept of "pixels" and "bins"
for the bivariate size-luminosity model distribution.
"""
model = bl.bivariate_RL(param,
limits,
pixdx,
'f435w',
'goods',
kgrid=None,
zdgrid=None,
mc=mci,
add_interloper=False)
def makedkcorr(c,
Mbincrit,
logrbincrit,
dropcrit,
dmz0,
dmz1,
ddmz,
drop,
z0=3.5,
z1=4.5):
"""
Calculates differential K-correction kernel and completeness for a given
M1500 bin.
The completeness is given as the function C(z), which is the completeness of dropout
selection as a function of redshift in a given magnitude bin.
"""
bincrit = logrbincrit & Mbincrit # to select objects in the mag & size bin
zcrit = (c.z_input >= z0) & (c.z_input < z1)
selcrit = (zcrit & bincrit & dropcrit)
# to select all dropouts in this M1500 bin
zarr = N.compress(
selcrit, c.z_input
) # the array of the redshifts of all dropouts in this M1500 bin
karr = N.zeros(len(zarr))
# zcent: reference redshift (zcent = 4.0 for B-dropouts; zcent = 5.0 for V-dropouts)
if drop == 'b':
mc = mconvert('M1500_to_i.txt')
zcent = 4.0
elif drop == 'v':
mc = mconvert('M1500_to_z.txt')
zcent = 5.0
#n_ztarget = sum(zcrit & bincrit & c.detect) # NOT including SE detection incompleteness
n_ztarget = sum(zcrit & bincrit) # including SE detection incompleteness
n_zselect = sum(zcrit & bincrit & dropcrit)
# calculate completeness as follows:
# completeness = (num. of objects that are detected and selected as dropouts) /
# (num. of INPUT objects in this bin)
if n_ztarget > 5:
completeness = float(n_zselect) / float(n_ztarget)
else:
completeness = 0.
# calculate the K-correction of all redshifts
for i in range(len(zarr)):
karr[i] = mc(zarr[i])
dkarr = karr - mc(zcent) # the differential K-correction
grid = N.arange(
dmz0, dmz1 + ddmz, ddmz
) # the grid over which differential K-correction will be calculated
if n_zselect > 4:
h = KPDF.UPDFOptimumBandwidth(dkarr)
p_dk = KPDF.UPDFEpanechnikov(dkarr, grid, h)
if sum(p_dk) > 0:
p_dk = p_dk / sum(p_dk)
else:
p_dk = N.zeros(len(grid))
else:
p_dk = N.zeros(len(grid))
# include zarr in the kernel for inspection purposes
return completeness, p_dk, n_ztarget, n_zselect, zarr
def __init__(self, parfile):
# reads in the parameter file that contains everything
#meanz_dic = {'b':4.0, 'v':5.0}
self.parfile = parfile
#c = parseconfig(parfile)
c = yaml.load(open(parfile, 'rb'))
for k in c.keys():
print k, '=', c[k]
for k in c.keys():
setattr(self, k, c[k])
# set all entries in the parameter file as attributes
self.parameters = array([self.ALPHA, self.MSTAR, self.LOGR0, self.SIGMA, self.BETA])
# IMPORTANT:
# WHENEVER A FITTING IS PERFORMED, ONE NEEDS TO INITIALIZE
# THE FOLLOWING ATTRIBUTES FOR THE FITTING TO WORK:
# self.datasets
# self.tfkgrid
# self.zdgrid
# self.limits
# These all should be dictionaries and cannot be easily included
# in the parameter files. However, here I use initiliaze them to
# null values.
self.datasets = {}
self.tfkgrid = {}
self.zdgrid = {}
self.limits = {}
for f in self.FIELDS:
self.datasets[f] = None
self.tfkgrid[f] = None
self.zdgrid[f] = None
self.limits[f] = None
self.nfields = len(self.FIELDS)
self.pixdx = self.PIXDX
self.drop = self.DROP
if hasattr(self, 'ALPHA'):
self.guess = array([self.ALPHA, self.MSTAR, self.LOGR0, self.SIGMA,
self.BETA])
self.M0 = self.M0
self.z_mean = self.Z_MEAN
if hasattr(self, 'MCFILE'):
# Same MCFILE for all fields
mc0 = bl.mconvert(self.MCFILE)
self.mc = {}
for f in self.FIELDS:
self.mc[f] = mc0
self.xout = array([])
# initialize arguments for bf.mlfunc
self.fft = 1 # use FFT for convolution
self.norm = -1. # Do not normalize the model b/c we'll use phistar
self.technique = self.TECHNIQUE
self.phistar = []
self.p_history = []
# models
self.sd = {}
self.intfrac = {}
self.mag_lolims = {}
# if self.ADD_INTERLOPER.lower() in ['yes','true']:
self.add_interloper = self.ADD_INTERLOPER
#self.sdfiles = self.SDFILES
#self.intfrac_files = self.INTFRAC_FILES
if self.add_interloper == False:
for f in self.FIELDS:
self.sd[f] = None
self.intfrac[f] = None
self.mag_lolims[f] = 21.0
self.nproc_model = self.NPROC_MODEL
# initialize the MCMC-related parameters
self.nwalkers = self.NWALKERS
self.nthreads_mcmc = self.NTHREADS_MCMC
self.nburn_mcmc = self.NBURN_MCMC
self.niter_mcmc = self.NITER_MCMC
#print "Total points used in fitting:", (len(mag1)+len(mag2))
####
self.sampler = None
self.verbose = 1
def bivmodel_MCMC_pymc(par, parfile, picklename, db='pickle', zlo=-1):
"""
par: best-fit parameter found by bivariate_fit.py
parfile: parameter file name of fitting run
"""
c = readpar(parfile)
# define 5 independent variables
alpha = pymc.Uniform(name='alpha', lower=-4.0, upper=0.0)
alpha.set_value(par[0])
Mstar = pymc.Uniform(name='Mstar', lower=-25.0, upper=-15.0)
Mstar.set_value(par[1])
logr0 = pymc.Uniform(name='logr0', lower=0.0, upper=5.0)
logr0.set_value(par[2])
sigma = pymc.Uniform(name='sigma', lower=0.0, upper=5.0)
sigma.set_value(par[3])
beta = pymc.Uniform(name='beta', lower=0.0, upper=10.0)
beta.set_value(par[4])
# define data
#if c['DROP'] == 'b': zlo = 3.0
#else: zlo = 4.0
mag1, re1, crit1 = cleandata(c['LBGCAT1'],
reerr_ratiolim=c['REERR_RATIOLIM1'],
chisqnulim=c['CHISQNULIM1'],
cullflags=c['CULLFLAGS'],
magautolim=c['MAGAUTOLIM1'],
limits=c['LIMITS1'],
drop=c['DROP'],
zlo=zlo)
data1 = array([mag1, log10(re1)])
print "shape(data1)", shape(data1)
n1 = shape(data1)[1] # n1 is the number of dropouts in the ACS dataset
mag2, re2, crit2 = cleandata(c['LBGCAT2'],
reerr_ratiolim=c['REERR_RATIOLIM2'],
chisqnulim=c['CHISQNULIM2'],
cullflags=c['CULLFLAGS'],
magautolim=c['MAGAUTOLIM2'],
limits=c['LIMITS2'],
drop=c['DROP'],
zlo=zlo)
data2 = array([mag2, log10(re2)])
print "shape(data2)", shape(data2)
kgrid1 = None
kgrid2 = None
if c['KERNEL1'] != "None":
f = open(c['KERNEL1'], 'rb')
kgrid1 = cPickle.load(f)
f.close()
if c['KERNEL2'] != "None":
f = open(c['KERNEL2'], 'rb')
kgrid2 = cPickle.load(f)
f.close()
pixdx = array([0.02, 0.02])
mc = bl.mconvert(c['MCFILE'])
if c['DROP'] == 'b':
meankcorr = mc(4.0)
else:
meankcorr = mc(5.0)
mlfunc_arg = [c['LIMITS1'], c['LIMITS2'], pixdx, kgrid1, kgrid2,\
c['WEIGHT1'], c['WEIGHT2'], c['M0'], c['ZDGRIDFILE1'],\
c['ZDGRIDFILE2'], c['MCFILE'], 1, meankcorr, 0, -1., 'phistar',\
c['DROP']]
# define data
logl0 = bf.mlfunc_RL(par, data1, data2, *mlfunc_arg)
value = concatenate([data1, data2], axis=1)
print shape(value) #, shape(value[:,0:n1]), shape(value[:,n1:])
@pymc.stochastic(observed=True)
def D(alpha=alpha,
Mstar=Mstar,
logr0=logr0,
sigma=sigma,
beta=beta,
value=value):
def logp(value, alpha, Mstar, logr0, sigma, beta):
logl = bf.mlfunc(array([alpha, Mstar, logr0, sigma, beta]),
value[:, :n1], value[:, n1:], *mlfunc_arg)
return (-1.) * (logl - logl0)
#return (-1.) * logl
if db == 'pickle':
kwarg = {'db': db, 'dbname': picklename}
else:
kwarg = {'db': db}
M = pymc.MCMC({'alpha':alpha, 'Mstar':Mstar, 'logr0':logr0, 'sigma':sigma, 'beta':beta,\
'D':D}, **kwarg)
M.use_step_method(pymc.Metropolis,
M.Mstar,
scale=1.,
proposal_sd=0.1,
proposal_distribution="Normal")
M.use_step_method(pymc.Metropolis,
M.alpha,
scale=1.,
proposal_sd=0.1,
proposal_distribution="Normal")
M.use_step_method(pymc.Metropolis,
M.logr0,
scale=1.,
proposal_sd=0.1,
proposal_distribution="Normal")
M.use_step_method(pymc.Metropolis,
M.sigma,
scale=1.,
proposal_sd=0.1,
proposal_distribution="Normal")
M.use_step_method(pymc.Metropolis,
M.beta,
scale=1.,
proposal_sd=0.1,
proposal_distribution="Normal")
return M, alpha, Mstar, logr0, sigma, beta, D
