class MockCatalog(object):
def __init__(self,iniFile,parDict,nemoOutputDir,noiseFile,params,parlist,mass_grid_log=None,z_grid=None,randoms=False):
Config = SafeConfigParser()
Config.optionxform=str
Config.read(iniFile)
if mass_grid_log:
logm_min,logm_max,logm_spacing = mass_grid_log
else:
logm_min = 12.7
logm_max = 15.72
logm_spacing = 0.04
if z_grid:
zmin,zmax,zdel = z_grid
else:
zmin = 0.0
zmax = 2.01
zdel = 0.1
self.fparams = {}
for (key, val) in Config.items('params'):
if ',' in val:
param, step = val.split(',')
self.fparams[key] = float(param)
else:
self.fparams[key] = float(val)
self.param_vals = alter_fparams(self.fparams,parlist,params)
bigDataDir = Config.get('general','bigDataDirectory')
self.clttfile = Config.get('general','clttfile')
self.constDict = dict_from_section(Config,'constants')
if mass_grid_log:
logm_min,logm_max,logm_spacing = mass_grid_log
else:
logm_min = 12.7
logm_max = 15.72
logm_spacing = 0.04
if z_grid:
zmin,zmax,zdel = z_grid
else:
zmin = 0.0
zmax = 2.01
zdel = 0.1
if randoms:
self.rand = 1
else:
self.rand = 0
self.mgrid = np.arange(logm_min,logm_max,logm_spacing)
self.zgrid = np.arange(zmin,zmax,zdel)
self.Medges = 10.**self.mgrid
self.Mcents = (self.Medges[1:]+self.Medges[:-1])/2.
self.Mexpcents = np.log10(self.Mcents)
self.zcents = (self.zgrid[1:]+self.zgrid[:-1])/2.
self.cc = ClusterCosmology(self.param_vals,self.constDict,clTTFixFile=self.clttfile)
self.HMF = Halo_MF(self.cc,self.mgrid,self.zgrid)
self.diagnosticsDir=nemoOutputDir+"diagnostics"
self.filteredMapsDir=nemoOutputDir+"filteredMaps"
self.tckQFit=simsTools.fitQ(parDict, self.diagnosticsDir, self.filteredMapsDir)
FilterNoiseMapFile = nemoOutputDir + noiseFile
MaskMapFile = self.diagnosticsDir + '/areaMask.fits'
self.rms_noise_map = read_MJH_noisemap(FilterNoiseMapFile,MaskMapFile)
self.wcs=astWCS.WCS(FilterNoiseMapFile)
self.fsky = 987.5/41252.9612 # in rads ACTPol D56-equ specific
self.scat_val = 0.2
self.seedval = np.int(np.round(time.time())) #1
def Total_clusters(self,fsky):
Nz = self.HMF.N_of_z()
print (np.trapz(self.HMF.N_of_z_500()*fsky,dx=np.diff(self.HMF.zarr)))
ans = np.trapz(Nz*fsky,dx=np.diff(self.HMF.zarr))
return ans
def Total_clusters_check(self,fsky):
Marr = np.outer(self.Mcents,np.ones(len(self.HMF.zarr)))
Medgearr = np.outer(self.Medges,np.ones(len(self.HMF.zarr)))
dn_dzdm = self.HMF.N_of_Mz(Marr,200)
N_z = np.zeros(self.HMF.zarr.size)
for i in range(self.HMF.zarr.size):
N_z[i] = np.dot(dn_dzdm[:,i],np.diff(Medgearr[:,i]))
N_z *= 4.*np.pi
ans = np.trapz(N_z*fsky,dx=np.diff(self.HMF.zarr))
#print ('test diff z',np.diff(self.HMF.zarr))
return ans
def create_basic_sample(self,fsky):
'''
Create simple mock catalog of Mass and Redshift by sampling the mass function
'''
#if (self.rand):
# Ntot100 = np.int32(np.ceil(self.Total_clusters(fsky))*4.) ## Note for randoms increasing the number density by factor of 100
#else:
# Ntot100 = np.int32(np.ceil(old_div(self.Total_clusters(fsky), 100.))) ## Note the default number of walkers in mcsample_mf is 100
#mlim = [np.min(self.mgrid),np.max(self.mgrid)]
#zlim = [np.min(self.zgrid),np.max(self.zgrid)]
#old MCMC
#samples = self.HMF.mcsample_mf(200.,Ntot100,mthresh = mlim,zthresh = zlim)
#return samples[:,0],samples[:,1]
Ntot100 = self.Total_clusters(fsky) # np.int32(np.ceil(self.Total_clusters(fsky)))
Ntot = np.int32(np.random.poisson(Ntot100))
print ("Mock cat gen internal counts and Poisson draw",Ntot100,Ntot)
zsamps, msamps = self.HMF.cpsample_mf(200.,Ntot)
#print (zsamps, msamps)
return zsamps, msamps
def create_basic_sample_Mat(self,fsky):
''' Create simple mock catalog of Mass and Redshift by sampling the mass function '''
nmzdensity = HMF.N_of_Mz(self.HMF.M200,200.)
Ndz = np.multiply(nmzdensity,np.diff(self.zgrid).reshape((1,self.zgrid.size-1)))
Nmz = np.multiply(Ndz,np.diff(10**self.mgrid).reshape((self.mgrid.size-1,1))) * 4.* np.pi
Ntot = Nmz.sum() * fsky
print ("fsky test",Ntot, np.int32(np.ceil(self.Total_clusters(fsky))))
np.random.seed(self.seedval)
nclusters = int(np.random.poisson(Ntot)) #if poisson else int(self.ntot)
mzs = np.zeros((nclusters,2),dtype=np.float32)
msamps = np.zeros((nclusters),dtype=np.float32)
zsamps = np.zeros((nclusters),dtype=np.float32)
print("Generating Nmz catalog...")
for i in range(nclusters):
linear_idx = np.random.choice(self.Nmz.size, p=self.Nmz.ravel()/float(self.Nmz.sum()))
x, y = np.unravel_index(linear_idx, self.Nmz.shape)
# mzs[i,0] = self.Mexpcents[x]
# mzs[i,1] = self.zcents[y]
msamps = np.random.uniform(self.Mexpcents[x].min(),self.Mexpcents[x].max())
zsamps = np.random.uniform(self.zcents[y].min(),self.zcents[y].max())
return zsamps, msamps
def plot_basic_sample(self,fname='default_mockcat.png',):
fsky = self.fsky
sampZ,sampM = self.create_basic_sample(fsky)
plt.figure()
plt.plot(sampZ,sampM,'x')
plt.savefig(fname, bbox_inches='tight',format='png')
return sampZ,sampM
def create_obs_sample(self,fsky):
#include observational effects like scatter and noise into the detection of clusters
sampZ,sampM = self.create_basic_sample(fsky)
nsamps = len(sampM)
Ytilde = sampM * 0.0
Om = old_div((self.param_vals['omch2'] + self.param_vals['ombh2']), (old_div(self.param_vals['H0'],100.))**2)
OL = 1.-Om
print("Omega_M", Om)
#the function call now includes cosmological dependences
for i in range(nsamps):
Ytilde[i], theta0, Qfilt = simsTools.y0FromLogM500(np.log10(self.param_vals['massbias']*10**sampM[i]/(old_div(self.param_vals['H0'],100.))), sampZ[i], self.tckQFit,sigma_int=self.param_vals['scat'],B0=self.param_vals['yslope'], H0 = self.param_vals['H0'], OmegaM0 = Om, OmegaL0 = OL)
#add scatter of 20% percent
np.random.seed(self.seedval)
ymod = np.exp(self.scat_val * np.random.randn(nsamps))
sampY0 = Ytilde*ymod
#calculate noise for a given object for a random place on the map and save coordinates
np.random.seed(self.seedval+1)
nmap = self.rms_noise_map #[::-1,:]
ylims = nmap.shape[0]
xlims = nmap.shape[1]
xsave = np.array([])
ysave = np.array([])
sampY0err = np.array([])
count = 0
while count < nsamps:
ytemp = np.int32(np.floor(np.random.uniform(0,ylims)))
xtemp = np.int32(np.floor(np.random.uniform(0,xlims)))
if nmap[ytemp,xtemp] > 0:
count += 1
xsave = np.append(xsave,xtemp)
ysave = np.append(ysave,ytemp)
sampY0err = np.append(sampY0err,nmap[ytemp,xtemp])
return xsave,ysave,sampZ,sampY0,sampY0err,old_div(sampY0,sampY0err),sampM
def plot_obs_sample(self,filename1='default_mockobscat',filename2='default_obs_mock_footprint'):
fsky = self.fsky
xsave,ysave,sampZ,sampY0,sampY0err,SNR,sampM = self.create_obs_sample(fsky)
ind = np.where(SNR >= 5.6)[0]
plt.figure()
plt.plot(sampZ,sampM,'x')
plt.plot(sampZ[ind],sampM[ind],'o')
plt.savefig(filename1+'.png', bbox_inches='tight',format='png')
nmap = self.rms_noise_map # np.flipud(self.rms_noise_map)#[::-1,:]
plt.figure(figsize=(40,6))
plt.imshow(nmap,cmap='Blues')
plt.plot(xsave[ind],ysave[ind],'ko')
plt.colorbar()
plt.savefig(filename2+'.png', bbox_inches='tight',format='png')
return xsave,ysave,sampZ,sampY0,sampY0err,SNR,sampM
def write_test_cat_toFits(self, filedir,filename):
'''
Write out the catalog
'''
f1 = filedir+filename+'_testsamp_mz'
sampZ,sampM = self.plot_basic_sample(f1)
sampZerr = sampZ * 0.0
sampMerr = sampM * 0.0
#ind = np.where(10**sampM >= 2.0*10**(np.min(self.mgrid)))[0]
clusterID = np.arange(len(sampM)).astype(str)
hdu = fits.BinTableHDU.from_columns(
[fits.Column(name='Cluster_ID', format='20A', array=clusterID),
fits.Column(name='redshift', format='E', array=sampZ),
fits.Column(name='redshift_err', format='E', array=sampZerr),
fits.Column(name='Mass', format='E', array=sampM),
fits.Column(name='Mass_err', format='E', array=sampMerr),])
hdu.writeto(filedir+filename+'.fits',overwrite=True)
return 0
def test_cat_samp(self, filedir,filename, mcut):
'''
Write out the catalog
'''
f1 = filedir+filename+'_testsamp_mz'
sampZ,sampM = self.plot_basic_sample(f1)
print (sampM)
return 0
def test_Mockcat_Nums(self, mmin):
'''
Quick write out of the number clusters in the mock catalog, for testing
'''
sampZ,sampM = self.create_basic_sample(self.fsky)
ind = np.where(sampM >= mmin)[0]
return len(ind)
def write_obs_cat_toFits(self, filedir,filename):
'''
Write out the catalog
'''
f1 = filedir+filename+'_mockobscat'
f2 = filedir+filename+'_obs_mock_footprint'
xsave,ysave,z,sampY0,sampY0err,SNR,sampM = self.plot_obs_sample(filename1=f1,filename2=f2)
ind = np.where(SNR >= 4.0)[0]
ind2 = np.where(SNR >= 5.6)[0]
print("number of clusters SNR >= 5.6", len(ind2), " SNR >= 4.0",len(ind))
RAdeg = xsave*0.0
DECdeg = ysave*0.0
count = 0
for xsv, ysv in zip(xsave,ysave):
ra,dec = self.wcs.pix2wcs(xsv,ysv)
RAdeg[count] = ra
DECdeg[count] = dec
count +=1
clusterID = ind.astype(str)
hdu = fits.BinTableHDU.from_columns(
[fits.Column(name='Cluster_ID', format='20A', array=clusterID),
fits.Column(name='x_ind', format='E', array=xsave[ind]),
fits.Column(name='y_ind', format='E', array=ysave[ind]),
fits.Column(name='RA', format='E', array=RAdeg[ind]),
fits.Column(name='DEC', format='E', array=DECdeg[ind]),
fits.Column(name='redshift', format='E', array=z[ind]),
fits.Column(name='redshiftErr', format='E', array=z[ind]*0.0),
fits.Column(name='fixed_y_c', format='E', array=sampY0[ind]*1e4),
fits.Column(name='err_fixed_y_c', format='E', array=sampY0err[ind]*1e4),
fits.Column(name='fixed_SNR', format='E', array=SNR[ind]),
fits.Column(name='M500', format='E', array=sampM[ind]),])
hdu.writeto(filedir+filename+'.fits',overwrite=True)
return 0
def Add_completeness(self,filedir,filename,compfile,zcut=False):
'''
Add in observing and filter completeness to the selection the mock catalogs
'''
fitsfile = filedir+filename+'.fits'
ID,x,y,ra,dec,z,zerr,Y0,Y0err,SNR,M500 = read_full_mock_cat(fitsfile)
rands = np.random.uniform(0,1,len(z))
compvals = np.load(compfile)
inter = interp2d(compvals['log10M500c'],compvals['z'],compvals['M500Completeness'],kind='cubic',fill_value=0)
use = 0.0*z
for ii in range(len(z)):
comp = inter(M500[ii],z[ii])
if (comp > rands[ii]):
use[ii] = 1
if (zcut):
ind = np.where((z < zcut)*(use > 0))[0]
print("number of clusters SNR >= 4.0 plus completeness",len(ind))
fitsout = filedir+filename+'_comp_added.fits'
hdu = fits.BinTableHDU.from_columns(
[fits.Column(name='Cluster_ID', format='20A', array=ID[ind]),
fits.Column(name='x_ind', format='E', array=x[ind]),
fits.Column(name='y_ind', format='E', array=y[ind]),
fits.Column(name='RA', format='E', array=ra[ind]),
fits.Column(name='DEC', format='E', array=dec[ind]),
fits.Column(name='redshift', format='E', array=z[ind]),
fits.Column(name='redshiftErr', format='E', array=zerr[ind]),
fits.Column(name='fixed_y_c', format='E', array=Y0[ind]),
fits.Column(name='err_fixed_y_c', format='E', array=Y0err[ind]),
fits.Column(name='fixed_SNR', format='E', array=SNR[ind]),
fits.Column(name='M500', format='E', array=M500[ind]),])
hdu.writeto(fitsout,overwrite=True)
return 0
bounds_error=False,
fill_value=np.inf)
hmf = Halo_MF(cc, Mexp_edges, z_edges)
SZProf = SZ_Cluster_Model(cc,
clusterDict,
rms_noises=noise,
fwhms=beam,
freqs=freq,
lknee=lknee,
alpha=alpha)
fsky = 0.4
N1 = hmf.N_of_z() * fsky
#hmf.sigN = np.loadtxt("temp.txt")
try:
hmf.sigN = np.loadtxt("tempSigN.txt")
N2 = hmf.N_of_z_SZ(SZProf) * fsky
except:
N2 = hmf.N_of_z_SZ(SZProf) * fsky
np.savetxt("tempSigN.txt", hmf.sigN)
pl = Plotter()
pl.plot2d(hmf.sigN)
pl.done(outDir + "signRefactor.png")
pl = Plotter(scaleY='log')
