def Build_TinkerCatalog(Mrcut=18):
''' Preprocess the group catalog data into a more python friendly format
with appropriate *little h* corrections!
'''
h = 0.7
M_cut = Tinker_Masscut(Mrcut)
# Read Group Catalog GalData
galdata_file = ''.join([
UT.dir_dat(), 'tinker2011catalogs/', 'clf_groups_M',
str(Mrcut), '_',
str(M_cut), '_D360.', 'galdata_corr.fits'
])
gal_data = mrdfits(galdata_file)
catalog = {}
for column in gal_data.__dict__.keys():
column_data = getattr(gal_data, column)
if column == 'stellmass':
# stellmass is in units of Msol/h^2
# why jeremy why?/?
column_data = column_data / h**2
catalog['mass'] = np.log10(column_data) # convert to log Mass
elif column == 'ssfr':
column_data += np.log10(h**2) # little h #*(@*#$
catalog['ssfr'] = column_data
elif column == 'cz': # convert to z
catalog['z'] = column_data / 299792.458
elif column in ['ra', 'dec']:
catalog[column] = column_data * 57.2957795
else:
catalog[column] = column_data
catalog['sfr'] = catalog['mass'] + catalog[
'ssfr'] # calculate SFR form mass and ssfr
# Read Group Catalog probability
prob_file = ''.join([
UT.dir_dat(), '/tinker2011catalogs/', 'clf_groups_M',
str(Mrcut), '_',
str(M_cut), '_D360.', 'prob.fits'
])
prob_data = mrdfits(prob_file) # import probability file
for column in prob_data.__dict__.keys():
catalog[column] = getattr(prob_data, column)
tinker_file = ''.join([
UT.dir_dat(), 'tinker2011catalogs/', 'GroupCat.Mr',
str(Mrcut), '.Mass',
str(M_cut), '.D360.hdf5'
])
f = h5py.File(tinker_file, 'w')
grp = f.create_group('data')
for key in catalog.keys():
grp.create_dataset(key, data=catalog[key])
f.close()
return None
def Read(self):
'''
'''
sfr_class = []
for sfq in ['star-forming', 'quiescent']:
file_dir = code_dir()+'dat/observations/envcount/'
if sfq == 'star-forming':
sfq_str = 'active'
else:
sfq_str = sfq
sdss_file = ''.join([file_dir,
'envcount_cylr2.5h35_thresh75_sdss_', sfq_str, '_z0.05_0.12_primuszerr.fits'])
primus_file = ''.join([file_dir,
'envcount_cylr2.5h35_thresh75_', sfq_str, '_z0.2_1.0_lit.fits'])
# redshift determines SDSS or PRIMUS sample
if self.redshift_bin < 0.2:
galdata = mrdfits(sdss_file)
else:
galdata = mrdfits(primus_file)
# environment cuts
if self.environment == 'no':
envcuts = (galdata.envcount == 0.)
else:
raise NotImplementedError
# redshift cuts
zlow = [0., 0.2, 0.4, 0.6, 0.8]
zhigh = [0.2, 0.4, 0.6, 0.8, 1.0]
i_z = int(np.floor(self.redshift_bin/0.2))
zcuts = (galdata.redshift >= zlow[i_z]) & (galdata.redshift < zhigh[i_z])
all_cuts = np.where(
envcuts & zcuts &
(galdata.mass > galdata.masslimit) &
(galdata.edgecut == 1)
)
for key in galdata.__dict__.keys():
try:
setattr(self, key,
np.concatenate([getattr(self,key), getattr(galdata, key)[all_cuts]]))
except AttributeError:
setattr(self, key, getattr(galdata, key)[all_cuts])
sfr_class_tmp = np.chararray(len(all_cuts[0]), itemsize=16)
sfr_class_tmp[:] = sfq
sfr_class.append(sfr_class_tmp)
setattr(self, 'sfr_class', np.concatenate(sfr_class))
return None
def _iSEDfitMatch(self):
''' Match the GroupCat galaxies with iSEDfit galaxy properties from
John Moustakas's MFData objects. The matching is done using PyDL's
spherematch
'''
# import SDSS MFdata catalog
mfdata = mrdfits(code_dir(), 'dat/observations/mfdata_all_supergrid01_sdss.fits.gz')
spherematch_time = time.time()
match1, match2, d_match = spherematch(
self.ra, self.dec, mfdata.ra, mfdata.dec, 0.001)
print 'spherematch took ', time.time() - spherematch_time
iSEDfit_mass = np.repeat(-999., len(self.ra))
iSEDfit_SFR = np.repeat(-999., len(self.ra))
iSEDfit_SSFR = np.repeat(-999., len(self.ra))
if np.max(self.z[match1] - mfdata.z[match2]) > 0.1:
raise ValueError
#wrong = np.argmax(self.z[match1] - mfdata.z[match2])
#print self.ra[match1[wrong]], self.dec[match1[wrong]], self.z[match1[wrong]]
#print mfdata.ra[match2[wrong]], mfdata.dec[match2[wrong]], mfdata.z[match2[wrong]]
iSEDfit_mass[match1] = mfdata.mass[match2]
iSEDfit_SFR[match1] = mfdata.sfr[match2]
iSEDfit_SSFR[match1] = iSEDfit_SFR[match1]-iSEDfit_mass[match1]
setattr(self, 'iSEDfit_mass', iSEDfit_mass)
setattr(self, 'iSEDfit_sfr', iSEDfit_SFR)
setattr(self, 'iSEDfit_ssfr', iSEDfit_SSFR)
return None
def _ReadGroupCat_GalData(self):
'''
'''
h = 0.7
galdata_file = ''.join([
code_dir(), 'dat/observations/',
'clf_groups_M', str(self.Mrcut), '_', str(self.masscut), '_D360.',
'galdata_corr.fits'])
gal_data = mrdfits(galdata_file)
#print gal_data.__dict__.keys()
for column in gal_data.__dict__.keys():
column_data = getattr(gal_data, column)
if column == 'stellmass':
# stellmass is in units of Msol/h^2
column_data = column_data / h**2
# convert to log Mass
setattr(gal_data, 'mass', np.log10(column_data))
elif column == 'ssfr':
column_data = column_data + np.log10(h**2)
# convert to log Mass
setattr(gal_data, 'ssfr', column_data)
elif column == 'cz':
# convert to z else:
setattr(gal_data, 'z', column_data/299792.458)
else:
pass
setattr(gal_data, 'sfr', gal_data.mass + gal_data.ssfr) # get sfr values
return gal_data
def _ReadGroupCat_Prob(self):
''' wrapper for reading in probability galaxy data
'''
file = ''.join([
code_dir(), 'dat/observations/',
'clf_groups_M', str(self.Mrcut), '_', str(self.masscut), '_D360.',
'prob.fits'])
prob_data = mrdfits(file) # import probability file
return prob_data
def Write(self):
''' Write into hdf5 file
'''
# Read in .fits file
drpall = Fits.mrdfits(self.fits_file)
# write to hdf5 file
f = h5py.File(self.hdf5_file, 'w')
nsa_grp = f.create_group('nsa_data')
for key in drpall.__dict__.keys():
nsa_grp.create_dataset(key, data=getattr(drpall, key))
f.close()
return None
def PreProcess(name, version):
''' PreProcess disparate data files in order to general some consistent
data file convenient for the fortran files to read in
notes
-----
* this code is messy because of all the different ways people distribute
catalogs. I have not figured out a dignified way to do this...
'''
CI = CatalogInventory()
catalog_info = CI.LoadCatalog(name, version)
for type in ['data', 'ran']:
if name == 'eBOSS_QSO': # eBOSS QSO catalogs
if version == 'v1.6_N': # version 1.6 North
# this particular catalog is for the P(k) comparison tests of
# https://trac.sdss.org/wiki/eBOSS/QGC/pk_comparison
file = ''.join([catalog_info['dir_data'], catalog_info['unprocessed_'+type+'_file']])
data_fits = mrdfits(file) # data from .fits file ... ugh
cuts = np.where((data_fits.z > np.float(catalog_info['z_min'])) &
(data_fits.z < np.float(catalog_info['z_max'])))
print 'impose redshift limit ', catalog_info['z_min'], ' < z < ', catalog_info['z_max']
print 1.-np.float(len(cuts[0]))/np.float(len(data_fits.z)), ' of galaxies removed in preprocessing'
# of the many data columns extract: ra, dec, z, n(z), wfkp, wsys, wnoz, wcp
columns = catalog_info['unprocessed_'+type+'_column']
data_list, data_fmt = [], [] # list of all the data, and list of data formats
for col in columns:
data_list.append(getattr(data_fits, col)[cuts])
if col == 'nz':
fmt = '%.5e'
else:
fmt = '%10.5f'
data_fmt.append(fmt)
data_hdr = 'columns : '+', '.join(columns)
# output file name
if type == 'data': # galaxy catalog
file_out = ''.join([catalog_info['dir_data'], catalog_info['file'], '.dat'])
elif type == 'ran': # random catalog
file_out = ''.join([catalog_info['dir_data'], catalog_info['file'], '.ran'])
else:
raise NotImplementedError
np.savetxt(file_out,
(np.vstack(np.array(data_list))).T,
fmt=data_fmt, delimiter='\t', header=data_hdr)
return None
def Build_MPAJHU_TinkerCatalog_ASCII(Mrcut=18):
''' Append MPA-JHU SSFR values to the Tinker et al. (2011) catalog.
The main purpose is to try to reproduce the Kauffmann et al. (2013) results.
Galaxies are matched to each other through spherematch.
'''
# import Tinker et al. (2011) catalog with specified Mr cut
catalog = TinkerCatalog(Mrcut=Mrcut)
# import MPA-JHU catalog
mpajhu_gals = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_info_dr7_v5_2.fit']))
# SFR total
mpajhu_sfrtot = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_totsfr_dr7_v5_2.fits']))
# SFR fiber
mpajhu_sfrfib = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_fibsfr_dr7_v5_2.fits']))
# SSFR total
mpajhu_ssfrtot = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_totspecsfr_dr7_v5_2.fits']))
# SSFR fiber
mpajhu_ssfrfib = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_fibspecsfr_dr7_v5_2.fits']))
# stellar mass total
mpajhu_masstot = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'totlgm_dr7_v5_2.fit']))
# stellar mass fiber
mpajhu_massfib = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'fiblgm_dr7_v5_2.fit']))
t_spherematch = time.time()
match = spherematch(catalog['ra'], catalog['dec'], mpajhu_gals.ra,
mpajhu_gals.dec, 0.000833333)
print 'Spherematch with matchlenght = ', 0.000833333
print 'takes ', time.time() - t_spherematch, 'seconds'
print 1. - np.float(len(match[0])) / np.float(len(
catalog['ra'])), 'of the VAGC galaxies'
print 'do not have matches, likely due to fiber collisions'
if len(match[0]) != len(np.unique(match[0])):
raise ValueError
# save the MPAJHU indices, jsut in case
catalog['mpajhu_index'] = np.repeat(-999, len(catalog['ra']))
catalog['mpajhu_index'][match[0]] = match[1]
# append SFR, SSFR, and mass values to catalog
for col in [
'sfr_tot_mpajhu', 'sfr_fib_mpajhu', 'ssfr_tot_mpajhu',
'ssfr_fib_mpajhu', 'mass_tot_mpajhu', 'mass_fib_mpajhu'
]: # initiate arrays
catalog[col] = np.repeat(-999., len(catalog['ra']))
catalog['sfr_tot_mpajhu'][match[0]] = mpajhu_sfrtot.median[match[1]]
catalog['sfr_fib_mpajhu'][match[0]] = mpajhu_sfrfib.median[match[1]]
catalog['ssfr_tot_mpajhu'][match[0]] = mpajhu_ssfrtot.median[match[1]]
catalog['ssfr_fib_mpajhu'][match[0]] = mpajhu_ssfrfib.median[match[1]]
catalog['mass_tot_mpajhu'][match[0]] = mpajhu_masstot.median[match[1]]
catalog['mass_fib_mpajhu'][match[0]] = mpajhu_massfib.median[match[1]]
print mpajhu_massfib.median[match[1]]
first_cols = [
'id_gal', 'ra', 'dec', 'z', 'mass', 'sfr', 'ssfr', 'mass_tot_mpajhu',
'mass_fib_mpajhu', 'sfr_tot_mpajhu', 'sfr_fib_mpajhu',
'ssfr_tot_mpajhu', 'ssfr_fib_mpajhu'
]
data_fmt = []
data_list = []
for i_key, key in enumerate(first_cols):
data_list.append(catalog[key])
if key == 'id_gal':
data_fmt.append('%i')
else:
data_fmt.append('%10.5f')
later_cols = []
for key in catalog.keys():
if key not in first_cols:
later_cols.append(key)
for key in later_cols:
data_list.append(catalog[key])
if 'id' in key:
data_fmt.append('%i')
elif 'index' in key:
data_fmt.append('%i')
elif key == 'n_sersic':
data_fmt.append('%i')
elif key == 'stellmass':
data_fmt.append('%1.5e')
else:
data_fmt.append('%10.5f')
str_header = ', '.join(first_cols + later_cols)
M_cut = Tinker_Masscut(Mrcut)
mpajhu_tinker_file = ''.join([
UT.dir_dat(), 'tinker2011catalogs/', 'GroupCat.Mr',
str(Mrcut), '.Mass',
str(M_cut), '.D360.MPAJHU.dat'
])
np.savetxt(mpajhu_tinker_file, (np.vstack(np.array(data_list))).T,
fmt=data_fmt,
delimiter='\t',
header=str_header)
return None
def Build_VAGCdr72_MPAJHU(Ascii=False):
''' Build VAGC dr72 with cross referenced MPAJHU stellar masses
and SSFRs.
'''
# import VAGC dr72bright34
vagc_dr72 = VAGCdr72bright34_Catalog()
print len(vagc_dr72['ra']), ', VAGC dr72bright34 galaxies'
# import MPA-JHU catalog
mpajhu_gals = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_info_dr7_v5_2.fit']))
# SFR total
mpajhu_sfrtot = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_totsfr_dr7_v5_2.fits']))
# SFR fiber
mpajhu_sfrfib = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_fibsfr_dr7_v5_2.fits']))
# SSFR total
mpajhu_ssfrtot = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_totspecsfr_dr7_v5_2.fits']))
# SSFR fiber
mpajhu_ssfrfib = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_fibspecsfr_dr7_v5_2.fits']))
# stellar mass total
mpajhu_masstot = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'totlgm_dr7_v5_2.fit']))
# stellar mass fiber
mpajhu_massfib = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'fiblgm_dr7_v5_2.fit']))
catalog = {}
catalog['ra'] = vagc_dr72['ra']
catalog['dec'] = vagc_dr72['dec']
catalog['z'] = vagc_dr72['z']
for i_band, band in enumerate(['u', 'g', 'r', 'i', 'z']):
catalog['M_' + band] = vagc_dr72['M_' + band]
t_spherematch = time.time()
match = spherematch(catalog['ra'], catalog['dec'], mpajhu_gals.ra,
mpajhu_gals.dec, 0.000833333)
print 'Spherematch with matchlenght = ', 0.000833333
print 'takes ', time.time() - t_spherematch, 'seconds'
print 1. - np.float(len(match[0])) / np.float(len(
catalog['ra'])), 'of the VAGC galaxies'
print 'do not have matches'
if len(match[0]) != len(np.unique(match[0])):
raise ValueError
# save the MPAJHU indices, jsut in case
catalog['mpajhu_index'] = np.repeat(-999, len(catalog['ra']))
catalog['mpajhu_index'][match[0]] = match[1]
# append SFR, SSFR, and mass values to catalog
for col in [
'sfr_tot', 'sfr_fib', 'ssfr_tot', 'ssfr_fib', 'mass_tot',
'mass_fib'
]: # initiate arrays
catalog[col] = np.repeat(-999., len(catalog['ra']))
catalog['sfr_tot'][match[0]] = mpajhu_sfrtot.median[match[1]]
catalog['sfr_fib'][match[0]] = mpajhu_sfrfib.median[match[1]]
catalog['ssfr_tot'][match[0]] = mpajhu_ssfrtot.median[match[1]]
catalog['ssfr_fib'][match[0]] = mpajhu_ssfrfib.median[match[1]]
catalog['mass_tot'][match[0]] = mpajhu_masstot.median[match[1]]
catalog['mass_fib'][match[0]] = mpajhu_massfib.median[match[1]]
mpajhu_file = ''.join(
[UT.dir_dat(), 'vagc/', 'VAGCdr72.MPAJHU.nocut.hdf5'])
f = h5py.File(mpajhu_file, 'w')
grp = f.create_group('data')
for key in catalog.keys():
grp.create_dataset(key, data=catalog[key])
f.close()
if Ascii: # write to Ascii (for jeremy)
mpajhu_file = ''.join(
[UT.dir_dat(), 'vagc/', 'VAGCdr72.MPAJHU.nocut.dat'])
column_order = [
'ra', 'dec', 'z', 'mass_tot', 'sfr_tot', 'ssfr_tot', 'mass_fib',
'sfr_fib', 'ssfr_fib'
]
data_list = []
data_fmt = ['%10.5f' for i in range(len(column_order))]
str_header = ''
for col in column_order:
data_list.append(catalog[col])
if 'mass' in col:
str_header += ' ' + col + ' (Msun),'
elif 'sfr' in col:
if 'ssfr' not in col:
str_header += ' ' + col + ' (Msun/yr),'
else:
str_header += ' ' + col + ','
else:
str_header += ' ' + col + ','
np.savetxt(mpajhu_file, (np.vstack(np.array(data_list))).T,
fmt=data_fmt,
delimiter='\t',
header=str_header)
return None
def Build_KauffmannParent():
''' Try to create the parent sample of Kauffmann et al.(2013)
'''
# import VAGC dr72bright34
vagc_dr72 = VAGCdr72bright34_Catalog()
# import MPA-JHU catalog
mpajhu_gals = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_info_dr7_v5_2.fit']))
# SFR total
mpajhu_sfrtot = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_totsfr_dr7_v5_2.fits']))
# SFR fiber
mpajhu_sfrfib = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_fibsfr_dr7_v5_2.fits']))
# SSFR total
mpajhu_ssfrtot = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_totspecsfr_dr7_v5_2.fits']))
# SSFR fiber
mpajhu_ssfrfib = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_fibspecsfr_dr7_v5_2.fits']))
# stellar mass total
mpajhu_masstot = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'totlgm_dr7_v5_2.fit']))
# stellar mass fiber
mpajhu_massfib = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'fiblgm_dr7_v5_2.fit']))
catalog = {}
catalog['ra'] = vagc_dr72['ra']
catalog['dec'] = vagc_dr72['dec']
catalog['z'] = vagc_dr72['z']
for i_band, band in enumerate(['u', 'g', 'r', 'i', 'z']):
catalog['M_' + band] = vagc_dr72['M_' + band]
# pre cut
cut_z = (catalog['z'] > 0.017) & (catalog['z'] < 0.03)
pre_cuts = np.where(cut_z) #& cut_stellarmass & cut_absmag)
for key in catalog.keys():
catalog[key] = catalog[key][pre_cuts]
t_spherematch = time.time()
match = spherematch(catalog['ra'], catalog['dec'], mpajhu_gals.ra,
mpajhu_gals.dec, 0.000833333)
print 'Spherematch with matchlenght = ', 0.000833333
print 'takes ', time.time() - t_spherematch, 'seconds'
print 1. - np.float(len(match[0])) / np.float(len(
catalog['ra'])), 'of the VAGC galaxies'
print 'do not have matches'
if len(match[0]) != len(np.unique(match[0])):
raise ValueError
# save the MPAJHU indices, jsut in case
catalog['mpajhu_index'] = np.repeat(-999, len(catalog['ra']))
catalog['mpajhu_index'][match[0]] = match[1]
# append SFR, SSFR, and mass values to catalog
for col in [
'sfr_tot_mpajhu', 'sfr_fib_mpajhu', 'ssfr_tot_mpajhu',
'ssfr_fib_mpajhu', 'mass_tot_mpajhu', 'mass_fib_mpajhu'
]: # initiate arrays
catalog[col] = np.repeat(-999., len(catalog['ra']))
catalog['sfr_tot_mpajhu'][match[0]] = mpajhu_sfrtot.median[match[1]]
catalog['sfr_fib_mpajhu'][match[0]] = mpajhu_sfrfib.median[match[1]]
catalog['ssfr_tot_mpajhu'][match[0]] = mpajhu_ssfrtot.median[match[1]]
catalog['ssfr_fib_mpajhu'][match[0]] = mpajhu_ssfrfib.median[match[1]]
catalog['mass_tot_mpajhu'][match[0]] = mpajhu_masstot.median[match[1]]
catalog['mass_fib_mpajhu'][match[0]] = mpajhu_massfib.median[match[1]]
# kauffmann et al.(2013) cuts
cut_stellarmass = (catalog['mass_tot_mpajhu'] > 9.25)
cut_absmag = (catalog['M_r'] < -16.) & (catalog['M_r'] > -24.)
cut_match = (catalog['mpajhu_index'] != -999)
final_cuts = np.where(cut_stellarmass & cut_absmag & cut_match)
for key in catalog.keys():
catalog[key] = catalog[key][final_cuts]
mpajhu_file = ''.join(
[UT.dir_dat(), 'vagc/', 'VAGCdr72.Kauff2013cut.hdf5'])
f = h5py.File(mpajhu_file, 'w')
grp = f.create_group('data')
for key in catalog.keys():
grp.create_dataset(key, data=catalog[key])
f.close()
return None
def Build_MPAJHU_TinkerCatalog(Mrcut=18):
''' Append MPA-JHU SSFR values to the Tinker et al. (2011) catalog.
The main purpose is to try to reproduce the Kauffmann et al. (2013) results.
Galaxies are matched to each other through spherematch.
'''
# import Tinker et al. (2011) catalog with specified Mr cut
catalog = TinkerCatalog(Mrcut=Mrcut)
# import MPA-JHU catalog
mpajhu_gals = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_info_dr7_v5_2.fit']))
# SFR total
mpajhu_sfrtot = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_totsfr_dr7_v5_2.fits']))
# SFR fiber
mpajhu_sfrfib = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_fibsfr_dr7_v5_2.fits']))
# SSFR total
mpajhu_ssfrtot = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_totspecsfr_dr7_v5_2.fits']))
# SSFR fiber
mpajhu_ssfrfib = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'gal_fibspecsfr_dr7_v5_2.fits']))
# stellar mass total
mpajhu_masstot = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'totlgm_dr7_v5_2.fit']))
# stellar mass fiber
mpajhu_massfib = mrdfits(''.join(
[UT.dir_dat(), 'mpa_jhu/', 'fiblgm_dr7_v5_2.fit']))
t_spherematch = time.time()
match = spherematch(catalog['ra'], catalog['dec'], mpajhu_gals.ra,
mpajhu_gals.dec, 0.000833333)
print 'Spherematch with matchlenght = ', 0.000833333
print 'takes ', time.time() - t_spherematch, 'seconds'
print 1. - np.float(len(match[0])) / np.float(len(
catalog['ra'])), 'of the VAGC galaxies'
print 'do not have matches, likely due to fiber collisions'
if len(match[0]) != len(np.unique(match[0])):
raise ValueError
# save the MPAJHU indices, jsut in case
catalog['mpajhu_index'] = np.repeat(-999, len(catalog['ra']))
catalog['mpajhu_index'][match[0]] = match[1]
# append SFR, SSFR, and mass values to catalog
for col in [
'sfr_tot_mpajhu', 'sfr_fib_mpajhu', 'ssfr_tot_mpajhu',
'ssfr_fib_mpajhu', 'mass_tot_mpajhu', 'mass_fib_mpajhu'
]: # initiate arrays
catalog[col] = np.repeat(-999., len(catalog['ra']))
catalog['sfr_tot_mpajhu'][match[0]] = mpajhu_sfrtot.median[match[1]]
catalog['sfr_fib_mpajhu'][match[0]] = mpajhu_sfrfib.median[match[1]]
catalog['ssfr_tot_mpajhu'][match[0]] = mpajhu_ssfrtot.median[match[1]]
catalog['ssfr_fib_mpajhu'][match[0]] = mpajhu_ssfrfib.median[match[1]]
catalog['mass_tot_mpajhu'][match[0]] = mpajhu_masstot.median[match[1]]
catalog['mass_fib_mpajhu'][match[0]] = mpajhu_massfib.median[match[1]]
# trim galaxies without matches
hasmatch = np.where(catalog['mpajhu_index'] != -999)
for key in catalog.keys():
key_val = catalog[key]
catalog[key] = key_val[hasmatch]
catalog['mpajhu_tinker_index'] = hasmatch[0]
M_cut = Tinker_Masscut(Mrcut)
mpajhu_tinker_file = ''.join([
UT.dir_dat(), 'tinker2011catalogs/', 'GroupCat.Mr',
str(Mrcut), '.Mass',
str(M_cut), '.D360.MPAJHU.hdf5'
])
f = h5py.File(mpajhu_tinker_file, 'w')
grp = f.create_group('data')
for key in catalog.keys():
grp.create_dataset(key, data=catalog[key])
f.close()
return None
def build(self):
''' Build Fibercollided mock catalogs using specific idl routines or by using the given fiber collision weights
Parameters
----------
cat_corr : catalog correction dictionary
Notes
-----
'''
catdict = (self.cat_corr)['catalog']
catalog_name = catdict['name'].lower()
data_cols = self.datacolumns()
data_fmts = self.datacols_fmt()
data_hdrs = self.datacols_header()
data_dir = direc('data', self.cat_corr)
if catalog_name == 'nseries':
# N-series mocks (high quality mocks with actual CMASS tiling)
# original file
orig_file = ''.join([
data_dir,
'CutskyN', str(catdict['n_mock']), '.rdzwc'
])
orig_ra, orig_dec, orig_z, orig_wfc, orig_zupw, orig_upw_index = np.loadtxt(
orig_file,
unpack = True,
usecols = [0,1,2,4,5,6]
)
# file with mask completeness
mask_file = ''.join([data_dir, 'CutskyN', str(catdict['n_mock']), '.mask_info'])
orig_wcomp = np.loadtxt(mask_file, unpack=True, usecols=[0])
coll = np.where(orig_wfc == 0.0)
# data column list
data_list = [
orig_ra,
orig_dec,
orig_z,
orig_wfc,
orig_wcomp,
orig_zupw,
orig_upw_index
]
# handle upweighted redshift/index discrepancies by simply ignoring them.
if not np.array(orig_z[orig_upw_index.astype(int)[coll]] == orig_zupw[coll]).all():
wrong_index = (coll[0])[np.where(orig_z[orig_upw_index.astype(int)[coll]] != orig_zupw[coll])[0]]
warn_message = ''.join([
'upweighted galaxy redshift and index data discrepancies in ',
self.file(),
' ',
str(len(wrong_index)),
' galaxies affected'
])
warnings.warn(warn_message, Warning)
if len(wrong_index) > 0:
for i_data, datum in enumerate(data_list):
data_list[i_data] = np.delete(datum, wrong_index)
elif catalog_name == 'qpm':
# Quick Particle Mesh mocks from Jeremy (quantity over quality mocks)
orig_file = ''.join([
'/mount/riachuelo2/rs123/BOSS/QPM/cmass/mocks/dr12d/ngc/data/',
'a0.6452_', str("%04d" % catdict['n_mock']), '.dr12d_cmass_ngc.rdz'])
ra, dec, z, wfc = np.loadtxt(orig_file, unpack=True, usecols=[0,1,2,4])
orig_info_file = ''.join([
'/mount/riachuelo2/rs123/BOSS/QPM/cmass/mocks/dr12d/ngc/data/',
'a0.6452_', str("%04d" % catdict['n_mock']), '.dr12d_cmass_ngc.rdz.info'])
# gal_id, comp, z_real, z_red, mass_halo, flag_sta, id_halo
comp = np.loadtxt(orig_info_file, unpack=True, skiprows=3, usecols=[1])
if catdict['n_mock'] in (44, 46, 52, 53, 54, 56, 61, 707, 756, 794, 819, 831, 835, 838):
orig_veto_file = ''.join([
'/mount/riachuelo1/hahn/data/QPM/dr12d/',
'a0.6452_', str("%04d" % catdict['n_mock']), '.dr12d_cmass_ngc.veto'])
else:
orig_veto_file = ''.join([
'/mount/riachuelo2/rs123/BOSS/QPM/cmass/mocks/dr12d/ngc/data/',
'a0.6452_', str("%04d" % catdict['n_mock']), '.dr12d_cmass_ngc.veto'])
veto = np.loadtxt(orig_veto_file)
n_gal = len(veto)
if len(ra) != n_gal:
print orig_file
print orig_veto_file
raise ValueError('veto mask doesnt match')
vetomask = np.where(veto == 0)
# data column list
data_list = [
ra[vetomask],
dec[vetomask],
z[vetomask],
wfc[vetomask],
comp[vetomask]
]
elif catalog_name == 'bigmd':
# Big MultiDark
P0 = 20000.0
# read original random catalog
data_dir = '/mount/riachuelo1/hahn/data/BigMD/'
if 'version' in catdict.keys():
if catdict['version'] == 'nowiggles':
# simulation with no wiggles
orig_file = ''.join([data_dir, 'nowiggles/BigMD-cmass-dr12v4-nowiggle-veto.dat'])
else:
raise NotImplementedError
#orig_file = ''.join([data_dir, 'bigMD-cmass-dr12v4-wcp-veto.dat']) # hardcoded
#orig_file = ''.join([data_dir, 'bigMD-cmass-dr12v4-RST-standardHAM-veto.dat'])
#orig_file = ''.join([data_dir, 'bigMD-cmass-dr12v4-RST-quadru-veto.dat'])
else: # default
orig_file = ''.join([data_dir, 'BigMD-cmass-dr12v4-RST-standHAM-Vpeak-veto.dat'])
# RA, Decl, Redhsift, veto
ra, dec, z, wfkp, veto, wfc = np.loadtxt(
orig_file,
unpack=True,
usecols=[0,1,2,3,4,5]
)
n_gal = len(ra)
#nbar = (1.0 / P0) * (1.0/wfkp - 1.0)
vetomask = np.where(veto == 1) # impose vetomask
data_list = [
ra[vetomask],
dec[vetomask],
z[vetomask],
wfc[vetomask]
]
elif catalog_name == 'tilingmock': # tiling mock
input_file = ''.join([
'/mount/riachuelo1/hahn/data/tiling_mocks/',
'cmass-boss5003sector-icoll012.fidcosmo.dat'])
output_file = self.file()
idl_cmd = ' '.join([
'idl', '-e', '"',
"build_wcp_assign, 'tilingmock', input_file='"+input_file+"', output_file='"+output_file+"'", '"'])
os.system(idl_cmd)
return None
elif 'cmass' in catalog_name:
if catalog_name == 'cmass':
# CMASS DR12v4 galaxy data
data_file = ''.join([
data_dir,
'cmass-dr12v4-N-Reid.dat.fits'
])
data = mrdfits(data_file) # fits data object
zlimit = np.where((data.z >= 0.43) & (data.z <= 0.7))
elif 'cmasslowz' in catalog_name:
# CMASS LOWZ DR12v5 combined sample
# for Ariel's sample has three separate
# set of sectors '', 'e2', and 'e3'
cmasslowz_str = ''
if 'e2' in catalog_name:
cmasslowz_str = 'E2'
elif 'e3' in catalog_name:
cmasslowz_str = 'E3'
# Divide combined sample in two
# two bins of redshift
if '_low' in catalog_name:
zmin, zmax = 0.2, 0.5
elif '_high' in catalog_name:
zmin, zmax = 0.5, 0.75
else:
raise NameError("redshift bin must be specified")
# .fits data files from mk_catalog pipeline
data_file = ''.join([
data_dir,
'galaxy_DR12v5_CMASSLOWZ', cmasslowz_str, '_North.fits.gz'
])
data = mrdfits(data_file)
zlimit = np.where((data.z >= zmin) & (data.z < zmax)) # redshift limit
else:
raise NameError()
data_list = [
(data.ra)[zlimit],
(data.dec)[zlimit],
(data.z)[zlimit],
(data.nz)[zlimit],
(data.weight_systot)[zlimit],
(data.weight_noz)[zlimit],
(data.weight_cp)[zlimit],
(data.comp)[zlimit]
]
# write to corrected file
output_file = self.file()
np.savetxt(
output_file,
(np.vstack(np.array(data_list))).T,
fmt=data_fmts,
delimiter='\t',
header=data_hdrs
)
return None