def anand():
dr8_elg = ft.read(
'/home/mehdi/data/formehdi/pixweight_ar-dr8-0.32.0-elgsv.fits')
nside = 256
npix = 12 * nside * nside
ss = [
'GALDEPTH_R', 'GALDEPTH_G', 'GALDEPTH_Z', 'PSFSIZE_R', 'PSFSIZE_G',
'PSFSIZE_Z', 'EBV', 'STARDENS'
]
sysmaps = {}
sysmaps['HPIX'] = np.arange(npix) #.astype('i8')
for ss_i in ss:
sysmaps[ss_i] = hp.reorder(dr8_elg[ss_i], n2r=True)
sysmaps['nran'] = hp.reorder(dr8_elg['FRACAREA'], n2r=True)
sysmaps['ngal'] = hp.reorder(
dr8_elg['SV'], n2r=True) * sysmaps['nran'] * hp.nside2pixarea(
256, degrees=True)
dataframe = pd.DataFrame(sysmaps)
dataframe.replace([np.inf, -np.inf], value=np.nan,
inplace=True) # replace inf
dataframe.to_hdf('/home/mehdi/data/formehdi/dr8_elgsv.h5',
'data',
overwrite=True)
hp.write_map('/home/mehdi/data/formehdi/dr8_elgsv_ngal.hp.256.fits',
dataframe.ngal,
fits_IDL=False,
dtype=np.float64)
mysample = dataframe[dataframe['nran'] > 0]
mysample.dropna(inplace=True)
mysample.shape
hd5_2_fits(mysample,
ss,
fitname='/home/mehdi/data/formehdi/dr8_elgsv.fits',
hpmask='/home/mehdi/data/formehdi/dr8_elgsv_mask.hp.256.fits',
hpfrac='/home/mehdi/data/formehdi/dr8_elgsv_frac.hp.256.fits',
fitnamekfold='/home/mehdi/data/formehdi/dr8_elgsv_5r.npy',
res=256,
k=5)
return 0
fitkfold = output_dir + f'/ngal_features_{cap}_{key_i}_{nside}.5r.npy'
mock.apply_zcut(zcuts[key_i])
mock.project2hp(nside)
mock.writehp(hpcat)
##random.apply_zcut(zcuts[key_i]) ## -- we don't cut randoms
random.project2hp(nside)
# --- append the galaxy and random density
dataframe_i = dataframe.copy()
dataframe_i['ngal'] = mock.galm
dataframe_i['nran'] = random.galm
dataframe_i['nran'][random.galm == 0] = np.nan
dataframe_i.dropna(inplace=True)
logger.info('df shape : {}'.format(dataframe_i.shape))
logger.info('columns : {}'.format(my_cols))
# --- write
hd5_2_fits(dataframe_i,
my_cols,
fitname,
hpmask,
fracgood,
fitkfold,
res=nside,
k=5)
def preparev7(dataname='/home/mehdi/data/eboss/sysmaps/SDSS_WISE_imageprop_HI_transformed_nside512.h5',
nside=512, transformed=True):
if transformed:
from LSSutils.catalogs.datarelease import cols_eboss_v7_qso as my_cols
else:
from LSSutils.catalogs.datarelease import cols_eboss_v6_qso_simp as my_cols
snside = str(nside)
dataframe = pd.read_hdf(dataname)
#dataframe = pd.read_hdf('/home/mehdi/data/eboss/sysmaps/'\
# +'SDSS_WISE_imageprop_HI_transformed_nside'+snside+'.h5')
#nside = 512
# zcuts = {'0.8': [0.8000000000000000, 1.1423845339193297],
# '1.1': [1.1423845339193297, 1.3862779004064971],
# '1.4': [1.3862779004064971, 1.6322502623999630],
# '1.6': [1.6322502623999630, 1.8829689752636356],
# '1.9': [1.8829689752636356, 2.2000000000000000]}
zcuts = {'low':[0.8, 1.508088732762684],
'high':[1.508088732762684, 2.2]}
path = '/home/mehdi/data/eboss/v7/'
fitname = lambda x:path + 'ngal_features_'+x+'.hp'+snside+'.fits'
hpmask = lambda x:path + 'mask_'+x+'.hp'+snside+'.fits'
fracgood = lambda x:path + 'frac_'+x+'.hp'+snside+'.fits'
fitkfold = lambda x:path + 'ngal_features_'+x+'.hp'+snside+'.5r.npy'
catname = lambda x:path + 'eBOSS_QSO_clustering_'+x+'_v7.dat.fits'
ranname = lambda x:path + 'eBOSS_QSO_clustering_'+x+'_v7.ran.fits'
hpcatname = lambda x:path + 'eBOSS_QSO_clustering_'+x+'_v7.hp'+snside+'.dat.fits'
#
for cap in ['NGC', 'SGC']:
catalogs = []
for i,key_i in enumerate(zcuts.keys()):
myframe = dataframe.copy()
mytag = cap+'_'+key_i
fitname_i = fitname(mytag)
catname_i = catname(cap)
ranname_i = ranname(cap)
hpcat_i = hpcatname(mytag)
hpmsk_i = hpmask(mytag)
hpfrac_i = fracgood(mytag)
fitkfld_i = fitkfold(mytag)
#print(mytag, fitname_i, catname_i, ranname_i, hpcat_i,
# hpmsk_i, hpfrac_i, fitkfld_i)
mycat = cf.EBOSSCAT([catname_i])
mycat.apply_zcut(zcuts[key_i])
mycat.project2hp(nside=nside)
mycat.writehp(hpcat_i)
myframe['ngal'] = mycat.galm.astype('f8')
myran = cf.EBOSSCAT([ranname_i])
#myran.apply_zcut(zcuts[key_i]) ## do not cut randoms
myran.project2hp(nside=nside)
myranhp = myran.galm.astype('f8')
myranhp[myranhp==0.] = np.nan
myframe['nran'] = myranhp
myfit = myframe.dropna()
#print('shape myfit {} {} {}'.format(cap, key_i, myfit.shape))
cf.hd5_2_fits(myfit, my_cols, fitname_i, hpmsk_i, hpfrac_i, fitkfld_i, res=nside, k=5)
def split_to_imag(path2sysmaps, mycols, CAP, nside, band, index_b, index_e, field):
sysmaps = pd.read_hdf(path2sysmaps)
sysmaps = sysmaps[mycols]
snside = str(nside)
print(index_b[band], field, nside, band, CAP)
cfull = ft.read('/home/mehdi/data/eboss/v6/eBOSS_QSO_full_'+CAP+'_v6.dat.fits')
cat = ft.read('/home/mehdi/data/eboss/v6/eBOSS_QSO_clustering_'+CAP+'_v6.dat.fits')
inc = np.in1d(cfull['QSO_ID'], cat['QSO_ID'])
mycat = cfull[inc]
hpix = ut.radec2hpix(nside, cat['RA'], cat['DEC'])
imag = mycat[field][:, index_b[band]] - index_e[band]*sysmaps['ebv'][hpix]
imag_percentiles = np.percentile(imag, [0, 25, 50, 75, 100])
print('percentiles : \n{}'.format(imag_percentiles))
ranname = '/home/mehdi/data/eboss/v6/eBOSS_QSO_clustering_NGC_v6.ran.fits'.replace('NGC', CAP)
myran = cf.EBOSSCAT([ranname])
myran.project2hp(nside=nside)
myranhp = myran.galm.astype('f8')
myranhp[myranhp==0.] = np.nan
tot = 0
for i in range(len(imag_percentiles)-1):
le = imag_percentiles[i]
ue = imag_percentiles[i+1]
if i==3:
mask = (imag>=le) & (imag<=ue)
else:
mask = (imag>=le) & (imag<ue)
#tot += mask.sum()
#print(tot)
mycat = cat[mask]
output_cat = '/home/mehdi/data/eboss/v6/imag_splits/'\
+ 'eBOSS_QSO_clustering_'+CAP+'_v6_'+str(i)+'.dat.fits'
ft.write(output_cat, mycat, clobber=True)
#
#
myframe = sysmaps.copy()
cap = CAP.lower()
key_i = str(i)
mytag = cap+'.'+key_i
fitname = '/home/mehdi/data/eboss/v6/imag_splits/ngal_features_ngc.fits'.replace('ngc', mytag)
fitkfld = '/home/mehdi/data/eboss/v6/imag_splits/qso.ngc.hp.'+snside+'.r.npy'
fitkfld = fitkfld.replace('ngc', mytag)
hpfrac = '/home/mehdi/data/eboss/v6/imag_splits/fracgood.ngc.hp.'+snside+'.fits'
hpfrac = hpfrac.replace('ngc', mytag)
hpmask = '/home/mehdi/data/eboss/v6/imag_splits/mask.ngc.hp.'+snside+'.fits'
hpmask = hpmask.replace('ngc', mytag)
myngalhp = ut.hpixsum(nside, mycat['RA'],
mycat['DEC'],
value=mycat['WEIGHT_NOZ']*mycat['WEIGHT_CP']).astype('f8')
output_hpngal = output_cat.replace('.dat.fits', '.dat.hp256.fits')
hp.write_map(output_hpngal, myngalhp, overwrite=True, fits_IDL=False)
myframe['ngal'] = myngalhp
myframe['nran'] = myranhp
myfit = myframe.dropna()
#print('shape myfit {} {} {}'.format(cap, key_i, myfit.shape))
#print(fitname, fitkfld, hpfrac, hpmask, end=3*'\n')
cf.hd5_2_fits(myfit, mycols, fitname, hpmask, hpfrac, fitkfld, res=nside, k=5)
print(10*'=', end=3*'\n')
if prepare_regression:
# path
version = '0.2'
path = '/home/mehdi/data/formehdi/' + version + '/'
if not os.path.isdir(path): os.makedirs(path)
# read
data = pd.read_hdf('/home/mehdi/data/dr8_combined256.h5')
mask = hp.read_map('/home/mehdi/data/formehdi/dr8_mask_eBOSS.hp256.fits',
verbose=False).astype('bool')
data['ngal'] = hp.read_map(
'/home/mehdi/data/formehdi/dr8_elgsv_ngal.hp.256.fits', verbose=False)
data['nran'] = hp.read_map(
'/home/mehdi/data/formehdi/dr8_elgsv_frac.hp.256.fits', verbose=False)
# split into NGC and SGC
caps = ['ngc', 'sgc']
ngc, sgc = ut.split2caps(mask)
for i, cap in enumerate([ngc, sgc]):
mysample = data[(data['nran'] > 0) & cap[data.index]]
mysample = mysample.dropna()
hd5_2_fits(mysample,
cols_dr8,
fitname=path + 'dr8_elgsv_' + caps[i] + '.fits',
hpmask=path + 'dr8_elgsv_mask_' + caps[i] + '.hp.256.fits',
hpfrac=None,
fitnamekfold=path + 'dr8_elgsv_' + caps[i] + '_5r.npy',
res=256,
k=5)
print(mysample.shape)