def anand(): dr8_elg = ft.read( '/home/mehdi/data/formehdi/pixweight_ar-dr8-0.32.0-elgsv.fits') nside = 256 npix = 12 * nside * nside ss = [ 'GALDEPTH_R', 'GALDEPTH_G', 'GALDEPTH_Z', 'PSFSIZE_R', 'PSFSIZE_G', 'PSFSIZE_Z', 'EBV', 'STARDENS' ] sysmaps = {} sysmaps['HPIX'] = np.arange(npix) #.astype('i8') for ss_i in ss: sysmaps[ss_i] = hp.reorder(dr8_elg[ss_i], n2r=True) sysmaps['nran'] = hp.reorder(dr8_elg['FRACAREA'], n2r=True) sysmaps['ngal'] = hp.reorder( dr8_elg['SV'], n2r=True) * sysmaps['nran'] * hp.nside2pixarea( 256, degrees=True) dataframe = pd.DataFrame(sysmaps) dataframe.replace([np.inf, -np.inf], value=np.nan, inplace=True) # replace inf dataframe.to_hdf('/home/mehdi/data/formehdi/dr8_elgsv.h5', 'data', overwrite=True) hp.write_map('/home/mehdi/data/formehdi/dr8_elgsv_ngal.hp.256.fits', dataframe.ngal, fits_IDL=False, dtype=np.float64) mysample = dataframe[dataframe['nran'] > 0] mysample.dropna(inplace=True) mysample.shape hd5_2_fits(mysample, ss, fitname='/home/mehdi/data/formehdi/dr8_elgsv.fits', hpmask='/home/mehdi/data/formehdi/dr8_elgsv_mask.hp.256.fits', hpfrac='/home/mehdi/data/formehdi/dr8_elgsv_frac.hp.256.fits', fitnamekfold='/home/mehdi/data/formehdi/dr8_elgsv_5r.npy', res=256, k=5) return 0
fitkfold = output_dir + f'/ngal_features_{cap}_{key_i}_{nside}.5r.npy' mock.apply_zcut(zcuts[key_i]) mock.project2hp(nside) mock.writehp(hpcat) ##random.apply_zcut(zcuts[key_i]) ## -- we don't cut randoms random.project2hp(nside) # --- append the galaxy and random density dataframe_i = dataframe.copy() dataframe_i['ngal'] = mock.galm dataframe_i['nran'] = random.galm dataframe_i['nran'][random.galm == 0] = np.nan dataframe_i.dropna(inplace=True) logger.info('df shape : {}'.format(dataframe_i.shape)) logger.info('columns : {}'.format(my_cols)) # --- write hd5_2_fits(dataframe_i, my_cols, fitname, hpmask, fracgood, fitkfold, res=nside, k=5)
def preparev7(dataname='/home/mehdi/data/eboss/sysmaps/SDSS_WISE_imageprop_HI_transformed_nside512.h5', nside=512, transformed=True): if transformed: from LSSutils.catalogs.datarelease import cols_eboss_v7_qso as my_cols else: from LSSutils.catalogs.datarelease import cols_eboss_v6_qso_simp as my_cols snside = str(nside) dataframe = pd.read_hdf(dataname) #dataframe = pd.read_hdf('/home/mehdi/data/eboss/sysmaps/'\ # +'SDSS_WISE_imageprop_HI_transformed_nside'+snside+'.h5') #nside = 512 # zcuts = {'0.8': [0.8000000000000000, 1.1423845339193297], # '1.1': [1.1423845339193297, 1.3862779004064971], # '1.4': [1.3862779004064971, 1.6322502623999630], # '1.6': [1.6322502623999630, 1.8829689752636356], # '1.9': [1.8829689752636356, 2.2000000000000000]} zcuts = {'low':[0.8, 1.508088732762684], 'high':[1.508088732762684, 2.2]} path = '/home/mehdi/data/eboss/v7/' fitname = lambda x:path + 'ngal_features_'+x+'.hp'+snside+'.fits' hpmask = lambda x:path + 'mask_'+x+'.hp'+snside+'.fits' fracgood = lambda x:path + 'frac_'+x+'.hp'+snside+'.fits' fitkfold = lambda x:path + 'ngal_features_'+x+'.hp'+snside+'.5r.npy' catname = lambda x:path + 'eBOSS_QSO_clustering_'+x+'_v7.dat.fits' ranname = lambda x:path + 'eBOSS_QSO_clustering_'+x+'_v7.ran.fits' hpcatname = lambda x:path + 'eBOSS_QSO_clustering_'+x+'_v7.hp'+snside+'.dat.fits' # for cap in ['NGC', 'SGC']: catalogs = [] for i,key_i in enumerate(zcuts.keys()): myframe = dataframe.copy() mytag = cap+'_'+key_i fitname_i = fitname(mytag) catname_i = catname(cap) ranname_i = ranname(cap) hpcat_i = hpcatname(mytag) hpmsk_i = hpmask(mytag) hpfrac_i = fracgood(mytag) fitkfld_i = fitkfold(mytag) #print(mytag, fitname_i, catname_i, ranname_i, hpcat_i, # hpmsk_i, hpfrac_i, fitkfld_i) mycat = cf.EBOSSCAT([catname_i]) mycat.apply_zcut(zcuts[key_i]) mycat.project2hp(nside=nside) mycat.writehp(hpcat_i) myframe['ngal'] = mycat.galm.astype('f8') myran = cf.EBOSSCAT([ranname_i]) #myran.apply_zcut(zcuts[key_i]) ## do not cut randoms myran.project2hp(nside=nside) myranhp = myran.galm.astype('f8') myranhp[myranhp==0.] = np.nan myframe['nran'] = myranhp myfit = myframe.dropna() #print('shape myfit {} {} {}'.format(cap, key_i, myfit.shape)) cf.hd5_2_fits(myfit, my_cols, fitname_i, hpmsk_i, hpfrac_i, fitkfld_i, res=nside, k=5)
def split_to_imag(path2sysmaps, mycols, CAP, nside, band, index_b, index_e, field): sysmaps = pd.read_hdf(path2sysmaps) sysmaps = sysmaps[mycols] snside = str(nside) print(index_b[band], field, nside, band, CAP) cfull = ft.read('/home/mehdi/data/eboss/v6/eBOSS_QSO_full_'+CAP+'_v6.dat.fits') cat = ft.read('/home/mehdi/data/eboss/v6/eBOSS_QSO_clustering_'+CAP+'_v6.dat.fits') inc = np.in1d(cfull['QSO_ID'], cat['QSO_ID']) mycat = cfull[inc] hpix = ut.radec2hpix(nside, cat['RA'], cat['DEC']) imag = mycat[field][:, index_b[band]] - index_e[band]*sysmaps['ebv'][hpix] imag_percentiles = np.percentile(imag, [0, 25, 50, 75, 100]) print('percentiles : \n{}'.format(imag_percentiles)) ranname = '/home/mehdi/data/eboss/v6/eBOSS_QSO_clustering_NGC_v6.ran.fits'.replace('NGC', CAP) myran = cf.EBOSSCAT([ranname]) myran.project2hp(nside=nside) myranhp = myran.galm.astype('f8') myranhp[myranhp==0.] = np.nan tot = 0 for i in range(len(imag_percentiles)-1): le = imag_percentiles[i] ue = imag_percentiles[i+1] if i==3: mask = (imag>=le) & (imag<=ue) else: mask = (imag>=le) & (imag<ue) #tot += mask.sum() #print(tot) mycat = cat[mask] output_cat = '/home/mehdi/data/eboss/v6/imag_splits/'\ + 'eBOSS_QSO_clustering_'+CAP+'_v6_'+str(i)+'.dat.fits' ft.write(output_cat, mycat, clobber=True) # # myframe = sysmaps.copy() cap = CAP.lower() key_i = str(i) mytag = cap+'.'+key_i fitname = '/home/mehdi/data/eboss/v6/imag_splits/ngal_features_ngc.fits'.replace('ngc', mytag) fitkfld = '/home/mehdi/data/eboss/v6/imag_splits/qso.ngc.hp.'+snside+'.r.npy' fitkfld = fitkfld.replace('ngc', mytag) hpfrac = '/home/mehdi/data/eboss/v6/imag_splits/fracgood.ngc.hp.'+snside+'.fits' hpfrac = hpfrac.replace('ngc', mytag) hpmask = '/home/mehdi/data/eboss/v6/imag_splits/mask.ngc.hp.'+snside+'.fits' hpmask = hpmask.replace('ngc', mytag) myngalhp = ut.hpixsum(nside, mycat['RA'], mycat['DEC'], value=mycat['WEIGHT_NOZ']*mycat['WEIGHT_CP']).astype('f8') output_hpngal = output_cat.replace('.dat.fits', '.dat.hp256.fits') hp.write_map(output_hpngal, myngalhp, overwrite=True, fits_IDL=False) myframe['ngal'] = myngalhp myframe['nran'] = myranhp myfit = myframe.dropna() #print('shape myfit {} {} {}'.format(cap, key_i, myfit.shape)) #print(fitname, fitkfld, hpfrac, hpmask, end=3*'\n') cf.hd5_2_fits(myfit, mycols, fitname, hpmask, hpfrac, fitkfld, res=nside, k=5) print(10*'=', end=3*'\n')
if prepare_regression: # path version = '0.2' path = '/home/mehdi/data/formehdi/' + version + '/' if not os.path.isdir(path): os.makedirs(path) # read data = pd.read_hdf('/home/mehdi/data/dr8_combined256.h5') mask = hp.read_map('/home/mehdi/data/formehdi/dr8_mask_eBOSS.hp256.fits', verbose=False).astype('bool') data['ngal'] = hp.read_map( '/home/mehdi/data/formehdi/dr8_elgsv_ngal.hp.256.fits', verbose=False) data['nran'] = hp.read_map( '/home/mehdi/data/formehdi/dr8_elgsv_frac.hp.256.fits', verbose=False) # split into NGC and SGC caps = ['ngc', 'sgc'] ngc, sgc = ut.split2caps(mask) for i, cap in enumerate([ngc, sgc]): mysample = data[(data['nran'] > 0) & cap[data.index]] mysample = mysample.dropna() hd5_2_fits(mysample, cols_dr8, fitname=path + 'dr8_elgsv_' + caps[i] + '.fits', hpmask=path + 'dr8_elgsv_mask_' + caps[i] + '.hp.256.fits', hpfrac=None, fitnamekfold=path + 'dr8_elgsv_' + caps[i] + '_5r.npy', res=256, k=5) print(mysample.shape)