def main(): # masking # galaxy density for all # plot (check) # galaxy density for one (plot) -> get weight # apply weight and calculate galaxy density for the next property (plot) # repeat # calling catalog from systematics import GalaxyDensity_Systematics,loadSystematicMaps, chisquare_dof, MatchHPArea, SysMapBadRegionMask, callingEliGoldMask #balrog_y1a1 = fitsio.read('result_cat/result_balrog_EMHUFF.fits') #balrog_SSPT = balrog_y1a1[balrog_y1a1['DEC'] < -3] #rand_clean = Cuts.keepGoodRegion(balrog_SSPT) # dmass result_y1a1 = fitsio.read('/n/des/lee.5922/data/y1a1_coadd/dmass_y1a1.fits') dmass_y1a1, _ = resampleWithPth(result_y1a1) resultpath = '/n/des/lee.5922/Dropbox/repositories/CMASS/code/result_cat/' result_gold_stripe82 = fitsio.read(resultpath+'result_gold_stripe82.fits') dmass_gold_stripe82, _ = resampleWithPth(result_gold_stripe82) resultpath = '/n/des/lee.5922/Dropbox/repositories/CMASS/code/result_cat/' result_gold_stripe82 = fitsio.read(resultpath+'result_gold_2_stripe82.fits') dmass_gold_stripe82, _ = resampleWithPth(result_gold_stripe82) clean_cmass_data_des = fitsio.read(resultpath+'clean_cmass_data_des_gold2.fits') # calling Eli mask GoldMask = callingEliGoldMask() GoldMask_st82 = GoldMask[ GoldMask['DEC'] > -3.0 ] angular_correlation(data = dmass_gold_stripe82, rand = GoldMask_st82, weight = [None, GoldMask_st82['FRAC']], suffix = '_gold_stripe82') # ---------------------------------------------------------------- kind = 'STRIPE82' nside = 4096 njack = 10 property = ['AIRMASS', 'SKYBRITE', 'FWHM', 'SKYSIGMA', 'NSTARS'] filter = ['g', 'r', 'i', 'z'] # Calling maps MaskDic = {} for i,p in enumerate(property): if p == 'NSTARS': filename = 'y1a1_gold_1.0.2_stars_nside1024.fits' sysMap = loadSystematicMaps( filename = filename, property = p, filter = 'g', nside = 1024 , kind = kind ) if kind is 'STRIPE82': sysMap = sysMap[sysMap['DEC'] > -3] mapname = 'sys_'+p+'_'+'g' MaskDic[mapname] = sysMap else : nside = 4096 filter = ['g', 'r', 'i', 'z'] filename = None for j,f in enumerate(filter): sysMap = loadSystematicMaps( filename = filename, property = p, filter = f, nside = nside , kind = kind) #goodIndmask = np.in1d( sysMap['PIXEL'], GoldMask['PIXEL']) #sysMap = sysMap[goodIndmask] #sysMap = Cuts.SpatialCuts(sysMap, ra =ra, ra2=ra2, dec=dec, dec2=dec2 ) mapname = 'sys_'+p+'_'+f MaskDic[mapname] = sysMap # basic cutoff mask cutvalue = {'AIRMASS':[1.4, 1.35, 1.35, 1.35], 'SKYBRITE':[150, 380, 1150, 2500], 'FWHM':[6.5, 5.5, 5.0, 4.5], 'SKYSIGMA':[6.5, 9.5, 18, 26], 'NSTARS':[5, 5, 5, 5 ]} cutvalue1 = {'AIRMASS':[1.45, 1.45, 1.45, 1.45], 'SKYBRITE':[3000, 3000, 3000, 3000], 'FWHM':[6.5, 5.0, 4.5, 4.0], 'SKYSIGMA':[40, 40, 40, 40], 'NSTARS':[501, 501, 501, 501 ]} for i,p in enumerate(property): correctMask = np.ones(dmass_gold_stripe82.size, dtype=bool) if p is 'NSTARS': nside = 1024 filter = ['g'] else : nside = 4096 filter = ['g', 'r', 'i', 'z'] for j,f in enumerate(filter): mapname = 'sys_'+p+'_'+f maskedsysMap, mask = SysMapBadRegionMask(dmass_gold_stripe82, MaskDic[mapname], nside = nside, cond = '<=', val = cutvalue[p][j]) correctMask = correctMask * mask MaskDic[mapname+'_masked'] = maskedsysMap MaskDic[p+'_mask'] = correctMask #angular_correlation(data = dmass_SSPT[correctMask], rand = rand_clean, weight = [None, None], suffix = '_'+kind+'_'+p+'_masked') print p+'_mask', np.sum(correctMask) TotalMask = MaskDic['AIRMASS_mask'] * MaskDic['FWHM_mask'] # * MaskDic['SKYSIGMA_mask'] * MaskDic['SKYBRITE_mask']# * MaskDic['NSTARS_mask'] clean_dmass = dmass_gold_stripe82[TotalMask] #angular_correlation(data = clean_dmass, rand = GoldMask, weight = [None, None], suffix = '_'+kind+'_masked') # pixel fraction weights for each gal from systematics import hpRaDecToHEALPixel import pandas as pd catHpInd = hpRaDecToHEALPixel(clean_dmass['RA'], clean_dmass['DEC'], nside=4096, nest = False) catHpInd = pd.DataFrame(catHpInd) catHpInd.columns = ['PIXEL'] GoldMask_pd = pd.DataFrame(GoldMask) frac_weight = pd.merge( catHpInd, GoldMask_pd, how = 'left', on = 'PIXEL') frac_weight = frac_weight.to_records() property = ['AIRMASS', 'SKYBRITE', 'FWHM', 'SKYSIGMA', 'NSTARS'] # calculating galaxy density and weights iterately from systematics import ReciprocalWeights, jksampling from systematics_module.corr import angular_correlation re_weights = np.ones(clean_dmass.size) #re_weights = frac_weight['FRAC'] #re_weights = GoldMask['FRAC'] rand_clean = None #correctMask = np.ones(clean_dmass.size, dtype=bool) re_weights_dic = {} for p in property: if p is 'NSTARS': nside = 1024 filter = ['g'] else : nside = 4096 filter = ['g', 'r', 'i', 'z'] for j,f in enumerate(filter): mapname = 'sys_'+p+'_'+f+'_masked' bins, Bdensity, Berr, Bf_area = GalaxyDensity_Systematics(clean_dmass, MaskDic[mapname], nside = nside, raTag = 'RA', decTag='DEC', property = p, filter = f, weight = re_weights) #bins = bins/np.sum(sysMap['SIGNAL']) *len(sysMap) #B_jkerr = jksampling(clean_dmass, MaskDic[mapname], property = p, nside = nside, njack = 30, raTag = 'RA', decTag = 'DEC' ) filename = 'data_txt/systematic_'+p+'_'+f+'_'+kind+'_masked.txt' #DAT = np.column_stack(( bins-(bins[1]-bins[0])*0.1, Bdensity, Berr, Bf_area, Bdensity, Berr, Bf_area )) DAT = np.column_stack(( bins, Bdensity, Berr, Bf_area, Bdensity, Berr, Bf_area )) np.savetxt(filename, DAT, delimiter = ' ', header = 'bins, Cdensity, Cerr, Cfarea, Bdensity, Berr, Bfarea, B_jkerr') print "saving data to ", filename #doVisualization_ngal(property = p, nside = nside, kind = kind, suffix='_masked') """ cutvalue[p][j] = 1.3 maskedsysMap, mask = SysMapBadRegionMask(dmass_SSPT, MaskDic[mapname], nside = nside, cond = '<=', val = cutvalue[p][j]) correctMask = correctMask * mask MaskDic[mapname+'_masked'] = maskedsysMap clean_dmass = dmass_SSPT[correctMask] """ re_weights = re_weights * ReciprocalWeights( catalog = clean_dmass, sysMap = MaskDic[mapname], property = p, filter = f, nside = nside, kind = kind ) re_weights_dic['re_weight_'+p] = re_weights # check bins, Bdensity, Berr, Bf_area = GalaxyDensity_Systematics(clean_dmass, MaskDic[mapname], nside = nside, raTag = 'RA', decTag='DEC', property = p, filter = f, weight = re_weights) filename = 'data_txt/systematic_'+p+'_'+f+'_'+kind+'_corrected.txt' DAT = np.column_stack(( bins, Bdensity, Berr, Bf_area, Bdensity, Berr, Bf_area )) np.savetxt(filename, DAT, delimiter = ' ', header = 'bins, Cdensity, Cerr, Cfarea, Bdensity, Berr, Bfarea') print "saving data to ", filename #doVisualization_ngal(property = p, nside = nside, kind = kind, suffix='_corrected') #doVisualization_ngal(property = p, nside = nside, kind = kind, suffix='_masked') #doVisualization_ngal(property = p, nside = nside, kind = kind, suffix='_corrected') #angular_correlation(data = clean_dmass, rand = rand_clean, weight = [re_weights, None], suffix = '_'+kind+'_w_'+p) angular_correlation(data = dmass_y1a1, rand = GoldMask, weight = [None, None], suffix = '_'+kind+'_noweight') angular_correlation(data = clean_dmass, rand = GoldMask, weight = [None, None], suffix = '_'+kind+'_masked') for p in property: doVisualization_ngal(property = p, nside = 4096, kind = kind, suffix='_masked') #doVisualization_ngal(property = p, nside = nside, kind = kind, suffix='_corrected') rw = re_weights_dic['re_weight_'+p] angular_correlation(data = clean_dmass, rand = GoldMask, weight = [rw, None], suffix = '_'+kind+'_w_'+p) # apply weights to correlation function property = [ 'AIRMASS', 'SKYBRITE', 'FWHM', 'SKYSIGMA', 'NSTARS'] kind = 'Y1A1' # corr com # corr comparison labels = ['y1a1'] linestyle = ['-']+['--' for i in labels[:-1]] fmt = ['.']+['o' for i in labels[:-2]]+['.'] color = ['red'] + [None for i in labels[:-1]] corr_txt = [np.loadtxt('data_txt/acf_comparison_'+s+'.txt') for s in labels] corr_txt2 = np.loadtxt('data_txt/acf_comparison_cmass_SGC.txt') thetaS, wS, Sjkerr = corr_txt2[:,0], corr_txt2[:,1], corr_txt2[:,2] fig, (ax, ax2) = plt.subplots(2,1, figsize = (10,15)) ax.errorbar( thetaS, wS, yerr = Sjkerr, label = 'CMASS SGC', color='black', alpha = 0.5) ax2.errorbar( thetaS, wS-wS, yerr = 10 * Sjkerr, label = 'CMASS SGC', color='black', alpha = 0.9) for i in range(len(labels)): thetaD, wD, Djkerr = corr_txt[i][:,0], corr_txt[i][:,1], corr_txt[i][:,2] if labels[i] == 'wnstar+mask': markersize = 12 else : markersize = 5 ax.errorbar( thetaD*(0.95+0.02*i), wD, yerr = Djkerr, fmt = fmt[i], linestyle = linestyle[i] ,label = labels[i], color = color[i], markersize = markersize) ax2.errorbar( thetaD*(0.95+0.05*i), 10 * (wD - wS), yerr = 10 *Djkerr, fmt = fmt[i], linestyle = linestyle[i], label = labels[i],color = color[i], markersize=markersize) ax.set_xlim(1e-2, 50) #ax.set_ylim(-0.02 , 0.5) ax.set_ylim(0.0001 , 5.0) ax.set_xlabel(r'$\theta(deg)$') ax.set_ylabel(r'${w(\theta)}$') ax.set_xscale('log') ax.set_yscale('log') ax.legend(loc = 'best') ax.set_title(' angular correlation ') ax2.axhline(y=0.0, color = 'black') ax2.set_xlim(1e-1, 10) ax2.set_ylim(-.2, .5) ax2.set_xscale('log') ax2.set_xlabel(r'$\theta(deg)$') ax2.set_ylabel(r'$10 \times$ $($ ${w}$ - ${w_{\rm{true}}}$ $)$') ax2.legend(loc='best') figname = 'figure/acf_comparison.png' fig.savefig(figname) print 'writing plot to ', figname
def main(): #ra, ra2, dec, dec2 = 350., 355, -1, 1 # callin all des data and make merged cat =============================== #des_gold = io.getDESY1A1catalogs(keyword = 'Y1A1_GOLD_STRIPE82_ALLCOLUMNS_EA', gold=True) #des_gold = Cuts.keepGoodRegion(des_gold) # ------ """ for j in np.arange(1,21): t1 = time.time() des_gold = io.getDESY1A1catalogs(keyword = 'Y1A1_GOLD_0000{:02}'.format(j), gold=True) merged_list = [] for i in np.arange(35): fds = io.getDESY1A1catalogs(keyword = 'Y1A1_COADD_OBJECTS_000{:02}'.format(i), sdssmask=False) merged = mergeCatalogsUsingPandas(des=fds, gold=des_gold, how='inner', key='COADD_OBJECTS_ID', suffixes = ['_DES','']) merged_list.append(merged) merged_list = np.hstack(merged_list) fitsio.write('/n/des/lee.5922/data/y1a1_coadd/'+'Y1A1_GOLD_merged_{:02}.fits'.format(j), merged_list) print time.time() - t1, ' s' resultpath = '/n/des/lee.5922/Dropbox/repositories/CMASS/code/result_cat/' clean_cmass_data_des = fitsio.read(resultpath+'clean_cmass_data_des_gold2.fits') prefix = 'gold_st82_2_' #ra, ra2, dec, dec2 = 350., 355, -1, 1 resultpath = '/n/des/lee.5922/Dropbox/repositories/CMASS/code/result_cat/' merged_des_s = fitsio.read(resultpath+'merged_des_s.fits') result_cat = [] for i in np.arange(1,350): fds = io.getDESY1A1catalogs(keyword = 'Y1A1_COADD_OBJECTS_000{:03}'.format(i), sdssmask=False) merged_des = mergeCatalogsUsingPandas(des=fds, gold=des_gold, how='inner', key='COADD_OBJECTS_ID', suffixes = ['_DES','']) merged_des1 = Cuts.doBasicCuts(merged_des, raTag = 'RA', decTag='DEC', object = None) result_gold = XDGMM_model(clean_cmass_data_des, clean_cmass_data_des, train=merged_des_s, test=merged_des1, prefix = prefix, mock=True, gold=True ) result_cat.append(result_gold) result_gold_cat = np.hstack(result_cat) """ # ======================================================================== import esutil #from systematics import mergeCatalogsUsingPandas from xd import doVisualization_1d from systematics_module.corr import angular_correlation, cross_angular_correlation,LensingSignal from utils import * # 1) gold, slr corrected """ full_des_data = io.getDESY1A1catalogs(keyword = 'STRIPE82_COADD', sdssmask=False) #des_gold = io.getDESY1A1catalogs(keyword = 'Y1A1_GOLD_00', gold=True) #des_gold = io.getDESY1A1catalogs(keyword = 'Y1A1_GOLD_STRIPE82_ALLCOLUMNS_EA', gold=True) des_gold = io.getDESY1A1catalogs(keyword = 'Y1A1_GOLD_STRIPE82_0', gold=True) merged_des = mergeCatalogsUsingPandas(des=full_des_data, gold=des_gold, key='COADD_OBJECTS_ID', suffixes = ['_DES','']) merged_des1 = Cuts.doBasicCuts(merged_des.copy(), raTag = 'RA', decTag='DEC', object = None) mags = ['MAG_MODEL', 'MAG_DETMODEL'] merged_des1 = getCorrectedMag( merged_des1, mags = mags, reddening = None ) merged_des2 = AddingReddening( merged_des1 ) merged_des1 = 0 merged_des2 = getCorrectedMag( merged_des2, mags = ['MAG_APER_4'], reddening = 'SLR' ) resultpath = '/n/des/lee.5922/Dropbox/repositories/CMASS/code/result_cat/' fitsio.write(resultpath+'merged_des_st82.fits', merged_des2) full_des_data, merged_des, des_gold = 0, 0, 0 """ # galaxy #merged_des1 = doBasicCuts(merged_des, raTag = 'RA', decTag = 'DEC', object = None) resultpath = '/n/des/lee.5922/Dropbox/repositories/CMASS/code/result_cat/' gold_final = fitsio.read(resultpath+'merged_des_st82.fits') gold_final_s = Cuts.SpatialCuts( gold_final, ra = 320.0, ra2=330.0 , dec= -1.0 , dec2=1.0 ) cmass = io.getSGCCMASSphotoObjcat() cmass = Cuts.keepGoodRegion(cmass) clean_cmass_data, clean_cmass_data_des = DES_to_SDSS.match( cmass, gold_final ) clean_cmass_data_des_s = Cuts.SpatialCuts( clean_cmass_data_des, ra = 320.0, ra2=330.0 , dec= -1.0 , dec2=1.0 ) #prefix = 'gold_small_' #prefix = 'gold_st82_' #prefix = 'gold_st82_2_' #prefix = 'gold_st82_3_' #prefix = 'gold_st82_SFD98_' #prefix = 'coadd_st82_SLR_' #prefix = 'gold_st82_5_' # the best one so far #prefix = 'gold_st82_6_' prefix = 'gold_st82_7_' prefix = 'gold_st82_9_' prefix = 'gold_st82_11_small_' (trainInd, testInd), (sdsstrainInd,sdsstestInd) = split_samples(gold_final, gold_final, [0.9,0.1], random_state=0) des_train = gold_final_s[trainInd] des_test = gold_final_s[testInd] result_gold_st82 = XDGMM_model(clean_cmass_data_des, clean_cmass_data_des, train=gold_final, test=gold_final, prefix = prefix, mock=True) resultpath = '/n/des/lee.5922/Dropbox/repositories/CMASS/code/result_cat/' fitsio.write(resultpath+'result_gold_st82_11_small.fits', result_gold_st82) fitsio.write(resultpath+'clean_cmass_data_des_gold_st82_11_small.fits', clean_cmass_data_des_s) #fitsio.write(resultpath+'clean_cmass_data_des_gold_st82_9_2.fits', clean_cmass_data_des) """ # 2) dmass, slr (test with coadd obects table ( use slrzeroshift instead of XDCORR )--------- full_des_data = io.getDESY1A1catalogs(keyword = 'STRIPE82_COADD', sdssmask=False) des_gold = io.getDESY1A1catalogs(keyword = 'Y1A1_GOLD_STRIPE82_0', gold=True) des = Cuts.doBasicCuts(full_des_data, raTag = 'RA', decTag='DEC', object = 'galaxy') des = AddingReddening(des) # add SLR_SHIFT column des = getCorrectedMag( des, reddening = 'SLR' ) des = addphotoz(des = des, im3shape=des_gold) full_des_data = 0 cmass = io.getSGCCMASSphotoObjcat() cmass = Cuts.keepGoodRegion(cmass) clean_cmass_data, clean_cmass_data_des = DES_to_SDSS.match( cmass, des ) prefix = 'coadd_st82_SLR_' (trainInd, testInd), (sdsstrainInd,sdsstestInd) = split_samples(des, des, [0.05,0.95], random_state=0) des_train = des[trainInd] des_test = des[testInd] clean_cmass_data, clean_cmass_data_des = DES_to_SDSS.match( cmass, des ) result_coadd_st82 = XDGMM_model(clean_cmass_data_des, clean_cmass_data_des, train=des_train, test=des_test, prefix = prefix, mock=True ) resultpath = '/n/des/lee.5922/Dropbox/repositories/CMASS/code/result_cat/' fitsio.write(resultpath+'result_coadd_st82_SLR.fits', result_coadd_st82) fitsio.write(resultpath+'clean_cmass_data_des_coadd_st82_SLR.fits', clean_cmass_data_des) Xtrue,_ = mixing_color( des ) fitsio.write(resultpath+prefix+'X_obs.fits', Xtrue) # 3) gold, SFD98 --------- full_des_data = io.getDESY1A1catalogs(keyword = 'STRIPE82_COADD', sdssmask=False) #des_gold = io.getDESY1A1catalogs(keyword = 'Y1A1_GOLD_STRIPE82_0', gold=True) des_gold = io.getDESY1A1catalogs(keyword = 'Y1A1_GOLD_STRIPE82_ALLCOLUMNS_EA', gold=True) des_gold = Cuts.keepGoodRegion(des_gold) # replace slr corrected gold mag to raw magnitude. merged_des = mergeCatalogsUsingPandas(des=full_des_data, gold=des_gold, key='COADD_OBJECTS_ID', suffixes = ['_DES','']) merged_des1 = Cuts.doBasicCuts(merged_des, raTag = 'RA', decTag='DEC', object = None) merged_des1 = getRawMag( merged_des1, reddening = 'SLR') merged_des1 = getCorrectedMag( merged_des1, reddening = 'SFD98') full_des_data, des_gold = 0, 0 cmass = io.getSGCCMASSphotoObjcat() cmass = Cuts.keepGoodRegion(cmass) clean_cmass_data, clean_cmass_data_des = DES_to_SDSS.match( cmass, merged_des1 ) (trainInd, testInd), (sdsstrainInd,sdsstestInd) = split_samples(merged_des1, merged_des1, [0.1,0.9], random_state=0) des_train = merged_des1[trainInd] des_test = merged_des1[testInd] result_gold_st82 = XDGMM_model(clean_cmass_data_des, clean_cmass_data_des, train=des_train, test=des_test, prefix = 'small_', mock=True) prefix = 'gold_st82_SFD98_2_' presuffix = 'gold_st82_SFD98_2' resultpath = '/n/des/lee.5922/Dropbox/repositories/CMASS/code/result_cat/' fitsio.write(resultpath+'result_'+presuffix+'.fits', result_gold_st82) fitsio.write(resultpath+'clean_cmass_data_des_'+presuffix+'.fits', clean_cmass_data_des) # 4) dmass, SFD98 (test with coadd obects table ( use slrzeroshift instead of XDCORR )--------- full_des_data = io.getDESY1A1catalogs(keyword = 'STRIPE82_COADD', sdssmask=False) des = Cuts.doBasicCuts(full_des_data, raTag = 'RA', decTag='DEC', object = 'galaxy') #des = AddingReddening(des) # add SLR_SHIFT column des = getCorrectedMag( des, reddening = 'SFD98' ) des_gold = io.getDESY1A1catalogs(keyword = 'Y1A1_GOLD_STRIPE82_0', gold=True) des = addphotoz( des = des, im3shape = des_gold) full_des_data = 0 cmass = io.getSGCCMASSphotoObjcat() cmass = Cuts.keepGoodRegion(cmass) clean_cmass_data, clean_cmass_data_des = DES_to_SDSS.match( cmass, des ) prefix = 'coadd_st82_SFD98_' (trainInd, testInd), (sdsstrainInd,sdsstestInd) = split_samples(des, des, [0.05,0.95], random_state=0) des_train = des[trainInd] des_test = des[testInd] clean_cmass_data, clean_cmass_data_des = DES_to_SDSS.match( cmass, des ) result_coadd_st82 = XDGMM_model(clean_cmass_data_des, clean_cmass_data_des, train=des_train, test=des_test, prefix = prefix, mock=True ) resultpath = '/n/des/lee.5922/Dropbox/repositories/CMASS/code/result_cat/' fitsio.write(resultpath+'result_coadd_st82_SFD98.fits', result_coadd_st82) fitsio.write(resultpath+'clean_cmass_data_des_coadd_st82_SFD98.fits', clean_cmass_data_des) #Xtrue,_ = mixing_color( des ) #fitsio.write(resultpath+prefix+'X_obs.fits', Xtrue) """ # Test resulted population in various aspects =================================== # hist in color spaces presuffix = 'coadd_st82_SLR' presuffix = 'gold_st82_SFD98' presuffix = 'coadd_st82_SFD98' presuffix = 'gold_st82_5' presuffix = 'gold_st82_9' # 340 - 360 presuffix = 'gold_st82_9_2' # stripe82 prefix = 'gold_st82_11_small_' # 320-330, -1-1 presuffix = 'gold_st82_11_small' resultpath = '/n/des/lee.5922/Dropbox/repositories/CMASS/code/result_cat/' clean_cmass_data_des = fitsio.read(resultpath+'clean_cmass_data_des_'+'gold_st82_9_2'+'.fits') #result_gold_st82 = fitsio.read(resultpath+'result_'+'gold_st82_9_2'+'.fits') result_gold_st82 = fitsio.read(resultpath+'result_'+presuffix+'.fits') result_gold_st82 = fitsio.read(resultpath+'result_gold_st82_11_small_2.fits') #result_gold_st82 = fitsio.read(resultpath+'result_gold_st82_nomax.fits') #result_gold_st82_s = Cuts.SpatialCuts(result_gold_st82, ra = 320, ra2 = 330, dec=-1, dec=1) probability_calibration( des = result_gold_st82, cmass_des = clean_cmass_data_des, prefix=prefix) result_gold_y1a1 = fitsio.read('/n/des/lee.5922/data/y1a1_coadd/result_gold_y1a1_all.fits') result_gold_st82 = result_gold_y1a1[result_gold_y1a1['DEC'] > -3 ] result_gold_spt = result_gold_y1a1[result_gold_y1a1['DEC'] < -3 ] gold_final = fitsio.read(resultpath+'merged_des_st82.fits') #result_st82 = addphotoz( des = result_st82, im3shape = gold_final) result_gold_st82 = result_gold_y1a1[result_gold_y1a1['DEC'] > -3] result_gold_spt = result_gold_y1a1[result_gold_y1a1['DEC'] < -3] dmass_gold_y1a1, _ = resampleWithPth(result_gold_y1a1, pstart = 0.2 ) dmass_gold_st82, _ = resampleWithPth(result_gold_st82, pstart = 0.05, pmax = 0.9 ) dmass_gold_spt, _ = resampleWithPth(result_gold_spt, pstart = 0.125 ) cmass_des, _= matchCatalogs( result_gold_st82, clean_cmass_data_des , tag = 'COADD_OBJECTS_ID') dmass_gold_st82 = resamplewithCmassHist(result_gold_st82, clean_cmass_data_des, des_st82 = result_gold_st82, pstart = 0.01 ) dmass_gold_spt = resamplewithCmassHist(result_gold_spt, clean_cmass_data_des, des_st82 = result_gold_st82, pstart = 0.125 ) probability_calibration( des = result_gold_st82, cmass_des = clean_cmass_data_des, prefix='gold_st82') from systematics_module.contCorrection import * from systematics_module import contCorrection def getavgbias( cat, pstart=0.1 ): mag = cat['MAG_MODEL_I_corrected'] z = cat['DESDM_ZP'] logL = logL_from_mag( mag = mag, z = z ) #logL = logL[logL > 9.0] avg_b = logL_to_galaxyBias(logL = logL) print 'pstart=',pstart, ' avg bias=',avg_b, ' sample size=', mag.size return avg_b avg_g = getavgbias( clean_cmass_data_des ) avg_g = getavgbias( dmass_gold_st82 ) avg_g = getavgbias( dmass_gold_spt ) ## getting bias for diferent pth_ from systematics_module.contCorrection import * from systematics_module import contCorrection #result_gold_st82 = fitsio.read(resultpath+'result_gold_st82_11_small_2.fits') #mags = ['MAG_DETMODEL', 'MAG_MODEL', 'MAG_APER_4'] #dmass_coadd_st82 = getCorrectedMag( dmass_coadd_st82, mags = mags, reddening = None , suffix='_corrected') def getavgbias( cat, pstart=0.1 ): mag = cat['MAG_MODEL_I_corrected'] z = cat['DESDM_ZP'] logL = logL_from_mag( mag = mag, z = z ) #logL = logL[logL > 9.0] avg_b = logL_to_galaxyBias(logL = logL) print 'pstart=',pstart, ' avg bias=',avg_b, ' sample size=', mag.size return avg_b import healpy as hp # call healpix map GoldMask = callingEliGoldMask() GoldMask_st82 = GoldMask[ GoldMask['DEC'] > -3.0 ] GoldMask_spt = GoldMask[ GoldMask['DEC'] < -3.0 ] pixarea = hp.nside2pixarea( 4096, degrees = True) sptnpix = GoldMask_spt['PIXEL'].size #hp.get_map_size( GoldMask_spt['PIXEL'] ) st82npix = GoldMask_st82['PIXEL'].size # hp.get_map_size( GoldMask_st82 ) SPTMaparea = pixarea * sptnpix ST82Maparea = pixarea * st82npix pbin = np.linspace(0.01, 0.2, 11 ) n_cmass = clean_cmass_data_des.size *1./ST82Maparea avg_bs = np.zeros(pbin.size) num_density = np.zeros(pbin.size) dmass_sample = [] for i, p in enumerate(pbin): sample, _ = resampleWithPth(result_gold_st82, pstart = p, pmax = 0.7) #sample = resamplewithCmassHist(result_gold_st82, clean_cmass_data_des, des_st82 = result_gold_st82, pstart = p ) dmass_sample.append(sample) avg_bs[i] = getavgbias( dmass_sample[i], pstart = p ) num_density[i] = dmass_sample[i].size * 1./ST82Maparea #print p, avg_bs[i], num_density[i] DAT = np.column_stack(( pbin, avg_bs, num_density )) np.savetxt( resultpath+'bias_pcut.txt', DAT, delimiter = ' ', header = 'pbin, avg_bs, num_density # dmass spt' ) fig, ax = plt.subplots() fig2, ax2 = plt.subplots() data = np.loadtxt( resultpath+'bias_pcut.txt') pbin, avg_bs, num_density = data[:,0], data[:,1], data[:,2] for i, p in enumerate(pbin): ax.scatter( p, avg_bs[i], marker = 'o', color = 'black') ax2.scatter( p, num_density[i], marker = 'o', color = 'black') #avg_b = getavgbias( dmass_coadd_st82_2, pstart = 0.01 ) #ax.axhline( y = avg_b, color = 'blue', linestyle = '--', label = 'coadd' ) #ax2.axhline( y = 11646 * 0.9 * 1./n_cmass, color = 'blue', linestyle = '--', label = 'coadd' ) avg_g = getavgbias( clean_cmass_data_des ) ax.axhline( y = 1.17090969569, color = 'red', label = 'cmass' ) ax2.axhline( y = 72.926946637338176, color = 'red', linestyle = '--', label = 'cmass' ) ax.set_xlabel('p start') ax.set_ylabel('bias') ax2.set_xlabel('p start') ax2.set_ylabel('number ratio') ax.legend(loc='best') ax2.legend(loc='best') fig.savefig('figure/bias.png') fig2.savefig('figure/numdensity.png') n_dmass = dmass_gold_spt.size * 1./SPTMaparea avg = getavgbias( dmass_gold_spt ) m1, m2 = esutil.numpy_util.match(dmass_coadd_st82['COADD_OBJECTS_ID'], gold_final['COADD_OBJECTS_ID']) not_in_gold_mask = np.ones(dmass_coadd_st82.size, dtype=bool) not_in_gold_mask[m1] = 0 not_in_gold = dmass_coadd_st82[not_in_gold_mask] #not_in_gold_z = addphotoz(des=not_in_gold, im3shape=gold_final) ## there must be bugs.. m, mm = esutil.numpy_util.match(not_in_gold['COADD_OBJECTS_ID'], gold_final['COADD_OBJECTS_ID']) clean_cmass_data_des = getRawMag( clean_cmass_data_des, reddening = 'SLR' ) mags = ['MAG_DETMODEL', 'MAG_MODEL'] clean_cmass_data_des = getCorrectedMag( clean_cmass_data_des, mags = mags, reddening = 'SFD98' ) not_in_gold = getCorrectedMag( not_in_gold, mags = mags, reddening = 'SFD98' ) A_not,_ = mixing_color(not_in_gold) A_true,_ = mixing_color(clean_cmass_data_des) doVisualization( A_not, A_true, labels = labels, nbins=100, ranges = ranges, prefix=presuffix+'test_') doVisualization_z( cats = [clean_cmass_data_des, not_in_gold], labels=['cmass', 'not'], suffix = 'test') # ----------- probability_calibration( des = result_gold_st82, cmass_des = clean_cmass_data_des, prefix = 'test' ) probability_calibration( des = result_st82, cmass_des = clean_cmass_data_des, prefix = 'test2' ) #result_y1a1 = fitsio.read('/n/des/lee.5922/data/y1a1_coadd/dmass_y1a1.fits') As_X, _ = mixing_color( dmass_gold_st82 ) Xtrue,_ = mixing_color( clean_cmass_data_des ) #Xall = fitsio.read(resultpath+presuffix+'_noisy_X_sample.fits') #X_obs = fitsio.read(resultpath+presuffix+'_X_obs_all.fits') presuffix = 'st82' labels = ['MAG_MODEL_R', 'MAG_MODEL_I', 'g-r', 'r-i', 'i-z'] ranges = [[17,22], [17,22], [0,2], [-.5,1.5], [0.0,.8]] #doVisualization_1d( Xtrue, As_X, labels = labels, ranges = ranges, nbins=100, prefix=presuffix+'_') doVisualization( As_X, Xtrue, labels = labels, nbins=100, name = ['dmass spt','cmass st82' ], ranges = ranges, prefix=presuffix+'_') #doVisualization( Xall, X_obs, labels = labels, nbins=100, ranges = ranges, prefix=presuffix+'_all_') doVisualization_z( cats = [clean_cmass_data_des, dmass_gold_st82], labels=['cmass', 'dmass'], suffix = presuffix) # angular from systematics import * from systematics_weight import doVisualization_Angcorr GoldMask = callingEliGoldMask() GoldMask_st82 = GoldMask[ GoldMask['DEC'] > -3.0 ] GoldMask_spt = GoldMask[ GoldMask['DEC'] < -3.0 ] we = GoldMask_spt['FRAC'] angular_correlation(data = dmass_gold_spt, rand = GoldMask_spt, weight = [None, we], suffix = '_gold_spt') clean_cmass_data_des = fitsio.read(resultpath+'clean_cmass_data_des_'+'gold_st82_9_2'+'.fits') angular_correlation(data = clean_cmass_data_des, rand = GoldMask_st82, weight = [None, we], suffix = '_gold_st82_9_2_cmass') random = fitsio.read('/n/des/lee.5922/data/cmass_cat/'+'random0_DR12v5_CMASS_South.fits.gz') clean_random = Cuts.keepGoodRegion(random) angular_correlation(data = dmass_gold_st82, rand = clean_random, weight = [None, None], suffix = '_gold_st82_9_2_cmass_random') labels = ['gold_st82_9_2', 'gold_spt', 'gold_st82_9_2_cmass'] name = ['dmass st82', 'dmass spt', 'cmass st82'] doVisualization_Angcorr( labels = labels, name = name )
def angular_correlation(self, data=None, rand=None, weight = None, suffix = ''): from systematics_module import corr w_data = corr.angular_correlation(data = data, rand = rand, weight = weight, suffix = suffix, out = True) return w_data