示例#1
0
def createBFITsample():

    # load the truth
    truth_array = tabletools.loadTable(table_name='truth_array',filepath=args.filepath_truth,dtype=dtype_table_truth)
    logger.info('truth array have %d galaxies' % len(truth_array))

    
    # load the results
    results_array = tabletools.loadTable(table_name='results_array',filepath=args.filepath_results,dtype=dtype_table_results)
    logger.info('results array have %d galaxies' % len(results_array))

    # load the stats - they contain the galaxies which passed
    stats_array = tabletools.loadTable(table_name='stats_sarray',filepath=args.filepath_stats,dtype=dtype_table_stats)
    logger.info('stats array have %d galaxies' % len(stats_array))

    results_array_bfit = numpy.zeros(1,dtype=dtype_table_results)
    truth_array_bfit = numpy.zeros(1,dtype=dtype_table_truth)

    for ig,g in enumerate(stats_array['cosmos_id']):
        select = truth_array['id_cosmos'] == g

        results_rows = results_array[select] 
        truth_rows = truth_array[select]

        if ig % 100 == 0 : 
            logger.info('passing galaxy %10d results id %10d truth id %10d' % (ig,results_rows['identifier'][0],truth_rows['id_unique'][0]))
            
        results_array_bfit = numpy.append(results_array_bfit,results_rows)
        truth_array_bfit = numpy.append(truth_array_bfit,truth_rows)

    # remove the last one
    results_array_bfit = results_array_bfit[1:]
    truth_array_bfit = truth_array_bfit[1:]
    n_gals = len(truth_array_bfit)

    logger.info('number of galaxies in bfit sample %d' % n_gals)

    n_gals = len(stats_array)

    filename_results_bfit = args.filepath_results.replace('results','bfit').replace('pp','fits')
    filename_truth_bfit = 'truth.%d.fits' % n_gals

    fits_results = tabletools.getFITSTable(results_array_bfit)
    fits_truth   = tabletools.getFITSTable(truth_array_bfit)
    n_gals_fits = fits_truth[1].data.shape
    logger.info('got fits table from numpy, with %d rows' % n_gals_fits)

    fits_results.writeto(filename_results_bfit,clobber=True)
    fits_truth.writeto(filename_truth_bfit,clobber=True)
    logger.info('saved %s %s' % (filename_truth_bfit,filename_results_bfit))

    # check

    results_array_bfit_loaded = pyfits.open(filename_results_bfit)
    n_loaded = len(results_array_bfit_loaded[1].data)
    logger.info('loaded n_gals %d' % n_loaded)
    logger.info('first ids %10d %10d' % (results_array_bfit_loaded[1].data[ 0]['identifier'] , results_array_bfit['identifier'][0]))
    logger.info('last  ids %10d %10d' % (results_array_bfit_loaded[1].data[-1]['identifier'] , results_array_bfit['identifier'][-1]))
    logger.info('first e1 % f % f' % (results_array_bfit_loaded[1].data['e1'][0]  , results_array_bfit['e1'][0]  ))
    logger.info('last  e1 % f % f' % (results_array_bfit_loaded[1].data['e1'][-1] , results_array_bfit['e1'][-1] ))
def main():

    description = 'filaments_fit'
    parser = argparse.ArgumentParser(description=description, add_help=True)
    parser.add_argument('-v', '--verbosity', type=int, action='store', default=2, choices=(0, 1, 2, 3 ), help='integer verbosity level: min=0, max=3 [default=2]')
    # parser.add_argument('-o', '--filename_output', default='test2.cat',type=str, action='store', help='name of the output catalog')
    # parser.add_argument('-c', '--filename_config', default='test2.yaml',type=str, action='store', help='name of the yaml config file')
    # parser.add_argument('-d', '--dry', default=False,  action='store_true', help='Dry run, dont generate data')

    args = parser.parse_args()
    # Parse the integer verbosity level from the command line args into a logging_level string
    logging_levels = { 0: logging.CRITICAL, 
                       1: logging.WARNING,
                       2: logging.INFO,
                       3: logging.DEBUG }
    logging_level = logging_levels[args.verbosity]
    logging.basicConfig(format="%(message)s", level=logging_level, stream=sys.stdout)
    log = logging.getLogger("filaments_fit") 
    log.setLevel(logging_level)

    id_pair = 7
    filename_shears = 'shears_bcc_g.%03d.fits' % id_pair
    filename_pairs = 'pairs_bcc.fits'
    filename_halo1 = 'pairs_bcc.halos1.fits'

    pairs_table = tabletools.loadTable(filename_pairs)
    shears_info = tabletools.loadTable(filename_shears)
    halo1_table = tabletools.loadTable(filename_halo1)


    fitobj = filaments_model_1hmc.modelfit()
    fitobj.sigma_g =  0.01
    fitobj.shear_g1 =  shears_info['g1sc'] + np.random.randn(len(shears_info['g1sc']))*fitobj.sigma_g
    fitobj.shear_g2 =  shears_info['g2sc'] + np.random.randn(len(shears_info['g1sc']))*fitobj.sigma_g
    fitobj.shear_u_arcmin =  shears_info['u_arcmin']
    fitobj.shear_v_arcmin =  shears_info['v_arcmin']
    fitobj.halo_u_arcmin =  pairs_table['u1_arcmin'][id_pair]
    fitobj.halo_v_arcmin =  pairs_table['v1_arcmin'][id_pair]
    fitobj.halo_z =  pairs_table['z'][id_pair]

    pair_info = pairs_table[id_pair]

    fitobj.run_mcmc()
    print fitobj.sampler
    print halo1_table['m200'][id_pair]

    pl.figure()
    pl.hist(fitobj.sampler.flatchain[:,0], 100, color="k", histtype="step")
    pl.figure
    pl.hist(fitobj.sampler.flatchain[:,1], 100, color="k", histtype="step")
    pl.show()


    import pdb; pdb.set_trace()
示例#3
0
def main():

    global log , config , args

    description = 'Get statistics and plot results of noise bias calibration runs'
    parser = argparse.ArgumentParser(description=description, add_help=True)
    parser.add_argument('-v', '--verbosity', type=int, action='store', default=2, choices=(0, 1, 2, 3 ), help='integer verbosity level: min=0, max=3 [default=2]')
    parser.add_argument('-i', '--filename_input', default='sha1-O1.cat',type=str, action='store', help='name of the output catalog')
    parser.add_argument('-c', '--filename_config', default='sha1.yaml',type=str, action='store', help='name of the yaml config file')
    parser.add_argument('-m', '--method_id', default='hsm',type=str, action='store', help='name of the yaml config file')
    
    args = parser.parse_args()
    # Parse the integer verbosity level from the command line args into a logging_level string
    logging_levels = { 0: logging.CRITICAL, 
                       1: logging.WARNING,
                       2: logging.INFO,
                       3: logging.DEBUG }
    logging_level = logging_levels[args.verbosity]
    
    log = logging.getLogger("nbc1_plots") 
    log.setLevel(logging_level)  
    log_formatter = logging.Formatter("%(asctime)s  %(name)s  %(levelname)s  %(message)s ")
    stream_handler = logging.StreamHandler(sys.stdout)
    stream_handler.setFormatter(log_formatter)
    log.addHandler(stream_handler)    

    log.info(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))

    config = yaml.load(open(args.filename_config))

    filename_results_sv_example = '/home/tomek/projects/131118_nbc1/sv-clusters-shears/results_sv_clusters.all.fits'
    filename_cal_all = '/home/tomek/projects/131118_nbc1/run-tiled-002/cleaned_calib.all.fits'

    global results_filename_fmt
    # results_filename_fmt = 'calib.v4.2013.12.20/cleaned_calib.v4.2013.12.20/nbc_%03d.fits.im3.cleaned.cat'
    results_filename_fmt = 'calib.v4.2014.01.24/cleaned_calib.v4.2014.01.24/nbc_%03d.fits.im3.cleaned.cat'

    global results_sv_example
    results_sv_example = tabletools.loadTable(filename_results_sv_example)

    global results_cal_all
    results_cal_all = tabletools.loadTable(filename_cal_all)

    # get_stats()  
    # get_weights_with_histograms()
    plot_mc_in_fwhm_ratio_bins()
    # plot_ellipticity_variance()
    # get_mc()
    # plot_mc()
    # plot_mc_cuts()

    log.info(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))
def estimate_snr():

    pairs_table = hp.get_pairs(Dxy=[6,18],Mstar=[3e13,1e16],zbin=[0.01,0.9],filename_halos='wide.fits',n_sigma=3)
    n_pairs=len(pairs_table)
    print 'n_pairs', n_pairs
    n_pairs_sdss = 200000
    n_eff_sdss = 0.5
    n_eff_cfht = 15
    n_sigma_sdss = 10
    kernel_gain_cfht = 3


    n_eff_pairs_sdss = n_pairs_sdss*n_eff_sdss
    print 'n_eff_pairs_sdss', n_eff_pairs_sdss
    n_eff_pairs_cfht = n_pairs*n_eff_cfht*kernel_gain_cfht
    print 'n_eff_pairs_cfht', n_eff_pairs_cfht

    sigma_single_filament = np.sqrt(n_eff_pairs_sdss)/n_sigma_sdss
    n_sigma_cfht = np.sqrt(n_eff_pairs_cfht)/sigma_single_filament

    print 'n_sigma_cfht', n_sigma_cfht

    filename_durret = 'wide.fits'
    durret_clusters = tabletools.loadTable(filename_durret)
    pl.hist(durret_clusters['snr'],bins=range(1,10))
    pl.show()
def get_shear_files_catalog():


    filelist_shears = np.loadtxt('filelist_shears.txt',dtype='a1024')  

    list_shearbase = []
    total_n_gals = 0
    for ix,fs in enumerate(filelist_shears):
        logger.info('%3d\t%s\t%1.2e' % ( ix, fs ,float(total_n_gals) ))
        shear_cat_full=tabletools.loadTable(fs)
        # use every 100 shear for speed
        shear_cat = shear_cat_full[::100]
        radius = shear_cat['ra']*0 + 1 # set to 1
        # xyz = cosmology.get_euclidian_coords(shear_cat['ra'], shear_cat['dec'] , radius)
        xs,ys,zs = cosmology.spherical_to_cartesian_deg(shear_cat['ra'], shear_cat['dec'] , radius)  
        x,y,z = np.mean(xs), np.mean(ys) , np.mean(zs)
        del(xs) 
        del(ys)
        del(zs)
        row = np.array([(ix, len(shear_cat_full),fs,x,y,z )],dtype=dtype_shearbase)        
        total_n_gals += len(shear_cat_full)
        list_shearbase.append(row)
        del(shear_cat)
        del(shear_cat_full)

    logger.info('total gals %d',total_n_gals)
    shearbase = np.concatenate(list_shearbase)
    tabletools.saveTable(filename_shearbase,shearbase)
    def get_bcc_pz(self,filename_lenscat):

        if self.prob_z == None:

            # filename_lenscat = os.environ['HOME'] + '/data/BCC/bcc_a1.0b/aardvark_v1.0/lenscats/s2n10cats/aardvarkv1.0_des_lenscat_s2n10.351.fit'
            # filename_lenscat = os.environ['HOME'] + '/data/BCC/bcc_a1.0b/aardvark_v1.0/lenscats/s2n10cats/aardvarkv1.0_des_lenscat_s2n10.351.fit'

            if 'fits' in filename_lenscat:
                lenscat = tabletools.loadTable(filename_lenscat)
                if 'z' in lenscat.dtype.names:
                    self.prob_z , _  = pl.histogram(lenscat['z'],bins=self.grid_z_edges,normed=True)
                elif 'z-phot' in lenscat.dtype.names:
                    self.prob_z , _  = pl.histogram(lenscat['z-phot'],bins=self.grid_z_edges,normed=True)

                if 'e1' in lenscat.dtype.names:

                    select = lenscat['star_flag'] == 0
                    lenscat = lenscat[select]
                    select = lenscat['fitclass'] == 0
                    lenscat = lenscat[select]
                    select = (lenscat['e1'] != 0.0) * (lenscat['e2'] != 0.0)
                    lenscat = lenscat[select]
                    self.sigma_ell = np.std(lenscat['e1']*lenscat['weight'],ddof=1)

            elif 'pp2' in filename_lenscat:

                pickle = tabletools.loadPickle(filename_lenscat,log=0)
                self.prob_z =  pickle['prob_z']
                self.grid_z_centers = pickle['bins_z']
                self.grid_z_edges = plotstools.get_bins_edges(self.grid_z_centers)
    def get_bcc_pz(self):

        if self.prob_z == None:


            filename_lenscat = os.environ['HOME'] + '/data/BCC/bcc_a1.0b/aardvark_v1.0/lenscats/s2n10cats/aardvarkv1.0_des_lenscat_s2n10.351.fit'
            lenscat = tabletools.loadTable(filename_lenscat)
            self.prob_z , _  = pl.histogram(lenscat['z'],bins=self.grid_z_edges,normed=True)
def get_calibration_bins():

    results_calib = tabletools.loadTable(filename_results_calib)  

    n_all = 0
    for isize,vsize in enumerate(bins_size[:-1]):
            for isnr,vsnr in enumerate(bins_snr[:-1]):

                bin_size_min = bins_size[isize]
                bin_size_max = bins_size[isize+1]
                bin_snr_min = bins_snr[isnr]
                bin_snr_max = bins_snr[isnr+1]
                select_size = (results_calib['size'] < bin_size_max) * (results_calib['size'] > bin_size_min)
                select_snr = (results_calib['snr'] < bin_snr_max) * (results_calib['snr'] > bin_snr_min)
                select = select_size * select_snr
                current_results = results_calib[select]
                # pl.subplot(1,3,1)
                # pl.hist(current_results['size'])
                # pl.subplot(1,3,2)
                # pl.hist(current_results['snr'])
                # pl.subplot(1,3,3)
                # pl.hist(current_results['isnr_true'])
                # pl.show()
                n_gals = len(current_results)
                n_all+=n_gals


                g1_tru = current_results['g1_true']
                g2_tru = current_results['g2_true']
                g1_est = current_results['g1']
                g2_est = current_results['g2']
                g1_err  = np.ones(n_gals)*np.std(current_results['g1'],ddof=1)
                g2_err  = np.ones(n_gals)*np.std(current_results['g2'],ddof=1)

                [c1,m1,C1cm] = fitting.get_line_fit(g1_tru,g1_est,g1_err)
                [c2,m2,C2cm] = fitting.get_line_fit(g2_tru,g2_est,g2_err)
                m1_std = np.sqrt(C1cm[1,1])
                m2_std = np.sqrt(C2cm[1,1])
                c1_std = np.sqrt(C1cm[0,0])
                c2_std = np.sqrt(C2cm[0,0])
                m_mean = (m1 + m2)/2.
                m_mean_std = np.sqrt((m1_std**2 + m2_std**2)/2.)

                calib_struct['bias_m'][isize][isnr] = m_mean
                calib_struct['bias_m_std'][isize][isnr] = m_mean_std

                del(select_size)
                del(select_snr)
                del(current_results)              

                log.info('size=%d [%5.2f,% 5.2f]\tsnr=%d [%5.2f,%5.2f]\tn_gals=%d\tm_mean=%2.4f\t(% 2.4f)' % (isize,bin_size_min,bin_size_max,isnr,bin_snr_min,bin_snr_max,n_gals,m_mean,m_mean_std))

    file_calibration = open(filename_calibration,'w')   
    pickle.dump(calib_struct,file_calibration)  
    file_calibration.close()  
    log.info('pickled %s' % filename_calibration)
def calibrate_results():

    filelist = np.loadtxt(filelist_svclusters,dtype='a')

    for ifile,vfile in enumerate(filelist):

        results_sv = tabletools.loadTable(vfile,dtype=dtype_table_results_sv,log=1)
        results_sv = add_calibration_columns(results_sv)

        filename_calibrated = vfile.replace('.cat','.nbc.cat')
        tabletools.saveTable(filename_calibrated,results_sv)
        log.info('calibrated file %3d %s' %(ifile,filename_calibrated)) 
    def get_bcc_pz(self,filename_lenscat):

        if self.prob_z == None:


            # filename_lenscat = os.environ['HOME'] + '/data/BCC/bcc_a1.0b/aardvark_v1.0/lenscats/s2n10cats/aardvarkv1.0_des_lenscat_s2n10.351.fit'
            # filename_lenscat = os.environ['HOME'] + '/data/BCC/bcc_a1.0b/aardvark_v1.0/lenscats/s2n10cats/aardvarkv1.0_des_lenscat_s2n10.351.fit'
            lenscat = tabletools.loadTable(filename_lenscat)

            if 'z' in lenscat.dtype.names:
                self.prob_z , _  = pl.histogram(lenscat['z'],bins=self.grid_z_edges,normed=True)
            elif 'z-phot' in lenscat.dtype.names:
                self.prob_z , _  = pl.histogram(lenscat['z-phot'],bins=self.grid_z_edges,normed=True)

            if 'e1' in lenscat.dtype.names:
                self.sigma_ell = np.std(lenscat['e1'],ddof=1)
def get_shears_for_single_pair(halo1,halo2,idp=0):

    global cfhtlens_shear_catalog
    if cfhtlens_shear_catalog == None:
        filename_cfhtlens_shears =  config['filename_cfhtlens_shears']

        cfhtlens_shear_catalog = tabletools.loadTable(filename_cfhtlens_shears)
        if 'star_flag' in cfhtlens_shear_catalog.dtype.names:
            select = cfhtlens_shear_catalog['star_flag'] == 0
            cfhtlens_shear_catalog = cfhtlens_shear_catalog[select]
            select = cfhtlens_shear_catalog['fitclass'] == 0
            cfhtlens_shear_catalog = cfhtlens_shear_catalog[select]
            logger.info('removed stars, remaining %d' , len(cfhtlens_shear_catalog))

            select = (cfhtlens_shear_catalog['e1'] != 0.0) * (cfhtlens_shear_catalog['e2'] != 0.0)
            cfhtlens_shear_catalog = cfhtlens_shear_catalog[select]
            logger.info('removed zeroed shapes, remaining %d' , len(cfhtlens_shear_catalog))

    # correcting additive systematics
    if 'e1corr' in cfhtlens_shear_catalog.dtype.names:       
        shear_g1 , shear_g2 = cfhtlens_shear_catalog['e1corr'] , -cfhtlens_shear_catalog['e2corr']
        shear_ra_deg , shear_de_deg , shear_z = cfhtlens_shear_catalog['ALPHA_J2000'] , cfhtlens_shear_catalog['DELTA_J2000'] ,  cfhtlens_shear_catalog['Z_B']
    else:
        shear_g1 , shear_g2 = cfhtlens_shear_catalog['e1'] , -(cfhtlens_shear_catalog['e2']  - cfhtlens_shear_catalog['c2'])
        shear_ra_deg , shear_de_deg , shear_z = cfhtlens_shear_catalog['ra'] , cfhtlens_shear_catalog['dec'] ,  cfhtlens_shear_catalog['z']

    halo1_ra_deg , halo1_de_deg = halo1['ra'],halo1['dec']
    halo2_ra_deg , halo2_de_deg = halo2['ra'],halo2['dec']

    pair_ra_deg,  pair_de_deg = cosmology.get_midpoint(halo1_ra_deg , halo1_de_deg , halo2_ra_deg , halo2_de_deg,unit='deg')
    pair_z = np.mean([halo1['z'],halo2['z']])

    pairs_shear , halos_coords , pairs_shear_full   = filaments_tools.create_filament_stamp(halo1_ra_deg, halo1_de_deg, 
                            halo2_ra_deg, halo2_de_deg, 
                            shear_ra_deg, shear_de_deg, 
                            shear_g1, shear_g2, shear_z, 
                            pair_z, lenscat=cfhtlens_shear_catalog , shear_bias_m=cfhtlens_shear_catalog['m'] , shear_weight=cfhtlens_shear_catalog['weight'] )

    if len(pairs_shear) < 100:
        logger.error('found only %d shears' % len(pairs_shear))
        return None , None , None

    return pairs_shear , halos_coords, pairs_shear_full
示例#12
0
def get_psf_index(ifwhm,ie1,ie2):

    filename_key = 'psf_key.fits'
    psf_table = tabletools.loadTable(filename_key)
    key_all , key_fwhm , key_e1 , key_e2  = psf_table['id_psf'] , psf_table['id_psf_fwhm'] , psf_table['id_psf_e1'] , psf_table['id_psf_e2']

    # for ii,vv in enumerate(psf_table):
    #     print 'psf_table index=%3d ifwhm=%d ie1=%d ie2=%d' % (key_all[ii] , key_fwhm[ii], key_e1[ii], key_e2[ii])


    indices = np.ones_like(ifwhm)
    for ii,vv in enumerate(ifwhm):
        select = (ifwhm[ii]==key_fwhm) * (ie1[ii]==key_e1) * (ie2[ii]==key_e2)
        nz = np.nonzero(select)[0][0]
        indices[ii] = key_all[nz] 
        # print 'index=%3d ifwhm=%d ie1=%d ie2=%d' % (indices[ii] , ifwhm[ii], ie1[ii], ie2[ii])



    # import pdb; pdb.set_trace()

    return indices
def run_all():

    bins_z = np.arange(0.025,3.5,0.05)

    filename_gals = '/home/kacprzak/data/CFHTLens/CFHTLens_2014-06-14.normalised.fits'
    filename_clusters = os.environ['HOME'] + '/data/CFHTLens/ClusterZ/clustersz.fits'

    cat_clusters = tabletools.loadTable(filename_clusters)
    cat_gals = tabletools.loadTable(filename_gals)

    gals_ra_deg = cat_gals['ALPHA_J2000']
    gals_de_deg = cat_gals['DELTA_J2000']
    gals_ra_rad , gals_de_rad = cosmology.deg_to_rad(gals_ra_deg, gals_de_deg)

    cylinder_radius_mpc=1

    pz_all=np.sum(cat_gals['PZ_full'],axis=0)
    pz_all=pz_all/np.sum(pz_all)

    n_brigthest = 40
    n_bins_hires = 10000
    bins_z_hires=np.linspace(bins_z.min(), bins_z.max(),n_bins_hires)
    new_z = np.zeros(len(cat_clusters))

 
    for ic in range(len(cat_clusters)):
    # for ic in range(2):

        cluster_ra_rad , cluster_de_rad = cosmology.deg_to_rad( cat_clusters[ic]['ra'] , cat_clusters[ic]['dec'] )
        cluster_z = cat_clusters['z_bad'][ic]
        
        gals_u_rad , gals_v_rad = cosmology.get_gnomonic_projection(gals_ra_rad , gals_de_rad , cluster_ra_rad , cluster_de_rad)
        gals_u_mpc , gals_v_mpc = cosmology.rad_to_mpc(gals_u_rad,gals_v_rad,cluster_z)

        select = (np.sqrt(gals_u_mpc**2 + gals_v_mpc**2) < cylinder_radius_mpc)*( np.abs(cat_gals['Z_B']-cluster_z) < 0.1 )
        # print 'selected %d gals in cylinder' % len(np.nonzero(select)[0])
        cylinder_gals = cat_gals[select]
        gals_u_mpc = gals_u_mpc[select]
        gals_v_mpc = gals_v_mpc[select]

        select1 = (cylinder_gals['MAG_i'] > 10) * (cylinder_gals['MAG_i'] < 27)
        select2 = (cylinder_gals['MAG_r'] > 10) * (cylinder_gals['MAG_r'] < 27)
        select3 = (cylinder_gals['MAG_g'] > 10) * (cylinder_gals['MAG_g'] < 27)
        select4 = (cylinder_gals['MAG_u'] > 10) * (cylinder_gals['MAG_u'] < 27)
        select5 = (cylinder_gals['MAG_y'] > 10) * (cylinder_gals['MAG_y'] < 27)
        select6 = (cylinder_gals['MAG_z'] > 10) * (cylinder_gals['MAG_z'] < 27)

        select = select2*select1*select3*select4*select6
        # print 'selected %d with good mags' % len(np.nonzero(select)[0])
        if len(np.nonzero(select)[0]) == 0:
            print '%d not enough gals' , ic
            new_z[ic] = cat_clusters['z'][ic]
            continue
        cylinder_gals = cylinder_gals[select]
        gals_u_mpc = gals_u_mpc[select]
        gals_v_mpc = gals_v_mpc[select]

        x1=cylinder_gals['MAG_r']-cylinder_gals['MAG_i']
        x2=cylinder_gals['MAG_g']-cylinder_gals['MAG_i']
        x3=cylinder_gals['MAG_u']-cylinder_gals['MAG_i']
        x4=cylinder_gals['MAG_y']-cylinder_gals['MAG_i']
        x5=cylinder_gals['MAG_z']-cylinder_gals['MAG_i']
        

        X=np.concatenate( [x1.astype('f4')[:,None], x2.astype('f4')[:,None], x3.astype('f4')[:,None],  x5.astype('f4')[:,None]] ,axis=1)

        from scipy.stats import gaussian_kde
        kde = gaussian_kde(X.T,bw_method=0.3) 
        w = kde(X.T)**3
        w = w/np.max(w)
        # pl.figure()
        # pl.scatter(X[:,0],X[:,1],s=50,c=w) ; pl.colorbar()
        # pl.figure()
        # pl.scatter(X[:,1],X[:,2],s=50,c=w) ; pl.colorbar()
        # pl.figure()
        # pl.scatter(X[:,0],X[:,2],s=50,c=w) ; pl.colorbar()
        # pl.figure()
        # pl.scatter(X[:,0],X[:,3],s=50,c=w) ; pl.colorbar()
        # # pl.figure()
        # # pl.scatter(X[:,0],X[:,4],s=50,c=w) ; pl.colorbar()
        # pl.show()

        # pl.figure()
        # pl.scatter(cylinder_gals[select_brightest]['MAG_r']-cylinder_gals[select_brightest]['MAG_i'],  cylinder_gals[select_brightest]['MAG_g']-cylinder_gals[select_brightest]['MAG_i'],c=cylinder_gals[select_brightest]['Z_B'])
        # pl.colorbar()
        # pl.figure()
        # pl.scatter(cylinder_gals[select_brightest]['MAG_i'],cylinder_gals[select_brightest]['MAG_r'],c=cylinder_gals[select_brightest]['Z_B'])
        # pl.colorbar()
        # pl.figure()
        # pl.scatter(cylinder_gals[select_brightest]['MAG_g'],cylinder_gals[select_brightest]['MAG_r'],c=cylinder_gals[select_brightest]['Z_B'])
        # pl.colorbar()
        # pl.figure()
        # pl.scatter(cylinder_gals[select_brightest]['MAG_u'],cylinder_gals[select_brightest]['MAG_i'],c=cylinder_gals[select_brightest]['Z_B'])
        # pl.colorbar()
        # pl.figure()
        # pl.scatter(cylinder_gals[select_brightest]['MAG_r'],cylinder_gals[select_brightest]['Z_B'],c=cylinder_gals[select_brightest]['Z_B'])
        # pl.colorbar()

        # pl.figure()
        # pl.scatter(gals_u_mpc,gals_v_mpc,c=cylinder_brightest['Z_B'],s=cylinder_brightest['MAG_r']*2)
        # pl.colorbar()

        # pl.show()

        # pz_hires = np.zeros([len(cylinder_gals),n_bins_hires])
        # for ib in range(len(cylinder_gals)):
        #     fz=interp1d(bins_z,cylinder_gals['PZ_full'][ib],'cubic')
        #     pz_hires[ib,:] = fz(bins_z_hires)
        #     pz_this = pz_hires[ib,:]/np.sum(pz_hires[ib,:])*w[ib]
        #     pl.plot(bins_z_hires,pz_this)
        #     # pl.plot(bins_z,cylinder_gals['PZ_full'][ib],'-');
        #     print 'interp' , ib, np.sum(pz_this)

        # pz_hires[pz_hires<0] = 1e-10
        # pz_prod = np.sum(np.log(pz_hires),axis=0)
        # pz_prod = pz_prod - pz_prod.max()
        # pz_cylinder=np.exp(pz_prod)
        # pz_cylinder=pz_cylinder/np.sum(pz_cylinder)

        # new_z[ic] = bins_z_hires[pz_cylinder.argmax()]
        new_z[ic] = np.sum(cylinder_gals['Z_B']*w)/np.sum(w)
        std_z=np.std(np.sqrt(((cylinder_gals['Z_B']*w - new_z[ic])**2)/np.sum(w)))
        print '%3d new_z=%.4f bad_z=%.4f naomi_z=%.4f n_eff=%2.4f n_cylinder_gals=%d std_z=%2.5f'  % (ic,new_z[ic],cluster_z,cat_clusters['z'][ic],np.sum(w),len(cylinder_gals),std_z)

    tabletools.appendColumn(arr=new_z,rec=cat_clusters,dtype='f4',name='z_est')
    filename_clusters_est = filename_clusters.replace('.fits','.update.fits')
    tabletools.saveTable(filename_clusters_est,cat_clusters)
def fit_single_filament():

    id_pair = 7
    # fitobj.parameters[0]['box']['min'] = 0.01
    # fitobj.parameters[0]['box']['max'] = 0.03
    # fitobj.parameters[1]['box']['min'] = 0.1
    # fitobj.parameters[1]['box']['max'] = 2.5


    filename_shears = 'shears_bcc_g.%03d.fits' % id_pair
    filename_pairs = 'pairs_bcc.fits'
    filename_halo1 = 'pairs_bcc.halos1.fits'
    filename_halo2 = 'pairs_bcc.halos2.fits'

    pairs_table = tabletools.loadTable(filename_pairs)
    shears_info = tabletools.loadTable(filename_shears)
    halo1_table = tabletools.loadTable(filename_halo1)
    halo2_table = tabletools.loadTable(filename_halo2)

    true_M200 = np.log10(halo1_table['m200'][id_pair])
    true_M200 = np.log10(halo2_table['m200'][id_pair])

    halo1_conc = halo1_table['r200'][id_pair]/halo1_table['rs'][id_pair]*1000.
    halo2_conc = halo2_table['r200'][id_pair]/halo2_table['rs'][id_pair]*1000.

    log.info( 'halo1 M200 %5.2e',halo1_table['m200'][id_pair] )
    log.info( 'halo2 M200 %5.2e',halo2_table['m200'][id_pair] )
    log.info( 'halo1 conc %5.2f',halo1_conc)
    log.info( 'halo2 conc %5.2f',halo2_conc)

    fitobj = filaments_model_1f.modelfit()
    fitobj.get_bcc_pz()
    fitobj.sigma_g =  0.02
    fitobj.shear_g1 =  shears_info['g1'] + np.random.randn(len(shears_info['g1']))*fitobj.sigma_g
    fitobj.shear_g2 =  shears_info['g2'] + np.random.randn(len(shears_info['g2']))*fitobj.sigma_g
    fitobj.shear_u_arcmin =  shears_info['u_arcmin']
    fitobj.shear_v_arcmin =  shears_info['v_arcmin']
    
    fitobj.halo1_u_arcmin =  pairs_table['u1_arcmin'][id_pair
    ]
    fitobj.halo1_v_arcmin =  pairs_table['v1_arcmin'][id_pair]
    fitobj.halo1_z =  pairs_table['z'][id_pair]
    fitobj.halo1_M200 = halo1_table['m200'][id_pair]
    fitobj.halo1_conc = halo1_conc

    fitobj.halo2_u_arcmin =  pairs_table['u2_arcmin'][id_pair]
    fitobj.halo2_v_arcmin =  pairs_table['v2_arcmin'][id_pair]
    fitobj.halo2_z =  pairs_table['z'][id_pair]
    fitobj.halo2_M200 = halo2_table['m200'][id_pair]
    fitobj.halo2_conc = halo2_conc

    fitobj.parameters[0]['box']['min'] = 0.0001
    fitobj.parameters[0]['box']['max'] = 0.1
    fitobj.parameters[1]['box']['min'] = 0.0001
    fitobj.parameters[1]['box']['max'] = 10
    
    # fitobj.plot_shears_mag(fitobj.shear_g1,fitobj.shear_g2)
    # pl.show()
    # fitobj.save_all_models=False
    log.info('running grid search')
    log_post , params, grid_kappa0, grid_radius = fitobj.run_gridsearch(n_grid=100)
    vmax_post , best_model_g1, best_model_g2 , limit_mask,  vmax_kappa0 , vmax_radius = fitobj.get_grid_max(log_post,params)

    scatter_size=10

    pl.figure()
    pl.subplot(1,2,1)
    prob_post = get_post_from_log(log_post)
    pl.scatter( params[:,0] , params[:,1] , scatter_size , log_post , lw=0)
    pl.colorbar()
    pl.subplot(1,2,2)
    pl.scatter( params[:,0] , params[:,1] , scatter_size , prob_post , lw=0)
    pl.plot( vmax_kappa0 , vmax_radius , 'ro' )
    pl.colorbar()
    filename_fig = 'post.png'
    pl.savefig(filename_fig, dpi=1000)
    

    # fitobj.plot_residual_whisker(best_model_g1, best_model_g2)
    # pl.suptitle('model post=% 10.4e kappa0=%5.2e radius=%2.4f' % (vmax_post,vmax_kappa0,vmax_radius) )
    # fitobj.plot_residual_g1g2(best_model_g1, best_model_g2)
    # pl.suptitle('model post=% 10.4e kappa0=%5.2e radius=%2.4f' % (vmax_post,vmax_kappa0,vmax_radius) )


    

    log.info('running mcmc')
    import pdb; pdb.set_trace()
    fitobj.n_samples=5000
    fitobj.run_mcmc()
    samples = fitobj.sampler.flatchain
    

    import pdb; pdb.set_trace()
    pl.figure()
    plotstools.plot_dist(samples)
    pl.show()
import pyfits
import numpy as np
import pylab as pl
import scipy.interpolate as interp
import cosmology
import tabletools
import yaml, argparse, sys, logging 
from sklearn.neighbors import BallTree as BallTree
import galsim


filename_big = 'big_halos.fits'
halocat = tabletools.loadTable(filename_big,table_name='big')
sorting = np.argsort(halocat['M200'])
biggest_halo = halocat[sorting[-3]]

biggest_halo['RA'],biggest_halo['DEC'] = 0. , 0.

dtheta=0.1
lenscat={}
lenscat['ra'] = np.random.uniform(biggest_halo['RA']-dtheta,biggest_halo['RA']+dtheta,1000)
lenscat['dec'] = np.random.uniform((biggest_halo['DEC']-dtheta),(biggest_halo['DEC']+dtheta),1000)
lenscat['z'] = lenscat['dec']*0 + biggest_halo['Z'] * 2.

conc=biggest_halo['RVIR']/(biggest_halo['RS']/1e3)


halo1_ra_arcsec, halo1_de_arcsec = cosmology.deg_to_arcsec(biggest_halo['RA'], biggest_halo['DEC']) 
shear_ra_arcsec, shear_de_arcsec = cosmology.deg_to_arcsec(lenscat['ra'], lenscat['dec']) 
nfw1=galsim.NFWHalo(conc=conc,redshift=biggest_halo['Z'],mass=biggest_halo['M200'],omega_m = cosmology.cospars.omega_m,halo_pos=galsim.PositionD(x=halo1_ra_arcsec,y=halo1_de_arcsec))
(g1,g2,_)=nfw1.getLensing(pos=(shear_ra_arcsec, shear_de_arcsec),z_s=lenscat['z'])
def self_fit():


    fixed_kappa  = 0.05
    fixed_radius = 2
    fixed_m200 = 14
    fixed_m200 = 14

    filename_pairs = 'pairs_cfhtlens_null1.fits'
    filename_halo1 = 'pairs_cfhtlens_null1.halos1.fits'
    filename_halo2 = 'pairs_cfhtlens_null1.halos2.fits'
    filename_shears = 'shears_cfhtlens_g_null1.fits'
    filename_selffit = 'shears_selftest_kappa%2.2f.fits' % fixed_kappa

    pairs_table = tabletools.loadTable(filename_pairs)
    halo1_table = tabletools.loadTable(filename_halo1)
    halo2_table = tabletools.loadTable(filename_halo2)

    sigma_g_add =  0.
    fitobj = filaments_model_2hf.modelfit()
    pz = fitobj.get_bcc_pz('cfhtlens_cat_sample.fits')
    prob_z = fitobj.prob_z

    id_pair = 0

    shears_info = tabletools.loadTable(filename_shears,hdu=id_pair+1)
    fitobj = filaments_model_2hf.modelfit()
    fitobj.prob_z = prob_z

    fitobj.halo1_z = 0.2
    fitobj.halo2_z = 0.2
    fitobj.halo1_u_arcmin = 20
    fitobj.halo1_v_arcmin = 0
    fitobj.halo2_u_arcmin = -20
    fitobj.halo2_v_arcmin = 0
    fitobj.shear_v_arcmin =  shears_info['v_arcmin']
    fitobj.shear_u_mpc =  shears_info['u_mpc']
    fitobj.shear_v_mpc =  shears_info['v_mpc']

    fitobj.halo1_u_arcmin =  pairs_table['u1_arcmin'][id_pair]
    fitobj.halo1_v_arcmin =  pairs_table['v1_arcmin'][id_pair]
    fitobj.halo1_u_mpc =  pairs_table['u1_mpc'][id_pair]
    fitobj.halo1_v_mpc =  pairs_table['v1_mpc'][id_pair]
    fitobj.halo1_z =  pairs_table['z'][id_pair]

    fitobj.halo2_u_arcmin =  pairs_table['u2_arcmin'][id_pair]
    fitobj.halo2_v_arcmin =  pairs_table['v2_arcmin'][id_pair]
    fitobj.halo2_u_mpc =  pairs_table['u2_mpc'][id_pair]
    fitobj.halo2_v_mpc =  pairs_table['v2_mpc'][id_pair]
    fitobj.halo2_z =  pairs_table['z'][id_pair]

    fitobj.pair_z  = (fitobj.halo1_z + fitobj.halo2_z) / 2.

    fitobj.filam = filament.filament()
    fitobj.filam.pair_z =fitobj.pair_z
    fitobj.filam.grid_z_centers = fitobj.grid_z_centers
    fitobj.filam.prob_z = fitobj.prob_z
    fitobj.filam.set_mean_inv_sigma_crit(fitobj.filam.grid_z_centers,fitobj.filam.prob_z,fitobj.filam.pair_z)

    fitobj.nh1 = nfw.NfwHalo()
    fitobj.nh1.z_cluster= fitobj.halo1_z
    fitobj.nh1.theta_cx = fitobj.halo1_u_arcmin
    fitobj.nh1.theta_cy = fitobj.halo1_v_arcmin 
    fitobj.nh1.set_mean_inv_sigma_crit(fitobj.grid_z_centers,fitobj.prob_z,fitobj.pair_z)

    fitobj.nh2 = nfw.NfwHalo()
    fitobj.nh2.z_cluster= fitobj.halo2_z
    fitobj.nh2.theta_cx = fitobj.halo2_u_arcmin
    fitobj.nh2.theta_cy = fitobj.halo2_v_arcmin 
    fitobj.nh2.set_mean_inv_sigma_crit(fitobj.grid_z_centers,fitobj.prob_z,fitobj.pair_z)

    fitobj.shear_u_arcmin =  shears_info['u_arcmin']
    fitobj.shear_v_arcmin =  shears_info['v_arcmin']

    shear_model_g1, shear_model_g2, limit_mask , _ , _ = fitobj.draw_model([fixed_kappa, fixed_radius, fixed_m200, fixed_m200])

    fitobj.shear_g1 =  shear_model_g1 + np.random.randn(len(shears_info['g1']))*sigma_g_add
    fitobj.shear_g2 =  shear_model_g2 + np.random.randn(len(shears_info['g2']))*sigma_g_add
    fitobj.sigma_g =  np.std(shear_model_g2,ddof=1)
    # fitobj.inv_sq_sigma_g = 1./sigma_g_add**2
    # logger.info('using sigma_g=%2.5f' , fitobj.sigma_g)

    fitobj.parameters[0]['box']['min'] = 0
    fitobj.parameters[0]['box']['max'] = 1
    fitobj.parameters[1]['box']['min'] = 1
    fitobj.parameters[1]['box']['max'] = 10
    fitobj.parameters[2]['box']['min'] = 14
    fitobj.parameters[2]['box']['max'] = 15
    fitobj.parameters[3]['box']['min'] = 14
    fitobj.parameters[3]['box']['max'] = 15

    # print 'halo1 m200' , halo1_table['m200'][id_pair]
    # print 'halo2 m200' , halo2_table['m200'][id_pair]

    shears_info['g1'] = fitobj.shear_g1
    shears_info['g2'] = fitobj.shear_g2
    fitobj.plot_shears(shears_info['g1'], shears_info['g2'],quiver_scale=0.1)
    pl.show()
    pl.scatter(shears_info['u_mpc'],shears_info['v_mpc'],c=np.abs(shears_info['g1'] + 1j*shears_info['g2'])); 
    pl.colorbar(); pl.show()

    
    tabletools.saveTable(filename_selffit, shears_info)
def add_nfw_to_random_points():
    
    import filaments_model_2hf, filament, nfw

    halo1 = tt.load(config['filename_pairs'].replace('.fits','.halos1.fits'))
    halo2 = tt.load(config['filename_pairs'].replace('.fits','.halos2.fits'))
    pairs_table = tabletools.loadTable(config['filename_pairs'])
    filename_shears_nfw = config['filename_shears'].replace('.pp2','.nfw.pp2')

    if os.path.isfile(filename_shears_nfw): 
        os.remove(filename_shears_nfw)
        logger.warning('overwriting file %s' , filename_shears_nfw)

    fitobj = filaments_model_2hf.modelfit()
    fitobj.get_bcc_pz(config['filename_pz'])
    prob_z = fitobj.prob_z
    grid_z_centers = fitobj.grid_z_centers
    grid_z_edges = fitobj.grid_z_edges

    for id_pair in range(len(pairs_table)):

        id_shear = pairs_table[id_pair]['ipair']
        logger.info('--------- pair %d shear %d --------' , id_pair, id_shear) 
        # now we use that
        id_pair_in_catalog = id_pair
        shears_info = tabletools.loadPickle(config['filename_shears'],pos=id_shear)
    
        fitobj = filaments_model_2hf.modelfit()
        fitobj.kappa_is_K = config['kappa_is_K']
        fitobj.prob_z = prob_z
        fitobj.grid_z_centers = grid_z_centers
        fitobj.grid_z_edges = grid_z_edges
        fitobj.shear_u_arcmin =  shears_info['u_arcmin']
        fitobj.shear_v_arcmin =  shears_info['v_arcmin']
        fitobj.shear_u_mpc =  shears_info['u_mpc']
        fitobj.shear_v_mpc =  shears_info['v_mpc']
        fitobj.shear_g1 =  shears_info['g1']
        fitobj.shear_g2 =  shears_info['g2']
        fitobj.shear_w =  shears_info['weight']
        fitobj.Dlos = pairs_table[id_pair]['Dlos']        
        fitobj.Dtot = np.sqrt(pairs_table[id_pair]['Dxy']**2+pairs_table[id_pair]['Dlos']**2)
        fitobj.boost = fitobj.Dtot/pairs_table[id_pair]['Dxy']
        fitobj.use_boost = config['use_boost']
        fitobj.R_start = config['R_start']

        # choose a method to add and account for noise
        if config['sigma_method'] == 'add':
            sigma_g_add =  config['sigma_add']
            fitobj.shear_g1 =  shears_info['g1'] + np.random.randn(len(shears_info['g1']))*sigma_g_add
            fitobj.shear_g2 =  shears_info['g2'] + np.random.randn(len(shears_info['g2']))*sigma_g_add
            fitobj.sigma_g =  np.std(fitobj.shear_g2,ddof=1)
            fitobj.sigma_ell = fitobj.sigma_g
            fitobj.inv_sq_sigma_g = 1./fitobj.sigma_g**2
            logger.info('added noise with level %f , using sigma_g=%2.5f' , sigma_g_add, fitobj.sigma_g)
        elif config['sigma_method'] == 'orig':
            fitobj.shear_n_gals = shears_info['n_gals']
            fitobj.inv_sq_sigma_g = fitobj.shear_w
            logger.info('using different sigma_g per pixel mean(inv_sq_sigma_g)=%2.5f len(inv_sq_sigma_g)=%d' , np.mean(fitobj.inv_sq_sigma_g) , len(fitobj.inv_sq_sigma_g))
        elif type(config['sigma_method'])==float:
            fitobj.shear_n_gals = shears_info['n_gals']
            fitobj.inv_sq_sigma_g =  shears_info['weight']**2 / ( shears_info['weight_sq'] * config['sigma_method']**2  )
            # remove infs
            fitobj.inv_sq_sigma_g[shears_info['weight_sq']<1e-8]=0
            logger.info('using constant sigma_g per pixel: sigma_e=%2.5f, mean(sigma_gp)=%2.5f  n_zeros=%d len(inv_sq_sigma_g)=%d n_nan=%d n_inf=%d' , config['sigma_method'], len(np.nonzero(shears_info['weight_sq']<1e-8)[0]),  np.mean(fitobj.inv_sq_sigma_g) , len(fitobj.inv_sq_sigma_g) , len( np.nonzero(np.isnan(fitobj.inv_sq_sigma_g))[0] ) , len( np.nonzero(np.isinf(fitobj.inv_sq_sigma_g))[0] ) )

                    
        fitobj.halo1_u_arcmin =  pairs_table['u1_arcmin'][id_pair_in_catalog]
        fitobj.halo1_v_arcmin =  pairs_table['v1_arcmin'][id_pair_in_catalog]
        fitobj.halo1_u_mpc =  pairs_table['u1_mpc'][id_pair_in_catalog]
        fitobj.halo1_v_mpc =  pairs_table['v1_mpc'][id_pair_in_catalog]
        fitobj.halo1_z =  pairs_table['z'][id_pair_in_catalog]

        fitobj.halo2_u_arcmin =  pairs_table['u2_arcmin'][id_pair_in_catalog]
        fitobj.halo2_v_arcmin =  pairs_table['v2_arcmin'][id_pair_in_catalog]
        fitobj.halo2_u_mpc =  pairs_table['u2_mpc'][id_pair_in_catalog]
        fitobj.halo2_v_mpc =  pairs_table['v2_mpc'][id_pair_in_catalog]
        fitobj.halo2_z =  pairs_table['z'][id_pair_in_catalog]

        fitobj.n_model_evals = 0

        fitobj.pair_z  = (fitobj.halo1_z + fitobj.halo2_z) / 2.

        fitobj.filam = filament.filament()
        fitobj.filam.pair_z =fitobj.pair_z
        fitobj.filam.grid_z_centers = fitobj.grid_z_centers
        fitobj.filam.prob_z = fitobj.prob_z
        fitobj.filam.set_mean_inv_sigma_crit(fitobj.filam.grid_z_centers,fitobj.filam.prob_z,fitobj.filam.pair_z)

        fitobj.nh1 = nfw.NfwHalo()
        fitobj.nh1.z_cluster= fitobj.halo1_z
        fitobj.nh1.theta_cx = fitobj.halo1_u_arcmin
        fitobj.nh1.theta_cy = fitobj.halo1_v_arcmin 
        fitobj.nh1.set_mean_inv_sigma_crit(fitobj.grid_z_centers,fitobj.prob_z,fitobj.pair_z)

        fitobj.nh2 = nfw.NfwHalo()
        fitobj.nh2.z_cluster= fitobj.halo2_z
        fitobj.nh2.theta_cx = fitobj.halo2_u_arcmin
        fitobj.nh2.theta_cy = fitobj.halo2_v_arcmin 
        fitobj.nh2.set_mean_inv_sigma_crit(fitobj.grid_z_centers,fitobj.prob_z,fitobj.pair_z)


        mock_m200_h1 = pairs_table['m200_h1_fit'][id_pair_in_catalog]/1e14
        mock_m200_h2 = pairs_table['m200_h2_fit'][id_pair_in_catalog]/1e14

        mock_kappa0 = 0
        mock_radius = 2
        shear_model_g1, shear_model_g2, limit_mask , _ , _  = fitobj.draw_model([mock_kappa0, mock_radius, mock_m200_h1, mock_m200_h2])

        # pl.scatter(fitobj.shear_u_mpc,fitobj.shear_v_mpc,c=shear_model_g2); pl.colorbar(); pl.show()

        shears_info['g1'] = shears_info['g1'] + shear_model_g1
        shears_info['g2'] = shears_info['g2'] + shear_model_g2

        tabletools.savePickle(filename_shears_nfw,shears_info,append=True)

        logger.info('noise mean_g1=%2.2f mean_g2=%2.2f std_g1=%2.2f std_g2=%2.2f',np.mean(shears_info['g1']),np.mean(shears_info['g2']),np.std(shears_info['g1'],ddof=1), np.std(shears_info['g2'],ddof=1))
示例#18
0
def get_weights_with_histograms():

    truth_cat = np.loadtxt(args.filename_input,dtype=dtype_table_cat)
    n_gals_total = sum(truth_cat['n_gals'])
    log.info('opened %s with %d rows and %d galaxies total' % (args.filename_input,len(truth_cat),n_gals_total))

    iall = 0
    
        
    for ipsf_fwhm,vpsf_fwhm in enumerate(config['grid']['psf_fwhm']):
           for isnr,vsnr in enumerate(config['grid']['snr']):

            select = np.array(truth_cat['ipsf_fwhm'] == ipsf_fwhm) * np.array(truth_cat['isnr'] == isnr)
            current_truth_cat = truth_cat[select]

            list_results = []
            for itruth,vtruth in enumerate(current_truth_cat):
                filename_result = results_filename_fmt % (vtruth['index'])
                current_res = tabletools.loadTable(filename_result,dtype=dtype_table_results_calib)
                list_results.append(current_res)

            results_ca_current = np.concatenate(list_results)

            current_snr = current_truth_cat['snr'][0]
            current_psf_fwhm = current_truth_cat['psf_fwhm'][0]

            select1 = (results_sv_example['fwhm_psf']*DES_PIXEL_SCALE < current_psf_fwhm + 0.05) * (results_sv_example['fwhm_psf']*DES_PIXEL_SCALE > current_psf_fwhm - 0.05)
            select2 = (results_sv_example['snr'] < current_snr + 2.5) * (results_sv_example['snr'] > current_snr - 2.5)
            results_sv_current = results_sv_example[select1*select2]
            n_current_gals = len(results_sv_current)
            log.info('snr=%2.2f psf_fwhm=%2.2f n_gals in that bin=%d' % (current_truth_cat['snr'][0],current_truth_cat['psf_fwhm'][0],n_current_gals))

            n_bins = 20
            bins_size=np.linspace(1.2,3,n_bins)
            # log.info('size of size bin %2.2f' % (bins_size[1]-bins_size[0]))
            title = 'snr=%2.2f psf_fwhm=%2.2f' % (current_truth_cat['snr'][0],current_truth_cat['psf_fwhm'][0])
            if n_current_gals > 2000:
                
                pl.subplot(1,2,1)
                h1,b1,_=pl.hist(results_ca_current['size'],bins=bins_size,histtype='step',normed=True,color='r',label='cal')
                h2,b2,_=pl.hist(results_sv_current['size'],bins=bins_size,histtype='step',normed=True,color='b',label='sv')
                pl.ylim([0,1.1*max([max(h1),max(h2)])])
                pl.legend()
                pl.xlabel('measured fwhm_ratio')

                pl.subplot(1,2,2)
                # abs_g_ca = np.abs(results_ca_current['g1']+1j*results_ca_current['g2'])
                # abs_g_sv = np.abs(results_sv_current['g1']+1j*results_sv_current['g2'])
                h1,b1,_=pl.hist(results_ca_current['g1'],bins=np.linspace(-1,1,n_bins),histtype='step',normed=True,color='r',label='cal')
                h2,b2,_=pl.hist(results_sv_current['g1'],bins=np.linspace(-1,1,n_bins),histtype='step',normed=True,color='b',label='sv')
                pl.legend()
                pl.xlabel('measured ellipticity')
                pl.ylim([0,1.1*max([max(h1),max(h2)])])
                
                pl.suptitle(title)
                filename_fig = 'figs/fig.hists.snr%02d.psf%02d.png' % (isnr,ipsf_fwhm)
                pl.savefig(filename_fig)
                pl.close()
                log.info('saved %s' % filename_fig)
               

            else:
                log.info('not enough gals to bother')
def self_fit():

    filename_pairs = 'pairs_bcc.fits'
    filename_halo1 = 'pairs_bcc.halos1.fits'
    filename_halo2 = 'pairs_bcc.halos2.fits'
    filename_shears = 'shears_bcc_g.fits' 

    pairs_table = tabletools.loadTable(filename_pairs)
    halo1_table = tabletools.loadTable(filename_halo1)
    halo2_table = tabletools.loadTable(filename_halo2)

    sigma_g_add =  0.1

    id_pair = 48
    shears_info = tabletools.loadTable(filename_shears,hdu=id_pair+1)

    fitobj = modelfit()
    fitobj.get_bcc_pz('aardvarkv1.0_des_lenscat_s2n10.351.fit')

    fitobj.halo1_z = 0.2
    fitobj.halo2_z = 0.2
    fitobj.halo1_u_arcmin = 20
    fitobj.halo1_v_arcmin = 0
    fitobj.halo2_u_arcmin = -20
    fitobj.halo2_v_arcmin = 0
    fitobj.shear_v_arcmin =  shears_info['v_arcmin']
    fitobj.shear_u_mpc =  shears_info['u_mpc']
    fitobj.shear_v_mpc =  shears_info['v_mpc']

    fitobj.halo1_u_arcmin =  pairs_table['u1_arcmin'][id_pair]
    fitobj.halo1_v_arcmin =  pairs_table['v1_arcmin'][id_pair]
    fitobj.halo1_u_mpc =  pairs_table['u1_mpc'][id_pair]
    fitobj.halo1_v_mpc =  pairs_table['v1_mpc'][id_pair]
    fitobj.halo1_z =  pairs_table['z'][id_pair]

    fitobj.halo2_u_arcmin =  pairs_table['u2_arcmin'][id_pair]
    fitobj.halo2_v_arcmin =  pairs_table['v2_arcmin'][id_pair]
    fitobj.halo2_u_mpc =  pairs_table['u2_mpc'][id_pair]
    fitobj.halo2_v_mpc =  pairs_table['v2_mpc'][id_pair]
    fitobj.halo2_z =  pairs_table['z'][id_pair]

    fitobj.pair_z  = (fitobj.halo1_z + fitobj.halo2_z) / 2.

    fitobj.filam = filament.filament()
    fitobj.filam.pair_z =fitobj.pair_z
    fitobj.filam.grid_z_centers = fitobj.grid_z_centers
    fitobj.filam.prob_z = fitobj.prob_z
    fitobj.filam.set_mean_inv_sigma_crit(fitobj.filam.grid_z_centers,fitobj.filam.prob_z,fitobj.filam.pair_z)

    fitobj.nh1 = nfw.NfwHalo()
    fitobj.nh1.z_cluster= fitobj.halo1_z
    fitobj.nh1.theta_cx = fitobj.halo1_u_arcmin
    fitobj.nh1.theta_cy = fitobj.halo1_v_arcmin 
    fitobj.nh1.set_mean_inv_sigma_crit(fitobj.grid_z_centers,fitobj.prob_z,fitobj.pair_z)

    fitobj.nh2 = nfw.NfwHalo()
    fitobj.nh2.z_cluster= fitobj.halo2_z
    fitobj.nh2.theta_cx = fitobj.halo2_u_arcmin
    fitobj.nh2.theta_cy = fitobj.halo2_v_arcmin 
    fitobj.nh2.set_mean_inv_sigma_crit(fitobj.grid_z_centers,fitobj.prob_z,fitobj.pair_z)

    fitobj.shear_u_arcmin =  shears_info['u_arcmin']

    shear_model_g1, shear_model_g2, limit_mask , _ , _  = fitobj.draw_model([0., 2., 14.5, 14.5])
    fitobj.plot_shears(shear_model_g1, shear_model_g2,quiver_scale=0.5)
    pl.show()

    fitobj.shear_g1 =  shear_model_g1 + np.random.randn(len(shears_info['g1']))*sigma_g_add
    fitobj.shear_g2 =  shear_model_g2 + np.random.randn(len(shears_info['g2']))*sigma_g_add
    fitobj.sigma_g =  np.std(shear_model_g2,ddof=1)
    fitobj.inv_sq_sigma_g = 1./sigma_g_add**2
    log.info('using sigma_g=%2.5f' , fitobj.sigma_g)

    fitobj.parameters[0]['box']['min'] = 0
    fitobj.parameters[0]['box']['max'] = 1

    fitobj.parameters[1]['box']['min'] = 1
    fitobj.parameters[1]['box']['max'] = 10

    fitobj.parameters[2]['box']['min'] = 14
    fitobj.parameters[2]['box']['max'] = 15

    fitobj.parameters[3]['box']['min'] = 14
    fitobj.parameters[3]['box']['max'] = 15

    print 'halo1 m200' , halo1_table['m200'][id_pair]
    print 'halo2 m200' , halo2_table['m200'][id_pair]

    # import pdb; pdb.set_trace()
    # fitobj.plot_shears_mag(fitobj.shear_g1,fitobj.shear_g2)
    # pl.show()

    fitobj.save_all_models=False
    log.info('running grid search')
    fitobj.run_gridsearch()
def test():

    filename_pairs = 'pairs_bcc.fits'
    filename_halo1 = 'pairs_bcc.halos1.fits'
    filename_halo2 = 'pairs_bcc.halos2.fits'
    filename_shears = 'shears_bcc_g.fits' 

    pairs_table = tabletools.loadTable(filename_pairs)
    halo1_table = tabletools.loadTable(filename_halo1)
    halo2_table = tabletools.loadTable(filename_halo2)

    sigma_g_add =  0.0001

    id_pair = 48
    shears_info = tabletools.loadTable(filename_shears,hdu=id_pair+1)

    fitobj = modelfit()
    fitobj.get_bcc_pz()
    fitobj.shear_u_arcmin =  shears_info['u_arcmin']
    fitobj.shear_v_arcmin =  shears_info['v_arcmin']
    fitobj.shear_u_mpc =  shears_info['u_mpc']
    fitobj.shear_v_mpc =  shears_info['v_mpc']
    fitobj.shear_g1 =  shears_info['g1'] + np.random.randn(len(shears_info['g1']))*sigma_g_add
    fitobj.shear_g2 =  shears_info['g2'] + np.random.randn(len(shears_info['g2']))*sigma_g_add
    fitobj.sigma_g =  np.std(shears_info['g2'],ddof=1)

    # fitobj.save_all_models = True

    log.info('using sigma_g=%2.5f' , fitobj.sigma_g)

    fitobj.halo1_u_arcmin =  pairs_table['u1_arcmin'][id_pair]
    fitobj.halo1_v_arcmin =  pairs_table['v1_arcmin'][id_pair]
    fitobj.halo1_u_mpc =  pairs_table['u1_mpc'][id_pair]
    fitobj.halo1_v_mpc =  pairs_table['v1_mpc'][id_pair]
    fitobj.halo1_z =  pairs_table['z'][id_pair]

    fitobj.halo2_u_arcmin =  pairs_table['u2_arcmin'][id_pair]
    fitobj.halo2_v_arcmin =  pairs_table['v2_arcmin'][id_pair]
    fitobj.halo2_u_mpc =  pairs_table['u2_mpc'][id_pair]
    fitobj.halo2_v_mpc =  pairs_table['v2_mpc'][id_pair]
    fitobj.halo2_z =  pairs_table['z'][id_pair]

    fitobj.parameters[0]['box']['min'] = 0
    fitobj.parameters[0]['box']['max'] = 1
    fitobj.parameters[1]['box']['min'] = 1
    fitobj.parameters[1]['box']['max'] = 10
    fitobj.parameters[2]['box']['min'] = 14
    fitobj.parameters[2]['box']['max'] = 15
    fitobj.parameters[3]['box']['min'] = 14
    fitobj.parameters[3]['box']['max'] = 15

    print 'halo1 m200' , halo1_table['m200'][id_pair]
    print 'halo2 m200' , halo2_table['m200'][id_pair]

    # import pdb; pdb.set_trace()

    # fitobj.plot_shears_mag(fitobj.shear_g1,fitobj.shear_g2)
    # pl.show()
    fitobj.save_all_models=False
    log.info('running grid search')
    n_grid=10
    log_post , params, grids = fitobj.run_gridsearch(n_grid=n_grid)

    vmax_post , best_model_g1, best_model_g2 , limit_mask,  vmax_params = fitobj.get_grid_max(log_post , params)
示例#21
0
def get_selection_split(selection_string, cols_res, cols_tru):

    results_filename_fmt = config['methods'][args.method]['filename_results']  

    truth_filename_fmt = config['filename_truth']  
    list_shears = []
    list_all_res = []
    list_all_tru = []

    n_all_loaded=0
    n_all_selected=0

    ia=0
    n_missing=0

    for ig,vg in enumerate(config['shear']):

        list_results = []

        id_first = args.first
        id_last = id_first + args.num
        # if id_last > 200: 
        #     id_last=200;
        #     warnings.warn('hard coded max number of files 200')

        for ip in range(id_first,id_last):
         
            filename_tru = truth_filename_fmt % (ip,ig)
            filename_res = results_filename_fmt % (ip,ig)
            try:
                    cat_tru_all = tabletools.loadTable(filename_tru,log=1,remember=False)
                    cat_tru = cat_tru_all
                    logger.debug('loaded %05d galaxies from file: %s' % (len(cat_tru_all),filename_tru))

            except Exception,errmsg:
                logger.error('file %s : %s' % (filename_tru,errmsg) )
                continue


            try:
                    cat_res_all = tabletools.loadTable(filename_res,log=1,remember=False)
                    cat_res = cat_res_all
                    logger.debug('loaded %05d galaxies from file: %s' % (len(cat_res_all),filename_res))

            except Exception,errmsg:
                logger.debug('sth wrong with file %s errmsg %s' % (filename_res,errmsg) )
                n_missing+=1
                continue

 

            if ('e1' in cols_res) & (args.method=='im3shape'):
                # cat_res['e1'] = cat_res['e1']*config['methods'][args.method]['flip_g1']
                cat_tru['g1_true'] = -1*cat_tru['g1_true']
                warnings.warn('flipping g1 in truth cat for method %s'%args.method)


            if len(cat_tru) != len(cat_res):
                cat_tru=cat_tru[cat_res['coadd_objects_id']]

            if args.method == 'ngmix':
                cat_res = rename_ngmix_cols(cat_res,cat_tru)
            elif args.method == 'im3shape':
                cat_res = rename_im3shape_cols(cat_res,cat_tru)
            
            cat_tru = rename_cols_truth(cat_tru)

            for col in cols_tru:
                if col not in cat_tru.dtype.names:
                    raise Exception('column %s not found in truth catalog %s' % (col,filename_tru))
            for col in cols_res:
                if col not in cat_res.dtype.names:
                    raise Exception('column %s not found in results catalog %s' % (col,filename_res))

            n_all_loaded+=len(cat_res)
            try:
                exec selection_string
            except Exception,errmsg:
                print errmsg
                import pdb; pdb.set_trace()
示例#22
0
def update_truth_table(update_snr=True , update_cosmos=True , update_hsm=True, update_fwhm=True):

    log.info('getting snr, flux and fwhm for the truth table')

    noise_std = config['des_pixel_noise_sigma']

    id_first = args.first
    id_last = id_first + args.num

    psf_images = None

    filename_cosmos_catalog = os.path.join(config['input']['real_catalog']['dir'],config['input']['real_catalog']['file_name'])
    filename_cosmos_catalog_fits = os.path.join(config['input']['real_catalog']['dir'],config['input']['real_catalog']['file_name']).replace('.fits','_fits.fits')
    cosmos_catalog = pyfits.getdata(filename_cosmos_catalog)
    cosmos_catalog_fits = pyfits.getdata(filename_cosmos_catalog_fits)
    n_cosmos_gals = len(cosmos_catalog)
    log.info('opened %s with %d images' , filename_cosmos_catalog, n_cosmos_gals)

    filename_great3_info = os.path.join(config['input']['real_catalog']['dir'],'real_galaxy_selection_info.fits')
    great3_info = np.array(pyfits.getdata(filename_great3_info))


    for ip in range(id_first,id_last):

        # all_snr=[]
   
        for il,vl in enumerate(config['shear']):

            list_normsq = []

            filename_cat = os.path.join(args.out_dir,'nbc.truth.%03d.g%02d.fits' % (ip,il))
            filename_meds = os.path.join(args.out_dir,'nbc.meds.%03d.g%02d.noisefree.fits' % (ip,il))


            log.info('part %d shear %d : getting snr, flux, hsm, and fwhm' , ip, il)
            log.debug('using %s %s', filename_meds, filename_cat)

            cat = tabletools.loadTable(filename_cat)
            n_gals = len(cat)

            # assure backwards compatibility
            if 'hsm_obs_g1'             not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='hsm_obs_g1',         arr=np.zeros(len(cat)), dtype='f8')
            if 'hsm_obs_g2'             not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='hsm_obs_g2',         arr=np.zeros(len(cat)), dtype='f8')
            if 'hsm_cor_g1'             not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='hsm_cor_g1',         arr=np.zeros(len(cat)), dtype='f8')
            if 'hsm_cor_g2'             not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='hsm_cor_g2',         arr=np.zeros(len(cat)), dtype='f8')
            if 'hsm_obs_sigma'          not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='hsm_obs_sigma',      arr=np.zeros(len(cat)), dtype='f8')
            if 'hsm_cor_sigma'          not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='hsm_cor_sigma',      arr=np.zeros(len(cat)), dtype='f8')
            if 'hsm_centroid_x'         not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='hsm_centroid_x',     arr=np.zeros(len(cat)), dtype='f8')
            if 'hsm_centroid_y'         not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='hsm_centroid_y',     arr=np.zeros(len(cat)), dtype='f8')
            if 'hsm_mom_amp'            not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='hsm_mom_amp',        arr=np.zeros(len(cat)), dtype='f8')
            if 'fwhm'                   not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='fwhm',               arr=np.zeros(len(cat)), dtype='f8')
            if 'sf_i'                   not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='sf_i',               arr=np.zeros(len(cat)), dtype='f8')    
            if 'sf_hlr'                 not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='sf_hlr',             arr=np.zeros(len(cat)), dtype='f8')    
            if 'sf_sersicn'             not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='sf_sersicn',         arr=np.zeros(len(cat)), dtype='f8')        
            if 'sf_q'                   not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='sf_q',               arr=np.zeros(len(cat)), dtype='f8')    
            if 'sf_boxiness'            not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='sf_boxiness',        arr=np.zeros(len(cat)), dtype='f8')        
            if 'sf_phi'                 not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='sf_phi',             arr=np.zeros(len(cat)), dtype='f8')    
            if 'zphot'                  not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='zphot',              arr=np.zeros(len(cat)), dtype='f8')   
            if 'psf_fwhm_measured'      not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='psf_fwhm_measured',  arr=np.zeros(len(cat)), dtype='f8')   
            if 'cosmos_mag_auto'        not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='cosmos_mag_auto',    arr=np.zeros(len(cat)), dtype='f8')   
            if 'cosmos_flux_radius'     not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='cosmos_flux_radius', arr=np.zeros(len(cat)), dtype='f8')   
            if 'mean_rgpp_rp'           not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='mean_rgpp_rp',       arr=np.zeros(len(cat)), dtype='f8')   
            if 'to_use'                 not in cat.dtype.names: cat=tabletools.appendColumn(rec=cat, name='to_use',             arr=np.zeros(len(cat)), dtype='i4')   

            for ig in range(n_gals):

                if update_snr == True:
                    noisless_gals = meds.MEDS(filename_meds)
                    n_gals = len(noisless_gals._cat)
                    img_gal = noisless_gals.get_cutout(ig,0)
                    normsq= np.sum( img_gal.flatten()**2 )
                    snr = np.sqrt(normsq)/noise_std
                    flux = np.sum(img_gal.flatten())
                    cat[ig]['snr'] = snr
                    cat[ig]['flux'] = flux

                if update_cosmos == True:
                    current_id_cosmos = cat[ig]['id_cosmos']
                    cat[ig]['sf_i']               = cosmos_catalog_fits[current_id_cosmos]['sersicfit'][0]
                    cat[ig]['sf_hlr']             = cosmos_catalog_fits[current_id_cosmos]['sersicfit'][1]*ACS_PIXEL_SCALE
                    cat[ig]['sf_sersicn']         = cosmos_catalog_fits[current_id_cosmos]['sersicfit'][2]
                    cat[ig]['sf_q']               = cosmos_catalog_fits[current_id_cosmos]['sersicfit'][3]
                    cat[ig]['sf_boxiness']        = cosmos_catalog_fits[current_id_cosmos]['sersicfit'][4]
                    cat[ig]['sf_phi']             = cosmos_catalog_fits[current_id_cosmos]['sersicfit'][7]
                    cat[ig]['zphot']              = cosmos_catalog_fits[current_id_cosmos]['zphot']
                    cat[ig]['cosmos_mag_auto']    = cosmos_catalog_fits[current_id_cosmos]['mag_auto']
                    cat[ig]['cosmos_flux_radius'] = cosmos_catalog_fits[current_id_cosmos]['flux_radius']
                    cat[ig]['to_use']             = great3_info[current_id_cosmos]['to_use']

                if update_hsm==True:
                    img_gal = noisless_gals.get_cutout(ig,0)
                    img_psf = pyfits.getdata(os.path.join(args.out_dir,'nbc.psf.lores.fits'),cat[ig]['id_psf'])
                    gs_img_gal = image_array_to_galsim(img_gal)
                    gs_img_psf = image_array_to_galsim(img_psf)

                    try:
                        shearobj1=galsim.hsm.EstimateShear(gs_img_gal,gs_img_psf)
                        cat[ig]['hsm_cor_g1'] = shearobj1.corrected_e1 / 2.
                        cat[ig]['hsm_cor_g2'] = shearobj1.corrected_e2 / 2.
                        cat[ig]['hsm_mom_amp'] = shearobj1.moments_amp
                    except:
                        log.error('HSM failed for object ig=%d ip=%d id_cosmos=%d psf_fwhm=%2.2f' , ig, ip , cat['id_cosmos'][ig] , cat['psf_fwhm'][ig])
                        cat[ig]['hsm_cor_g1'] = -99
                        cat[ig]['hsm_cor_g1'] = -99
                        cat[ig]['hsm_mom_amp'] = -99

                    try:
                        shearobj2=galsim.hsm.FindAdaptiveMom(gs_img_gal)
                        cat[ig]['hsm_obs_g1'] = shearobj2.observed_shape.g1
                        cat[ig]['hsm_obs_g2'] = shearobj2.observed_shape.g2
                        cat[ig]['hsm_obs_sigma'] = shearobj2.moments_sigma
                        cat[ig]['hsm_cor_sigma'] = shearobj2.moments_sigma
                        cat[ig]['hsm_centroid_x'] = shearobj2.moments_centroid.x
                        cat[ig]['hsm_centroid_y'] = shearobj2.moments_centroid.y
                    except:
                        cat[ig]['hsm_obs_g1'] = -99
                        cat[ig]['hsm_obs_g2'] = -99
                        cat[ig]['hsm_obs_sigma'] = -99
                        cat[ig]['hsm_cor_sigma'] = -99
                        cat[ig]['hsm_centroid_x'] = -99
                        cat[ig]['hsm_centroid_y'] = -99


                if update_fwhm==True:

                    try:
                        noisless_gals = meds.MEDS(filename_meds)
                        img_gal = noisless_gals.get_cutout(ig,0)
                        import mathstools
                        cat[ig]['fwhm'] = mathstools.get_2D_fwhm(img_gal)
                    except:
                        log.error('getting FWHM failed for galaxy %d in %s' , ig , filename_meds )
                        cat[ig]['fwhm'] = 666

                    try:
                        # if psf_images == None: psf_images = pyfits.open(os.path.join(args.out_dir,'nbc.psf.hires.fits'))
                        # img_hires_psf = psf_images[cat[ig]['id_psf']].data
                        # cat[ig]['psf_fwhm_measured'] = mathstools.get_2D_fwhm(img_hires_psf)
                        cat[ig]['psf_fwhm_measured'] = cat[ig]['psf_fwhm'] 


                    except:
                        log.error('getting PSF FWHM failed for galaxy %d in %s' , ig , filename_meds )
                        cat[ig]['psf_fwhm_measured'] = 666

                    if (cat[ig]['fwhm'] != 666) & (cat[ig]['psf_fwhm_measured'] != 666):
                        cat[ig]['mean_rgpp_rp'] = cat[ig]['fwhm']/cat[ig]['psf_fwhm_measured']
                    else:
                        cat[ig]['mean_rgpp_rp'] = 666

                                
                if ig % 100 == 0: log.debug('getting snr, flux, hsm and fwhm of galaxy %d' , ig)


            tabletools.saveTable(filename_cat, cat)
def get_shears_for_single_pair(halo1,halo2,idp=0):

        logger.debug('ra=(%2.2f,%2.2f) dec=(%2.2f,%2.2f) ' % (halo1['ra'],halo2['ra'],halo1['dec'],halo2['dec']))

        shear_base = tabletools.loadTable(filename_shearbase,dtype=dtype_shearbase)
       
        redshift_offset = 0.2
        # pair_dra = np.abs(halo1['ra'] - halo2['ra'])
        # pair_ddec = np.abs(halo1['dec'] - halo2['dec'])

        halo1_ra_deg , halo1_de_deg = halo1['ra'],halo1['dec']
        halo2_ra_deg , halo2_de_deg = halo2['ra'],halo2['dec']

        pair_ra_deg,  pair_de_deg = cosmology.get_midpoint_deg(halo1_ra_deg , halo1_de_deg , halo2_ra_deg , halo2_de_deg)
        pair_z = np.mean([halo1['z'],halo2['z']])
        
        # find the corresponding files

        radius = 1.
        # pair_xyz = cosmology.get_euclidian_coords( pair_ra_deg , pair_de_deg , radius)
        x,y,z = cosmology.spherical_to_cartesian_deg( pair_ra_deg , pair_de_deg , radius)
        pair_xyz = np.array([x , y , z])
        box_coords_x = shear_base['x']
        box_coords_y = shear_base['y']
        box_coords_z = shear_base['z']

        box_coords_xyz = np.concatenate([ box_coords_x[:,None], box_coords_y[:,None], box_coords_z[:,None] ] , axis=1)
        logger.info('getting Ball Tree for 3D')
        BT = BallTree(box_coords_xyz, leaf_size=5)
        n_connections=5
        bt_dx,bt_id = BT.query(pair_xyz,k=n_connections)

        list_set = []

        for iset, vset in enumerate( shear_base[bt_id]['file'][0] ):
            # vset=vset.replace('kacprzak','tomek')
            lenscat=tabletools.loadTable(vset)
            # prelim cut on z
            select =  lenscat['z'] > (pair_z + redshift_offset)
            lenscat=lenscat[select]

            list_set.append(lenscat)
            logger.debug('opened %s with %d gals mean_ra=%2.2f, mean_de=%2.2f' % (vset,len(lenscat),np.mean(lenscat['ra']),np.mean(lenscat['dec'])))

        shear1_col = config['shear1_col']
        shear2_col = config['shear2_col']

        lenscat_all = np.concatenate(list_set)
        shear_g1 , shear_g2 = -lenscat_all[shear1_col] , lenscat_all[shear2_col] 
        shear_ra_deg , shear_de_deg , shear_z = lenscat_all['ra'] , lenscat_all['dec'] ,  lenscat_all['z']

        # import pylab as pl
        # pl.figure()
        # pl.scatter(pair_ra_deg      , pair_de_deg  , 100, 'r' , marker='x')
        # pl.scatter(halo1_ra_deg     , halo1_de_deg , 100, 'c' , marker='o')
        # pl.scatter(halo2_ra_deg     , halo2_de_deg , 100, 'm' , marker='o')
        # select = np.random.permutation(len(shear_ra_deg))[:10000]
        # pl.scatter(shear_ra_deg[select]  , shear_de_deg[select] ,1,  'm' , marker='.')
        # pl.show()

        pairs_shear , halos_coords , pairs_shear_full   = filaments_tools.create_filament_stamp(halo1_ra_deg, halo1_de_deg, 
                                halo2_ra_deg, halo2_de_deg, 
                                shear_ra_deg, shear_de_deg, 
                                shear_g1, shear_g2, shear_z, 
                                pair_z, lenscat_all )

        if len(pairs_shear) < 100:
            logger.error('found only %d shears' % len(pairs_shear))
            return None , None

        return pairs_shear , halos_coords, pairs_shear_full
def get_closeby_shear(shear_ra_arcmin,shear_de_arcmin,pair):

    filename_pairs =  config['filename_pairs']                                   # pairs_bcc.fits'
    filename_halo1 =  config['filename_pairs'].replace('.fits' , '.halos1.fits') # pairs_bcc.halos1.fits'
    filename_halo2 =  config['filename_pairs'].replace('.fits' , '.halos2.fits') # pairs_bcc.halos2.fits'
    filename_shears = config['filename_shears']                                  # args.filename_shears 
    filename_halos  = config['filename_halos']                                  # args.filename_shears 

    pairs_table = tabletools.loadTable(filename_pairs)
    halo1_table = tabletools.loadTable(filename_halo1)
    halo2_table = tabletools.loadTable(filename_halo2)
    halos_table = tabletools.loadTable(filename_halos)

    limit_deg=2
    shear_g1_removed , shear_g2_removed = np.zeros_like(shear_ra_arcmin) , np.zeros_like(shear_ra_arcmin)
    n_close = 0

    for ih,vh in enumerate(halos_table):

        if (ih == pair['ih1']) | (ih == pair['ih2']):
            continue

        if (vh['m200_fit']<1e13):
            continue

        x , y = cosmology.get_gnomonic_projection(vh['ra'], vh['dec'] , pair['ra_mid'], pair['dec_mid'] , unit='deg')
        dist_sky_deg1 = cosmology.get_angular_separation(x,y,pair['u1_arcmin']/60.,0,unit='deg')
        dist_sky_deg2 = cosmology.get_angular_separation(x,y,pair['u2_arcmin']/60.,0,unit='deg')

        if ((dist_sky_deg1 > limit_deg) | (dist_sky_deg2 > limit_deg)):
            continue
        else:
            logger.info('closeby halo % 5d\tdist_sky1=%7.2f\tdist_sky2=%7.2f\tm200=%2.2e sig=%2.2f' , ih , dist_sky_deg1 , dist_sky_deg2 , vh['m200_fit'] , vh['m200_sig'])

            fitobj = filaments_model_1h.modelfit()
            fitobj.shear_u_arcmin =  shear_ra_arcmin
            fitobj.shear_v_arcmin =  shear_de_arcmin
            fitobj.halo_u_arcmin = x*60.
            fitobj.halo_v_arcmin = y*60.
            fitobj.halo_z = vh['z']
            fitobj.get_bcc_pz(config['filename_pz'])

            fitobj.nh = nfw.NfwHalo()
            fitobj.nh.z_cluster= fitobj.halo_z
            fitobj.nh.theta_cx = fitobj.halo_u_arcmin
            fitobj.nh.theta_cy = fitobj.halo_v_arcmin 
            fitobj.nh.set_mean_inv_sigma_crit(fitobj.grid_z_centers,fitobj.prob_z,fitobj.halo_z)

            model_g1 , model_g2 , limit_mask , Delta_Sigma , kappa = fitobj.draw_model([vh['m200_fit']])

            shear_g1_removed , shear_g2_removed = shear_g1_removed + model_g1, shear_g2_removed + model_g2

            n_close +=1 


    # pl.figure()
    # pl.scatter(shear_ra_arcmin,shear_de_arcmin,c=shear_g1_removed,lw=0)
    # # pl.hist(shear_g1-shear_g1_removed,100); pl.show()
    # pl.colorbar()
    # pl.scatter(pair['u1_arcmin'],pair['v1_arcmin'],s=100)
    # pl.scatter(pair['u2_arcmin'],pair['v2_arcmin'],s=100)
    # # pl.xlim([pair['u2_arcmin'],pair['u1_arcmin']])

    # pl.show()

    logger.info('n_close=%d' , n_close)


    return shear_g1_removed, shear_g2_removed
def select_halos():

    halocat = tabletools.loadTable('DR7-Full.fits')

    # for now just save all LRGs
    tabletools.saveTable(filename_halos, halocat)
示例#26
0
import tabletools
import pylab as pl
import scipy.stats
import numpy as np

import plotstools




cosmos=tabletools.loadTable('cosmos_acs_shera_may2011.fits.gz')
cos=cosmos[(cosmos['FWHM_IMAGE']<100) * (cosmos['ZPHOT']>0) * (cosmos['ZPHOT']<3) ]

npix = 50
pixel_size=0.05
plotstools.plot_dist(X=[cos['FWHM_IMAGE']*pixel_size,cos['ZPHOT'],cos['MODD']],bins=[30,30,range(0,31)],labels=['FWHM [arcsec]' , 'ZPHOT' , 'MODD'])
filename_fig = 'histograms_size_zphot_modd.eps'

left  = 0.1  # the left side of the subplots of the figure
right = 0.9    # the right side of the subplots of the figure
bottom = 0.1   # the bottom of the subplots of the figure
top = 0.9      # the top of the subplots of the figure
wspace = 0.0   # the amount of width reserved for blank space between subplots
hspace = 0.0   # the amount of height reserved for white space between subplots
pl.subplots_adjust(left=left, bottom=bottom, right=right, top=top, wspace=wspace, hspace=hspace)
pl.savefig(filename_fig)


print 'saved' , filename_fig

示例#27
0
def runIm3shape():

    # open the RGC
    if 'real_catalog' in config['input']:
        filepath_rgc = os.path.join(config['input']['real_catalog']['dir'],config['input']['real_catalog']['file_name'])
        rgc = pyfits.open(filepath_rgc)[1]

    # open the ring test catalog
    filename_cat = os.path.join(config['input']['catalog']['dir'],config['input']['catalog']['file_name'])
    # truth_cat = numpy.loadtxt(filename_cat,dtype=dtype_table_truth)
    truth_cat = tabletools.loadTable(filepath=filename_cat,table_name='truth_cat',dtype=dtype_table_truth,logger=logger)
    logger.info('loaded %s' % filename_cat)

    n_objects = truth_cat.shape[0]

    # get im3shape
    dirpath_im3shape = os.path.join(os.environ['IM3SHAPE'],'python')
    sys.path.append(dirpath_im3shape)
    import im3shape

    # get images
    img_gals = getGalaxyImages()

    # get options
    n_pix = config['image']['size']
    pixel_scale = config['image']['pixel_scale']
    i3_options = im3shape.I3_options()
    i3_options.read_ini_file(config['args'].filepath_ini)
    logger.info('loaded im3shape ini file %s' % config['args'].filepath_ini)  
    i3_options.stamp_size = n_pix

    # get the file 
    filename_results = 'results.%s.%012d.cat' % (config['args'].name_config,config['args'].obj_num)
    file_results = open(filename_results,'w')

    # create PSF from Moffat parameters
    # get i3 images - get first i3_galaxy to initialise the PSF - kind of strange, but hey..
    i3_galaxy = im3shape.I3_image(n_pix, n_pix)
    psf_beta = float(config['psf']['beta'])
    psf_fwhm = float(config['psf']['fwhm'])/float(pixel_scale)
    psf_e1 = float(config['psf']['ellip']['g1'])
    psf_e2 = float(config['psf']['ellip']['g2'])
    i3_psf = i3_galaxy.make_great10_psf(psf_beta, psf_fwhm, psf_e1, psf_e2, i3_options)

    obj_num = config['args'].obj_num

    # loop over all created images
    for ig,img_gal in enumerate(img_gals):

        # get i3 images
        i3_galaxy = im3shape.I3_image(n_pix, n_pix)
        scale = img_gal.array.sum()
        i3_galaxy.from_array(img_gal.array)  
        i3_galaxy.scale(1.0/scale)


        # this is a workaround - the array for model bias real images had id_unique,
        # but the results files have 'identifier' - so we use one or the other id they exist
        # get the unique_id
        if 'id_unique' in truth_cat.dtype.names:

            id_global = ig
            id_object = (obj_num+ig) % n_objects
            id_cosmos = truth_cat['id_cosmos'][ id_object ]
            id_unique = truth_cat['id_unique'][ id_object ]

        elif 'identifier' in truth_cat.dtype.names:
            
            id_global = ig
            id_object = (obj_num+ig) % n_objects
            id_unique = truth_cat['identifier'][ id_object ]
            id_cosmos = int(truth_cat['identifier'][ id_object ]//10000)

        
        i3_result, i3_best_fit = im3shape.i3_analyze(i3_galaxy, i3_psf, i3_options, ID=id_unique)


        saveResult(file_results,i3_result,id_global,id_object,id_unique,id_cosmos)
        printResult(i3_result,id_global)
        if 'e1' in truth_cat.dtype.names: printTruth(i3_result,truth_cat[ig])


        # save residual plots
        if config['args'].verbosity > 2:

            i1 = i3_best_fit.array/sum(i3_best_fit.array.flatten())
            i2 = img_gal.array/sum(img_gal.array.flatten())          

            import pylab
            pylab.subplot(1,5,1)
            pylab.imshow(i1,interpolation='nearest')
            pylab.title('best fit')

            pylab.subplot(1,5,2)
            pylab.imshow(i2,interpolation='nearest')
            pylab.title('galaxy')

            pylab.subplot(1,5,3)
            pylab.imshow(i1-i2,interpolation='nearest')
            pylab.title('residuals')

            pylab.subplot(1,5,4)
            pylab.imshow(i3_psf.array,interpolation='nearest')
            pylab.title('PSF')

            pylab.subplot(1,5,5)
            pylab.imshow(img_gal.array,interpolation='nearest')
            pylab.colorbar()
            pylab.title('best fit')


            filename_fig = 'debug/fig.residual.%09d.png' % id_unique
            pylab.savefig(filename_fig)
            logger.info('saved %s' % filename_fig)
            pylab.close()

    file_results.close()
    logger.info('saved %s' % filename_results)
def test_overlap():

    filename_pairs =  config['filename_pairs']                                   # pairs_bcc.fits'
    filename_halo1 =  config['filename_pairs'].replace('.fits' , '.halos1.fits') # pairs_bcc.halos1.fits'
    filename_halo2 =  config['filename_pairs'].replace('.fits' , '.halos2.fits') # pairs_bcc.halos2.fits'
    filename_shears = config['filename_shears']                                  # args.filename_shears 
    filename_shears_overlap = filename_shears.replace('.pp2','.overlap.pp2')
    filename_pairs_overlap = filename_pairs.replace('.fits','.overlap.fits')
    filename_halo1_overlap = filename_halo1.replace('.halos1.fits','.overlap.halos1.fits')
    filename_halo2_overlap = filename_halo2.replace('.halos2.fits','.overlap.halos2.fits')

    if os.path.isfile(filename_shears_overlap):
        os.remove(filename_shears_overlap)
        logger.warning('overwriting file %s' , filename_shears_overlap)

    pairs_table = tabletools.loadTable(filename_pairs)
    halo1_table = tabletools.loadTable(filename_halo1)
    halo2_table = tabletools.loadTable(filename_halo2)

    sigma_g_add =  0.0001

    id_pair = 3
    shears_info = tabletools.loadPickle(filename_shears,id_pair)

    overlapping_halo_m200 = 2 # x 1e14
    overlapping_halo_z = 0.3
    no_m200 = 1e-8

    n_pairs = len(grid_x) * len(grid_y)

    pairs_table_overlap = pairs_table[np.ones([n_pairs],dtype=np.int32)*id_pair]
    halo1_table_overlap = halo1_table[np.ones([n_pairs],dtype=np.int32)*id_pair]
    halo2_table_overlap = halo2_table[np.ones([n_pairs],dtype=np.int32)*id_pair]

    pairs_table_overlap['ipair'] = range(len(pairs_table_overlap))

    tabletools.saveTable(filename_pairs_overlap,pairs_table_overlap)
    tabletools.saveTable(filename_halo1_overlap,halo1_table_overlap)
    tabletools.saveTable(filename_halo2_overlap,halo2_table_overlap)

    for x_pos in grid_x:
        for y_pos in grid_y:

            logger.info('dx = %2.2f dy = %2.2f' % (x_pos,y_pos))

            fitobj = filaments_model_2hf.modelfit()
            fitobj.get_bcc_pz(config['filename_pz'])
            fitobj.kappa_is_K = False
            fitobj.R_start = config['R_start']
            fitobj.Dlos = pairs_table[id_pair]['Dlos']        
            fitobj.Dtot = np.sqrt(pairs_table[id_pair]['Dxy']**2+pairs_table[id_pair]['Dlos']**2)
            fitobj.boost = fitobj.Dtot/pairs_table[id_pair]['Dxy']
            fitobj.use_boost = config['use_boost']

            fitobj.shear_v_arcmin =  shears_info['v_arcmin']
            fitobj.shear_u_arcmin =  shears_info['u_arcmin']
            fitobj.shear_u_mpc =  shears_info['u_mpc']
            fitobj.shear_v_mpc =  shears_info['v_mpc']

            fitobj.halo1_u_arcmin =  pairs_table['u1_arcmin'][id_pair]
            fitobj.halo1_v_arcmin =  pairs_table['v1_arcmin'][id_pair]
            fitobj.halo1_u_mpc =  pairs_table['u1_mpc'][id_pair]
            fitobj.halo1_v_mpc =  pairs_table['v1_mpc'][id_pair]
            fitobj.halo1_z =  pairs_table['z'][id_pair]

            fitobj.halo2_u_arcmin =  pairs_table['u2_arcmin'][id_pair]
            fitobj.halo2_v_arcmin =  pairs_table['v2_arcmin'][id_pair]
            fitobj.halo2_u_mpc =  pairs_table['u2_mpc'][id_pair]
            fitobj.halo2_v_mpc =  pairs_table['v2_mpc'][id_pair]
            fitobj.halo2_z =  pairs_table['z'][id_pair]

            fitobj.pair_z  = (fitobj.halo1_z + fitobj.halo2_z) / 2.

            fitobj.filam = filament.filament()
            fitobj.filam.pair_z =fitobj.pair_z
            fitobj.filam.grid_z_centers = fitobj.grid_z_centers
            fitobj.filam.prob_z = fitobj.prob_z
            fitobj.filam.set_mean_inv_sigma_crit(fitobj.filam.grid_z_centers,fitobj.filam.prob_z,fitobj.filam.pair_z)

            fitobj.nh1 = nfw.NfwHalo()
            fitobj.nh1.z_cluster= fitobj.halo1_z
            fitobj.nh1.theta_cx = fitobj.halo1_u_arcmin
            fitobj.nh1.theta_cy = fitobj.halo1_v_arcmin 
            fitobj.nh1.set_mean_inv_sigma_crit(fitobj.grid_z_centers,fitobj.prob_z,fitobj.pair_z)

            fitobj.nh2 = nfw.NfwHalo()
            fitobj.nh2.z_cluster= fitobj.halo2_z
            fitobj.nh2.theta_cx = fitobj.halo2_u_arcmin
            fitobj.nh2.theta_cy = fitobj.halo2_v_arcmin
            fitobj.nh2.set_mean_inv_sigma_crit(fitobj.grid_z_centers,fitobj.prob_z,fitobj.pair_z)
            shear_model_g1, shear_model_g2, limit_mask , _ , _  = fitobj.draw_model([filament_ds, filament_radius, no_m200, no_m200])
            fitobj.nh2.theta_cy = fitobj.halo2_v_arcmin
            fitobj.nh2.theta_cx = fitobj.halo2_u_arcmin

            # second fitobj ---------- overlapping halo
            fitobj2 = filaments_model_2hf.modelfit()
            fitobj2.get_bcc_pz(config['filename_pz'])
            fitobj2.use_boost = True
            fitobj2.kappa_is_K = False
            fitobj2.R_start = config['R_start']
            fitobj2.Dlos = pairs_table[id_pair]['Dlos']        
            fitobj2.Dtot = np.sqrt(pairs_table[id_pair]['Dxy']**2+pairs_table[id_pair]['Dlos']**2)
            fitobj2.boost = fitobj.Dtot/pairs_table[id_pair]['Dxy']
            fitobj2.use_boost = config['use_boost']

            fitobj2.shear_v_arcmin =  shears_info['v_arcmin']
            fitobj2.shear_u_arcmin =  shears_info['u_arcmin']
            fitobj2.shear_u_mpc =  shears_info['u_mpc']
            fitobj2.shear_v_mpc =  shears_info['v_mpc']

            fitobj2.halo1_z =  overlapping_halo_z
            fitobj2.halo1_u_arcmin = x_pos/cosmology.get_ang_diam_dist(overlapping_halo_z) * 180. / np.pi * 60.
            fitobj2.halo1_v_arcmin = y_pos/cosmology.get_ang_diam_dist(overlapping_halo_z) * 180. / np.pi * 60.
            fitobj2.halo1_u_mpc =  x_pos
            fitobj2.halo1_v_mpc =  y_pos

            fitobj2.halo2_u_arcmin =  pairs_table['u2_arcmin'][id_pair]
            fitobj2.halo2_v_arcmin =  pairs_table['v2_arcmin'][id_pair]
            fitobj2.halo2_u_mpc =  pairs_table['u2_mpc'][id_pair]
            fitobj2.halo2_v_mpc =  pairs_table['v2_mpc'][id_pair]
            fitobj2.halo2_z =  pairs_table['z'][id_pair]


            fitobj2.pair_z  = (fitobj2.halo1_z + fitobj2.halo2_z) / 2.

            fitobj2.filam = filament.filament()
            fitobj2.filam.pair_z =fitobj2.pair_z
            fitobj2.filam.grid_z_centers = fitobj2.grid_z_centers
            fitobj2.filam.prob_z = fitobj2.prob_z
            fitobj2.filam.set_mean_inv_sigma_crit(fitobj2.filam.grid_z_centers,fitobj2.filam.prob_z,fitobj2.filam.pair_z)

            fitobj2.nh1 = nfw.NfwHalo()
            fitobj2.nh1.z_cluster= fitobj2.halo1_z
            fitobj2.nh1.theta_cx = fitobj2.halo1_u_arcmin
            fitobj2.nh1.theta_cy = fitobj2.halo1_v_arcmin 
            fitobj2.nh1.set_mean_inv_sigma_crit(fitobj2.grid_z_centers,fitobj2.prob_z,fitobj2.pair_z)

            fitobj2.nh2 = nfw.NfwHalo()
            fitobj2.nh2.z_cluster= fitobj2.halo2_z
            fitobj2.nh2.theta_cx = fitobj2.halo2_u_arcmin
            fitobj2.nh2.theta_cy = fitobj2.halo2_v_arcmin 
            fitobj2.nh2.set_mean_inv_sigma_crit(fitobj2.grid_z_centers,fitobj2.prob_z,fitobj2.pair_z)

            shear_model_g1_neighbour, shear_model_g2_neighbour, limit_mask , _ , _  = fitobj2.draw_model([0.0, 0.5, overlapping_halo_m200, no_m200 ])

            do_plot=False
            if do_plot:

                cmax  = np.max([ np.abs(shear_model_g1_neighbour.min()),np.abs(shear_model_g1_neighbour.min()) , np.abs(shear_model_g1_neighbour.max()),np.abs(shear_model_g1_neighbour.max()),np.abs(shear_model_g1.max()),np.abs(shear_model_g1.max()),np.abs(shear_model_g1.min()),np.abs(shear_model_g1.min())])
                pl.figure(figsize=(20,10))
                pl.scatter( fitobj.shear_u_mpc  , fitobj.shear_v_mpc  , s=100, c=shear_model_g1+shear_model_g1_neighbour, lw=0)
                pl.clim(-cmax,cmax)
                pl.colorbar()
                pl.scatter( fitobj2.halo1_u_mpc , fitobj2.halo1_v_mpc , s=100, lw=0)
                pl.scatter( fitobj.halo1_u_mpc , fitobj.halo1_v_mpc , s=100)
                pl.scatter( fitobj.halo2_u_mpc , fitobj.halo2_v_mpc , s=100)
                pl.axis('equal')

                pl.show()

            fitobj.shear_g1 =  shear_model_g1 + np.random.randn(len(fitobj.shear_u_arcmin))*sigma_g_add
            fitobj.shear_g2 =  shear_model_g2 + np.random.randn(len(fitobj.shear_u_arcmin))*sigma_g_add
            fitobj.sigma_g =  np.std(shear_model_g2,ddof=1)
            fitobj.inv_sq_sigma_g = 1./sigma_g_add**2

            shears_info['g1'] = fitobj.shear_g1
            shears_info['g2'] = fitobj.shear_g2
            shears_info['weight'] = fitobj.inv_sq_sigma_g

            tabletools.savePickle(filename_shears_overlap,shears_info,append=True)          
示例#29
0
def test_calibration_procedure():

    filelist = np.loadtxt(filelist_svclusters,dtype='a')
    calib_struct = pickle.load(open(filename_calibration))


    for ifile,filename_results in enumerate(filelist):

        filename_results_calibrated = filename_results.replace('.cat','.nbc.cat')
        results_sv = tabletools.loadTable(filename_results_calibrated,dtype=dtype_table_results_sv_calibrated,log=1)
        select = results_sv['flag'] == 0
        results_sv = results_sv[select]

        # analyse_population(results_sv)

        g1_mean = np.mean(results_sv['g1'])
        g1_bias_m = np.mean(results_sv['nbc_m'])
        g1_bias_m_std = get_m_std_for_sample(results_sv)
        g1_mean_calibrated = g1_mean / g1_bias_m

        log.info( 'g1_mean            %10.4f' % g1_mean  )
        log.info( 'g1_mean_calibrated %10.4f' % g1_mean_calibrated )
        log.info( 'g1_bias_m          %10.4f' % g1_bias_m )
        log.info( 'g1_bias_m_std      %10.4f' % g1_bias_m_std )

        marg_size_m, marg_size_m_std, marg_size_centers = get_marg('size',results_sv)
        marg_snr_m, marg_snr_m_std, marg_snr_centers = get_marg('snr',results_sv)

        pl.figure()
        pl.suptitle('%s\ntotal_calibration = %0.4f +/- %0.4f' % (filename_results,g1_bias_m,g1_bias_m_std))
        pl.subplot(1,2,1)
        hh,bh,_=pl.hist(results_sv['size'],bins=calib_struct['bins_size'],histtype='step')
        # hh,bh,_=pl.hist(results_sv['size'],bins=np.linspace(1.2,3,20),histtype='step')
        pl.xlabel('FWHM_RATIO')
        pl.ylabel('histogram')
        ax2=pl.gca().twinx()
        ax2.errorbar(marg_size_centers,marg_size_m,yerr=marg_size_m_std,fmt='md')
        ax2.set_ylabel('multiplicative bias m')

        pl.subplot(1,2,2)
        # pl.hist(results_sv['snr'],bins=calib_struct['bins_snr'],histtype='step')
        pl.hist(results_sv['snr'],bins=np.linspace(0,60,20),histtype='step')
        pl.xlabel('SNR')
        pl.ylabel('histogram')
        ax2=pl.gca().twinx()
        ax2.errorbar(marg_snr_centers,marg_snr_m,yerr=marg_snr_m_std,fmt='cd')
        ax2.set_ylabel('multiplicative bias m')

        left  = 0.125  # the left side of the subplots of the figure
        right = 0.9    # the right side of the subplots of the figure
        bottom = 0.1   # the bottom of the subplots of the figure
        top = 0.9      # the top of the subplots of the figure
        wspace = 0.7   # the amount of width reserved for blank space between subplots
        hspace = 0.5   # t
        pl.subplots_adjust(left=left, bottom=bottom, right=right, top=top, wspace=wspace, hspace=hspace)

        filename_fig = ('figs/fig.marg.%s.png' % filename_results ).replace('.fits.im3.cleaned.wcs.cat','').replace('sv-clusters-shears/','')
        pl.savefig(filename_fig)
        pl.close()
        log.info('saved %s' % filename_fig)

        
        # pl.show()

        # n_gals = len(results_sv)

        # n_gals_select = int(n_gals*0.25)
        # select = np.random.permutation(n_gals)[0:n_gals_select]
        # n_selected = len(select)

        select = results_sv['snr'] < 15
        g1_mean = np.mean(results_sv[select]['g1'])
        g1_bias_m = np.mean(results_sv[select]['nbc_m'])
        g1_bias_m_std = get_m_std_for_sample(results_sv[select])
        g1_mean_calibrated = g1_mean / g1_bias_m

        print 'g1_mean' , g1_mean 
        print 'g1_mean_calibrated' , g1_mean_calibrated
        print 'g1_bias_m' , g1_bias_m
        print 'g1_bias_m_std' , g1_bias_m_std

        # import pdb; pdb.set_trace()


        # h,b,_=pl.hist(results_sv['nbc_m'][select],histtype='step')
        # pl.ylim([-10,max(h)])
        # pl.yscale('log')
        # pl.hist(results_sv['nbc_m'],histtype='step')
        # results_sv['nbc_m'].sort()
        # pl.plot(results_sv['nbc_m'])
        # pl.figure()
        # pl.scatter(results_sv['nbc_m'],results_sv['size'])
        # pl.plot()
        # pl.figure()
        # pl.scatter(results_sv['nbc_m'],results_sv['snr'])

        pl.show()
def run_test():

    bins_z = np.arange(0.025,3.5,0.05)

    filename_gals = '/home/kacprzak/data/CFHTLens/CFHTLens_2014-06-14.normalised.fits'
    filename_clusters = os.environ['HOME'] + '/data/CFHTLens/ClusterZ/clustersz.fits'
    filename_lrgclus = 'halos_cfhtlens_lrgsclus.fits'

    cat_clusters = tabletools.loadTable(filename_clusters)
    cat_lrgclus = tabletools.loadTable(filename_lrgclus)
    cat_gals = tabletools.loadTable(filename_gals)
    cat_clusters = cat_clusters[cat_lrgclus['id']]

    # i=1
    # x=bins_z[20:30]
    # y=cat_gals['PZ_full'][i][20:30]
    # pl.plot(x,y,'o-'); 
    # pl.plot(bins_z,cat_gals['PZ_full'][i],'rx-');

    # xx=np.linspace(x.min(), x.max(),100)
    # f=interp1d(x,y,'cubic')
    # yy=f(xx)
    # pl.plot(xx,yy,'d'); pl.show()

    # import pdb; pdb.set_trace()

    # perm = np.random.permutation(len(cat_gals))[:10000]
    # pl.scatter(cat_gals['ALPHA_J2000'][perm],cat_gals['DELTA_J2000'][perm])

    gals_ra_deg = cat_gals['ALPHA_J2000']
    gals_de_deg = cat_gals['DELTA_J2000']
    gals_ra_rad , gals_de_rad = cosmology.deg_to_rad(gals_ra_deg, gals_de_deg)

    cylinder_radius_mpc=1

    pz_all=np.sum(cat_gals['PZ_full'],axis=0)
    pz_all=pz_all/np.sum(pz_all)

    n_brigthest = 40
    n_bins_hires = 10000
    bins_z_hires=np.linspace(bins_z.min(), bins_z.max(),n_bins_hires)
    new_z = np.zeros(len(cat_lrgclus))

    print 'len(cat_lrgclus)', len(cat_lrgclus)

    for ic in range(len(cat_lrgclus)):

        cluster_ra_rad , cluster_de_rad = cosmology.deg_to_rad( cat_clusters[ic]['ra'] , cat_clusters[ic]['dec'] )
        cluster_z = cat_clusters['z'][ic]
        cluster_zspec = cat_lrgclus['z'][ic]

        gals_u_rad , gals_v_rad = cosmology.get_gnomonic_projection(gals_ra_rad , gals_de_rad , cluster_ra_rad , cluster_de_rad)
        gals_u_mpc , gals_v_mpc = cosmology.rad_to_mpc(gals_u_rad,gals_v_rad,cluster_z)

        select = (np.sqrt(gals_u_mpc**2 + gals_v_mpc**2) < cylinder_radius_mpc)*( np.abs(cat_gals['Z_B']-cluster_z) < 0.1 )
        # print 'selected %d gals in cylinder' % len(np.nonzero(select)[0])
        cylinder_gals = cat_gals[select]
        gals_u_mpc = gals_u_mpc[select]
        gals_v_mpc = gals_v_mpc[select]

        select1 = (cylinder_gals['MAG_i'] > 10) * (cylinder_gals['MAG_i'] < 27)
        select2 = (cylinder_gals['MAG_r'] > 10) * (cylinder_gals['MAG_r'] < 27)
        select3 = (cylinder_gals['MAG_g'] > 10) * (cylinder_gals['MAG_g'] < 27)
        select4 = (cylinder_gals['MAG_u'] > 10) * (cylinder_gals['MAG_u'] < 27)
        select5 = (cylinder_gals['MAG_y'] > 10) * (cylinder_gals['MAG_y'] < 27)
        select6 = (cylinder_gals['MAG_z'] > 10) * (cylinder_gals['MAG_z'] < 27)

        select = select2*select1*select3*select4*select6
        # print 'selected %d with good mags' % len(np.nonzero(select)[0])
        if len(np.nonzero(select)[0]) == 0:
            continue
        cylinder_gals = cylinder_gals[select]
        gals_u_mpc = gals_u_mpc[select]
        gals_v_mpc = gals_v_mpc[select]

        # select_brightest_i = np.ones(len(cylinder_gals))[np.argsort(cylinder_gals['MAG_i'])[:n_brigthest]] == True
        # select_brightest_r = np.ones(len(cylinder_gals))[np.argsort(cylinder_gals['MAG_r'])[:n_brigthest]] == True
        # select_brightest_u = np.ones(len(cylinder_gals))[np.argsort(cylinder_gals['MAG_u'])[:n_brigthest]] == True
        # select_brightest_g = np.ones(len(cylinder_gals))[np.argsort(cylinder_gals['MAG_g'])[:n_brigthest]] == True
        # select_brightest = select_brightest_r 
        # cylinder_brightest = cylinder_gals[select_brightest]
        # gals_u_mpc = gals_u_mpc[select_brightest]
        # gals_v_mpc = gals_v_mpc[select_brightest]
        # print 'using %d gals' % len(cylinder_brightest)

        x1=cylinder_gals['MAG_r']-cylinder_gals['MAG_i']
        x2=cylinder_gals['MAG_g']-cylinder_gals['MAG_i']
        x3=cylinder_gals['MAG_u']-cylinder_gals['MAG_i']
        x4=cylinder_gals['MAG_y']-cylinder_gals['MAG_i']
        x5=cylinder_gals['MAG_z']-cylinder_gals['MAG_i']
        

        X=np.concatenate( [x1.astype('f4')[:,None], x2.astype('f4')[:,None], x3.astype('f4')[:,None],  x5.astype('f4')[:,None]] ,axis=1)

        from scipy.stats import gaussian_kde
        kde = gaussian_kde(X.T,bw_method=0.3) 
        w = kde(X.T)**3
        w = w/np.max(w)
        # pl.figure()
        # pl.scatter(X[:,0],X[:,1],s=50,c=w) ; pl.colorbar()
        # pl.figure()
        # pl.scatter(X[:,1],X[:,2],s=50,c=w) ; pl.colorbar()
        # pl.figure()
        # pl.scatter(X[:,0],X[:,2],s=50,c=w) ; pl.colorbar()
        # pl.figure()
        # pl.scatter(X[:,0],X[:,3],s=50,c=w) ; pl.colorbar()
        # # pl.figure()
        # # pl.scatter(X[:,0],X[:,4],s=50,c=w) ; pl.colorbar()
        # pl.show()

        # pl.figure()
        # pl.scatter(cylinder_gals[select_brightest]['MAG_r']-cylinder_gals[select_brightest]['MAG_i'],  cylinder_gals[select_brightest]['MAG_g']-cylinder_gals[select_brightest]['MAG_i'],c=cylinder_gals[select_brightest]['Z_B'])
        # pl.colorbar()
        # pl.figure()
        # pl.scatter(cylinder_gals[select_brightest]['MAG_i'],cylinder_gals[select_brightest]['MAG_r'],c=cylinder_gals[select_brightest]['Z_B'])
        # pl.colorbar()
        # pl.figure()
        # pl.scatter(cylinder_gals[select_brightest]['MAG_g'],cylinder_gals[select_brightest]['MAG_r'],c=cylinder_gals[select_brightest]['Z_B'])
        # pl.colorbar()
        # pl.figure()
        # pl.scatter(cylinder_gals[select_brightest]['MAG_u'],cylinder_gals[select_brightest]['MAG_i'],c=cylinder_gals[select_brightest]['Z_B'])
        # pl.colorbar()
        # pl.figure()
        # pl.scatter(cylinder_gals[select_brightest]['MAG_r'],cylinder_gals[select_brightest]['Z_B'],c=cylinder_gals[select_brightest]['Z_B'])
        # pl.colorbar()

        # pl.figure()
        # pl.scatter(gals_u_mpc,gals_v_mpc,c=cylinder_brightest['Z_B'],s=cylinder_brightest['MAG_r']*2)
        # pl.colorbar()

        # pl.show()

        # pz_hires = np.zeros([len(cylinder_gals),n_bins_hires])
        # for ib in range(len(cylinder_gals)):
        #     fz=interp1d(bins_z,cylinder_gals['PZ_full'][ib],'cubic')
        #     pz_hires[ib,:] = fz(bins_z_hires)
        #     pz_this = pz_hires[ib,:]/np.sum(pz_hires[ib,:])*w[ib]
        #     pl.plot(bins_z_hires,pz_this)
        #     pl.plot(bins_z,cylinder_gals['PZ_full'][ib],'x');
        #     print 'interp' , ib, np.sum(pz_this)

        # pz_hires[pz_hires<0] = 1e-10
        # pz_prod = np.sum(np.log(pz_hires),axis=0)
        # pz_prod = pz_prod - pz_prod.max()
        # pz_cylinder=np.exp(pz_prod)
        # pz_cylinder=pz_cylinder/np.sum(pz_cylinder)

        # new_z[ic] = bins_z_hires[pz_cylinder.argmax()]
        new_z[ic] = np.sum(cylinder_gals['Z_B']*w)/np.sum(w)
        std_z=np.std(np.sqrt(((cylinder_gals['Z_B']*w - new_z[ic])**2)/np.sum(w)))
        print '%3d new_z=%.4f zspec=%.4f bad_z=%.4f naomi_z=%.4f new-zpec=% .4f  naomi-zspec=% .4f n_eff=%2.4f n_cylinder_gals=%d std_z=%2.5f'  % (ic,new_z[ic],cluster_zspec,cluster_z,cat_clusters['z'][ic],new_z[ic]-cluster_zspec,cat_clusters['z'][ic]-cluster_zspec,np.sum(w),len(cylinder_gals),std_z)

        # pl.figure()
        # pl.plot(bins_z_hires,pz_cylinder,'kd');
        # # pl.plot(bins_z,pz_all); 
        # pl.axvline(cluster_z,color='b')
        # pl.axvline(cluster_zspec,color='c')
        # pl.axvline(cluster_z+0.1,color='r')
        # pl.axvline(cluster_z-0.1,color='r')
        # pl.xlim([0,1])
        # pl.show()

    pl.hist(new_z-cat_lrgclus['z'],np.linspace(-0.1,0.1,20),histtype='step',label='new z',normed=True);
    pl.hist(cat_clusters['z']-cat_lrgclus['z'],np.linspace(-0.1,0.1,20),histtype='step',label='naomi z',normed=True);
    pl.hist(cat_clusters['z_bad']-cat_lrgclus['z'],np.linspace(-0.1,0.1,20),histtype='step',label='old z',normed=True);
    pl.xlabel('z_estimated - z_spec')
    pl.legend()
    pl.show()
    import pdb; pdb.set_trace()