def clusters_in_pix(clusters, pix, nside, search_nside=None): import pandas as pd import healpy as hp import numpy as np from DS_healpix_fragmentation import radec2pix df = pd.read_csv(clusters) cl_pix = radec2pix(df['RA'], df['DEC'], nside) df = df[cl_pix == pix] df.index = np.arange(df.shape[0]) if not (search_nside is None): search_pix = radec2pix(df['RA'], df['DEC'], search_nside) df['search'] = search_pix return df
def pixels_with_clusters(clusters_files, big_pixels, nside, min_rad=0.62): from astropy.coordinates import SkyCoord from astropy import units as u import pandas as pd import healpy as hp import numpy as np from DS_healpix_fragmentation import radec2pix df = [pd.read_csv(clusters_file) for clusters_file in clusters_files] df = pd.concat(df, ignore_index=True) pix2 = radec2pix(df['RA'], df['DEC'], 2) df = df[np.in1d(pix2, big_pixels)] df.index = np.arange(df.shape[0]) small_pixels = set() sc = SkyCoord(ra=np.array(df['RA']) * u.degree, dec=np.array(df['DEC']) * u.degree, frame='icrs') vecs = hp.ang2vec(theta=sc.galactic.l.degree, phi=sc.galactic.b.degree, lonlat=True) for i in range(df.shape[0]): small_pixels = small_pixels.union( hp.query_disc(nside, vecs[i], np.radians(min_rad), nest=True)) small_pixels = np.array(list(small_pixels)) return small_pixels, df
def mark_cat(cat, d): import numpy as np import pandas as pd from DS_healpix_fragmentation import radec2pix pix2 = radec2pix(cat['RA'], cat['DEC'], 2) cat['pix2'] = '' for name in d: cat['pix2'].loc[np.in1d(pix2, d[name])] = name return cat
def gen_data_for_class( tp_coef, n, cats, pix2, dirname='/home/rt2122/Data/clusters/', fp_file='/home/rt2122/Data/class/fp_coords/fp_pz_pnz_act.csv', radius=7.5 / 60, class_len=50): import os import numpy as np import pandas as pd from DS_healpix_fragmentation import radec2pix df_cat = pd.concat( [pd.read_csv(os.path.join(dirname, cat + '.csv')) for cat in cats], ignore_index=True) df_cat['pix2'] = radec2pix(df_cat['RA'], df_cat['DEC'], 2) df_cat = df_cat[np.in1d(df_cat['pix2'], pix2)] df_cat.index = np.arange(len(df_cat)) n_tp = int(tp_coef * n) n_fp = n - n_tp tp_df = df_cat.sample(n=n_tp)[['RA', 'DEC']] fp_df = pd.read_csv(fp_file) fp_df['pix2'] = radec2pix(fp_df['RA'], fp_df['DEC'], 2) fp_df = fp_df[np.in1d(fp_df['pix2'], pix2)].sample(n=n_fp)[['RA', 'DEC']] tp_df['y'] = True fp_df['y'] = False df = pd.concat([tp_df, fp_df], ignore_index=True) X = np.stack([ radec2line_class([df['RA'].iloc[i], df['DEC'].iloc[i]], radius=radius, class_len=class_len).flatten() for i in range(len(df)) ]) y = np.array(df['y']) return X, y
def radec2pred_ind(scan_name, input_name, output_name, mode='fits'): from DS_healpix_fragmentation import radec2pix import os import numpy as np import pandas as pd in_df = get_df(input_name, mode=mode) in_df['healpix'] = radec2pix(in_df['RA'], in_df['DEC'], 2**11) scan = np.load(scan_name) in_df['pred_ind'] = scan[in_df['healpix']] in_df.index.name = 'index' in_df.to_csv(output_name)
def stat_split_cats(files, big_pix=list(range(48))): import pandas as pd import numpy as np from DS_healpix_fragmentation import radec2pix res_df = [] comp_df = [] for model in files: df = pd.read_csv(files[model]) df = df.iloc[np.in1d(radec2pix(df['RA'], df['DEC'], 2), big_pix)] df.index = np.arange(len(df)) line = {} line_c = {} for cat in ['planck_z', 'planck_no_z', 'mcxcwp', 'actwp']: cur_df = df[df['catalog'] == cat] cur_df.index = np.arange(len(cur_df)) line_c[cat] = np.count_nonzero(cur_df['status'] == 'tp') line[cat] = line_c[cat] / len(cur_df) line['fp'] = np.count_nonzero(df['status'] == 'fp') res_df.append(pd.DataFrame(line, index=[model])) comp_df.append(pd.DataFrame(line_c, index=[model])) res_df = pd.concat(res_df) comp_df = pd.concat(comp_df) return res_df, comp_df
def gen_pics_for_detection( ipix, model, big_nside=2, step=64, size=64, depth=10, mask_radius=15 / 60, clusters_dir='/home/rt2122/Data/clusters/', planck_dirname='/home/rt2122/Data/Planck/normalized/', data_type=np.float64, only=False, only_idx=100): from DS_healpix_fragmentation import one_pixel_fragmentation, pix2radec, radec2pix from DS_Planck_Unet import draw_pic_with_mask, draw_pic import pandas as pd import numpy as np import healpy as hp import os true_clusters = pack_all_catalogs(clusters_dir) clusters_pix = radec2pix(true_clusters['RA'], true_clusters['DEC'], 2) true_clusters = true_clusters[clusters_pix == ipix] true_clusters.index = np.arange(true_clusters.shape[0]) big_matr = one_pixel_fragmentation(big_nside, ipix, depth) big_pic, big_mask = draw_pic_with_mask(center=None, matr=big_matr, mask_radius=mask_radius, clusters_arr=np.array( true_clusters[['RA', 'DEC']]), dirname=planck_dirname) big_pic = big_pic.astype(data_type) pics, matrs, masks = [], [], [] pic_idx = [] starts = [] for k in range(2): x_st = [ i for i in range(0, big_matr.shape[k], step) if i + size <= big_matr.shape[k] ] + [big_matr.shape[k] - size] starts.append(x_st) for i in starts[0]: for j in starts[1]: pic = big_pic[i:i + size, j:j + size, :] mask = big_mask[i:i + size, j:j + size, :] matr = big_matr[i:i + size, j:j + size] if pic.shape == (size, size, pic.shape[-1]): pics.append(pic) pic_idx.append((i, j)) matrs.append(matr) masks.append(mask) ans = None if only: ans = [] for i in range(0, len(pics), only_idx): ans.append(model.predict(np.array(pics[i:i + only_idx]))) ans = np.concatenate(ans, axis=0) else: ans = model.predict(np.array(pics)) if only: return {'ans': ans, 'pic_idx': pic_idx} return { 'true_clusters': true_clusters, 'pics': pics, 'matrs': matrs, 'masks': masks, 'ans': ans, 'pic_idx': pic_idx }