def clusters_in_pix(clusters, pix, nside, search_nside=None):
import pandas as pd
import healpy as hp
import numpy as np
from DS_healpix_fragmentation import radec2pix
df = pd.read_csv(clusters)
cl_pix = radec2pix(df['RA'], df['DEC'], nside)
df = df[cl_pix == pix]
df.index = np.arange(df.shape[0])
if not (search_nside is None):
search_pix = radec2pix(df['RA'], df['DEC'], search_nside)
df['search'] = search_pix
return df
def pixels_with_clusters(clusters_files, big_pixels, nside, min_rad=0.62):
from astropy.coordinates import SkyCoord
from astropy import units as u
import pandas as pd
import healpy as hp
import numpy as np
from DS_healpix_fragmentation import radec2pix
df = [pd.read_csv(clusters_file) for clusters_file in clusters_files]
df = pd.concat(df, ignore_index=True)
pix2 = radec2pix(df['RA'], df['DEC'], 2)
df = df[np.in1d(pix2, big_pixels)]
df.index = np.arange(df.shape[0])
small_pixels = set()
sc = SkyCoord(ra=np.array(df['RA']) * u.degree,
dec=np.array(df['DEC']) * u.degree,
frame='icrs')
vecs = hp.ang2vec(theta=sc.galactic.l.degree,
phi=sc.galactic.b.degree,
lonlat=True)
for i in range(df.shape[0]):
small_pixels = small_pixels.union(
hp.query_disc(nside, vecs[i], np.radians(min_rad), nest=True))
small_pixels = np.array(list(small_pixels))
return small_pixels, df
def mark_cat(cat, d):
import numpy as np
import pandas as pd
from DS_healpix_fragmentation import radec2pix
pix2 = radec2pix(cat['RA'], cat['DEC'], 2)
cat['pix2'] = ''
for name in d:
cat['pix2'].loc[np.in1d(pix2, d[name])] = name
return cat
def gen_data_for_class(
tp_coef,
n,
cats,
pix2,
dirname='/home/rt2122/Data/clusters/',
fp_file='/home/rt2122/Data/class/fp_coords/fp_pz_pnz_act.csv',
radius=7.5 / 60,
class_len=50):
import os
import numpy as np
import pandas as pd
from DS_healpix_fragmentation import radec2pix
df_cat = pd.concat(
[pd.read_csv(os.path.join(dirname, cat + '.csv')) for cat in cats],
ignore_index=True)
df_cat['pix2'] = radec2pix(df_cat['RA'], df_cat['DEC'], 2)
df_cat = df_cat[np.in1d(df_cat['pix2'], pix2)]
df_cat.index = np.arange(len(df_cat))
n_tp = int(tp_coef * n)
n_fp = n - n_tp
tp_df = df_cat.sample(n=n_tp)[['RA', 'DEC']]
fp_df = pd.read_csv(fp_file)
fp_df['pix2'] = radec2pix(fp_df['RA'], fp_df['DEC'], 2)
fp_df = fp_df[np.in1d(fp_df['pix2'], pix2)].sample(n=n_fp)[['RA', 'DEC']]
tp_df['y'] = True
fp_df['y'] = False
df = pd.concat([tp_df, fp_df], ignore_index=True)
X = np.stack([
radec2line_class([df['RA'].iloc[i], df['DEC'].iloc[i]],
radius=radius,
class_len=class_len).flatten() for i in range(len(df))
])
y = np.array(df['y'])
return X, y
def radec2pred_ind(scan_name, input_name, output_name, mode='fits'):
from DS_healpix_fragmentation import radec2pix
import os
import numpy as np
import pandas as pd
in_df = get_df(input_name, mode=mode)
in_df['healpix'] = radec2pix(in_df['RA'], in_df['DEC'], 2**11)
scan = np.load(scan_name)
in_df['pred_ind'] = scan[in_df['healpix']]
in_df.index.name = 'index'
in_df.to_csv(output_name)
def stat_split_cats(files, big_pix=list(range(48))):
import pandas as pd
import numpy as np
from DS_healpix_fragmentation import radec2pix
res_df = []
comp_df = []
for model in files:
df = pd.read_csv(files[model])
df = df.iloc[np.in1d(radec2pix(df['RA'], df['DEC'], 2), big_pix)]
df.index = np.arange(len(df))
line = {}
line_c = {}
for cat in ['planck_z', 'planck_no_z', 'mcxcwp', 'actwp']:
cur_df = df[df['catalog'] == cat]
cur_df.index = np.arange(len(cur_df))
line_c[cat] = np.count_nonzero(cur_df['status'] == 'tp')
line[cat] = line_c[cat] / len(cur_df)
line['fp'] = np.count_nonzero(df['status'] == 'fp')
res_df.append(pd.DataFrame(line, index=[model]))
comp_df.append(pd.DataFrame(line_c, index=[model]))
res_df = pd.concat(res_df)
comp_df = pd.concat(comp_df)
return res_df, comp_df
def gen_pics_for_detection(
ipix,
model,
big_nside=2,
step=64,
size=64,
depth=10,
mask_radius=15 / 60,
clusters_dir='/home/rt2122/Data/clusters/',
planck_dirname='/home/rt2122/Data/Planck/normalized/',
data_type=np.float64,
only=False,
only_idx=100):
from DS_healpix_fragmentation import one_pixel_fragmentation, pix2radec, radec2pix
from DS_Planck_Unet import draw_pic_with_mask, draw_pic
import pandas as pd
import numpy as np
import healpy as hp
import os
true_clusters = pack_all_catalogs(clusters_dir)
clusters_pix = radec2pix(true_clusters['RA'], true_clusters['DEC'], 2)
true_clusters = true_clusters[clusters_pix == ipix]
true_clusters.index = np.arange(true_clusters.shape[0])
big_matr = one_pixel_fragmentation(big_nside, ipix, depth)
big_pic, big_mask = draw_pic_with_mask(center=None,
matr=big_matr,
mask_radius=mask_radius,
clusters_arr=np.array(
true_clusters[['RA', 'DEC']]),
dirname=planck_dirname)
big_pic = big_pic.astype(data_type)
pics, matrs, masks = [], [], []
pic_idx = []
starts = []
for k in range(2):
x_st = [
i for i in range(0, big_matr.shape[k], step)
if i + size <= big_matr.shape[k]
] + [big_matr.shape[k] - size]
starts.append(x_st)
for i in starts[0]:
for j in starts[1]:
pic = big_pic[i:i + size, j:j + size, :]
mask = big_mask[i:i + size, j:j + size, :]
matr = big_matr[i:i + size, j:j + size]
if pic.shape == (size, size, pic.shape[-1]):
pics.append(pic)
pic_idx.append((i, j))
matrs.append(matr)
masks.append(mask)
ans = None
if only:
ans = []
for i in range(0, len(pics), only_idx):
ans.append(model.predict(np.array(pics[i:i + only_idx])))
ans = np.concatenate(ans, axis=0)
else:
ans = model.predict(np.array(pics))
if only:
return {'ans': ans, 'pic_idx': pic_idx}
return {
'true_clusters': true_clusters,
'pics': pics,
'matrs': matrs,
'masks': masks,
'ans': ans,
'pic_idx': pic_idx
}