def add_kids_sdss_columns(data):
if 'ID' in data.columns:
kids_x_sdss = read_fits_to_pandas(
os.path.join(DATA_PATH, 'KiDS/DR4/KiDS.DR4.x.SDSS.DR14.fits'), [
'ID', 'MAG_GAAP_r', 'Z_B', 'Z_ML', 'Flag', 'IMAFLAGS_ISO',
'MASK', 'CLASS_STAR'
])
kids_x_sdss = process_bitmaps(kids_x_sdss)
return data.merge(kids_x_sdss, on=['ID'])
else:
warnings.warn('KiDS columns not added due to lack of KiDS ID column')
return data
def test_gaia(catalog,
catalog_x_gaia_path,
class_column='CLASS_PHOTO',
id_column='ID',
save=False):
print('catalog size: {}'.format(catalog.shape[0]))
print(get_column_desc(catalog[class_column]))
catalog_x_gaia = read_fits_to_pandas(catalog_x_gaia_path)
movement_mask = ~catalog_x_gaia[['parallax', 'pmdec', 'pmra'
]].isnull().any(axis=1)
catalog_x_gaia_movement = catalog_x_gaia.loc[movement_mask]
catalog_x_gaia_movement = clean_gaia(catalog_x_gaia_movement)
test_against_external_catalog(catalog_x_gaia_movement,
catalog,
class_column=class_column,
id_column=id_column,
title='GAIA',
save=save)
def test_external_qso(catalog, save=False, plot=True):
print('catalog size: {}'.format(catalog.shape[0]))
for name, file_path, columns in EXTERNAL_QSO:
external_catalog = read_fits_to_pandas(file_path, columns=columns)
# Limit minimum proba for Galex QSOs
if name == 'x DiPompeo 2015':
external_catalog = external_catalog.loc[
external_catalog['PQSO'] > 0.7]
# Take only QSOs for 2QZ/6QZ
if 'id1' in external_catalog.columns:
external_catalog = process_2df(external_catalog)
external_catalog = external_catalog.loc[external_catalog['id1'] ==
'QSO']
title = os.path.basename(file_path)[:-4]
test_against_external_catalog(external_catalog,
catalog,
title=title,
plot=plot,
save=save)
import os
from utils import save_fits
from data import DATA_PATH, read_fits_to_pandas
ctlg_clf_path = os.path.join(
DATA_PATH,
'KiDS/DR4/catalogs/KiDS_DR4_x_SDSS_DR14_ann_clf_f-all__2020-06-08_17:07:35.fits'
)
ctlg_z_qso_path = os.path.join(
DATA_PATH,
'KiDS/DR4/catalogs/KiDS_DR4_x_SDSS_DR14_ann_z_f-all_spec-qso__2020-06-08_16:22:38.fits'
)
ctlg_clf = read_fits_to_pandas(ctlg_clf_path)
ctlg_z_qso = read_fits_to_pandas(ctlg_z_qso_path)
print(ctlg_clf.shape)
ctlg_clf[['QSO_Z_PHOTO',
'QSO_Z_PHOTO_STDDEV']] = ctlg_z_qso[['Z_PHOTO', 'Z_PHOTO_STDDEV']]
ctlg_clf = ctlg_clf[[
'ID', 'RAJ2000', 'DECJ2000', 'GALAXY_PHOTO', 'QSO_PHOTO', 'STAR_PHOTO',
'QSO_Z_PHOTO', 'QSO_Z_PHOTO_STDDEV'
]]
# Save FITS
catalog_path = os.path.join(
DATA_PATH, 'KiDS/DR4/catalogs/KiDS_DR4_x_SDSS_DR14_ann__2020-06-08.fits')
save_fits(ctlg_clf, catalog_path)
def plot_external_qso_consistency(catalog):
step = 0.001
thresholds = np.arange(0.3, 1.0, step)
color_palette = get_cubehelix_palette(len(EXTERNAL_QSO))
# Read data
data_dict = OrderedDict(
(data_name, read_fits_to_pandas(data_path, columns=columns))
for data_name, data_path, columns in EXTERNAL_QSO)
# TODO: Ugly work around, extract process_DiPompeo function
# Limit galex to minimum QSO proba
data_dict['x DiPompeo 2015'] = data_dict['x DiPompeo 2015'].loc[
data_dict['x DiPompeo 2015']['PQSO'] > 0.7]
# Take only QSOs for 2QZ/6QZ
data_tmp = process_2df(data_dict['x 2QZ/6QZ'])
data_dict['x 2QZ/6QZ'] = data_tmp.loc[data_tmp['id1'] == 'QSO']
plt.figure()
for i, (external_qso_name, external_qso) in enumerate(data_dict.items()):
id_column_ext = 'ID' if 'ID' in external_qso else 'ID_1'
catalog_int = catalog.loc[catalog['ID'].isin(
external_qso[id_column_ext])]
threshold_data_arr = [
catalog_int.loc[
catalog_int[['QSO_PHOTO', 'STAR_PHOTO', 'GALAXY_PHOTO']].max(
axis=1) >= thr][['ID', 'CLASS_PHOTO']]
for thr in thresholds
]
qso_data_arr = [
int_data.loc[int_data['CLASS_PHOTO'] == 'QSO']
for int_data in threshold_data_arr
]
agreement_arr = [
qso_data_arr[i].shape[0] / float(threshold_data_arr[i].shape[0])
for i, _ in enumerate(qso_data_arr)
]
# # TODO: Ugly work around
# external_qso_name = get_external_qso_short_name(external_qso_name)
# external_qso_name += ' QSO'
plt.plot(thresholds,
agreement_arr,
label=external_qso_name,
linestyle=get_line_style(i),
alpha=1.0,
color=color_palette[i])
plt.xlabel(r'KiDS minimum photo probability')
plt.ylabel(r'KiDS $\rm QSO_{cand}$ contribution')
legend = plt.legend(loc='lower left', framealpha=1.0)
plt.tight_layout()
ax = plt.axes()
ax.yaxis.grid(True)
# Customized legend place
ax = plt.gca()
bounding_box = legend.get_bbox_to_anchor().inverse_transformed(
ax.transAxes)
y_offset = 0.13
bounding_box.y0 += y_offset
bounding_box.y1 += y_offset
legend.set_bbox_to_anchor(bounding_box, transform=ax.transAxes)
plt.show()