def get_object(alpha, delta, input_d):
"""
:param alpha:
:param delta:
:return:
"""
o_df_galaxies = check_source(input_d['galaxies'],
alpha,
delta,
keys=['ra', 'dec'])
o_df_stars = check_source(input_d['stars'],
alpha,
delta,
keys=['RA2000(Gaia)', 'DEC2000(Gaia)'])
if o_df_galaxies.empty is not True:
return 'galaxies'
elif o_df_stars.empty is not True:
return 'star'
else:
return 'SSO'
def test_check_out_source(self):
"""
create_configurations should return:
[[30, 0.01, 1.5, 4, 'models/gauss_2.0_5x5.conv']], 1
prfs_d = extract_settings_elvis()
# How many times the cross search radius will be increased
tolerance_factor = 2
df = df[df[keys[0]] + prfs_d['tolerance'] * tolerance_factor > i_alpha]
df = df[i_alpha > df[keys[0]] - prfs_d['tolerance'] * tolerance_factor]
df = df[df[keys[1]] + prfs_d['tolerance'] * tolerance_factor > i_delta]
df = df[i_delta > df[keys[1]] - prfs_d['tolerance'] * tolerance_factor]
:return:
"""
misc.extract_settings_elvis = MagicMock()
misc.extract_settings_elvis.return_value = {'tolerance': 0.000138889}
i_alpha = 2.0
i_delta = 2.0
keys = ['ALPHA_J2000', 'DELTA_J2000']
test = misc.check_source(self.df_, i_alpha, i_delta, keys)
return self.assertTrue(test.empty)
def extract_stats_filt(self, filt_cat, input_df):
"""
:param filt_cat:
:param input_df:
:return:
"""
# Unique sources (?)
unique_sources = list(set(input_df['SOURCE'].tolist()))
ok = 0
no = 0
filt_cat_size = filt_cat['SOURCE_NUMBER'].size
total_input_size = input_df['SOURCE'].size
print('sources in input catalogue: {}'.format(total_input_size))
print('sources in filtered catalogue {}'.format(filt_cat_size))
for source_ in unique_sources:
source_df = input_df[input_df['SOURCE'].isin([source_])]
print(source_df.columns)
for i, row in enumerate(source_df.itertuples(), 1):
dither_df = source_df[source_df['DITHER'].isin([row.DITHER])]
i_alpha = float(dither_df['RA'].iloc[0])
i_delta = float(dither_df['DEC'].iloc[0])
out_df = check_source(filt_cat,
i_alpha,
i_delta,
keys=['ALPHA_J2000', 'DELTA_J2000'])
if out_df.empty:
no += 1
else:
ok += 1
print('detected: {}'.format(ok))
print('not detected: {}'.format(no))
def splits_data(self, idx_l, unique_sources, filt_cat, input_df):
""" Splits filtered catalogue due object type.
This method creates a dictionary with three different keys.
Each one (SSOs, stars, galaxies) it is a Dataframe with all valuable
data.
:param unique_sources:
:param filt_cat:
:param input_df:
:return:
"""
print('Creating new catalogues from filtered catalogue due type')
print('Total sources: {} of thread: {}'.format(len(unique_sources),
idx_l))
for source_ in unique_sources:
source_df = filt_cat[filt_cat['SOURCE_NUMBER'].isin([source_])]
for i, row in enumerate(source_df.itertuples(), 1):
dither_n = get_dither(int(row.CATALOG_NUMBER))
# Checks object type
alpha = source_df['ALPHA_J2000'].iloc[0]
delta = source_df['DELTA_J2000'].iloc[0]
keys = ['RA', 'DEC']
test_sso = check_source(input_df[dither_n]['SSOs'],
alpha, delta, keys)
keys = ['X_WORLD', 'Y_WORLD']
test_star = check_source(input_df[dither_n]['stars'],
alpha, delta, keys)
keys = ['X_WORLD', 'Y_WORLD']
test_galaxy = check_source(input_df[dither_n]['galaxies'],
alpha, delta, keys)
if test_sso.empty is not True:
key_ = 'ssos'
elif test_star.empty is not True:
key_ = 'stars'
elif test_galaxy.empty is not True:
key_ = 'galaxies'
else:
key_ = 'lost'
a_image = row.MEDIAN_A_IMAGE
self.data_d[key_]['median_a_image'].append(a_image)
erra_image = row.MEDIAN_ERRA_IMAGE
self.data_d[key_]['median_erra_image'].append(erra_image)
b_image = row.MEDIAN_B_IMAGE
self.data_d[key_]['median_b_image'].append(b_image)
errb_image = row.MEDIAN_ERRB_IMAGE
self.data_d[key_]['median_errb_image'].append(errb_image)
class_star = row.MEDIAN_CLASS_STAR
self.data_d[key_]['median_class_star'].append(class_star)
ellipticity = row.ELLIPTICITY
self.data_d[key_]['ellipticity'].append(ellipticity)
mag_iso = row.MEDIAN_MAG_ISO
self.data_d[key_]['median_mag_iso'].append(mag_iso)
magerr_iso = row.MEDIAN_MAGERR_ISO
self.data_d[key_]['median_magerr_iso'].append(magerr_iso)
flux_iso = row.MEDIAN_FLUX_ISO
self.data_d[key_]['median_flux_iso'].append(flux_iso)
pm = row.PM
self.data_d[key_]['pm'].append(pm)
pmerr = row.PMERR
self.data_d[key_]['pmerr'].append(pmerr)
fwhm_image = row.FWHM_IMAGE
self.data_d[key_]['fwhm_image'].append(fwhm_image)
for type_key in self.data_d.keys():
data_df = DataFrame(self.data_d[type_key])
data_df.to_csv('cat_{}_{}.csv'.format(idx_l, type_key))
def create_stars_catalog_thread(idx_l, sub_list, stars_df, full_d, scamp_df):
"""
:param idx_l:
:param sub_list:
:param stars_df:
:param full_d:
:return:
"""
keys = ['ALPHA_J2000', 'DELTA_J2000']
cat_d = create_empty_catalog_dict()
total_thread = len(sub_list)
stdout.write('total stars {} of thread {}\n'.format(total_thread, idx_l))
for idx, star in enumerate(sub_list):
source_df = stars_df[stars_df['IDX'].isin([star])]
alpha = source_df['RA2000(Gaia)'].iloc[0]
delta = source_df['DEC2000(Gaia)'].iloc[0]
source_d = create_empty_catalog_dict()
for dither in range(1, 5, 1):
sex_df = check_source(full_d[dither], alpha, delta, keys)
# Should check source in Scamp too!
if sex_df.empty is not True:
# Returns the index of the closest found source
index = check_distance(sex_df, alpha, delta)
sex_df = sex_df.iloc[[index]]
cat_number = sex_df['CATALOG_NUMBER'].iloc[0]
df = scamp_df[scamp_df['CATALOG_NUMBER'].isin([cat_number])]
df = check_source(df, alpha, delta, keys)
if df.empty is not True:
source_d['DITHER'].append(dither)
catalog_number = int(sex_df['CATALOG_NUMBER'].iloc[0])
source_d['CATALOG_NUMBER'].append(catalog_number)
x_world = float(sex_df['X_WORLD'].iloc[0])
source_d['X_WORLD'].append(x_world)
y_world = float(sex_df['Y_WORLD'].iloc[0])
source_d['Y_WORLD'].append(y_world)
mag_auto = float(sex_df['MAG_AUTO'].iloc[0])
source_d['MAG_AUTO'].append(mag_auto)
magerr_auto = float(sex_df['MAGERR_AUTO'].iloc[0])
source_d['MAGERR_AUTO'].append(magerr_auto)
a_image = float(sex_df['A_IMAGE'].iloc[0])
source_d['A_IMAGE'].append(a_image)
b_image = float(sex_df['B_IMAGE'].iloc[0])
source_d['B_IMAGE'].append(b_image)
theta_image = float(sex_df['THETA_IMAGE'].iloc[0])
source_d['THETA_IMAGE'].append(theta_image)
erra_image = float(sex_df['ERRA_IMAGE'].iloc[0])
source_d['ERRA_IMAGE'].append(erra_image)
errb_image = float(sex_df['ERRB_IMAGE'].iloc[0])
source_d['ERRB_IMAGE'].append(errb_image)
erra_world = float(sex_df['ERRA_WORLD'].iloc[0])
source_d['ERRA_WORLD'].append(erra_world)
errb_world = float(sex_df['ERRB_WORLD'].iloc[0])
source_d['ERRB_WORLD'].append(errb_world)
errtheta_world = float(sex_df['ERRTHETA_WORLD'].iloc[0])
source_d['ERRTHETA_WORLD'].append(errtheta_world)
class_star = float(sex_df['CLASS_STAR'].iloc[0])
source_d['CLASS_STAR'].append(class_star)
pm = float(df['PM'].iloc[0])
source_d['PM'].append(pm)
pmerr = float(df['PMERR'].iloc[0])
source_d['PMERR'].append(pmerr)
if len(source_d['DITHER']) != 0:
for key_ in source_d.keys():
for value_ in source_d[key_]:
cat_d[key_].append(value_)
cat_df = DataFrame(cat_d,
columns=[
'DITHER', 'CATALOG_NUMBER', 'X_WORLD', 'Y_WORLD',
'MAG_AUTO', 'MAGERR_AUTO', 'A_IMAGE', 'B_IMAGE',
'THETA_IMAGE', 'ERRA_IMAGE', 'ERRB_IMAGE',
'ERRA_WORLD', 'ERRB_WORLD', 'ERRTHETA_WORLD',
'CLASS_STAR', 'PM', 'PMERR'
])
cat_df.to_csv('tmp_stars/stars_{}.csv'.format(idx_l))
def create_galaxies_catalog_thread(idx_l, sub_list, galaxies_df, full_d):
"""
:param idx_l:
:param sub_list:
:param galaxies_df:
:param full_d:
:return:
"""
save = True
keys = ['ALPHA_J2000', 'DELTA_J2000']
cat_d = create_empty_catalog_dict()
total_thread = len(sub_list)
stdout.write('total galaxies {} of thread {}\n'.format(
total_thread, idx_l))
for idx, galaxy in enumerate(sub_list):
source_df = galaxies_df[galaxies_df['IDX'].isin([galaxy])]
alpha = source_df['ra'].iloc[0]
delta = source_df['dec'].iloc[0]
source_d = create_empty_catalog_dict()
for dither in range(1, 5, 1):
o_df = check_source(full_d[dither], alpha, delta, keys)
if o_df.empty is not True:
# Returns the index of the closest found source
index = check_distance(o_df, alpha, delta)
o_df = o_df.iloc[[index]]
source_d['DITHER'].append(dither)
catalog_number = int(o_df['CATALOG_NUMBER'].iloc[0])
source_d['CATALOG_NUMBER'].append(catalog_number)
x_world = float(o_df['X_WORLD'].iloc[0])
source_d['X_WORLD'].append(x_world)
y_world = float(o_df['Y_WORLD'].iloc[0])
source_d['Y_WORLD'].append(y_world)
mag_auto = float(o_df['MAG_AUTO'].iloc[0])
source_d['MAG_AUTO'].append(mag_auto)
magerr_auto = float(o_df['MAGERR_AUTO'].iloc[0])
source_d['MAGERR_AUTO'].append(magerr_auto)
a_image = float(o_df['A_IMAGE'].iloc[0])
source_d['A_IMAGE'].append(a_image)
b_image = float(o_df['B_IMAGE'].iloc[0])
source_d['B_IMAGE'].append(b_image)
theta_image = float(o_df['THETA_IMAGE'].iloc[0])
source_d['THETA_IMAGE'].append(theta_image)
erra_image = float(o_df['ERRA_IMAGE'].iloc[0])
source_d['ERRA_IMAGE'].append(erra_image)
errb_image = float(o_df['ERRB_IMAGE'].iloc[0])
source_d['ERRB_IMAGE'].append(errb_image)
erra_world = float(o_df['ERRA_WORLD'].iloc[0])
source_d['ERRA_WORLD'].append(erra_world)
errb_world = float(o_df['ERRB_WORLD'].iloc[0])
source_d['ERRB_WORLD'].append(errb_world)
errtheta_world = float(o_df['ERRTHETA_WORLD'].iloc[0])
source_d['ERRTHETA_WORLD'].append(errtheta_world)
class_star = float(o_df['CLASS_STAR'].iloc[0])
source_d['CLASS_STAR'].append(class_star)
if len(source_d['DITHER']) != 0:
for key_ in source_d.keys():
for value_ in source_d[key_]:
cat_d[key_].append(value_)
# for key_ in cat_d.keys():
# print(key_, len(cat_d[key_]))
cat_df = DataFrame(cat_d,
columns=[
'DITHER', 'CATALOG_NUMBER', 'X_WORLD', 'Y_WORLD',
'MAG_AUTO', 'MAGERR_AUTO', 'A_IMAGE', 'B_IMAGE',
'THETA_IMAGE', 'ERRA_IMAGE', 'ERRB_IMAGE',
'ERRA_WORLD', 'ERRB_WORLD', 'ERRTHETA_WORLD',
'CLASS_STAR'
])
if save:
cat_df.to_csv('tmp_galaxies/galaxies_{}.csv'.format(idx_l))
def create_stars_catalog_thread(idx_l, sub_list, inputs_d, full_d):
"""
:param idx_l:
:param sub_list:
:param inputs_d:
:param full_d:
:return:
"""
save = True
keys = ['ALPHA_J2000', 'DELTA_J2000']
cat_d = create_empty_catalog_dict()
total_thread = len(sub_list)
stdout.write('total stars {} of thread {}\n'.format(total_thread, idx_l))
for idx, star in enumerate(sub_list):
stars_df = inputs_d['stars']
source_df = stars_df[stars_df['IDX'].isin([star])]
alpha = source_df['RA2000(Gaia)'].iloc[0]
delta = source_df['DEC2000(Gaia)'].iloc[0]
source_d = create_empty_catalog_dict()
for dither in range(1, 5, 1):
o_df = check_source(full_d[dither], alpha, delta, keys)
if o_df.empty is not True:
# Returns the index of the closest found source
index = check_distance(o_df, alpha, delta)
o_df = o_df.iloc[[index]]
source_d['DITHER'].append(dither)
catalog_number = int(o_df['CATALOG_NUMBER'].iloc[0])
source_d['CATALOG_NUMBER'].append(catalog_number)
x_world = float(o_df['X_WORLD'].iloc[0])
source_d['X_WORLD'].append(x_world)
y_world = float(o_df['Y_WORLD'].iloc[0])
source_d['Y_WORLD'].append(y_world)
mag_auto = float(o_df['MAG_AUTO'].iloc[0])
source_d['MAG_AUTO'].append(mag_auto)
magerr_auto = float(o_df['MAGERR_AUTO'].iloc[0])
source_d['MAGERR_AUTO'].append(magerr_auto)
erra_world = float(o_df['ERRA_WORLD'].iloc[0])
source_d['ERRA_WORLD'].append(erra_world)
errb_world = float(o_df['ERRB_WORLD'].iloc[0])
source_d['ERRB_WORLD'].append(errb_world)
errtheta_world = float(o_df['ERRTHETA_WORLD'].iloc[0])
source_d['ERRTHETA_WORLD'].append(errtheta_world)
if len(source_d['DITHER']) != 0:
for key_ in source_d.keys():
for value_ in source_d[key_]:
cat_d[key_].append(value_)
cat_df = DataFrame(cat_d,
columns=[
'DITHER', 'CATALOG_NUMBER', 'X_WORLD', 'Y_WORLD',
'MAG_AUTO', 'MAGERR_AUTO', 'ERRA_WORLD',
'ERRB_WORLD', 'ERRTHETA_WORLD'
])
if save:
cat_df.to_csv('tmp_stars/stars_{}.csv'.format(idx_l))
def create_ssos_catalog_thread(idx_l, sub_list, ssos_df, full_d):
"""
:param idx_l:
:param sub_list:
:param ssos_df:
:param full_d:
:return:
"""
keys = ['ALPHA_J2000', 'DELTA_J2000']
# Creates an empty catalog for all detected sources
cat_d = create_output_catalog_dict()
total_thread = len(sub_list)
stdout.write('total SSOs {} of thread {}\n'.format(total_thread, idx_l))
for idx, sso in enumerate(sub_list):
source_df = ssos_df[ssos_df['SOURCE'].isin([sso])]
for dither_ in source_df['DITHER'].tolist():
dither_df = source_df[source_df['DITHER'].isin([dither_])]
# Gets alpha/delta of actual source
alpha = float(dither_df['RA'].iloc[0])
delta = float(dither_df['DEC'].iloc[0])
o_df = check_source(full_d[dither_], alpha, delta, keys)
if o_df.empty is not True:
# Returns the index of the closest found source
index = check_distance(o_df, alpha, delta)
o_df = o_df.iloc[[index]]
idx_ = int(dither_df['IDX'].iloc[0])
cat_d['IDX'].append(idx_)
source = int(dither_df['SOURCE'].iloc[0])
cat_d['SOURCE'].append(source)
cat_d['DITHER'].append(dither_)
alpha_j2000 = float(o_df['ALPHA_J2000'].iloc[0])
cat_d['RA'].append(alpha_j2000)
delta_j2000 = float(o_df['DELTA_J2000'].iloc[0])
cat_d['DEC'].append(delta_j2000)
vel = float(dither_df['VEL'].iloc[0])
cat_d['VEL'].append(vel)
abmag = float(dither_df['ABMAG'].iloc[0])
cat_d['ABMAG'].append(abmag)
theta = float(dither_df['THETA'].iloc[0])
cat_d['THETA'].append(theta)
mag_auto = float(o_df['MAG_AUTO'].iloc[0])
cat_d['MAG_AUTO'].append(mag_auto)
a_image = float(o_df['A_IMAGE'].iloc[0])
cat_d['A_IMAGE'].append(a_image)
b_image = float(o_df['B_IMAGE'].iloc[0])
cat_d['B_IMAGE'].append(b_image)
theta_image = float(o_df['THETA_IMAGE'].iloc[0])
cat_d['THETA_IMAGE'].append(theta_image)
erra_image = float(o_df['ERRA_IMAGE'].iloc[0])
cat_d['ERRA_IMAGE'].append(erra_image)
errb_image = float(o_df['ERRB_IMAGE'].iloc[0])
cat_d['ERRB_IMAGE'].append(errb_image)
magerr_auto = float(o_df['MAGERR_AUTO'].iloc[0])
cat_d['MAGERR_AUTO'].append(magerr_auto)
erra_world = float(o_df['ERRA_WORLD'].iloc[0])
cat_d['ERRA_WORLD'].append(erra_world)
errb_world = float(o_df['ERRB_WORLD'].iloc[0])
cat_d['ERRB_WORLD'].append(errb_world)
errtheta_world = float(o_df['ERRTHETA_WORLD'].iloc[0])
cat_d['ERRTHETA_WORLD'].append(errtheta_world)
class_star = float(o_df['CLASS_STAR'].iloc[0])
cat_d['CLASS_STAR'].append(class_star)
cat_df = DataFrame(cat_d,
columns=[
'IDX', 'SOURCE', 'DITHER', 'RA', 'DEC', 'VEL',
'ABMAG', 'THETA', 'MAG_AUTO', 'A_IMAGE', 'B_IMAGE',
'THETA_IMAGE', 'ERRA_IMAGE', 'ERRB_IMAGE',
'MAGERR_AUTO', 'ERRA_WORLD', 'ERRB_WORLD',
'ERRTHETA_WORLD', 'CLASS_STAR'
])
cat_df.to_csv('tmp_ssos/ssos_{}.csv'.format(idx_l))
def extract_stats(self, filt_cat, input_df):
"""
:param filt_cat:
:param input_df:
:return:
"""
# Unique sources (?)
unique_sources = list(set(filt_cat['SOURCE_NUMBER'].tolist()))
print('Creating new catalogues from filtered catalogue due type')
print('Total sources: {}'.format(filt_cat['SOURCE_NUMBER'].size))
for idx_source_, source_ in enumerate(unique_sources):
source_df = filt_cat[filt_cat['SOURCE_NUMBER'].isin([source_])]
# Takes the first value of MAG Series
o_mag_bin = get_norm_mag(source_df['MEDIAN_MAG_AUTO'].iloc[0])
# Takes the first value of PM Series
o_pm_norm = get_norm_speed(source_df['PM'].iloc[0])
# Takes the error of the proper motion calculation
o_pm_err = source_df['PMERR'].iloc[0]
# Takes the median value of CLASS_STAR
o_class_star = source_df['CLASS_STAR'].iloc[0]
for i, row in enumerate(source_df.itertuples(), 1):
alpha = row.ALPHA_J2000
delta = row.DELTA_J2000
dither_n = get_dither(int(row.CATALOG_NUMBER))
# Checks object type
keys = ['RA', 'DEC'] # Catalogue version 2
test_sso = check_source(input_df[dither_n]['SSOs'], alpha,
delta, keys)
if test_sso.empty is not True:
self.right_positives[dither_n]['SOURCE'].append(source_)
self.right_positives[dither_n]['RA'].append(alpha)
self.right_positives[dither_n]['DEC'].append(delta)
self.right_positives[dither_n]['MAG'].append(o_mag_bin)
i_pm_norm = get_norm_speed(float(test_sso['VEL'].iloc[0]))
self.right_positives[dither_n]['PM'].append(i_pm_norm)
self.right_positives[dither_n]['PMERR'].append(o_pm_err)
self.right_positives[dither_n]['CLASS'].append(
o_class_star)
object_type = get_object(alpha, delta, self.input_d)
self.right_positives[dither_n]['OBJECT'].append(
object_type)
else:
self.false_positives[dither_n]['SOURCE'].append(source_)
self.false_positives[dither_n]['RA'].append(alpha)
self.false_positives[dither_n]['DEC'].append(delta)
self.false_positives[dither_n]['MAG'].append(o_mag_bin)
self.false_positives[dither_n]['PM'].append(o_pm_norm)
self.false_positives[dither_n]['PMERR'].append(o_pm_err)
self.false_positives[dither_n]['CLASS'].append(
o_class_star)
object_type = get_object(alpha, delta, self.input_d)
self.false_positives[dither_n]['OBJECT'].append(
object_type)
# Regions creation
for dither_ in self.false_positives.keys():
alpha_list = self.false_positives[dither_]['RA']
alpha_serie = Series(alpha_list, name='ALPHA_J2000')
delta_list = self.false_positives[dither_]['DEC']
delta_serie = Series(delta_list, name='DELTA_J2000')
mag_list = self.false_positives[dither_]['MAG']
mag_serie = Series(mag_list, name='MAG_AUTO')
pm_list = self.false_positives[dither_]['PM']
pm_serie = Series(pm_list, name='PM')
object_list = self.false_positives[dither_]['OBJECT']
object_serie = Series(object_list, name='OBJECT')
output = concat([alpha_serie, delta_serie, pm_serie], axis=1)
for pm_ in [0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0]:
output_pm = output[output['PM'].isin([pm_])]
cat_name = 'false_positives/false_{}_{}.reg'.format(
pm_, dither_)
output_pm.to_csv(cat_name, index=False, header=False, sep=" ")
# Catalogue creation for false sources
source_total_list = []
dither_total_list = []
alpha_total_list = []
delta_total_list = []
mag_total_list = []
pm_total_list = []
pmerr_total_list = []
class_total_list = []
object_total_list = []
dither_list = [1, 2, 3, 4]
for dither_ in dither_list:
alpha_list = self.false_positives[dither_]['RA']
for alpha_ in alpha_list:
dither_total_list.append(dither_)
alpha_total_list.append(alpha_)
delta_list = self.false_positives[dither_]['DEC']
for delta_ in delta_list:
delta_total_list.append(delta_)
source_list = self.false_positives[dither_]['SOURCE']
for source_ in source_list:
source_total_list.append(source_)
mag_list = self.false_positives[dither_]['MAG']
for mag_ in mag_list:
mag_total_list.append(mag_)
pm_list = self.false_positives[dither_]['PM']
for pm_ in pm_list:
pm_total_list.append(pm_)
pmerr_list = self.false_positives[dither_]['PMERR']
for pmerr_ in pmerr_list:
pmerr_total_list.append(pmerr_)
class_list = self.false_positives[dither_]['CLASS']
for class_ in class_list:
class_total_list.append(class_)
object_list = self.false_positives[dither_]['OBJECT']
for object_ in object_list:
object_total_list.append(object_)
source_serie = Series(source_total_list, name='SOURCE')
dither_serie = Series(dither_total_list, name='DITHER')
alpha_serie = Series(alpha_total_list, name='ALPHA_J2000')
delta_serie = Series(delta_total_list, name='DELTA_J2000')
mag_serie = Series(mag_total_list, name='MAG_AUTO')
pm_serie = Series(pm_total_list, name='PM')
pmerr_serie = Series(pmerr_total_list, name='PMERR')
class_serie = Series(class_total_list, name='CLASS_STAR')
object_serie = Series(object_total_list, name='OBJECT')
output = concat([
source_serie, dither_serie, alpha_serie, delta_serie, mag_serie,
pm_serie, pmerr_serie, class_serie, object_serie
],
axis=1)
for pm_ in [0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0]:
output_pm = output[output['PM'].isin([pm_])]
cat_name = 'false_positives/false_{}.csv'.format(pm_)
output_pm.to_csv(cat_name)
# Catalogue creation for right sources
source_total_list = []
dither_total_list = []
alpha_total_list = []
delta_total_list = []
mag_total_list = []
pm_total_list = []
pmerr_total_list = []
class_total_list = []
object_total_list = []
dither_list = [1, 2, 3, 4]
for dither_ in dither_list:
alpha_list = self.right_positives[dither_]['RA']
for alpha_ in alpha_list:
dither_total_list.append(dither_)
alpha_total_list.append(alpha_)
delta_list = self.right_positives[dither_]['DEC']
for delta_ in delta_list:
delta_total_list.append(delta_)
source_list = self.right_positives[dither_]['SOURCE']
for source_ in source_list:
source_total_list.append(source_)
mag_list = self.right_positives[dither_]['MAG']
for mag_ in mag_list:
mag_total_list.append(mag_)
pm_list = self.right_positives[dither_]['PM']
for pm_ in pm_list:
pm_total_list.append(pm_)
pmerr_list = self.right_positives[dither_]['PMERR']
for pmerr_ in pmerr_list:
pmerr_total_list.append(pmerr_)
class_list = self.right_positives[dither_]['CLASS']
for class_ in class_list:
class_total_list.append(class_)
object_list = self.right_positives[dither_]['OBJECT']
for object_ in object_list:
object_total_list.append(object_)
source_serie = Series(source_total_list, name='SOURCE')
dither_serie = Series(dither_total_list, name='DITHER')
alpha_serie = Series(alpha_total_list, name='ALPHA_J2000')
delta_serie = Series(delta_total_list, name='DELTA_J2000')
mag_serie = Series(mag_total_list, name='MAG_AUTO')
pm_serie = Series(pm_total_list, name='PM')
pmerr_serie = Series(pmerr_total_list, name='PMERR')
class_serie = Series(class_total_list, name='CLASS_STAR')
object_serie = Series(object_total_list, name='OBJECT')
output = concat([
source_serie, dither_serie, alpha_serie, delta_serie, mag_serie,
pm_serie, pmerr_serie, class_serie, object_serie
],
axis=1)
for pm_ in [0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0]:
output_pm = output[output['PM'].isin([pm_])]
cat_name = 'false_positives/right_{}.csv'.format(pm_)
output_pm.to_csv(cat_name)
def extract_stats_ccds(self, cats_d):
"""
:param cats_d:
:param filt_cat:
:return:
"""
# Unique sources (?)
test_dict = {
'PM': [],
'A_IMAGE': [],
'B_IMAGE': [],
'CLASS_STAR': [],
'MAG_ISO': [],
'ELLIPTICITY': []
}
idx = 0
borders_d = get_borders()
print('total {}'.format(self.input_df['DITHER'].size))
for i, row in enumerate(self.input_df.itertuples(), 1):
print('idx: {}'.format(idx))
idx += 1
dither = int(row.DITHER)
i_alpha = float(row.X_WORLD)
i_delta = float(row.Y_WORLD)
pm = ''
a_image = ''
b_image = ''
class_star = ''
mag_iso = ''
ellipticity = ''
test = False
# Looks for where the source should be
source_coords = SkyCoord(ra=i_alpha * degree,
dec=i_delta * degree,
equinox='J2021.5')
source_coords_ecliptic = source_coords.barycentrictrueecliptic
lon_e = float(source_coords_ecliptic.lon.degree)
lat_e = float(source_coords_ecliptic.lat.degree)
for ccd_ in borders_d[dither].keys():
borders = borders_d[dither][ccd_]
lon_comp = borders['below_lon'] < lon_e < borders['above_lon']
lat_comp = borders['below_lat'] < lat_e < borders['above_lat']
comp = lon_comp and lat_comp
if comp:
catalogue = '{}.cat'.format(ccd_[:-5])
out_df = check_source(cats_d[dither][catalogue],
i_alpha,
i_delta,
keys=['ALPHA_J2000', 'DELTA_J2000'])
if out_df.empty is not True:
alpha = float(out_df['ALPHA_J2000'].iloc[0])
delta = float(out_df['DELTA_J2000'].iloc[0])
pm_df = check_source(
self.filt_cat,
alpha,
delta,
keys=['ALPHA_J2000', 'DELTA_J2000'])
if pm_df.empty is not True:
test = True
pm = float(pm_df['PM'].iloc[0])
a_image = float(pm_df['MEDIAN_A_IMAGE'].iloc[0])
b_image = float(pm_df['MEDIAN_B_IMAGE'].iloc[0])
class_star = float(
pm_df['MEDIAN_CLASS_STAR'].iloc[0])
mag_iso = float(pm_df['MEDIAN_MAG_ISO'].iloc[0])
ellipticity = float(
pm_df['MEDIAN_ELLIPTICITY'].iloc[0])
if test:
test_dict['PM'].append(pm)
test_dict['A_IMAGE'].append(a_image)
test_dict['B_IMAGE'].append(b_image)
test_dict['CLASS_STAR'].append(class_star)
test_dict['MAG_ISO'].append(mag_iso)
test_dict['ELLIPTICITY'].append(ellipticity)
pm_list = Series(test_dict['PM'], name='PM')
a_image_list = Series(test_dict['A_IMAGE'], name='A_IMAGE')
b_image_list = Series(test_dict['B_IMAGE'], name='B_IMAGE')
class_star_list = Series(test_dict['CLASS_STAR'], name='CLASS_STAR')
mag_iso_list = Series(test_dict['MAG_ISO'], name='MAG_ISO')
ellipticity_list = Series(test_dict['ELLIPTICITY'], name='ELLIPTICITY')
positions_table = concat([
pm_list, a_image_list, b_image_list, class_star_list, mag_iso_list,
ellipticity_list
],
axis=1)
positions_table.to_csv('false_catalogues/galaxies_catalogue.csv')
positions_table.to_csv('true_catalogues/galaxies_catalogue.csv')
def extract_stats_ccds(self, cats_d):
"""
:param cats_d:
:param filt_cat:
:return:
"""
# Unique sources (?)
test_dict = {
'PM': [],
'A_IMAGE': [],
'B_IMAGE': [],
'CLASS_STAR': [],
'MAG_ISO': [],
'ELLIPTICITY': []
}
idx = 0
print('total {}'.format(self.input_df['DITHER'].size))
for i, row in enumerate(self.input_df.itertuples(), 1):
print('idx: {}'.format(idx))
idx += 1
i_alpha = float(row.X_WORLD)
i_delta = float(row.Y_WORLD)
pm = ''
a_image = ''
b_image = ''
class_star = ''
mag_iso = ''
ellipticity = ''
test = False
for cat_ in cats_d[row.DITHER]:
out_df = check_source(cats_d[row.DITHER][cat_],
i_alpha,
i_delta,
keys=['ALPHA_J2000', 'DELTA_J2000'])
if out_df.empty is not True:
alpha = float(out_df['ALPHA_J2000'].iloc[0])
delta = float(out_df['DELTA_J2000'].iloc[0])
pm_df = check_source(self.filt_cat,
alpha,
delta,
keys=['ALPHA_J2000', 'DELTA_J2000'])
if pm_df.empty is not True:
test = True
pm = float(pm_df['PM'].iloc[0])
a_image = float(pm_df['MEDIAN_A_IMAGE'].iloc[0])
b_image = float(pm_df['MEDIAN_B_IMAGE'].iloc[0])
class_star = float(pm_df['MEDIAN_CLASS_STAR'].iloc[0])
mag_iso = float(pm_df['MEDIAN_MAG_ISO'].iloc[0])
ellipticity = float(
pm_df['MEDIAN_ELLIPTICITY'].iloc[0])
if test:
test_dict['PM'].append(pm)
test_dict['A_IMAGE'].append(a_image)
test_dict['B_IMAGE'].append(b_image)
test_dict['CLASS_STAR'].append(class_star)
test_dict['MAG_ISO'].append(mag_iso)
test_dict['ELLIPTICITY'].append(ellipticity)
pm_list = Series(test_dict['PM'], name='PM')
a_image_list = Series(test_dict['A_IMAGE'], name='A_IMAGE')
b_image_list = Series(test_dict['B_IMAGE'], name='B_IMAGE')
class_star_list = Series(test_dict['CLASS_STAR'], name='CLASS_STAR')
mag_iso_list = Series(test_dict['MAG_ISO'], name='MAG_ISO')
ellipticity_list = Series(test_dict['ELLIPTICITY'], name='ELLIPTICITY')
positions_table = concat([
pm_list, a_image_list, b_image_list, class_star_list, mag_iso_list,
ellipticity_list
],
axis=1)
# positions_table.to_csv('catalogues_not_detectd/stars_catalogue.csv')
positions_table.to_csv('catalogues_detected/stars_catalogue.csv')
def error(self, logger, mag, sex_cf):
"""
:param logger:
:param mag:
:param sex_cf:
:return:
"""
self.mag = mag
errors_a_stars = []
errors_b_stars = []
errors_a_galaxies = []
errors_b_galaxies = []
errors_a_ssos = []
errors_b_ssos = []
keys = ['ALPHA_J2000', 'DELTA_J2000']
# Input sources
input_ssos_d = {}
for d in range(1, 5, 1):
cat_loc = '{}/{}/Catalogs'.format(self.prfs_d['fits_dir'], mag)
cat_name = '{}/Cat_{}_d{}'.format(cat_loc, self.mag, d)
input_ssos_d[d] = '{}.dat'.format(cat_name)
input_ssos_d = Create_regions(input_ssos_d).check_luca(
self.mag, False, True)
# Creates a DataFrame from an input dictionary
input_ssos_l = []
for key_ in input_ssos_d.keys():
input_ssos_l.append(input_ssos_d[key_])
i_ssos_df = concat(input_ssos_l, axis=0)
i_ssos_df = i_ssos_df.reset_index(drop=True)
# Input stars
input_stars_d = {}
for d in range(1, 5, 1):
cat_loc = '{}/{}/Catalogs'.format(self.prfs_d['fits_dir'], mag)
cat_name = '{}/Cat_20-21_d{}'.format(cat_loc, d)
input_stars_d[d] = '{}.dat'.format(cat_name)
input_stars_d = Create_regions(input_stars_d).check_luca(
mag, False, True)
# Creates a DataFrame from an input dictionary
input_stars_l = []
for key_ in input_stars_d.keys():
input_stars_l.append(input_stars_d[key_])
i_stars_df = concat(input_stars_l, axis=0)
i_stars_df = i_stars_df.reset_index(drop=True)
# Input stars
input_galaxies_d = {}
for d in range(1, 5, 1):
cat_loc = '{}/{}/Catalogs'.format(self.prfs_d['fits_dir'], mag)
cat_name = '{}/Cat_20-21_d{}'.format(cat_loc, d)
input_galaxies_d[d] = '{}.dat'.format(cat_name)
input_galaxies_d = Create_regions(input_galaxies_d).check_luca(
mag, False, True)
# Creates a DataFrame from an input dictionary
input_galaxies_l = []
for key_ in input_galaxies_d.keys():
input_galaxies_l.append(input_galaxies_d[key_])
i_galaxies_df = concat(input_galaxies_l, axis=0)
i_galaxies_df = i_galaxies_df.reset_index(drop=True)
# Creates an input dictionary with all input sources
logger.debug('checking performance for {}'.format(sex_cf))
opts = [[0, 0], [0, 1], [0, 2], [1, 0], [1, 1], [1, 2], [2, 0], [2, 1],
[2, 2]]
for opt in opts:
for dither in range(1, 5, 1):
# Lists definition
errors_a_stars = []
errors_b_stars = []
errors_a_galaxies = []
errors_b_galaxies = []
errors_a_ssos = []
errors_b_ssos = []
cat_n = 'mag_{}_CCD_x{}_y{}_d{}.cat'.format(
mag, opt[0], opt[1], dither)
cat_o_n = '{}/{}/CCDs/{}/{}'.format(self.prfs_d['fits_dir'],
mag, sex_cf, cat_n)
hdu_list = fits.open(cat_o_n)
o_cat = Table(hdu_list[2].data).to_pandas()
i_stars_d_df = i_stars_df[i_stars_df['dither_values'].isin(
[dither])]
i_galaxies_d_df = i_galaxies_df[
i_galaxies_df['dither_values'].isin([dither])]
i_ssos_d_df = i_ssos_df[i_ssos_df['dither_values'].isin(
[dither])]
for idx, row in o_cat.iterrows():
i_alpha = row['ALPHA_J2000']
i_delta = row['DELTA_J2000']
o_df = check_source(i_stars_d_df, i_alpha, i_delta, keys)
if o_df.empty is not True:
errors_a_stars.append(row['ERRA_IMAGE'])
errors_b_stars.append(row['ERRB_IMAGE'])
o_df = check_source(i_galaxies_d_df, i_alpha, i_delta,
keys)
if o_df.empty is not True:
errors_a_galaxies.append(row['ERRA_IMAGE'])
errors_b_galaxies.append(row['ERRB_IMAGE'])
o_df = check_source(i_ssos_d_df, i_alpha, i_delta, keys)
if o_df.empty is not True:
errors_a_ssos.append(row['ERRA_IMAGE'])
errors_b_ssos.append(row['ERRB_IMAGE'])
mean_a_stars = array(errors_a_stars).mean()
mean_b_stars = array(errors_b_stars).mean()
mean_a_galaxies = array(errors_a_galaxies).mean()
mean_b_galaxies = array(errors_b_galaxies).mean()
mean_a_ssos = array(errors_a_ssos).mean()
mean_b_ssos = array(errors_b_ssos).mean()
std_a_stars = array(errors_a_stars).std()
std_b_stars = array(errors_b_stars).std()
std_a_galaxies = array(errors_a_galaxies).std()
std_b_galaxies = array(errors_b_galaxies).std()
std_a_ssos = array(errors_a_ssos).std()
std_b_ssos = array(errors_b_ssos).std()
stats_d = {
'conf': sex_cf,
'm_a_stars': mean_a_stars,
'm_b_stars': mean_b_stars,
'm_a_gals': mean_a_galaxies,
'm_b_gals': mean_b_galaxies,
'm_a_ssos': mean_a_ssos,
'm_b_ssos': mean_b_ssos,
's_a_stars': std_a_stars,
's_b_stars': std_b_stars,
's_a_gals': std_a_galaxies,
's_b_gals': std_b_galaxies,
's_a_ssos': std_a_ssos,
's_b_ssos': std_b_ssos
}
return stats_d
def check(self, logger, mag, scmp_cf, sex_cf, confidence_):
"""
:param logger:
:param mag:
:param scmp_cf:
:param sex_cf:
:param confidence_:
:return:
"""
input_pm = []
output_pm = []
# Creates an input dictionary with all input sources
logger.debug('checking performance for {} and {}'.format(
scmp_cf, sex_cf))
input_d = {}
for d in range(1, 5, 1):
input_d[d] = '{}/Cat_20-21_d{}.dat'.format(
self.prfs_d['input_ref'], d)
input_d = Create_regions(input_d).check_luca(mag, True, True)
# Creates a DataFrame from an input dictionary
input_l = []
for key_ in input_d.keys():
input_l.append(input_d[key_])
i_df = concat(input_l, axis=0)
# Look for < 3 coincidences
i_df = concat(g for _, g in i_df.groupby('source') if len(g) >= 3)
i_df = i_df.reset_index(drop=True)
i_df.to_csv('input.csv')
# Open particular file!
filt_n = 'filt_{}_{}_4.csv'.format(scmp_cf, mag)
filter_o_n = '{}/{}/{}/{}'.format(self.prfs_d['filter_dir'], sex_cf,
scmp_cf, filt_n)
# Cross with filtered data - Opens datafile
o_cat = read_csv('{}'.format(filter_o_n), index_col=0)
# Gets unique sources from input data
unique_sources = list(set(i_df['source'].tolist()))
# Loops over input data
for idx_source, source_ in enumerate(unique_sources):
# Gets associated data in input catalog
cat = i_df[i_df['source'].isin([source_])]
# cat has, at least, three or more rows, one for each dither
# Creates lists for each source
boolean_l = []
tmp_catalog = []
tmp_source = []
# tmp lists
tmp_input_pm = []
tmp_output_pm = []
# Iterate over each detection of each source
for i, row in enumerate(cat.itertuples(), 1):
catalog_n = row.catalog
i_pm = row.pm_values
i_alpha = row.alpha_j2000
i_delta = row.delta_j2000
# Checks if there is a source closed to input one
# o_cat contains data from output (filtered) catalog
o_df = check_source(catalog_n, o_cat, i_alpha, i_delta)
# If there is one saves data from input data
if o_df.empty is not True and o_df['PM'].size == 1:
pm_mask = self.pm_filter(o_df, i_pm, confidence_)
if pm_mask:
if o_df['SOURCE_NUMBER'].size != 1:
boolean_l.append('False')
else:
boolean_l.append('True')
tmp_catalog.append(catalog_n)
tmp_source.append(o_df['SOURCE_NUMBER'].iloc[0])
tmp_input_pm.append(i_pm)
o_pm = o_df['PM'].iloc[0]
tmp_output_pm.append(o_pm)
else:
boolean_l.append('False')
# check
if len(tmp_input_pm) >= 3:
input_pm.append(i_pm)
output_pm.append(o_pm)
pm_d = {}
for key_ in list(set(input_pm)):
pm_d[key_] = []
for idx, o_pm in enumerate(output_pm):
i_pm = input_pm[idx]
pm_d[i_pm].append(o_pm)
stats_d = {
'pm': [],
'mean': [],
'median': [],
'std': [],
'max': [],
'min': [],
'detected': []
}
# mean and std determination
for key_ in pm_d.keys():
stats_d['pm'].append(key_)
stats_d['mean'].append(array(pm_d[key_]).mean())
stats_d['median'].append(median(array(pm_d[key_])))
stats_d['std'].append(array(pm_d[key_]).std())
stats_d['max'].append(max(pm_d[key_]))
stats_d['min'].append(min(pm_d[key_]))
stats_d['detected'].append(len(pm_d[key_]))
# print(key_, len(pm_d[key_]))
# print(array(pm_d[key_]).mean())
# print(median(array(pm_d[key_])))
# print(array(pm_d[key_]).std())
# print(max(pm_d[key_]))
# print(min(pm_d[key_]))
# print(" ")
if self.plot:
if not self.plot_comp(sex_cf, scmp_cf, input_pm, output_pm,
confidence_):
raise Exception
return stats_d
def extract_stats_ccds(self, cats_d, input_df, filt_cat):
"""
:param cats_d:
:param input_df:
:param filt_cat:
:return:
"""
ssos_d = creates_ssos_dict()
# Unique sources (?)
unique_sources = list(set(input_df['SOURCE'].tolist()))
test_dict = {
'PM': [],
'A_IMAGE': [],
'B_IMAGE': [],
'CLASS_STAR': [],
'MAG_ISO': [],
'ELLIPTICITY': []
}
print('total {}'.format(len(unique_sources)))
for idx_source_, source_ in enumerate(unique_sources):
print('idx {}'.format(idx_source_))
source_df = input_df[input_df['SOURCE'].isin([source_])]
test_number = 0 # Counter of right detections
mag_norm = 0 # Just to clear the code
pm_norm = 0 # Just to clear the code
# Loops over CCD catalogues
for i, row in enumerate(source_df.itertuples(), 1):
# dither_df = source_df[source_df['DITHER'].isin([row.DITHER])]
i_alpha = float(row.RA)
i_delta = float(row.DEC)
pm_norm = get_norm_speed(row.VEL)
mag_norm = get_norm_mag(row.ABMAG)
pm = ''
a_image = ''
b_image = ''
class_star = ''
mag_iso = ''
ellipticity = ''
test = False
for cat_ in cats_d[row.DITHER]:
out_df = check_source(cats_d[row.DITHER][cat_],
i_alpha,
i_delta,
keys=['ALPHA_J2000', 'DELTA_J2000'])
if out_df.empty is not True:
alpha = float(out_df['ALPHA_J2000'].iloc[0])
delta = float(out_df['DELTA_J2000'].iloc[0])
pm_df = check_source(
filt_cat,
alpha,
delta,
keys=['ALPHA_J2000', 'DELTA_J2000'])
if pm_df.empty is not True:
test = True
pm = float(pm_df['PM'].iloc[0])
a_image = float(pm_df['MEDIAN_A_IMAGE'].iloc[0])
b_image = float(pm_df['MEDIAN_B_IMAGE'].iloc[0])
class_star = float(
pm_df['MEDIAN_CLASS_STAR'].iloc[0])
mag_iso = float(pm_df['MEDIAN_MAG_ISO'].iloc[0])
ellipticity = float(
pm_df['MEDIAN_ELLIPTICITY'].iloc[0])
test_number += 1 # Test number
if test:
test_dict['PM'].append(pm)
test_dict['A_IMAGE'].append(a_image)
test_dict['B_IMAGE'].append(b_image)
test_dict['CLASS_STAR'].append(class_star)
test_dict['MAG_ISO'].append(mag_iso)
test_dict['ELLIPTICITY'].append(ellipticity)
if test_number >= 3:
ssos_d[mag_norm][pm_norm] += 1
for mag_key in [
'20-21', '21-22', '22-23', '23-24', '24-25', '25-26', '26-27'
]:
for pm_key in [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]:
print(mag_key, pm_key, ssos_d[mag_key][pm_key])
pm_list = Series(test_dict['PM'], name='PM')
a_image_list = Series(test_dict['A_IMAGE'], name='A_IMAGE')
b_image_list = Series(test_dict['B_IMAGE'], name='B_IMAGE')
class_star_list = Series(test_dict['CLASS_STAR'], name='CLASS_STAR')
mag_iso_list = Series(test_dict['MAG_ISO'], name='MAG_ISO')
ellipticity_list = Series(test_dict['ELLIPTICITY'], name='ELLIPTICITY')
positions_table = concat([
pm_list, a_image_list, b_image_list, class_star_list, mag_iso_list,
ellipticity_list
],
axis=1)
# positions_table.to_csv('catalogues_not_detected/ssos_catalogue.csv')
positions_table.to_csv('catalogues_detected/ssos_catalogue.csv')
