def make_photerror_plot(tile):
"""
This function produces a photometry error plot for bands J, H and Ks using given tile.
:param tile:
:return:
"""
file = tile.get_file(data_dir)
table = read_fits_table(file)
fig, axs = plt.subplots(3, sharex=True, sharey=True, gridspec_kw={'hspace': 0})
fig.suptitle(f'Photometric error for tile {tile.name}')
kargs_plot = dict(markersize=1.0, alpha=0.1)
axs[0].plot(table['mag_J'], table['er_J'], '.', **kargs_plot, label='J', color='b')
axs[1].plot(table['mag_H'], table['er_H'], '.', **kargs_plot, label='H', color='g')
axs[2].plot(table['mag_Ks'], table['er_Ks'], '.', **kargs_plot, label='Ks', color='r')
axs[2].set_xlabel('Magnitudes')
# Hide x labels and tick labels for all but bottom plot. Tweak some default plot configuration
for ax in axs:
ax.label_outer()
ax.set_ylim(-0.05, 0.4)
ax.set_xlim(9.9, 22.1)
ax.legend(loc='upper left', markerscale=0, markerfirst=False, framealpha=0.00)
ax.set_ylabel(r'$\sigma$')
fig.savefig(f'figphoterr_{tile.name}.png', overwrite=True)
fig.clf()
def make_photerror_plot(tile):
"""
This function produces a photometry error plot for bands J, H and Ks using given tile.
:param tile:
:return:
"""
file = tile.get_file(data_dir)
table = read_fits_table(file)
fig = plt.figure()
ax1 = fig.add_subplot(311, projection='scatter_density')
ax2 = fig.add_subplot(312, projection='scatter_density')
ax3 = fig.add_subplot(313, projection='scatter_density')
fig.subplots_adjust(hspace=0)
fig.suptitle(f'Photometric error for tile {tile.name}')
norm = ImageNormalize(vmin=0., vmax=1000, stretch=LogStretch())
ax1.scatter_density(table['mag_J'],
table['er_J'],
color='blue',
norm=norm,
label='J')
ax2.scatter_density(table['mag_H'],
table['er_H'],
color='green',
norm=norm,
label='H')
ax3.scatter_density(table['mag_Ks'],
table['er_Ks'],
color='red',
norm=norm,
label='Ks')
ax3.set_xlabel('Magnitudes')
# Hide x labels and tick labels for all but bottom plot. Tweak default plot configuration
for ax, lbl in zip([ax1, ax2, ax3], ['J', 'H', 'Ks']):
ax.label_outer()
ax.set_ylim(-0.05, 0.4)
ax.set_xlim(9.9, 22.1)
red_patch = mpatches.Patch(label=lbl, alpha=0.00)
ax.legend(handles=[red_patch],
loc='upper left',
markerscale=0,
markerfirst=False,
framealpha=0.00)
ax.set_ylabel(r'$\sigma$')
fig.savefig(f'figphoterr_{tile.name}_v2.png', overwrite=True)
fig.clf()
def read_catalog(input_catalog, times=2.0):
table = read_fits_table(input_catalog)
date_time = datetime.utcnow()
metadata = {
'FILE': input_catalog,
'TIMES': times,
'STAGE': 'read_catalog',
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S')
}
table.meta.update(metadata)
return table
tile_number += 1
if write_fits:
log_table.write(path.join(output_dir,
f'log_tiling_bf{partitioning_id}.ecsv'),
format='ascii.ecsv')
tiles_objects_dict = dict()
for t in tile_list:
tiles_objects_dict.update({t[0]: models.Tessera(*t)})
return tiles_objects_dict
table = read_fits_table(
path.join(dirconfig.test_tiling, 'complete_region.fits'))
l_min_roi = table['l'].min()
l_max_roi = table['l'].max()
b_min_roi = table['b'].min()
b_max_roi = table['b'].max()
tile_size = 4.0 / 60 # tiles of 4 arcmin (deg)
# Grid aligned to the left top corner of the roi
l_grid_0 = np.arange(l_min_roi, l_max_roi + tile_size, tile_size)
b_grid_0 = np.arange(b_min_roi, b_max_roi + tile_size, tile_size)
tiles_0 = rectangular_tiling(table, l_grid_0, b_grid_0, 0, True)
# Grid aligned to the left top corner of the roi
def tile_routine(tile_file, output_dir, space_param='Mini-alternative',):
"""
This function implement a routine to be used with separate tiles files, including read file, add pseudo-color
column,
Parameters
----------
space_param
tile_file
output_dir
Returns
-------
"""
table = read_fits_table(tile_file)
tile_name = path.splitext(path.basename(tile_file))[0]
# Check if file exists, if so return False
expected_filename = path.join(output_dir, tile_name)
if glob(expected_filename + '*'):
print(f'Tile {tile_name} already processed. Skipping...')
return False
table.meta.update({'FILE': path.basename(tile_file)})
table.meta.update({'TILENAME': tile_name})
print('Processing', tile_name)
add_pseudocolor(table, color_excess=1.8)
scores = perform_grid_score(table,
mcs_range=(5, 50),
ms_range=(5, 50),
space_param=space_param,
cols=None,
cluster_selection_method='leaf',
noise_cluster=False,
make_plots=False,
out_dir=output_dir)
# In case that clustering was not successfully return False
if len(scores) == 0:
print('-' * 20)
print('No clusters found in tile: ', tile_name)
print('-' * 20)
return False
score_filepath = path.join(output_dir, 'scores_' + tile_name + '.ecsv')
scores.write(score_filepath, format='ascii.ecsv')
summarized_scores = summarize_score(scores)
score_filepath = path.join(output_dir, 'summary_' + tile_name + '.ecsv')
summarized_scores.write(score_filepath, format='ascii.ecsv')
best_mcs = summarized_scores['mcs_start'][0]
best_ms = summarized_scores['ms'][0]
ctools.do_hdbscan(table,
space_param=space_param,
cols=None,
min_cluster_size=int(best_mcs),
min_samples=int(best_ms),
cluster_selection_method='leaf')
cplots.plot_clustered_data(table, output_dir, summarized_scores)
return True
from os import path
import time
"""
The purpose of this script is to measure how much time take to process a complete tile
using a grid of hyper-parameters
"""
# '/home/jorge/Documents/DATA/test/neighbors/bigtile_with_tiling_v2.fits' # 2048
# '/home/jorge/Documents/DATA/test/neighbors/bigtile_with_tiling_v3.fits' # 1024
# '/home/jorge/Documents/DATA/test/neighbors/bigtile_with_tiling_v4.fits' # 512
# '/home/jorge/Documents/DATA/test/neighbors/bigtile_with_tiling_v5.fits' # 768
out_dir = path.join(dirconfig.test_tiling, '2048')
make_dir(out_dir)
catalog_filename = '/home/jorge/Documents/DATA/test/neighbors/bigtile_with_tiling_v2.fits' # 2048
table = read_fits_table(catalog_filename)
tile_selection = table[table['tile'] == 100]
add_pseudocolor(tile_selection, color_excess=1.8)
start = time.time()
scores = perform_grid_score(tile_selection,
mcs_range=(5, 50),
ms_range=(5, 50),
space_param='Mini-alternative',
cols=None,
cluster_selection_method='leaf',
noise_cluster=False,
make_plots=False,
out_dir=out_dir,
memory='/home/jorge/Documents/DATA/memory')
end = time.time()
def make_completeness_plot(tile, data_dir):
"""
This function produces a completeness histograms for all J, H and Ks bands for a given tile.
:param data_dir:
:param tile:
:return:
"""
file = tile.get_file(data_dir)
table = read_fits_table(file)
fig, axs = plt.subplots(3,
sharex=True,
sharey=True,
gridspec_kw={'hspace': 0})
fig.suptitle(f'Histogram of magnitudes per band for tile {tile.name}')
kwargs = dict(histtype='stepfilled', alpha=0.5, ec="k")
start = 4
stop = 21
bin_width = 0.25
hbins = np.arange(start, stop, bin_width)
axs[0].hist(table['mag_J'],
bins=hbins,
label='J (VVV/2MASS)',
color='b',
**kwargs)
axs[0].hist(table['mag_J'][table['catalog'] == '2MASS'],
bins=hbins,
label='J (2MASS)',
color='darkblue',
**kwargs)
axs[1].hist(table['mag_H'],
bins=hbins,
label='H (VVV/2MASS)',
color='g',
**kwargs)
axs[1].hist(table['mag_H'][table['catalog'] == '2MASS'],
bins=hbins,
label='H (2MASS)',
color='darkgreen',
**kwargs)
axs[2].hist(table['mag_Ks'],
bins=hbins,
label='Ks (VVV/2MASS)',
color='r',
**kwargs)
axs[2].hist(table['mag_Ks'][table['catalog'] == '2MASS'],
bins=hbins,
label='Ks (2MASS)',
color='darkred',
**kwargs)
axs[2].set_xlabel('Magnitudes')
# Hide x labels and tick labels for all but bottom plot. Tweak some default plot configuration
for ax in axs:
ax.label_outer()
ax.set_yscale('log')
ax.set_ylim(0.5, 590400)
#ax.set_xlim(9.9, 22.1)
ax.legend(loc='upper left',
markerscale=0,
markerfirst=True,
framealpha=0.00)
#ax.set_ylabel(r'$\sigma$')
fig.savefig(f'fighist_{tile.name}.png', overwrite=True)
fig.clf()
from apolo.tiling.tools import join_tiles
from apolo.data import dirconfig
from apolo.catalog_proc.utils import read_fits_table, write_fits_table
from os import path
path.join(dirconfig.cross_vvv_combis_gaia, 't067_vvv-2mass-combi-gaia_clean.fits')
t067 = read_fits_table(path.join(dirconfig.cross_vvv_combis_gaia, 't067_vvv-2mass-combi-gaia_clean.fits'))
t0105 = read_fits_table(path.join(dirconfig.cross_vvv_combis_gaia,'t105_vvv-2mass-combi-gaia_clean.fits'))
t067_t105 = join_tiles(t067, t0105)
file_t067_t105 = path.join(dirconfig.test_tiling, 't067_t105.fits')
write_fits_table(t067_t105, file_t067_t105)
t068 = read_fits_table(path.join(dirconfig.cross_vvv_combis_gaia,'t068_vvv-2mass-combi-gaia_clean.fits'))
t0106 = read_fits_table(path.join(dirconfig.cross_vvv_combis_gaia,'t106_vvv-2mass-combi-gaia_clean.fits'))
t068_t106 = join_tiles(t068, t0106)
file_t068_t106 = path.join(dirconfig.test_tiling, 't068_t106.fits')
write_fits_table(t068_t106, file_t068_t106)
t069 = read_fits_table(path.join(dirconfig.cross_vvv_combis_gaia,'t069_vvv-2mass-combi-gaia_clean.fits'))
t0107 = read_fits_table(path.join(dirconfig.cross_vvv_combis_gaia,'t107_vvv-2mass-combi-gaia_clean.fits'))
t069_t107 = join_tiles(t069, t0107)
file_t069_t107 = path.join(dirconfig.test_tiling, 't069_t107.fits')
write_fits_table(t069_t107, file_t069_t107)
t070 = read_fits_table(path.join(dirconfig.cross_vvv_combis_gaia,'t070_vvv-2mass-combi-gaia_clean.fits'))
t0108 = read_fits_table('t108_vvv-2mass-combi-gaia_clean.fits')
t070_t108 = join_tiles(t070, t0108)
file_t070_t108 = path.join(dirconfig.test_tiling, 't070_t108.fits')
write_fits_table(t070_t108, file_t070_t108)
def add_proper_motions(phot_file, pm_file, out_dir=dirconfig.test_knowncl):
"""
Function that match proper motion catalog and VVV clean catalogs.
:param pm_file:
:param phot_file:
:param out_dir:
:return:
"""
# Check if files exist
if files_exist(pm_file, phot_file):
print(f'Processing files:', phot_file, pm_file)
# Read tables
tbl_phot = read_fits_table(phot_file)
tbl_pm = read_fits_table(pm_file)
# Check if tile numbers match
if not tbl_phot.meta['TILE'] == tbl_pm.meta['TILE']:
raise ValueError(f'Files do not correspond to the same tile')
# Cross-match
cphot = SkyCoord(tbl_phot['ra'], tbl_phot['dec'])
cpm = SkyCoord(tbl_pm['ra'], tbl_pm['dec'])
idx, d2d, d3d = cphot.match_to_catalog_sky(cpm)
match = d2d < 0.34 * u.arcsec
# Remove duplicated matches
# In this case we prefer to omit sources with duplicated matches
unique_idx, count = np.unique(idx[match], return_counts=True)
duplicated_idxs = unique_idx[count > 1]
for i in duplicated_idxs:
match[idx == i] = False
# join table of matched sources
join_table = hstack([tbl_phot, tbl_pm[idx]],
uniq_col_name='{col_name}{table_name}',
table_names=['', '_pm'])
match_table = join_table[match]
# Setup names and output file
tile_number = tbl_phot.meta['TILE']
catype = tbl_phot.meta['CATYPE'] + '-' + tbl_pm.meta['CATYPE']
date_time = datetime.utcnow()
match_table.meta = {
'TILE': tile_number,
'FCOMBI': pm_file,
'FPHOT': phot_file,
'STAGE': 'add_proper_motions',
'CATYPE': catype,
'NPHOT': len(tbl_phot),
'NCOMBI': len(tbl_pm),
'NDUPL': len(idx[match]) - len(np.unique(idx[match])),
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S'),
'AUTHOR': 'Jorge Anais'
}
# Save file
fname = f't{tile_number:03d}_{catype}.fits'
outfile = path.join(out_dir, fname)
write_fits_table(match_table, outfile)
def gaia_cleaning(fname_phot,
fname_gaia,
clean_dir=dirconfig.cross_vvv_gaia,
cont_dir=dirconfig.cross_vvv_gaia_cont,
save_contam=True,
distance=1.0):
"""
This function matches gaia sources against VVV sources. Sources with a distance
less than 1 kpc are considered contaminants and are removed from vvv catalog.
This function generate two tables, one with the cleaned table and the other
with the contaminants.
:param fname_phot: String, path to the catalog to be cleaned
:param fname_gaia: String, path to the gaia catalog
:param clean_dir: String, output dir
:param cont_dir: String, output dir for contaminants
:param save_contam: Boolean
:param distance: Float, distance in kpc
:return:
"""
# Check if files exist
if files_exist(fname_phot, fname_gaia):
print(f'Processing files: ', fname_phot, fname_gaia)
# Load tables
tbl_phot = read_fits_table(fname_phot)
tbl_gaia = read_fits_table(fname_gaia)
# Check if tile match
if not tbl_phot.meta['TILE'] == tbl_gaia.meta['TILE']:
raise ValueError(f'Files do not correspond to the same tile')
tile_number = tbl_phot.meta['TILE']
# Apply threshold to gaia data
threshold = 1.0 / distance
match_parallax = tbl_gaia['parallax'] >= threshold
tbl_gaia_par = tbl_gaia[match_parallax]
# Cross-match
cphot = SkyCoord(tbl_phot['ra'], tbl_phot['dec'])
cgaia = SkyCoord(tbl_gaia_par['ra'], tbl_gaia_par['dec'])
idx, d2d, d3d = cphot.match_to_catalog_sky(cgaia)
match = d2d < 0.34 * u.arcsec
# Remove duplicated matches
# We only consider sources with 1 to 1 match
unique_idx, count = np.unique(idx[match], return_counts=True)
duplicated_idxs = unique_idx[count > 1]
for i in duplicated_idxs:
match[idx == i] = False
# join table of matched sources
join_table = hstack([tbl_phot, tbl_gaia_par[idx]])
# Catalog with contaminants (objects that are closer than "distance")
contam_table = join_table[match]
# Add metadata
catype = tbl_phot.meta['CATYPE'] + '-' + tbl_gaia.meta['CATYPE']
date_time = datetime.utcnow()
contam_table.meta = {
'TILE': int(tile_number),
'FGAIA': fname_gaia,
'FPHOT': fname_phot,
'STAGE': 'gaia_cleaning',
'CATYPE': catype + 'CONT',
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S'),
'DIST': distance,
'SELECT': 'contaminants',
'NDUPL': len(idx[match]) - len(np.unique(idx[match])),
'AUTHOR': 'Jorge Anais'
}
# Cleaned catalog
clean_catalog = tbl_phot[~match]
clean_catalog.meta = {
'TILE': int(tile_number),
'FGAIA': fname_gaia,
'FPHOT': fname_phot,
'STAGE': 'gaia_cleaning',
'CATYPE': catype,
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S'),
'DIST': distance,
'SELECT': 'clean',
'AUTHOR': 'Jorge Anais'
}
# Save clean catalog to a fits file
filename = f't{tile_number:03d}_{catype}'
path_out = path.join(clean_dir, filename + '_clean.fits')
write_fits_table(clean_catalog, path_out)
# Save contaminants
if save_contam:
path_out = path.join(cont_dir, filename + '_contaminants.fits')
write_fits_table(contam_table, path_out)
def combine_vvv_2mass(vvvpsf_file,
twomass_file,
out_dir=dirconfig.cross_vvv_2mass,
max_error=1.00):
"""
This function add 2MASS sources to the VVV-PSF catalog
:param twomass_file: string
:param vvvpsf_file: string
:param out_dir: string
:param max_error: number
:return:
"""
# Check if files exist
if files_exist(twomass_file, vvvpsf_file):
print('Combining: ', vvvpsf_file, twomass_file)
# Read catalogs
twomass_table = read_fits_table(twomass_file)
vvvpsf_table = read_fits_table(vvvpsf_file)
# Check if tile match
if not twomass_table.meta['TILE'] == vvvpsf_table.meta['TILE']:
raise ValueError(f'Files do not correspond to the same tile')
# Cross-match
c2mass = SkyCoord(twomass_table['RAJ2000'],
twomass_table['DEJ2000'],
unit='deg')
cvvv = SkyCoord(vvvpsf_table['ra'], vvvpsf_table['dec'], unit='deg')
idx, d2d, d3d = c2mass.match_to_catalog_sky(cvvv)
match = d2d > max_error * u.arcsec
# In this case repeated sources are not removed (otherwise they will be included in the output catalog)
unpaired_2mass_sources = twomass_table[match]
# Create a new table to store combined data
unp_table = Table()
# Add unpaired 2MASS sources to new_catalog
unp_table['ra'] = unpaired_2mass_sources['RAJ2000']
unp_table['dec'] = unpaired_2mass_sources['DEJ2000']
unp_table['l'] = unpaired_2mass_sources['l']
unp_table['b'] = unpaired_2mass_sources['b']
unp_table['mag_J'] = unpaired_2mass_sources['J_vista']
unp_table['eJ'] = unpaired_2mass_sources['e_Jmag']
unp_table['mag_H'] = unpaired_2mass_sources['H_vista']
unp_table['eH'] = unpaired_2mass_sources['e_Hmag']
unp_table['mag_Ks'] = unpaired_2mass_sources['Ks_vista']
unp_table['eKs'] = unpaired_2mass_sources['e_Kmag']
unp_table['H-Ks'] = unpaired_2mass_sources[
'H_vista'] - unpaired_2mass_sources['Ks_vista']
unp_table['J-Ks'] = unpaired_2mass_sources[
'J_vista'] - unpaired_2mass_sources['Ks_vista']
unp_table['J-H'] = unpaired_2mass_sources[
'J_vista'] - unpaired_2mass_sources['H_vista']
unp_table['catalog'] = [
'2MASS' for _ in range(len(unpaired_2mass_sources))
]
unp_table['id'] = unpaired_2mass_sources['id']
# Aux catalog for VVV-PSF sources
aux_table = Table()
# Add VVV-PSF sources to new_catalog
aux_table['ra'] = vvvpsf_table['ra']
aux_table['dec'] = vvvpsf_table['dec']
aux_table['l'] = vvvpsf_table['l']
aux_table['b'] = vvvpsf_table['b']
aux_table['mag_Z'] = Table.MaskedColumn(vvvpsf_table['mag_Z'].data,
mask=np.isnan(
vvvpsf_table['mag_Z'].data))
aux_table['er_Z'] = Table.MaskedColumn(vvvpsf_table['er_Z'].data,
mask=np.isnan(
vvvpsf_table['er_Z'].data))
aux_table['mag_Y'] = Table.MaskedColumn(vvvpsf_table['mag_Y'].data,
mask=np.isnan(
vvvpsf_table['mag_Y'].data))
aux_table['er_Y'] = Table.MaskedColumn(vvvpsf_table['er_Y'].data,
mask=np.isnan(
vvvpsf_table['er_Y'].data))
aux_table['mag_J'] = vvvpsf_table['mag_J']
aux_table['eJ'] = vvvpsf_table['er_J']
aux_table['mag_H'] = vvvpsf_table['mag_H']
aux_table['eH'] = vvvpsf_table['er_H']
aux_table['mag_Ks'] = vvvpsf_table['mag_Ks']
aux_table['eKs'] = vvvpsf_table['er_Ks']
aux_table['H-Ks'] = vvvpsf_table['H-Ks']
aux_table['J-Ks'] = vvvpsf_table['J-Ks']
aux_table['J-H'] = vvvpsf_table['J-H']
aux_table['catalog'] = ['PSF-VVV' for _ in range(len(vvvpsf_table))]
aux_table['id'] = vvvpsf_table['id']
output_table = vstack([unp_table, aux_table])
# Add metadata to the new file
date_time = datetime.utcnow()
tile = vvvpsf_table.meta['TILE']
catype = vvvpsf_table.meta['CATYPE'] + '-' + twomass_table.meta['CATYPE']
output_table.meta = {
'TILE': tile,
'F2MASS': twomass_file,
'N2MASS': len(unp_table),
'FVVV': vvvpsf_file,
'NVVV': len(vvvpsf_table),
'STAGE': 'combine_vvv_2mass',
'CATYPE': catype,
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S'),
'AUTHOR': 'Jorge Anais'
}
# Write output table
fname = f't{tile:03d}_{catype}.fits'
output_file = path.join(out_dir, fname)
write_fits_table(output_table, output_file)