def process_2mass_vot(input_file, out_dir=dirconfig.proc_2mass):
"""
This function reads the votable 2MASS catalogs and extract sources that are
in the region of the respective tile and also select sources with Qflag = AAA.
:type out_dir: string
:param input_file: path to the 2MASS file (in vot format)
:param out_dir: output path
:return:
"""
# Check if files exist
if files_exist(input_file):
print(f'Processing file {input_file}')
table = Table.read(input_file, format='votable')
# Extract tile name and number
filename = path.splitext(path.split(input_file)[-1])[0]
tile_name = filename.split("_")[0]
tile_num = tile_name.replace('t', '')
tile = objects.all_tiles[tile_name]
# Unique Object ID
object_id = np.arange(len(table), dtype=np.int32) + 1
table['id'] = [f'2mass-{tile_name}_{oid:07d}' for oid in object_id]
# Add columns with magnitudes in VVV photometric system
transformation_2mass_to_vista(table)
# Select objects in the area of interest
table['RAJ2000'].unit = u.deg
table['DEJ2000'].unit = u.deg
aux = SkyCoord(ra=table['RAJ2000'], dec=table['DEJ2000']).galactic
table['l'] = aux.l
table['b'] = aux.b
lmin, lmax = tile.lmin, tile.lmax
bmin, bmax = tile.bmin, tile.bmax
l_filter = (table['l'] >= lmin) * (table['l'] <= lmax)
b_filter = (table['b'] >= bmin) * (table['b'] <= bmax)
q_filter = table['Qflg'] == 'AAA'
match = l_filter * b_filter * q_filter
date_time = datetime.utcnow()
table.meta = {'TILE': int(tile_num),
'F2MASS': input_file,
'STAGE': 'process_2mass_vot',
'CATYPE': '2mass',
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S'),
'AUTHOR': 'Jorge Anais'}
filename += '.fits'
out_path = path.join(out_dir, filename)
write_fits_table(table[match], out_path)
def process_gaia_vot(input_file, out_dir=dirconfig.proc_gaia, features=None):
"""
This function transform extract relevant features from vo-table (from gaia query)
in order to produce more easily manageable files.
:param features: A list with desired features from Gaia
:param input_file:
:param out_dir:
"""
# Check if files exist
if files_exist(input_file):
print(f'Processing file {input_file}')
# Read table
tbl = Table.read(input_file, format='votable')
tile_name = path.basename(input_file).replace('_gaia.vot.gz', '')
tile_num = tile_name.replace('t', '')
# Unique Object ID
object_id = np.arange(len(tbl), dtype=np.int32) + 1
tbl['id'] = [f'gaia-{tile_name}_{oid:07d}' for oid in object_id]
# Extract only sources with parallax
mask = ~tbl['parallax'].mask
tbl = tbl[mask]
# Default features to be extracted from gaia votable
if features is None:
features = ['id', 'ra', 'ra_error', 'dec', 'dec_error', 'l', 'b',
'parallax', 'parallax_error', 'parallax_over_error',
'pmra', 'pmra_error', 'pmdec', 'pmdec_error',
'phot_g_mean_flux', 'phot_g_mean_flux_error', 'phot_g_mean_flux_over_error', 'phot_g_mean_mag',
'phot_bp_mean_flux', 'phot_bp_mean_flux_error', 'phot_bp_mean_flux_over_error', 'phot_bp_mean_mag',
'phot_rp_mean_flux', 'phot_rp_mean_flux_error', 'phot_rp_mean_flux_over_error', 'phot_rp_mean_mag',
'phot_bp_rp_excess_factor', 'bp_rp', 'bp_g', 'g_rp',
'radial_velocity', 'radial_velocity_error', 'source_id']
filtered_tbl = tbl[features]
filename_out = path.basename(input_file).replace('.vot.gz', '') + '.fits'
filename_out = path.join(out_dir, filename_out)
print(f'Writing file: {filename_out}')
date_time = datetime.utcnow()
filtered_tbl.meta = {'TILE': int(tile_num),
'FGAIA': input_file,
'STAGE': 'process_gaia_vot',
'CATYPE': 'gaia',
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S'),
'AUTHOR': 'Jorge Anais'}
write_fits_table(filtered_tbl, filename_out)
def process_combis_csv(input_file, out_dir=dirconfig.proc_combis, combis_phot=False):
"""
This function simply transform combis catalogs (with proper motions) from a csv file
to a fits file.
:param combis_phot: Consider only data with complete photometrical information in bands mj, mh and mk
:param input_file: String. Path to the combi catalog (*.csv file)
:param out_dir: String. Output directory
:return:
"""
# Check if files exist
if files_exist(input_file):
print(f'Processing file {input_file}')
table = Table.read(input_file, format='csv')
mask_nan_values(table)
# Filename and tile number
filename = path.splitext(path.basename(input_file))[0]
tile_num = filename.split('_')[0].replace('d', '')
# Unique Object ID
object_id = np.arange(len(table), dtype=np.int32) + 1
table['id'] = [f'combi-t{tile_num}_{oid:07d}' for oid in object_id]
# Create l and b columns
table['ra'].unit = u.deg
table['dec'].unit = u.deg
aux = SkyCoord(ra=table['ra'], dec=table['dec']).galactic
table['l'] = aux.l
table['b'] = aux.b
# Remove sources with missing data
mask = ~table['pmra'].mask * ~table['pmdec'].mask
if combis_phot:
# Special case, Only consider rows with complete photometric info
mask *= ~table['mj'].mask * ~table['mh'].mask * ~table['mk'].mask
aux = table[mask]
# Create colors
aux['mh-mk'] = aux['mh'] - aux['mk']
aux['mj-mk'] = aux['mj'] - aux['mk']
aux['mj-mh'] = aux['mj'] - aux['mh']
if combis_phot:
# Replace column names with vvv-like ones
original_col_names = ('mj', 'mh', 'mk', 'mh-mk', 'mj-mk', 'mj-mh')
vvvlike_col_names = ('mag_J', 'mag_H', 'mag_Ks', 'H-Ks', 'J-Ks', 'J-H')
for original_col_name, vvvlike_col_name in zip(original_col_names, vvvlike_col_names):
aux.rename_column(original_col_name, vvvlike_col_name)
out = path.join(out_dir, filename + '.fits')
date_time = datetime.utcnow()
aux.meta = {'TILE': int(tile_num),
'FCOMBI': input_file,
'STAGE': 'process_combis_csv',
'CATYPE': 'combi',
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S'),
'AUTHOR': 'Jorge Anais'}
if combis_phot:
aux.meta.update({'CATYPE': 'combisphot'})
write_fits_table(aux, out)
def process_vvv_cals(input_file, out_dir=dirconfig.proc_vvv):
"""
This function take a raw PSF tile in plain text format (.cals) and transform it to a fits file,
removing sources that does not contain all J, H and Ks data.
Fits files are handled with much better performance in python than plain-text files.
For some reason, reading .cals files uses a lot of memory (~6 gb).
:param input_file: String. Path to the .cals file
:param out_dir: output directory
:return:
"""
# Check if files exist
if files_exist(input_file):
print(f'Processing file {input_file}')
table = Table.read(input_file, format='ascii')
# Check if all the columns exist in table
expected_cols = ['ra', 'dec', 'mag_Z', 'er_Z', 'mag_Y', 'er_Y', 'mag_J', 'er_J', 'mag_H', 'er_H', 'mag_Ks', 'er_Ks']
if not all(_ in table.columns for _ in expected_cols):
raise KeyError(f'Table does not contain all expected columns: {expected_cols}')
# Filename and tile number
input_filename = path.basename(input_file)
tile_num = path.splitext(input_filename.replace('zyjhk', ''))[0]
# Unique Object ID
object_id = np.arange(len(table), dtype=np.int32) + 1
table['id'] = [f'vvv-t{tile_num}_{oid:07d}' for oid in object_id]
# Create l and b columns
table['ra'].unit = u.deg
table['dec'].unit = u.deg
aux = SkyCoord(ra=table['ra'], dec=table['dec']).galactic
table['l'] = aux.l
table['b'] = aux.b
# Print some stats of the tile (before removing sources)
# table.info('stats')
# Create colors
mask = ~table['mag_J'].mask * ~table['mag_H'].mask * ~table['mag_Ks'].mask
aux = table[mask]
aux['H-Ks'] = aux['mag_H'] - aux['mag_Ks']
aux['J-Ks'] = aux['mag_J'] - aux['mag_Ks']
aux['J-H'] = aux['mag_J'] - aux['mag_H']
# print(f'Number of remaining sources: {len(aux)}')
# Save aux table as fits file with metadata
date_time = datetime.utcnow()
aux.meta = {'TILE': int(tile_num),
'FVVV': input_file,
'STAGE': 'VVV cals file to fits',
'CATYPE': 'vvv',
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S'),
'AUTHOR': 'Jorge Anais'}
out_fn = 't' + tile_num + '_vvv.fits'
out_path = path.join(out_dir, out_fn)
output_table = aux['id', 'ra', 'dec', 'l', 'b', 'mag_Z', 'er_Z', 'mag_Y', 'er_Y', 'mag_J', 'er_J',
'mag_H', 'er_H', 'mag_Ks', 'er_Ks', 'H-Ks', 'J-Ks', 'J-H']
write_fits_table(output_table, out_path)
def add_proper_motions(phot_file, pm_file, out_dir=dirconfig.test_knowncl):
"""
Function that match proper motion catalog and VVV clean catalogs.
:param pm_file:
:param phot_file:
:param out_dir:
:return:
"""
# Check if files exist
if files_exist(pm_file, phot_file):
print(f'Processing files:', phot_file, pm_file)
# Read tables
tbl_phot = read_fits_table(phot_file)
tbl_pm = read_fits_table(pm_file)
# Check if tile numbers match
if not tbl_phot.meta['TILE'] == tbl_pm.meta['TILE']:
raise ValueError(f'Files do not correspond to the same tile')
# Cross-match
cphot = SkyCoord(tbl_phot['ra'], tbl_phot['dec'])
cpm = SkyCoord(tbl_pm['ra'], tbl_pm['dec'])
idx, d2d, d3d = cphot.match_to_catalog_sky(cpm)
match = d2d < 0.34 * u.arcsec
# Remove duplicated matches
# In this case we prefer to omit sources with duplicated matches
unique_idx, count = np.unique(idx[match], return_counts=True)
duplicated_idxs = unique_idx[count > 1]
for i in duplicated_idxs:
match[idx == i] = False
# join table of matched sources
join_table = hstack([tbl_phot, tbl_pm[idx]],
uniq_col_name='{col_name}{table_name}',
table_names=['', '_pm'])
match_table = join_table[match]
# Setup names and output file
tile_number = tbl_phot.meta['TILE']
catype = tbl_phot.meta['CATYPE'] + '-' + tbl_pm.meta['CATYPE']
date_time = datetime.utcnow()
match_table.meta = {
'TILE': tile_number,
'FCOMBI': pm_file,
'FPHOT': phot_file,
'STAGE': 'add_proper_motions',
'CATYPE': catype,
'NPHOT': len(tbl_phot),
'NCOMBI': len(tbl_pm),
'NDUPL': len(idx[match]) - len(np.unique(idx[match])),
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S'),
'AUTHOR': 'Jorge Anais'
}
# Save file
fname = f't{tile_number:03d}_{catype}.fits'
outfile = path.join(out_dir, fname)
write_fits_table(match_table, outfile)
def gaia_cleaning(fname_phot,
fname_gaia,
clean_dir=dirconfig.cross_vvv_gaia,
cont_dir=dirconfig.cross_vvv_gaia_cont,
save_contam=True,
distance=1.0):
"""
This function matches gaia sources against VVV sources. Sources with a distance
less than 1 kpc are considered contaminants and are removed from vvv catalog.
This function generate two tables, one with the cleaned table and the other
with the contaminants.
:param fname_phot: String, path to the catalog to be cleaned
:param fname_gaia: String, path to the gaia catalog
:param clean_dir: String, output dir
:param cont_dir: String, output dir for contaminants
:param save_contam: Boolean
:param distance: Float, distance in kpc
:return:
"""
# Check if files exist
if files_exist(fname_phot, fname_gaia):
print(f'Processing files: ', fname_phot, fname_gaia)
# Load tables
tbl_phot = read_fits_table(fname_phot)
tbl_gaia = read_fits_table(fname_gaia)
# Check if tile match
if not tbl_phot.meta['TILE'] == tbl_gaia.meta['TILE']:
raise ValueError(f'Files do not correspond to the same tile')
tile_number = tbl_phot.meta['TILE']
# Apply threshold to gaia data
threshold = 1.0 / distance
match_parallax = tbl_gaia['parallax'] >= threshold
tbl_gaia_par = tbl_gaia[match_parallax]
# Cross-match
cphot = SkyCoord(tbl_phot['ra'], tbl_phot['dec'])
cgaia = SkyCoord(tbl_gaia_par['ra'], tbl_gaia_par['dec'])
idx, d2d, d3d = cphot.match_to_catalog_sky(cgaia)
match = d2d < 0.34 * u.arcsec
# Remove duplicated matches
# We only consider sources with 1 to 1 match
unique_idx, count = np.unique(idx[match], return_counts=True)
duplicated_idxs = unique_idx[count > 1]
for i in duplicated_idxs:
match[idx == i] = False
# join table of matched sources
join_table = hstack([tbl_phot, tbl_gaia_par[idx]])
# Catalog with contaminants (objects that are closer than "distance")
contam_table = join_table[match]
# Add metadata
catype = tbl_phot.meta['CATYPE'] + '-' + tbl_gaia.meta['CATYPE']
date_time = datetime.utcnow()
contam_table.meta = {
'TILE': int(tile_number),
'FGAIA': fname_gaia,
'FPHOT': fname_phot,
'STAGE': 'gaia_cleaning',
'CATYPE': catype + 'CONT',
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S'),
'DIST': distance,
'SELECT': 'contaminants',
'NDUPL': len(idx[match]) - len(np.unique(idx[match])),
'AUTHOR': 'Jorge Anais'
}
# Cleaned catalog
clean_catalog = tbl_phot[~match]
clean_catalog.meta = {
'TILE': int(tile_number),
'FGAIA': fname_gaia,
'FPHOT': fname_phot,
'STAGE': 'gaia_cleaning',
'CATYPE': catype,
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S'),
'DIST': distance,
'SELECT': 'clean',
'AUTHOR': 'Jorge Anais'
}
# Save clean catalog to a fits file
filename = f't{tile_number:03d}_{catype}'
path_out = path.join(clean_dir, filename + '_clean.fits')
write_fits_table(clean_catalog, path_out)
# Save contaminants
if save_contam:
path_out = path.join(cont_dir, filename + '_contaminants.fits')
write_fits_table(contam_table, path_out)
def combine_vvv_2mass(vvvpsf_file,
twomass_file,
out_dir=dirconfig.cross_vvv_2mass,
max_error=1.00):
"""
This function add 2MASS sources to the VVV-PSF catalog
:param twomass_file: string
:param vvvpsf_file: string
:param out_dir: string
:param max_error: number
:return:
"""
# Check if files exist
if files_exist(twomass_file, vvvpsf_file):
print('Combining: ', vvvpsf_file, twomass_file)
# Read catalogs
twomass_table = read_fits_table(twomass_file)
vvvpsf_table = read_fits_table(vvvpsf_file)
# Check if tile match
if not twomass_table.meta['TILE'] == vvvpsf_table.meta['TILE']:
raise ValueError(f'Files do not correspond to the same tile')
# Cross-match
c2mass = SkyCoord(twomass_table['RAJ2000'],
twomass_table['DEJ2000'],
unit='deg')
cvvv = SkyCoord(vvvpsf_table['ra'], vvvpsf_table['dec'], unit='deg')
idx, d2d, d3d = c2mass.match_to_catalog_sky(cvvv)
match = d2d > max_error * u.arcsec
# In this case repeated sources are not removed (otherwise they will be included in the output catalog)
unpaired_2mass_sources = twomass_table[match]
# Create a new table to store combined data
unp_table = Table()
# Add unpaired 2MASS sources to new_catalog
unp_table['ra'] = unpaired_2mass_sources['RAJ2000']
unp_table['dec'] = unpaired_2mass_sources['DEJ2000']
unp_table['l'] = unpaired_2mass_sources['l']
unp_table['b'] = unpaired_2mass_sources['b']
unp_table['mag_J'] = unpaired_2mass_sources['J_vista']
unp_table['eJ'] = unpaired_2mass_sources['e_Jmag']
unp_table['mag_H'] = unpaired_2mass_sources['H_vista']
unp_table['eH'] = unpaired_2mass_sources['e_Hmag']
unp_table['mag_Ks'] = unpaired_2mass_sources['Ks_vista']
unp_table['eKs'] = unpaired_2mass_sources['e_Kmag']
unp_table['H-Ks'] = unpaired_2mass_sources[
'H_vista'] - unpaired_2mass_sources['Ks_vista']
unp_table['J-Ks'] = unpaired_2mass_sources[
'J_vista'] - unpaired_2mass_sources['Ks_vista']
unp_table['J-H'] = unpaired_2mass_sources[
'J_vista'] - unpaired_2mass_sources['H_vista']
unp_table['catalog'] = [
'2MASS' for _ in range(len(unpaired_2mass_sources))
]
unp_table['id'] = unpaired_2mass_sources['id']
# Aux catalog for VVV-PSF sources
aux_table = Table()
# Add VVV-PSF sources to new_catalog
aux_table['ra'] = vvvpsf_table['ra']
aux_table['dec'] = vvvpsf_table['dec']
aux_table['l'] = vvvpsf_table['l']
aux_table['b'] = vvvpsf_table['b']
aux_table['mag_Z'] = Table.MaskedColumn(vvvpsf_table['mag_Z'].data,
mask=np.isnan(
vvvpsf_table['mag_Z'].data))
aux_table['er_Z'] = Table.MaskedColumn(vvvpsf_table['er_Z'].data,
mask=np.isnan(
vvvpsf_table['er_Z'].data))
aux_table['mag_Y'] = Table.MaskedColumn(vvvpsf_table['mag_Y'].data,
mask=np.isnan(
vvvpsf_table['mag_Y'].data))
aux_table['er_Y'] = Table.MaskedColumn(vvvpsf_table['er_Y'].data,
mask=np.isnan(
vvvpsf_table['er_Y'].data))
aux_table['mag_J'] = vvvpsf_table['mag_J']
aux_table['eJ'] = vvvpsf_table['er_J']
aux_table['mag_H'] = vvvpsf_table['mag_H']
aux_table['eH'] = vvvpsf_table['er_H']
aux_table['mag_Ks'] = vvvpsf_table['mag_Ks']
aux_table['eKs'] = vvvpsf_table['er_Ks']
aux_table['H-Ks'] = vvvpsf_table['H-Ks']
aux_table['J-Ks'] = vvvpsf_table['J-Ks']
aux_table['J-H'] = vvvpsf_table['J-H']
aux_table['catalog'] = ['PSF-VVV' for _ in range(len(vvvpsf_table))]
aux_table['id'] = vvvpsf_table['id']
output_table = vstack([unp_table, aux_table])
# Add metadata to the new file
date_time = datetime.utcnow()
tile = vvvpsf_table.meta['TILE']
catype = vvvpsf_table.meta['CATYPE'] + '-' + twomass_table.meta['CATYPE']
output_table.meta = {
'TILE': tile,
'F2MASS': twomass_file,
'N2MASS': len(unp_table),
'FVVV': vvvpsf_file,
'NVVV': len(vvvpsf_table),
'STAGE': 'combine_vvv_2mass',
'CATYPE': catype,
'CDATE': date_time.strftime('%Y-%m-%d'),
'CTIME': date_time.strftime('%H:%M:%S'),
'AUTHOR': 'Jorge Anais'
}
# Write output table
fname = f't{tile:03d}_{catype}.fits'
output_file = path.join(out_dir, fname)
write_fits_table(output_table, output_file)