def pstarrs(obsids_table, data_folder, moc_folder, nir_moc=None, radius=15 * u.arcmin, moc_order=16, overwrite=True): """ Get Pan-STARRS data using astroquery and Vizier. For each observation in obsids_table, saves a fits file with name 'OBS_ID.fits' in 'data_folder/groups'. The function sends a Vizier query and selects all sources within 'radius' arcmin of the RA,DEC of the observation, then it filters the result selecting the sources in the corresponding MOC stored in 'moc_folder/mocs' (moc_order must be consistent with the order used to calculate the MOC). If overwrite is True, always create a new fits file. If False, checks for an existing file and uses it to calculate the number of Pan-STARRS sources in the field. If it doesn't exist, creates the file. The function returns obsids_table with an additional column 'NSRC_PS' with the number of sources in the field. """ # Groups folder if nir_moc is None: groups_folder = os.path.join(data_folder, 'groups') else: root, _ = os.path.splitext(os.path.basename(nir_moc)) survey = root.split('_')[-1] groups_folder = os.path.join(data_folder, 'groups_' + survey) moc_nirsurvey = MOC() read_moc_fits(moc_nirsurvey, nir_moc) if not os.path.exists(groups_folder): os.makedirs(groups_folder) moc_folder = os.path.join(moc_folder, 'mocs') nsources_field = np.full((len(obsids_table), ), np.nan) hp = HEALPix(nside=2**moc_order, order='nested', frame=ICRS()) v = Vizier(columns=[ 'objID', 'RAJ2000', 'DEJ2000', 'e_RAJ2000', 'e_DEJ2000', 'Nd', 'Qual' ], column_filters={"Nd": ">1"}, row_limit=np.inf, timeout=6000) for i, row in enumerate(tqdm(obsids_table, desc="Making Pan-STARRS groups")): ## Group file name field_table_file = os.path.join(data_folder, groups_folder, '{}.fits'.format(row['OBS_ID'])) is_field_table = os.path.exists(field_table_file) if overwrite or (not overwrite and not is_field_table): ## Select all sources in the field field_coords = SkyCoord(ra=row['RA'] * u.deg, dec=row['DEC'] * u.deg) vrsp = v.query_region_async(field_coords, radius=radius, catalog='II/349', return_type='asu-tsv') # Fix bug in the vizier response # (returns the objID as a short int and fails to load # properly as an astropy table) with open('/tmp/tmp.tab', 'wb') as tmpfile: tmpfile.write(vrsp.content) src_table = Table.read('/tmp/tmp.tab', format='ascii.tab') src_table = src_table[2:] objid = np.array(src_table['objID']).astype(np.int64) ra = np.array(src_table['RAJ2000']).astype(np.float) * u.deg dec = np.array(src_table['DEJ2000']).astype(np.float) * u.deg err_ra = np.array(src_table['e_RAJ2000']) err_ra[err_ra == ' '] = '-1' err_ra = err_ra.astype(np.float) * u.arcsec err_ra[err_ra == -1] = np.nan err_dec = np.array(src_table['e_DEJ2000']) err_dec[err_dec == ' '] = '-1' err_dec = err_dec.astype(np.float) * u.arcsec err_dec[err_dec == -1] = np.nan flag = np.array(src_table['Qual']).astype(np.int32) src_table = Table([objid, ra, dec, err_ra, err_dec, flag], names=[ 'objID', 'RAJ2000', 'DEJ2000', 'e_RAJ2000', 'e_DEJ2000', 'Qual' ]) # Filter table msk_good = (src_table['Qual'] & 16) != 0 src_table_new = src_table[msk_good] ## Select sources in the non-overlaping area moc_field = MOC() read_moc_fits( moc_field, os.path.join(moc_folder, '{}.moc'.format(row['OBS_ID']))) if nir_moc is not None: moc_field = moc_nirsurvey.intersection(moc_field) inmoc_table = sources_inmoc(src_table_new, hp, moc_field, moc_order=moc_order, ra='RAJ2000', dec='DEJ2000') ## Save sources inmoc_table.remove_columns(['Qual']) inmoc_table.meta['description'] = 'Pan-STARRS' inmoc_table.write(field_table_file, overwrite=True) else: inmoc_table = Table.read(field_table_file) nsources_field[i] = len(inmoc_table) colsrc = Table.Column(nsources_field, name='NSRC_PS') obsids_table.add_column(colsrc) return obsids_table
def tmass(obsids_table, data_folder, moc_folder, opt_moc=None, radius=15 * u.arcmin, moc_order=16, overwrite=True): """ Get 2MASS data using astroquery and Vizier For each observation in obsids_table, saves a fits file with name 'OBS_ID.fits' in 'data_folder/groups'. The function sends a Vizier query and selects all sources within 'radius' arcmin of the RA,DEC of the observation, then it filters the result selecting the sources in the corresponding MOC stored in 'moc_folder/mocs' (moc_order must be consistent with the order used to calculate the moc). If overwrite is True, always create a new fits file. If False, checks for an existing file and uses it to calculate the number of WISE sources in the field. If it doesn't exist, creates the file. The function returns obsids_table with an additional column 'NSRC_2M' with the number of sources in the field. """ # Groups folder groups_folder = os.path.join(data_folder, 'groups') if not os.path.exists(groups_folder): os.makedirs(groups_folder) moc_folder = os.path.join(moc_folder, 'mocs') if opt_moc is not None: moc_optsurvey = MOC() read_moc_fits(moc_optsurvey, opt_moc) nsources_field = np.full((len(obsids_table), ), np.nan) hp = HEALPix(nside=2**moc_order, order='nested', frame=ICRS()) v = Vizier(columns=[ 'Cntr', 'RAJ2000', 'DEJ2000', 'errMaj', 'errMin', 'errPA', 'Qflg' ], row_limit=np.inf, timeout=6000) for i, row in enumerate(tqdm(obsids_table, desc="Making 2MASS groups")): ## Group file name field_table_file = os.path.join(data_folder, groups_folder, '{}.fits'.format(row['OBS_ID'])) is_field_table = os.path.exists(field_table_file) if overwrite or (not overwrite and not is_field_table): ## Select all sources in the field field_coords = SkyCoord(ra=row['RA'] * u.deg, dec=row['DEC'] * u.deg) vrsp = v.query_region_async(field_coords, radius=radius, catalog='II/246/out', return_type='asu-tsv') # Fix bug in the vizier response # (returns the id as a short int and fails to load # properly as an astropy table) with open('/tmp/tmp.tab', 'wb') as tmpfile: tmpfile.write(vrsp.content) src_table = Table.read('/tmp/tmp.tab', format='ascii.tab') src_table = src_table[2:] objid = np.array(src_table['Cntr']).astype(np.int64) ra = np.array(src_table['RAJ2000']).astype(np.float) * u.deg dec = np.array(src_table['DEJ2000']).astype(np.float) * u.deg errMaj = np.array(src_table['errMaj']).astype(np.float) * u.arcsec # err_ra[err_ra == ' '] = '-1' # err_ra = err_ra.astype(np.float) * u.arcsec # err_ra[err_ra == -1] = np.nan errMin = np.array(src_table['errMin']).astype(np.float) * u.arcsec # err_dec[err_dec == ' '] = '-1' # err_dec = err_dec.astype(np.float) * u.arcsec # err_dec[err_dec == -1] = np.nan errPA = np.array(src_table['errPA']).astype(np.float) * u.deg flag = np.array(src_table['Qflg']) src_table = Table([objid, ra, dec, errMaj, errMin, errPA, flag], names=[ 'objID', 'RAJ2000', 'DEJ2000', 'errMaj', 'errMin', 'errPA', 'Qflg' ]) # Filter table # Sources detected with SNR>=5 in J, H or K flgJ = [f[0] in ['A', 'B', 'C'] for f in src_table['Qflg']] flgH = [f[1] in ['A', 'B', 'C'] for f in src_table['Qflg']] flgK = [f[2] in ['A', 'B', 'C'] for f in src_table['Qflg']] msk_good = np.logical_and(flgJ, np.logical_and(flgH, flgK)) src_table_new = src_table[msk_good] ## Select sources in the non-overlaping area moc_field = MOC() read_moc_fits( moc_field, os.path.join(moc_folder, '{}.moc'.format(row['OBS_ID']))) if opt_moc is not None: moc_field = moc_optsurvey.intersection(moc_field) inmoc_table = sources_inmoc(src_table_new, hp, moc_field, moc_order=moc_order, ra='RAJ2000', dec='DEJ2000') ## Save sources field_table_file = os.path.join(data_folder, groups_folder, '{}.fits'.format(row['OBS_ID'])) inmoc_table.meta['description'] = '2MASS' inmoc_table.remove_column('Qflg') inmoc_table.write(field_table_file, overwrite=True) else: inmoc_table = Table.read(field_table_file) nsources_field[i] = len(inmoc_table) colsrc = Table.Column(nsources_field, name='NSRC_2M') obsids_table.add_column(colsrc) return obsids_table
def query_vizier(name, type1, config, RA="_RAJ2000", DEC="_DEJ2000", z="Redshift"): """ Use astroquery to query the Vizier catalogue database Input: name: name of the source to query region for type1: redshift of velocity query config: configuration RA, DEC: optional, coordinates of RA and DEC columns; defaults to vizier names which point to RA and DEC homogenized to deg and J2000 Return: Final table containing 4 columns, RA, DEC, redshift and origin of redshift measurement, compiled from all data available of Vizier """ # Setup the column and keywords used for the selection; calculate # homogenized RA and DEC and return unlimited rows if type1 == "velocity": UCD = VELOCITY_SRC KEYS = VELOCITY_KEYS if type1 == "redshift": UCD = REDSHIFT_SRC KEYS = REDSHIFT_KEYS v = Vizier( columns=["**", RA, DEC], ucd=UCD, row_limit=-1, timeout=timeout, ) # Query a region using source name, return a XML response and process # through Astropy as VOTABLE cat_vot = parse( BytesIO( v.query_region_async(name, radius=config.radius).text.encode()), pedantic=False, invalid="mask", ) # Make a preliminary selection of columns to keep only RA, DEC and the # possible redshift columns cat_list = prelim_selection(cat_vot, type1, RA, DEC, KEYS) # Initialize a list of table to be appended; These will be tables that have # a redshift measurement table_list = [] # Find whether the catalogue contains relevant information and save the # column with the relevant data for cat in cat_list: final_cat = process_catalog(type1, cat, config, RA, DEC) if final_cat is not None: table_list.append(final_cat) # Stack all the final catalogues with the relevant data if table_list: grand_table = vstack(table_list) # Remove potential photoz's by removing measurements with little precision grand_table = remove_potential_photoz(grand_table, z) # Return results of the vizier search return grand_table else: return None