def xmatch(coordinates, catalog, radius): """ Perform cross-match with a catalog using the CDS XMatch parameters: coordinates, catalog, radius: coordinates: astropy table with RA, DEC of all detected sources catalog: Vizier identifier of the catalog radius in arcsecond returns: astropy.table object Vizier catalog identifiers: Gaia DR2: I/345/gaia2 SDSS DR12: V/147/sdss12 2MASS: II/246/out USNO B1: I/284/out USNO A2: I/252/out GLADE 2: VII/281/glade2 Panstarrs DR1: II/349/ps1 """ matched_stars = XMatch.query(coordinates, cat2='vizier:%s' % catalog, max_distance=radius * u.arcsec, colRA1='_RAJ2000', colDec1='_DEJ2000') return matched_stars
def xmatch_cds_from_csv(file, ra_name='RA', dec_name='Dec', dist=5, cat='src'): """ do a CDS XMatch between targets in file and Gaia; return astropy table """ if cat == 'tgas': gaia_cat = 'vizier:I/337/tgas' # DR1 TGAS elif cat == 'src': gaia_cat = 'vizier:I/337/gaia' # DR1 src elif cat == 'dr2': gaia_cat = 'vizier:I/345/gaia2' # DR2 print("starting xmatch between {0} and {1} cat".format(file, cat)) table = XMatch.query(cat1=open(file), cat2=gaia_cat, max_distance=dist * u.arcsec, colRA1=ra_name, colDec1=dec_name) print("xmatch complete") # Assign units to columns where possible for col in table.colnames: #if col not in gaia_col_keep and col not in gaia_unit_map: # don't care about this quantity # table.remove_column(col) if col in gaia_unit_map: if not isinstance(gaia_unit_map[col], u.UnrecognizedUnit): # unit is valid table[col].unit = gaia_unit_map[col] table.remove_columns([ra_name, dec_name]) # avoid later conflicts return table
def get_denis_xmatch(c, _id=None, mag=None, drop_duplicates=1): """ Given J2000 coordinate(s), search for the DENIS crossmatch(es). Does this via the CDS XMatch service, which goes through the Vizier tables and does a spatial crossmatch. args: c: astropy SkyCoord. Can contain many. _id: identifier string for the star(s). mag: optional magnitude(s) to compute a difference against. drop_duplicates: whether to use angular distance to do a true "left-join". returns: denis_xm (pandas DataFrame): containing searched coordinates, identifier, magnitudes, the default DENIS columns, and crucially `angDist` and `mag_diff`. """ t = Table() t['RAJ2000'] = c.ra t['DEJ2000'] = c.dec if mag is not None: t['mag'] = mag if _id is not None: t['_id'] = _id # # NOTE: simplest crossmatching approach fails, b/c no easy merging. # # however, astroquery.XMatch, which queries the CDS XMatch service # # <http://cdsxmatch.u-strasbg.fr/xmatch> # Vizier.ROW_LIMIT = -1 # v = Vizier(columns=["*", "+_r"], catalog="B/denis") # denis_xm = v.query_region(t, radius="3s")[0] denis_xm = XMatch.query(cat1=t, cat2='vizier:B/denis/denis', max_distance=3 * u.arcsec, colRA1='RAJ2000', colDec1='DEJ2000', colRA2='RAJ2000', colDec2='DEJ2000') if mag is not None: denis_xm['mag_diff'] = denis_xm['mag'] - denis_xm['Imag'] # Take the closest (proper motion and epoch-corrected) angular distance as # THE single match. get_leftjoin_xm = lambda _t: (_t.to_pandas().sort_values(by='angDist'). drop_duplicates(subset='_id', keep='first')) if drop_duplicates: return get_leftjoin_xm(denis_xm) else: return denis_xm.to_pandas()
def download_xmatch(cat1, cat2, outfile_name): """Download a cross-match from VizieR.""" if not proceed_checkfile(outfile_name): return result = XMatch.query(cat1=cat1, cat2=cat2, max_distance=5 * units.arcsec) io.ascii.write(result, outfile_name, format='csv')
def _gaia_galex_xmatch(cats, ra, dec, radius): galex = cats['II/312/ais'] coord = SkyCoord(ra=ra * u.deg, dec=dec * u.deg, frame='icrs') region = CircleSkyRegion(coord, radius=radius) xm = XMatch.query(cat1='vizier:I/345/gaia2', cat2=galex, colRA2='RAJ2000', colDec2='DEJ2000', area=region, max_distance=radius) xm.sort('angDist') return xm
def _gaia_mermilliod_xmatch(cats, ra, dec, radius): mermilliod = cats['II/168/ubvmeans'] coord = SkyCoord(ra=ra * u.deg, dec=dec * u.deg, frame='icrs') region = CircleSkyRegion(coord, radius=radius) xm = XMatch.query(cat1='vizier:I/345/gaia2', cat2=mermilliod, colRA2='_RA', colDec2='_DE', area=region, max_distance=radius) xm.sort('angDist') return xm
def _gaia_hauck_xmatch(cats, ra, dec, radius): mermilliod = cats['J/A+A/580/A23/catalog'] coord = SkyCoord(ra=ra * u.deg, dec=dec * u.deg, frame='icrs') region = CircleSkyRegion(coord, radius=radius) xm = XMatch.query(cat1='vizier:I/345/gaia2', cat2=mermilliod, colRA2='_RA.icrs', colDec2='_DE.icrs', area=region, max_distance=radius) xm.sort('angDist') return xm
def download_xmatch(cat1: str, cat2: str, path: Path) -> None: """Download a cross-match from VizieR.""" if not proceed_checkfile(path): return result = XMatch.query( cat1=cat1, cat2=cat2, max_distance=5 * units.arcsec, ) io_ascii.write(result, path, format='csv')
def Vizier_xmatch(self, viz_cat, cat_name, ra_col='_RAJ2000', dec_col='_DEJ2000', radius='', group=True): """ Use astroquery to pull in and cross match a catalog with sources in self.catalog Parameters ---------- viz_cat: str The catalog string from Vizier (e.g. 'II/246' for 2MASS PSC) cat_name: str A name for the imported catalog (e.g. '2MASS') radius: astropy.units.quantity.Quantity The matching radius """ # Make sure sources have been grouped if self.catalog.empty: print('Please run group_sources() before cross matching.') return if self._catalog_check(cat_name): # Verify the cat_name viz_cat = "vizier:{}".format(viz_cat) # Prep the current catalog as an astropy.QTable tab = at.Table.from_pandas(self.catalog) # Crossmatch with Vizier print("Cross matching {} sources with {} catalog. Please be patient...".format(len(tab), viz_cat)) data = XMatch.query(cat1=tab, cat2=viz_cat, max_distance=radius or self.xmatch_radius*q.deg, colRA1='ra', colDec1='dec', colRA2=ra_col, colDec2=dec_col) # Ingest the data self.ingest_data(data, cat_name, 'id', ra_col=ra_col, dec_col=dec_col) # Regroup if group: self.group_sources(self.xmatch_radius)
def match_catalog(self, file_name, ra_keyword="ALPHA_J2000", dec_keyword="DELTA_J2000", catalogue='I/345/gaia2', plot=False, catalog_output=False, max_sources=None): """ Basic circular aperture photometry of given images. Parameters ---------- file_name : file object File name to be source extracted. ra_keyword : str RA keyword in catalogue. Default is "ALPHA_J2000" dec_keyword : str DEC keyword in catalogue. Default is "DEC_J2000" catalogue : vizer catalogue object Vizer catalogue name Default is 'I/345/gaia2'. plot: boolean Shell we plot the correlation? Returns ------- 'A dict object' Examples -------- >>> from astrolib import catalog >>> from astrolib import astronomy >>> from astropy.table import Table >>> co = catalog.Query() >>> ac = astronomy.AstCalc() >>> c = ac.radec2wcs("21:05:15.250", "+07:52:6.734") >>> ra_list_in_deg = [c.ra.degree] >>> dec_list_in_deg = [c.dec.degree] >>> phot_object = Table([ra_list_in_deg, dec_list_in_deg], names=("ALPHA_J2000", "DELTA_J2000")) >>> co.match_catalog("file.fits", catalogue="II/336/apass9", filter="Vmag", phot_object=phot_object, plot=True) """ fo = FitsOps(file_name) ds = fo.source_extract() table = XMatch.query(cat1=ds, cat2='vizier:{}'.format(catalogue), max_distance=5 * u.arcsec, colRA1=ra_keyword, colDec1=dec_keyword) if max_sources is not None: table = table[:max_sources] if catalog_output is True: cat_file = os.path.splitext(file_name)[0] ascii.write(table, "{}.csv".format(cat_file), format='csv', fast_writer=False) if plot is True: data = fo.hdu[0].data.astype(float) splt = StarPlot() splt.star_plot(data, table) return table
# create an astropy table with RA and Dec as column ra = [] dec = [] for coord in data: ra.append(coord.ra.value) dec.append(coord.dec.value) tab = Table([ra, dec], names=('RA', 'Dec')) # define the maximim distance for the cross match xmatch_distance = 20 * u.arcsec # cross match local file with Simbad through the Xmatch service tab.write('orig_coord.csv', overwrite=True) xmatch_tab = XMatch.query(cat1=open('orig_coord.csv'), cat2='SIMBAD', max_distance=xmatch_distance, colRA1='RA', colDec1='Dec') # xmatch_tab will only have entries for those sources that have a cross # match in Simbad, so to get a lists of those coordinates that do not have # a cross match in Simbad we use the cross matching fuctionalities of # astropy (https://docs.astropy.org/en/stable/coordinates/matchsep.html#astropy-coordinates-matching) orig_coord = SkyCoord(ra=tab['RA']*u.degree, dec=tab['Dec']*u.degree) xmatch_coord = SkyCoord(ra=xmatch_tab['RA']*u.degree, dec=xmatch_tab['Dec']*u.degree) nearest_source_index, dist_2d, dist_3d = orig_coord.match_to_catalog_sky(xmatch_coord) # this procedure provides the shortest distance (dist_2d) for all entries from # the XMatch result table to entries in the original table of all coordinates # if we now just select those entires with short shortest distances we get # a table of all coordinates with a cross match in Simbad, if we select those
def rota(self, image_path=None, object_name=None, ephemeris_file=None, odate=None, radius=None, srg_radius=10, time_travel=1, min_mag=0, max_mag=17.0, circle_color='yellow', arrow_color='red', invert_yaxis="True"): """ Moving object trajectory plotter. Parameters ---------- ephemeris_file: file object Ephemeris file. object_name : str Asteroid or moving object name. odate : str Ephemeris date of observation in date. min_mag : list or float Faintest magnitude to be plotted. Default is '20.0'. max_mag : float Brightest magnitude to be plotted. Default is '15.0'. circle_color : str Moving object mark color arrow_color : Trajectory color Returns ------- 'A table object or file' """ from .catalog import Query # filename = get_pkg_data_filename(image_path) rcParams['figure.figsize'] = [12., 12.] # rcParams.update({'font.size': 10}) fo = FileOps() srg = fo.srg_ephemeris_reader( "/Users/ykilic/Downloads/RTT-150_20200109-1708_20200110-0423.txt") data_len = len(srg) mid = int(len(srg) / 2) data_mid_date = srg['Date-Time'][mid] ra = srg['RA2000'][mid] dec = srg['DECL2000'][mid] odate = data_mid_date if radius is None: ra_first = srg['RA2000'][0] dec_first = srg['DECL2000'][0] ra_last = srg['RA2000'][data_len - 1] dec_last = srg['DECL2000'][data_len - 1] c_first = coordinates.SkyCoord(ra_first, dec_first, unit=(u.hourangle, u.deg), frame='icrs') c_last = coordinates.SkyCoord(ra_last, dec_last, unit=(u.hourangle, u.deg), frame='icrs') radius = c_first.separation(c_last) radius = radius.arcmin if image_path is not None: hdu = fits.open(image_path)[0] elif (image_path is None) and ra and dec and odate: co = coordinates.SkyCoord('{0} {1}'.format(ra, dec), unit=(u.hourangle, u.deg), frame='icrs') print('Target Coordinates:', co.to_string(style='hmsdms', sep=':'), 'in {0} arcmin'.format(radius)) try: print(co) server_img = SkyView.get_images(position=co, survey=['DSS'], radius=radius * u.arcmin) hdu = server_img[0][0] except Exception as e: print("SkyView could not get the image from DSS server.") print(e) raise SystemExit fig = aplpy.FITSFigure(hdu, figsize=(12, 12)) srg_c = coordinates.SkyCoord(srg['RA2000'], srg['DECL2000'], unit=(u.hourangle, u.deg), frame='icrs') srg['APLHA_J2000'] = srg_c.ra.degree srg['DELTA_J2000'] = srg_c.dec.degree table = XMatch.query(cat1=srg['APLHA_J2000', 'DELTA_J2000', 'RA2000', 'DECL2000'], cat2='vizier:{}'.format("I/345/gaia2"), max_distance=srg_radius * u.arcsec, colRA1='APLHA_J2000', colDec1='DELTA_J2000') table_pd = table.to_pandas() table_pd_masked = table_pd[(table_pd['phot_g_mean_mag'] >= min_mag) & (table_pd['phot_g_mean_mag'] <= max_mag)] # fig.show_markers(srg['APLHA_J2000'], srg['DELTA_J2000'], edgecolor='green') fig.show_markers(table_pd_masked['APLHA_J2000'], table_pd_masked['DELTA_J2000'], edgecolor='red') # fig.show_markers(srg['APLHA_J2000'], srg['DELTA_J2000'], edgecolor='red') srg_pd = srg.to_pandas() for i, element in enumerate(table_pd_masked['RA2000']): srg_pd = srg_pd[srg_pd.RA2000 != element] srg_best_positions = srg_pd fig.show_markers(srg_best_positions['APLHA_J2000'], srg_best_positions['DELTA_J2000'], edgecolor='blue') fig.show_grayscale(invert=True) fig.add_colorbar() fig.add_grid() fig.set_title("{} {}".format(ra, dec)) return srg_best_positions
def crossmatch_tables(d1, d2, fout=None, join_type="1 and 2", ra1='RA_deg', dec1="DEC_deg", ra2='RA_deg', vizier=False, dec2="DEC_deg", conrad=3.0, verbose=False, d2_unicol=None, match_sel="best for both", mark_groups=True): """ Crossmatch two astropy tables by coordinates. At the moment all matches are considered not only the closest. If the vizier flag is set, then d1 has to be the local table, while d2 has to be the vizier ID of the online table (in this case, obviously, the options "1 or 2" and "2 not 1" do not work). """ # --- make sure the tables support masking dN = T.Table(d1, masked=True) nN = len(dN) ncolsN = len(dN.columns) # print(dN.columns) join_type = join_type.lower() match_sel = match_sel.lower() joins = [ "1 and 2", "1 not 2", "2 not 1", "1 or 2", "all from 1", "all from 2" ] matches = ["best for both", "best for 1", "best for 2", "all"] if join_type not in joins: print("CROSSMATCH_TABLES: ERROR: selected join type not valid: ", join_type) print(" - available types are: ", joins) return (-1) if match_sel not in matches: print("CROSSMATCH_TABLES: ERROR: selected match selection not valid: ", match_sel) print(" - available selections are: ", matches) return (-1) if not vizier: dC = T.Table(d2, masked=True) nC = len(dC) ncolsC = len(dC.columns) if np.sum(dC[ra2].mask) + np.sum(dC[dec2].mask) > 0: print( "CROSSMATCH_TABLES: ERROR: Table 2 contains empty coordinates! Aborting..." ) return (-1) coordsN = coordinates.SkyCoord(ra=dN[ra1] * u.degree, dec=dN[dec1] * u.degree) coordsC = coordinates.SkyCoord(ra=dC[ra2] * u.degree, dec=dC[dec2] * u.degree) if verbose: print(timestamp() + ": CROSSMATCH_TABLES: Input parameters: ") print(" - input rows (left): ", nN) if not vizier: print(" - input rows (right): ", nC) print(" - join type: ", join_type) print(" - match selection: ", match_sel) print(" - cone radius: ", conrad) print(" - vizier XMATCH: ", vizier) print(" - RA 1: ", ra1) print(" - DEC 1: ", dec1) print(" - RA 2: ", ra2) print(" - DEC 2: ", dec2) print(" - output file: ", fout) # --- the crossmatch can not handle masked coordinates! if np.sum(dN[ra1].mask) + np.sum(dN[dec1].mask) > 0: print( "CROSSMATCH_TABLES: ERROR: Table 1 contains empty coordinates! Aborting..." ) return (-1) if verbose: print(timestamp() + ": CROSSMATCH_TABLES: Doing the crossmatch...") sys.stdout.flush() # --- in case of the online XMATCH with vizier table: if vizier: if join_type == "1 or 2" or join_type == "2 not 1": print( "CROSSMATCH_TABLES: ERROR: For XMATCH with VISIR '1 or 2' and '2 not 1' are not available! Aborting..." ) return (-1) # --- crerate small table for upload (the columns can not be of masked # type either) dtemp = T.Table({ "UniID": range(nN), ra1: np.array(dN[ra1]), dec1: np.array(dN[dec1]) }) dC = XMatch.query(cat1=dtemp, cat2=d2, max_distance=conrad * u.arcsec, colRA1=ra1, colDec1=dec1) idxl = np.array(dC["UniID"]) idxr = np.array(range(len(dC))) sep = dC["angDist"] nC = len(dC) del dC[ra1] del dC[dec1] del dC["UniID"] del dC["angDist"] ncolsC = len(dC.columns) # --- now we need to reconstruct the right side table if d2_unicol is not None: d2IDs = np.array(dC[d2_unicol]) d2unique = np.unique(d2IDs) nrunique = len(d2unique) else: d2IDs = None else: idxr, idxl, sep, _ = coordsN.search_around_sky(coordsC, conrad * u.arcsec) nrunique = len(np.unique(idxr)) sep = sep.value * 3600 d2IDs = None nmatch = len(idxr) #nmatch_vol = np.sum(dN['NEDredshift'][idxr] < zlim) lunique = np.unique(idxl) nlunique = len(lunique) if verbose: print( timestamp() + ": CROSSMATCH_TABLES: Total number of found matches: ", nmatch) print( " CROSSMATCH_TABLES: Number of unique left sources match: ", nlunique) print( " CROSSMATCH_TABLES: Number of unique right sources match: ", nrunique) # --- if we are only interested in the non-matches we are done here: if join_type == "1 not 2": # --- ids of those not in the match idNnomatch = [x for x in range(nN) if x not in idxl] dout = dN[idNnomatch] if fout is not None: dout.write(fout, delimiter=',', format='ascii', fill_values=[(ascii.masked, '')], overwrite=True) if verbose: print("CROSSMATCH_TABLES: Number of non-matches from 1: ", len(dout)) return (dout) elif join_type == "2 not 1": # --- ids of those not in the match idCnomatch = [x for x in range(nC) if x not in idxr] dout = dC[idCnomatch] if fout is not None: dout.write(fout, delimiter=',', format='ascii', fill_values=[(ascii.masked, '')], overwrite=True) if verbose: print("CROSSMATCH_TABLES: Number of non-matches from 2: ", len(dout)) return (dout) # --- Groups and Duplicates # --- get allleft side duplicates id_uni_l, lcounts = np.unique(idxl, return_counts=True) id_dupl_l = np.array(id_uni_l[lcounts > 1]) n_dupl_l = len(id_dupl_l) # --- get all right side duplicates if vizier and d2_unicol is not None: id_uni_r, rcounts = np.unique(d2IDs, return_counts=True) id_dupl_r = np.array(id_uni_r[rcounts > 1]) n_dupl_r = len(id_dupl_r) else: id_uni_r, rcounts = np.unique(idxr, return_counts=True) id_dupl_r = id_uni_r[rcounts > 1] n_dupl_r = len(id_dupl_r) # --- In case of best matches only select correspondingly select = np.ones(nmatch, dtype=bool) if match_sel == "best for both" or match_sel == "best for 1": if verbose: print(timestamp() + "CROSSMATCH_TABLES: Finding best match on the left side...") for i in tqdm(range(n_dupl_l)): ids = np.where(idxl == id_dupl_l[i])[0] exclude = np.where(sep[ids] > np.nanmin(sep[ids]))[0] keep = np.where(sep[ids] == np.nanmin(sep[ids]))[0] if len(keep) > 1: if verbose: print( " - WARNING: more than 1 object at minimum separation: ", len(keep)) select[ids[exclude]] = False # print(i, idxl[ids], np.nanmin(sep[idxl[ids]]), exclude, idxl[ids[exclude]]) if match_sel == "best for both" or match_sel == "best for 2": if verbose: print(timestamp() + "CROSSMATCH_TABLES: Finding best match on the right side...") for i in tqdm(range(n_dupl_r)): ids = np.where(idxr == id_dupl_r[i])[0] exclude = np.where(sep[ids] > np.nanmin(sep[ids]))[0] select[ids[exclude]] = False if verbose: print("CROSSMATCH_TABLES: Number of excluded matches: ", np.sum(np.invert(select))) idxl = idxl[select] idxr = idxr[select] sep = sep[select] if vizier: # idxr = np.array(range(len(idxr))) d2IDs = d2IDs[select] # --- Group business is only requierd if multiple matches are allowed if match_sel != "best for both" and mark_groups: # --- Add the columns of the tables to each other if verbose: print(timestamp() + ": CROSSMATCH_TABLES: Adding columns...") for i in range(ncolsN): if dN.columns[i].name not in dC.columns: dC.add_column( T.MaskedColumn(np.ma.zeros(nC, dtype=dN.columns[i].dtype), name=dN.columns[i].name)) dC[dN.columns[i].name] = np.ma.masked # print(" Added column: ", dN.columns[i].name) dC.add_column( T.MaskedColumn(np.ma.zeros(nC, dtype=float), name="separation_as")) dC["separation_as"] = np.ma.masked dC.add_column( T.MaskedColumn(np.ma.zeros(nC, dtype=int), name="groupID")) dC.add_column( T.MaskedColumn(np.ma.zeros(nC, dtype=int), name="groupsize")) for i in range(ncolsC): if dC.columns[i].name not in dN.columns: dN.add_column( T.MaskedColumn(np.ma.zeros(nN, dtype=dC.columns[i].dtype), name=dC.columns[i].name)) dN[dC.columns[i].name] = np.ma.masked # print(dN.columns) dN.add_column( T.MaskedColumn(np.ma.zeros(nN, dtype=float), name="separation_as")) dN["separation_as"] = np.ma.masked dN.add_column( T.MaskedColumn(np.ma.zeros(nN, dtype=int), name="groupID")) dN.add_column( T.MaskedColumn(np.ma.zeros(nN, dtype=int), name="groupsize")) # --- now identify the groups if verbose: print(timestamp() + ": CROSSMATCH_TABLES: Identifying groups...") lgroupids, lgroupsizes, rgroupids, rgroupsizes = _identify_groups( idxl, idxr, nN, nC, match_sel=match_sel, d2IDs=d2IDs, verbose=verbose) dN['groupID'][idxl] = lgroupids[idxl] dC['groupID'][idxr] = rgroupids[idxr] dN['groupsize'][idxl] = lgroupsizes[idxl] dC['groupsize'][idxr] = rgroupsizes[idxr] # --- then we build a table with the matches dmatch = dN[idxl] for i in range(ncolsC): dmatch[dC.columns[i].name] = dC[dC.columns[i].name][idxr] # --- fill in the separations dmatch['separation_as'] = sep #for i in range(len(dmatch)): print(dmatch['NEDname'][i], " <--> ", dmatch['CDSname'][i]) if verbose: print(timestamp() + ": CROSSMATCH_TABLES: Preparing output...") # --- now the output options: if join_type == "1 or 2": # --- ids of those not in the match idNnomatch = [x for x in range(nN) if x not in idxl] idCnomatch = [x for x in range(nC) if x not in idxr] dout = T.vstack([dN[idNnomatch], dmatch, dC[idCnomatch]]) elif join_type == "all from 1": # --- ids of those not in the match idNnomatch = [x for x in range(nN) if x not in idxl] dout = T.vstack([dN[idNnomatch], dmatch]) elif join_type == "all from 2": # --- ids of those not in the match idCnomatch = [x for x in range(nC) if x not in idxr] dout = T.vstack([dC[idCnomatch], dmatch]) else: # out == "1 and 2" dout = dmatch # --- mask the empy values if 'groupID' in dout.colnames: idnull = dout['groupID'] == 0 dout['groupID'][idnull] = np.ma.masked dout['groupsize'][idnull] = np.ma.masked if fout is not None: dout.write(fout, delimiter=',', format='ascii', fill_values=[(ascii.masked, '')], overwrite=True) if verbose: print(timestamp() + ": CROSSMATCH_TABLES: Number of output lines: ", len(dout)) return (dout, idxl, idxr)
'extinction_error' not in config['mapping']: print('Adding extinction data') data.add_column( Column(name='pix', data=hp.ang2pix(512, data['l'], data['b'], lonlat=True))) extinction_data = Table.read( '/home/mints/data/extinction/lambda_sfd_ebv_Ak.fits') data = astropy_join(data, extinction_data, keys=('pix'), join_type='left') if '2MASS' not in has_matches and '2MASS' in need_matches: print('XMatching with 2MASS') data = XMatch.query(cat1=data, cat2='vizier:II/246/out', max_distance=3 * u.arcsec, colRA1=ra, colDec1=dec, responseformat='votable', selection='best') data.remove_columns( ['angDist', 'errHalfMaj', 'errHalfMin', 'errPosAng', 'X', 'MeasureJD']) clean(data) if 'AllWISE' not in has_matches and 'AllWISE' in need_matches: print('XMatching with AllWISE') data = XMatch.query(cat1=data, cat2='vizier:II/328/allwise', max_distance=3 * u.arcsec, colRA1=ra, colDec1=dec, responseformat='votable',
cat1.rename_column('col1', 'ra_rfc') cat1.rename_column('col2', 'dec_rfc') cat1.rename_column('col3', 'id_rfc') cat1.info() if survey_xmatch == 'VHS': result = Table.read(infile2) itest = np.unique(result['UPLOAD_ID']) if survey_xmatch != 'VHS': print('cat1:', cat1) print('Vizier xmatch:', cat2) result = XMatch.query(cat1=cat1, cat2=cat2, max_distance=1.0 * u.arcsec, colRA1='ra_rfc', colDec1='dec_rfc') itest = np.unique(result['id_rfc']) print(len(itest), len(result)) print('Elapsed time(secs): ', time.time() - t0) print(type(result), len(result)) result.info() result.info('stats') #sys.exit() if survey_xmatch == 'VHS': ra_rfc = result['UPLOAD_RA'] dec_rfc = result['UPLOAD_DEC']
def match_catalog_and_phot(self, file_name, ra_keyword="ALPHA_J2000", dec_keyword="DELTA_J2000", catalogue='I/345/gaia2', filter=None, phot_object=None, phot_method=None, plot=False, catalog_output=False): """ Basic circular aperture photometry of given images. Parameters ---------- file_name : file object File name to be source extracted. ra_keyword : str RA keyword in catalogue. Default is "ALPHA_J2000" dec_keyword : str DEC keyword in catalogue. Default is "DEC_J2000" catalogue : vizer catalogue object Vizer catalogue name Default is 'I/345/gaia2'. filter : str Filter of image. Default is 'Vmag'. phot_object : astropy table Astropy table of objetcs that contains "ALPHA_J2000" and "DEC_J2000" columns. Default is None. phot_method : astropy table MAG_AUTO or None Default is None. plot: boolean Shell we plot the correlation? Returns ------- 'A dict object' Examples -------- >>> from astrolib import catalog >>> from astrolib import astronomy >>> from astropy.table import Table >>> co = catalog.Query() >>> ac = astronomy.AstCalc() >>> c = ac.radec2wcs("21:05:15.250", "+07:52:6.734") >>> ra_list_in_deg = [c.ra.degree] >>> dec_list_in_deg = [c.dec.degree] >>> phot_object = Table([ra_list_in_deg, dec_list_in_deg], names=("ALPHA_J2000", "DELTA_J2000")) >>> phot_method = "MAG_AUTO" >>> co.match_catalog("file.fits", catalogue="II/336/apass9", filter="Vmag", phot_object=phot_object, phot_method="MAG_AUTO", plot=True) """ fo = FitsOps(file_name) object_name = fo.get_header("OBJECT") exptime = fo.get_header("EXPTIME") telescope = fo.get_header("TELESCOP") if filter is None: filter = fo.get_header("FILTER") if "T100" in telescope: if "u" in filter: filter = "upmag" elif "g" in filter: filter = "gpmag" elif "r" in filter: filter = "rpmag" elif "i" in filter: filter = "ipmag" elif "z" in filter: filter = "zpmag" elif "U" in filter: filter = "Umag" elif "B" in filter: filter = "Bmag" elif "V" in filter: filter = "Vmag" elif "R" in filter: filter = "Rmag" elif "I" in filter: filter = "Imag" else: raise SystemExit('Filter not found!') ds = fo.source_extract() table = XMatch.query(cat1=ds, cat2='vizier:{}'.format(catalogue), max_distance=5 * u.arcsec, colRA1=ra_keyword, colDec1=dec_keyword) print(table.colnames) if phot_method is None: phot_method = "MAG_AUTO" table['deltaMag'] = table[filter] - table["MAG_AUTO"] else: table['deltaMag'] = table[filter] - table[phot_method] mean, median, stddev = stats.sigma_clipped_stats(table['deltaMag'], sigma=2, maxiters=5) linear_zero_point = None linear_r2 = None ransac_r2 = None ransac_zero_point = None linear_calibrated_mag = [] ransac_calibrated_mag = [] if "FLUX" in phot_method: X = np.log10(np.asarray(table[phot_method]).reshape(-1, 1)) else: X = np.asarray(table[phot_method]).reshape(-1, 1) y_ = np.asarray(table[filter]).reshape(-1, 1) imputer = SimpleImputer() y = imputer.fit_transform(y_) # Fit line using all data lr = linear_model.LinearRegression() lr.fit(X, y) # Robustly fit linear model with RANSAC algorithm ransac = linear_model.RANSACRegressor() ransac.fit(X, y) inlier_mask = ransac.inlier_mask_ outlier_mask = np.logical_not(inlier_mask) # Predict data of estimated models line_X = np.arange(X.min(), X.max())[:, np.newaxis] line_y = lr.predict(line_X) line_y_ransac = ransac.predict(line_X) # Compare estimated coefficients # linear_zero_point = lr.intercept_ # linear_slope = lr.coef_ ransac_slope = ransac.estimator_.coef_ ransac_zero_point = ransac.estimator_.intercept_ if phot_object is not None: phot_object[ra_keyword].unit = u.deg phot_object[dec_keyword].unit = u.deg for object in phot_object: c = coord.SkyCoord(object[ra_keyword], object[dec_keyword], frame="icrs", unit="deg") catalog = coord.SkyCoord(ds[ra_keyword], ds[dec_keyword], frame="icrs", unit="deg") max_sep = 1.0 * u.arcsec idx, d2d, d3d = c.match_to_catalog_3d(catalog) sep_constraint = d2d < max_sep to_be_calibrated_table = ds[idx] # linear_calibrated_mag.append( # lr.predict(np.asarray(to_be_calibrated_table[phot_method]).reshape(-1, 1))) ransac_calibrated_mag.append( ransac.predict(np.asarray(to_be_calibrated_table[phot_method]).reshape(-1, 1))) else: linear_calibrated_mag = [None] ransac_calibrated_mag = [None] if plot is True: lw = 2 rcParams['figure.figsize'] = 8, 8 plt.scatter(X[inlier_mask], y[inlier_mask], color='yellowgreen', marker='.', label='Inliers') plt.scatter(X[outlier_mask], y[outlier_mask], color='gold', marker='.', label='Outliers') # plt.plot(line_X, line_y, color='navy', linewidth=lw, label='Linear regressor') plt.plot(line_X, line_y_ransac, color='cornflowerblue', linewidth=lw, label='RANSAC regressor') plt.legend(loc='lower right') plt.title('{} ({} - {}), {} seconds'.format(object_name, catalogue, filter, exptime)) if "FLUX" in phot_method: plt.xlabel(phot_method + " (log)") else: plt.xlabel(phot_method) plt.ylabel(filter) # plt.xscale('log') plt.show() if catalog_output is True: cat_file = os.path.splitext(file_name)[0] np.savetxt("{}_ransac_inst_vs_cat.csv".format(cat_file), np.concatenate((X[inlier_mask], y[inlier_mask]), axis=1), delimiter=",") ascii.write(table, "{}.csv".format(cat_file), format='csv', fast_writer=False) try: ransac_calibrated_mag_result = ransac_calibrated_mag[0][0][0] except TypeError: ransac_calibrated_mag_result = None return ({'table': table[phot_method, "MAGERR_AUTO", filter], 'astropy_zero_point': median, 'ransac_zero_point': ransac_zero_point[0], 'ransac_slope': ransac_slope[0][0], 'stddev': stddev, 'std_error': stddev / math.sqrt(len(X[inlier_mask])), 'ransac_calibrated_mag': ransac_calibrated_mag_result, 'ransac_calibrated_mag_err': "{}".format(stddev / math.sqrt(len(X[inlier_mask]))), 'ransac_equation': "{}X + {}".format(ransac_slope[0][0], ransac_zero_point[0]), })
delta_find_sources=objects[:,DELTA_J2000] #t = fits.BinTableHDU.from_columns([c1,c2,c3,c4,c5,c6,c7,c8],name='valores') #t.writeto('a.fits',overwrite=True) t = fits.BinTableHDU.from_columns([c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13],name='valores') t.writeto('CFC_configuration/Intermediate_files/parametros.fits',overwrite=True) hdul = fits.open('CFC_configuration/Intermediate_files/parametros.fits') delta=hdul['VALORES'] t = Table.read(delta) t.write('CFC_configuration/Intermediate_files/parametros.vot', table_id='updated_table', format='votable',overwrite=True) sdss_key=0 catalog = XMatch.query(cat1=open('CFC_configuration/Intermediate_files/parametros.vot'), cat2='vizier:V/147/sdss12', max_distance=2 * u.arcsec, colRA1='ALPHA', colDec1='DELTA') if len(catalog)==0: print('No SSDS field, working with APASS DR9 catalog') catalog = XMatch.query(cat1=open('CFC_configuration/Intermediate_files/parametros.vot'), cat2='vizier:II/336/apass9', max_distance=2 * u.arcsec, colRA1='ALPHA', colDec1='DELTA') sdss_key=1 if len(catalog)<6: if len(catalog)==0: motivo='no SDSS/APASS coverage' print('no SDSS/APASS coverage')
def cal_image(img_path): image_path = os.path.join(basepath, img_path) # load img with WCS if not '.new' in image_path: img = image_path.strip('.fits') + '.new' else: img = image_path fits_file = fits.open(img, mode='update') hdr = fits_file[0].header #fits.getheader(img) data = fits_file[0].data # fits.getdata(img) wcs = WCS(hdr) # Obtain image center crval_ra= hdr['CRVAL1'] crval_dec = hdr['CRVAL2'] crval_x = hdr['CRPIX1'] crval_y = hdr['CRPIX2'] # Get sources in image mask = make_source_mask(data, snr=2, npixels=5, dilate_size=11) mask_zeros = data==0 mask = mask | mask_zeros mean, median, std = sigma_clipped_stats(data, sigma=3.0, mask=mask) sigma_clip = SigmaClip(sigma=3., iters=10) bkg_estimator = MedianBackground() bkg_estimator = SExtractorBackground() bkg = Background2D(data, (50, 50), filter_size=(3, 3), sigma_clip=sigma_clip, bkg_estimator=bkg_estimator, mask=mask) back = bkg.background * ~mask daofind = DAOStarFinder(fwhm=5.0, threshold=5.*std) sources = daofind(data - median) #print(sources) positions = (sources['xcentroid'], sources['ycentroid']) xypos = np.array([sources['xcentroid'], sources['ycentroid']]).T sky_pos = wcs.all_pix2world(xypos, 0) skycoords = coord.SkyCoord(sky_pos*u.deg, frame='icrs') radii = [5.*u.arcsec, 7.*u.arcsec, 10.*u.arcsec] annulus = [(6.*u.arcsec, 8.*u.arcsec), (9.*u.arcsec, 11.*u.arcsec), (11.*u.arcsec, 12.*u.arcsec)] apertures = [SkyCircularAperture(skycoords, r=r).to_pixel(wcs) for r in radii] annulii = [SkyCircularAnnulus(skycoords, r_in=ri, r_out=ro).to_pixel(wcs) for ri, ro in annulus] apers = apertures + annulii #~ tables = [aperture_photometry(data-back, aper) for aper in apers] phot_table = aperture_photometry(data-back, apers) bkg_sum_0 = apertures[0].area()*phot_table['aperture_sum_3']/annulii[0].area() bkg_sum_1 = apertures[0].area()*phot_table['aperture_sum_4']/annulii[1].area() bkg_sum_2 = apertures[0].area()*phot_table['aperture_sum_5']/annulii[2].area() final_sum_0 = phot_table['aperture_sum_0'] - bkg_sum_0 final_sum_1 = phot_table['aperture_sum_1'] - bkg_sum_1 final_sum_2 = phot_table['aperture_sum_2'] - bkg_sum_2 phot_table['resid_aper_sum_0'] = final_sum_0 phot_table['resid_aper_sum_1'] = final_sum_1 phot_table['resid_aper_sum_2'] = final_sum_2 phot_table['cmag_0'] = -2.5 * np.log10(final_sum_0) phot_table['cmag_1'] = -2.5 * np.log10(final_sum_1) phot_table['cmag_2'] = -2.5 * np.log10(final_sum_2) phot_table['mag_0'] = -2.5 * np.log10(phot_table['aperture_sum_0']) phot_table['mag_1'] = -2.5 * np.log10(phot_table['aperture_sum_1']) phot_table['mag_2'] = -2.5 * np.log10(phot_table['aperture_sum_2']) phot_table['RA'] = skycoords.ra phot_table['Dec'] = skycoords.dec # match the catalogs # attempt a xmatch phot_table.write('/tmp/sourcecat.csv', overwrite=True) try: from astroquery.xmatch import XMatch XMatch.TIMEOUT = 600 try: table = XMatch.query(cat1=open('/tmp/sourcecat.csv'), #cat2='vizier:II/336/apass9', #apass cat2='vizier:I/322A/out', # UCAC4 #cat2='vizier:II/246/out', # 2MASS max_distance=5 * u.arcsec, colRA1='RA', colDec1='Dec', colRA2='RAJ2000', colDec2='DEJ2000') print 'Using UCAC4' print table.colnames if 'Rmag' not in table.colnames and 'rmag' in table.colnames: table['Rmag'] = table['rmag'] except: try: table = XMatch.query(cat1=open('/tmp/sourcecat.csv'), cat2='vizier:II/336/apass9', #apass #cat2='vizier:I/322A/out', # UCAC4 #cat2='vizier:II/246/out', # 2MASS max_distance=5 * u.arcsec, colRA1='RA', colDec1='Dec', colRA2='RAJ2000', colDec2='DEJ2000') print 'Using APASS' print table.colnames if 'Rmag' not in table.colnames and 'rmag' in table.colnames: table['Rmag'] = table['rmag'] except: try: table = XMatch.query(cat1=open('/tmp/sourcecat.csv'), #cat2='vizier:II/336/apass9', #apass #cat2='vizier:I/322A/out', # UCAC4 cat2='vizier:II/246/out', # 2MASS max_distance=5 * u.arcsec, colRA1='RA', colDec1='Dec', colRA2='RAJ2000', colDec2='DEJ2000') print 'Using 2MASS' print table.colnames if 'Rmag' not in table.colnames and 'rmag' in table.colnames: table['Rmag'] = table['rmag'] except: print('\ntable 1 avaliable?:') print XMatch.is_table_available('vizier:II/336/apass9') print('\ntable 2 avaliable?:') print XMatch.is_table_available('vizier:I/322A/out') print('\ntable 3 avaliable?:') print XMatch.is_table_available('vizier:II/246/out') raise except: # query PanSTARRS DR1 #~ catalog = panstarrs_query(crval_ra, crval_dec, 0.5, maxsources=300) #~ if len(ps1_tab) < 20: #~ from astroquery.vizier import Vizier #~ catalog = Vizier.query_region(coord.SkyCoord(crval_ra*u.deg, #~ crval_dec*u.deg, #~ frame='icrs'), #~ radius=coord.Angle(0.3, "deg"), #~ catalog='UCAC4')[0] #~ ra = catalog['RAJ2000'] #~ dec = catalog['DEJ2000'] #~ cat = coord.SkyCoord(ra, dec, frame='icrs') #~ # cat to sources #~ idx_c2s, d2d_c2s, _ = skycoords.match_to_catalog_sky(cat) #~ # sources to cat #~ idx_s2c, d2d_s2c, _ = cat.match_to_catalog_sky(skycoords) #~ matches = cat[idx_c2s] # same length than skycoords #~ idx_c2s[idx_s2c] raise from sklearn import linear_model if 'UCAC4' in table.colnames: table['color'] = table['Bmag'] - table['Vmag'] if '2MASS' in table.colnames: table['color'] = table['Jmag'] - table['Kmag'] table['Bmag'] = table['Jmag'] # V Mag plt.subplot(221) reg = linear_model.LinearRegression() sub = table[['mag_1', 'Vmag']].to_pandas() sub = sub.dropna(axis=0, how='any') X = sub[['mag_1']].as_matrix() Y = sub['Vmag'].as_matrix() reg.fit(X,Y) print reg.coef_, reg.intercept_ regV = reg table['pred_Vmag'] = reg.coef_*table['mag_1']+reg.intercept_ plt.plot(table['mag_1'], table['Vmag'], 'r.') plt.plot(table['mag_1'], table['pred_Vmag'], 'k-', label='linear fit') plt.xlabel('mag_1') plt.ylabel('Vmag') plt.grid() # B mag plt.subplot(222) reg = linear_model.LinearRegression() sub = table[['mag_1', 'Bmag']].to_pandas() sub = sub.dropna(axis=0, how='any') X = sub[['mag_1']].as_matrix() Y = sub['Bmag'].as_matrix() reg.fit(X,Y) print reg.coef_, reg.intercept_ regB = reg table['pred_Bmag'] = reg.coef_*table['mag_1']+reg.intercept_ plt.plot(table['mag_1'], table['Bmag'], 'r.') plt.plot(table['mag_1'], table['pred_Bmag'], 'k-', label='linear fit') plt.xlabel('mag_1') plt.ylabel('Bmag') plt.grid() # R mag plt.subplot(223) reg = linear_model.LinearRegression() sub = table[['mag_1', 'Rmag']].to_pandas() sub = sub.dropna(axis=0, how='any') X = sub[['mag_1']].as_matrix() Y = sub['Rmag'].as_matrix() reg.fit(X,Y) print reg.coef_, reg.intercept_ regR = reg table['pred_Rmag'] = reg.coef_*table['mag_1']+reg.intercept_ plt.plot(table['mag_1'], table['Rmag'], 'r.') plt.plot(table['mag_1'], table['pred_Rmag'], 'k-', label='linear fit') plt.xlabel('mag_1') plt.ylabel('Rmag') plt.grid() # Residuals vs B-V plt.subplot(224) plt.plot(table['color'], table['pred_Vmag']-table['Vmag'], 'g.', label='Vmag') plt.plot(table['color'], table['pred_Bmag']-table['Bmag'], 'b.', label='Bmag') plt.plot(table['color'], table['pred_Rmag']-table['Rmag'], 'r.', label='Rmag') plt.xlabel('color') plt.ylabel('Residual') plt.legend(loc='best') plt.grid() plt.tight_layout() plt.savefig(image_path.strip('.new')+'_calibration.png') plt.clf() hdr['ZP_R'] = regR.intercept_ hdr['ZP_B'] = regB.intercept_ hdr['ZP_V'] = regV.intercept_ hdr['SLOPE_R'] = regR.coef_[0] hdr['SLOPE_B'] = regB.coef_[0] hdr['SLOPE_V'] = regV.coef_[0] chosen = {'band': None, 'res': 10} for band in ['R', 'B', 'V']: res = table['pred_'+band+'mag']-table[band+'mag'] residual = np.sqrt(np.sum(np.square(res))) if residual < chosen['res']: chosen['res'] = residual chosen['band'] = band hdr['CAL_BAND'] = chosen['band'] hdr['CAL_RES'] = chosen['res'] fits_file.flush() fits_file.close() band = chosen['band'] plt.hist(table['pred_'+band+'mag']) plt.xlabel('instrumental '+band+' mag') plt.savefig(image_path.strip('.new')+'_magnitudes.png') table.write(image_path.strip('.new')+'_sources.csv', overwrite=True) with open('calibrations.txt', 'a') as cals: cals.write(image_path) cals.write(' ') cals.write('{}'.format(np.max(table['pred_'+band+'mag']))) cals.write(' ') cals.write('{}'.format(np.std(table['pred_'+band+'mag']))) cals.write(' ') cals.write('{}'.format(hdr['ZP_R'])) cals.write(' ') cals.write('{}'.format(hdr['ZP_B'])) cals.write(' ') cals.write('{}'.format(hdr['ZP_V'])) cals.write(' ') cals.write('{}'.format(hdr['SLOPE_R'])) cals.write(' ') cals.write('{}'.format(hdr['SLOPE_B'])) cals.write(' ') cals.write('{}'.format(hdr['SLOPE_V'])) cals.write(' ') cals.write('{}'.format(hdr['CAL_BAND'])) cals.write(' ') cals.write('{}'.format(hdr['CAL_RES'])) cals.write('\n') print 'finished'
"input_source_id": "source_id", "input_RAJ2000": "RAJ2000", "input_DEJ2000": "DEJ2000" }, inplace=True) #------------------------------------- #--------- set index ----------------- df.set_index("source_id", inplace=True) #------------------------------------- for cat in catalogues: #---------- Match --------------------- result = XMatch.query(cat1=tab, cat2=cat["reference"], max_distance=cat["max_distance"], colRA1='input_RAJ2000', colDec1='input_DEJ2000') #-------------------------------------- #-------- To pandas ----- tmp = result.to_pandas() #------------------------ #--------- Filter ----------------------- for key, value in cat["filters"].items(): tmp = tmp.loc[tmp[key] > value] #---------------------------------------- #-------- Drop duplicates ------------------ tmp.drop_duplicates(subset=["input_source_id"], keep="first", inplace=True)
#!/usr/bin/env python2 # -*- coding: utf-8 -*- """ Created on Fri Sep 20 11:59:50 2019 @author: rmk2432 """ from astropy import units as u from astroquery.xmatch import XMatch table = XMatch.query(cat1=open('BigCatalogs/GALAH_DR2_FEH.csv'), cat2='vizier:I/347/gaia2dis', max_distance=1 * u.arcsec, colRA1='ra', colDec1='dec') #table = XMatch.query(cat1=open('BigCatalogs/GALAH_DR2_FEH_TR.csv'), # cat2='vizier:I/347/gaia2dis',max_distance=1 * u.arcsec,colRA1='ra',colDec1='dec')
def xmatch(self, srcTable, raColName, decColName): return XMatch.query(cat1=srcTable, cat2='vizier:' + self.catFull, max_distance=5 * u.arcsec, colRA1=raColName, colDec1=decColName, colRA2='RAJ2000', colDec2='DEJ2000')