def _queryGaia(ID=None, coords=None, radius=2):
""" Query Gaia archive for bp-rp
Sends an ADQL query to the Gaia archive to look up a requested target ID or
set of coordinates.
If the query is based on coordinates a cone search will be performed and the
closest target is returned. Provided coordinates must be astropy.Skycoords.
Parameters
----------
ID : str
Gaia source ID to search for.
coord : astropy.Skycoords
An Astropy Skycoords object with the target coordinates. Must only
contain one target.
radius : float, optional
Radius in arcseconds to use for the sky cone search. Default is 20".
Returns
-------
bp_rp : float
Gaia bp-rp value of the requested target from the Gaia archive.
"""
from astroquery.gaia import Gaia
if ID is not None:
print('Querying Gaia archive for bp-rp values by target ID.')
adql_query = "select * from gaiadr2.gaia_source where source_id=%s" % (
ID)
try:
job = Gaia.launch_job(adql_query).get_results()
except:
return None
return float(job['bp_rp'][0])
elif coords is not None:
print('Querying Gaia archive for bp-rp values by target coordinates.')
ra = coords.ra.value
dec = coords.dec.value
adql_query = f"SELECT DISTANCE(POINT('ICRS', {ra}, {dec}), POINT('ICRS', ra, dec)) AS dist, * FROM gaiaedr3.gaia_source WHERE 1=CONTAINS( POINT('ICRS', {ra}, {dec}), CIRCLE('ICRS', ra, dec,{radius})) ORDER BY dist ASC"
try:
job = Gaia.launch_job(adql_query).get_results()
except:
return None
return float(job['bp_rp'][0])
else:
raise ValueError(
'No ID or coordinates provided when querying the Gaia archive.')
def get_gaia_data(self, index):
# get data about star
row = self.stars.iloc[index]
ls_id = row.ls_id
# get gaia id of star from decals
q = """SELECT
ra1, dec1, id1, ra2, dec2, id2, distance
FROM
ls_dr9.x1p5__tractor__gaia_edr3__gaia_source
WHERE
id1 = {} """.format(ls_id)
res = qc.query(sql=q)
decals_data = convert(res, 'pandas')
# get gaia id (or input id = 0 if no gaia data)
gaia_id = 0 if len(decals_data.id2) == 0 else decals_data.id2
# use gaia id to get info about it
query = """SELECT
source_id, ra, ra_error, dec, dec_error, parallax, pmra, pmra_error, pmdec, pmdec_error
FROM
gaiadr2.gaia_source
WHERE
source_id = {} """.format(int(gaia_id))
gaia_data = Gaia.launch_job(query).get_results().to_pandas()
return gaia_data
def get_bc_from_gaia(gaia_dr2_id, jd, rvabs=0):
"""
wrapper routine for using barycorrpy with Gaia DR2 coordinates
"""
# use 2015.5 as an epoch (Gaia DR2)
epoch = 2457206.375
# gaia_data = Gaia.query_object_async(coordinate=coord, width=width, height=height)
# q = Gaia.launch_job_async('SELECT * FROM gaiadr2.gaia_source WHERE source_id = ' + str(gaia_dr2_id))
q = Gaia.launch_job(
'SELECT * FROM gaiadr2.gaia_source WHERE source_id = ' +
str(gaia_dr2_id))
gaia_data = q.results
bc = barycorrpy.get_BC_vel(JDUTC=jd,
ra=gaia_data['ra'],
dec=gaia_data['dec'],
pmra=gaia_data['pmra'],
pmdec=gaia_data['pmdec'],
px=gaia_data['parallax'],
rv=rvabs * 1e3,
epoch=epoch,
obsname='AAO',
ephemeris='de430')
# bc = barycorrpy.get_BC_vel(JDUTC=utmjd, ra=ra, dec=dec, pmra=gaia_data['pmra'], pmdec=gaia_data['pmdec'],
# px=gaia_data['parallax'], rv=gaia_data['radial_velocity']*1e3, obsname='AAO', ephemeris='de430')
# bc = barycorrpy.get_BC_vel(JDUTC=utmjd, ra=ra, dec=dec, pmra=pmra, pmdec=pmdec,
# px=px, rv=rv, obsname='AAO', ephemeris='de430')
return bc[0][0]
def query_GaiaDR1(self, ID=None):
from astroquery.gaia import Gaia
key = 'Gaia DR1'
if ID is None:
ID = self.IDs[key]
tbl = Table(names = ('source_id',), dtype = (int,))
for i, gid in tqdm(enumerate(ID)):
if not isinstance(gid, str):
tbl.add_row(None)
tbl[-1][key] = int(re.sub(r"\D", "", gid))
elif len(gid) == 0:
tbl.add_row(None)
else:
gid = int(gid.replace(key+' ', ''))
adql_query = "select * from gaiadr1.gaia_source where source_id=%i" % (gid)
job = Gaia.launch_job(adql_query).get_results()
idx = np.where(job['source_id'].quantity == gid)[0]
if len(idx) > 0:
tbl = avstack([tbl, job[idx]])
else:
tbl.add_row(None)
tbl[-1][key] = int(re.sub(r"\D", "", gid))
self.GDR1 = tbl
return self.GDR1
def get_gaia_data(n_sources=2000):
"""Build a data structure for the Gaia data.
Paper on bands, colors, and equivalencies for GAIA:
arxiv.org/pdf/1008.0815.pdf'
Basically, since GAIA uses longer wavelength light, their colors don't map
to UBVRI bandpasses (p. 4, Table 1). This paper refers us to use B-R for
color, and since we used V magnitude when considering B-V color, I chose
to use the R band magnitude for my apparent magnitude values.
"""
print "Building GAIA dataframe for {} sources.".format(n_sources)
gaia_str = "SELECT top {} * FROM gaiadr2.gaia_source \
WHERE pmra between {} and {} \
AND pmdec between {} and {} \
AND ra between {} and {} \
AND dec between {} and {} \
AND parallax between {} and {} \
AND parallax_error < 2 \
AND parallax_over_error > 5 \
".format(n_sources, pmra_lims[0], pmra_lims[1], pmdec_lims[0],
pmdec_lims[1], posra_lims[0], posra_lims[1],
posdec_lims[0], posdec_lims[1], plx_lims[0],
plx_lims[1])
job = Gaia.launch_job(gaia_str) # , dump_to_file=True)
# job = Gaia.launch_job(gaia_str)
gaia_results_raw = job.get_results()
# gaia_results_raw['phot_rp_mean_mag'].description
gaia_results = gaia_results_raw.to_pandas()
# gaia_cols = sorted(gaia_results.columns)
print "Acquired data; now building dataframe..."
gaia_df = pd.DataFrame()
gaia_df['Distance'] = (gaia_results['parallax'] * 1e-3)**(-1)
gaia_df['Proper Motion (RA)'] = gaia_results['pmra']
gaia_df['Proper Motion (Dec)'] = gaia_results['pmdec']
gaia_df['mag'] = gaia_results['phot_rp_mean_mag']
gaia_df['Color'] = gaia_results['bp_rp']
gaia_df['Absolute Magnitude'] = gaia_df['mag'] - \
5 * (np.log10(gaia_df['Distance']) - 1)
gaia_df['T Effective'] = gaia_results['teff_val']
gaia_df['Parallax'] = gaia_results['parallax']
gaia_df['Plx. Error'] = gaia_results['parallax_error']
gaia_df['Confidence'] = 1 - gaia_results['parallax_error'] / max(
gaia_results['parallax_error'])
pleiades_gaia = {
'Survey': 'Gaia',
'Mean Distance': round(np.mean(gaia_df['Distance']), 1),
'Number of Stars': len(gaia_df['Distance']),
'Data': gaia_df,
'Full Results': gaia_results,
'Full Table': gaia_results_raw
}
return pleiades_gaia
def download_reino_tgas_full():
r = Gaia.launch_job("""
select * from TAP_UPLOAD.reino
left join gaiadr1.tgas_source tgas
on tgas.source_id = reino.source_id""",
upload_resource=Table.from_pandas(reino_tgas[['index','source_id']].astype(int)),
upload_table_name='reino').get_results()
return r.to_pandas()
def get_gaia_data_for_id(gaia_id, **kwargs):
j = Gaia.launch_job(
"select * from gaiadr2.gaia_source where source_id={0}".format(
gaia_id))
r = j.get_results()
if not len(r):
raise ValueError("no matches found")
return _parse_gaia_data(r[0], **kwargs)
def run_gaia2mass_cross_match(self,
upload_table='dummy.vot',
verbose=True,
add_input_cols="",
qpar_SN=10,
qpar_vis=7,
qpar_ruwe=1.40):
"""
Run a crossmatch between Gaia, 2MASS
"""
# 1.- Search input table ===============================
upload_resource = glob.glob(upload_table)
upload_table_name = "input_table"
if len(upload_resource) == 0:
raise Exception('Upload Table not found')
if add_input_cols != "":
add_input_cols = "," + add_input_cols + " "
# 2.- Write ADQL query =================================
query = (
"SELECT input_table.col2mass, " + self.gaia_cols +
",sqrt(gaia.astrometric_chi2_al/(gaia.astrometric_n_good_obs_al-5)) as unit_weight_e, g_ruwe.ruwe as ruwe "
"FROM tap_upload.input_table as input_table "
"LEFT OUTER JOIN gaiadr2.tmass_best_neighbour AS xmatch ON input_table.col2mass = xmatch.original_ext_source_id "
"LEFT OUTER JOIN gaiadr2.gaia_source AS gaia ON xmatch.source_id = gaia.source_id "
"LEFT OUTER JOIN gaiadr2.ruwe AS g_ruwe ON gaia.source_id = g_ruwe.source_id "
f"WHERE gaia.parallax/gaia.parallax_error >{qpar_SN} AND gaia.visibility_periods_used >{qpar_vis} AND g_ruwe.ruwe <{qpar_ruwe}"
)
# 3.- Run ADQL query ===================================
warnings.simplefilter('ignore', category=AstropyWarning)
print(f'RUNNING ADQL SYNCRHRONOUS QUERY ' + '=' * 58)
job = Gaia.launch_job(query=query,
upload_resource=upload_resource[0],
upload_table_name=upload_table_name,
verbose=verbose)
self.cat = job.get_results()
flag_psn, flag_vis, flag_ruwe = '', '', ''
if qpar_SN == 10: flag_psn = '(Default)'
if qpar_vis == 7: flag_vis = '(Default)'
if qpar_ruwe == 1.40: flag_ruwe = '(Default)'
if verbose:
print('=' * 90)
print(
f'Selection Criteria in Parallax S/N: Parallax S/N > {qpar_SN} {flag_psn}'
)
print(
f'Selection Criteria in Visibility Periods Used: Vis > {qpar_vis} {flag_vis}'
)
print(
f'Selection Criteria in RUWE: RUWE < {qpar_ruwe} {flag_ruwe}'
)
print()
print(f'SAMPLE OUTPUT N_els = {len(self.cat):3.0f}')
print('=' * 63)
def query_gaiadr3_names_from_dr2(input_table):
gaiadr3_query_string = "SELECT * FROM gaiaedr3.dr2_neighbourhood " \
"INNER JOIN tap_upload.upload_table ON " \
"gaiaedr3.dr2_neighbourhood.dr2_source_id = tap_upload.upload_table.source_id"
job_gaiadr3_query = Gaia.launch_job(gaiadr3_query_string,
upload_resource=input_table,
upload_table_name="upload_table",
verbose=VERBOSE)
gaiadr3_names = job_gaiadr3_query.get_results()
print("length of eDR3 names table: ", len(gaiadr3_names))
# Find DR2 sources with only one EDR3 entry
gaiadr3_unique = table.unique(gaiadr3_names,
keys='dr2_source_id',
keep='none')
print("Number of unique sources: ", len(gaiadr3_unique))
# Find the DR2 sources with multiple EDR3 matches
dr3_dupes = setdiff(gaiadr3_names, gaiadr3_unique, keys='dr2_source_id')
dr3_dupes_grouped = dr3_dupes.group_by('dr2_source_id')
# Find the best EDR3 match for DR2 sources with multiple matches
for group in dr3_dupes_grouped.groups:
# Find the EDR3 matches with the smallest magnitude difference (if one exists) and angular distance.
if not np.ma.is_masked(min(abs(group['magnitude_difference']))):
min_mag_index = abs(group['magnitude_difference']).tolist().index(
min(abs(group['magnitude_difference'])))
min_angdist_index = group['angular_distance'].tolist().index(
min(group['angular_distance']))
if min_angdist_index == min_mag_index:
# Source with smallest mag difference is the same as the closest source
verboseprint(
'best one found \n', group['dr2_source_id',
'dr3_source_id'][min_angdist_index],
'\n')
gaiadr3_unique.add_row(group[min_mag_index])
elif np.ma.is_masked(min(abs(group['magnitude_difference']))):
# Magnitude differences aren't available for all sources so just using closest one
verboseprint(
'best one found just with angular distance \n',
group['dr2_source_id',
'dr3_source_id'][min_angdist_index], '\n')
gaiadr3_unique.add_row(group[min_mag_index])
else:
verboseprint(
'no choice for \n',
group['dr2_source_id', 'dr3_source_id', 'magnitude_difference',
'angular_distance'], '\n')
verboseprint("Number of unique sources after dupe fix:",
len(gaiadr3_unique))
return gaiadr3_unique
def coords_from_gaia(gaia_id):
"""Returns table of Gaia DR2 data given a source_id."""
from astroquery.gaia import Gaia
import warnings
warnings.filterwarnings('ignore', module='astropy.io.votable.tree')
adql = 'SELECT gaia.source_id, ra, dec FROM gaiadr2.gaia_source AS gaia WHERE gaia.source_id={0}'.format(gaia_id)
job = Gaia.launch_job(adql)
table = job.get_results()
return table
def augment_with_gaia(cluster):
new_cols = ['phot_bp_mean_mag', 'phot_rp_mean_mag']
cols = 'source_id, ' + ', '.join(new_cols)
query = 'SELECT ' + cols + ' FROM gaiadr2.gaia_source WHERE source_id = '
query += ' OR source_id = '.join(list(map(str, cluster['Source'])))
newjob = Gaia.launch_job(query=query)
results = newjob.get_results()
cluster.add_columns([results[col] for col in new_cols])
return cluster
def synchronousQuery():
query1 = """SELECT
TOP 10
source_id, ref_epoch, ra, dec, parallax
FROM gaiadr2.gaia_source"""
job1 = Gaia.launch_job(query1)
print(job1)
results1 = job1.get_results()
results1.pprint_all()
def xmatch_gaia_to_Tmag_relations(stats, projid='1301'):
"""
using gaia IDs. get matches in tmass_best_neighbour table. use the
resulting Gmag and 2mass photometry values, + the relations given by
Stassun, to get Tmags.
(simpler than spatial+magnitude crossmatch)
"""
catdir = '/home/luke/local/tess-trex/catalogs/proj1301'
reformedcatalogfile = os.path.join(
catdir,
'GAIADR2-RA1.2049336110863-DEC-26.0582248258775-SIZE24.reformed_catalog'
)
columns = 'id,ra,dec,xi,eta,G,Rp,Bp,plx,pmra,pmdec'
columns = columns.split(',')
catarr = np.genfromtxt(reformedcatalogfile,
comments='#',
usecols=list(range(len(columns))),
names=columns,
dtype='U19,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8',
delimiter=' ')
catarr_w_lcs = catarr[np.in1d(catarr['id'], stats['lcobj'])]
# match catarr to whatever "stats" lightcurve exist TODO
# i have gaia IDs. i will get 2mass information via... the gaia to 2mass
# crossmatch table
tmass_ids = []
for source_id in tqdm(stats['lcobj']):
querystr = ("select top 1 * from gaiadr2.tmass_best_neighbour "
"where source_id = {}".format(source_id))
job = Gaia.launch_job(querystr)
res = job.get_results()
if len(res) == 0:
tmass_ids.append(-1)
elif len(res) == 1:
tmass_ids.append(
res['original_ext_source_id'].data.data[0].decode('utf-8'))
else:
raise AssertionError
outdf = pd.DataFrame({
'gaia_dr2_id': stats['lcobj'],
'tmass_id': tmass_ids
})
outpath = (
'../data/rms_vs_mag/proj{}_xmatch_gaia_to_tmass.csv'.format(projid))
outdf.to_csv(outpath, index=False)
print('saved {}'.format(outpath))
def get_local_data(n_sources=10000):
"""Build a data structure for the Gaia data.
Paper on bands, colors, and equivalencies for GAIA:
arxiv.org/pdf/1008.0815.pdf'
Basically, since GAIA uses longer wavelength light, their colors don't map
to UBVRI bandpasses (p. 4, Table 1). This paper refers us to use B-R for
color, and since we used V magnitude when considering B-V color, I chose
to use the R band magnitude for my apparent magnitude values.
"""
print "Building local dataframe for {} sources.".format(n_sources)
local_str = "SELECT top 20000 * FROM gaiadr2.gaia_source \
WHERE parallax between 1 and 20"
job = Gaia.launch_job(local_str) # , dump_to_file=True)
# job = Gaia.launch_job(gaia_str)
local_results_raw = job.get_results()
# local_results_raw['phot_rp_mean_mag'].description
local_results = local_results_raw.to_pandas()
# local_cols = sorted(local_results.columns)
print "Acquired data; now building dataframe..."
local_df = pd.DataFrame()
local_df['Distance'] = (local_results['parallax'] * 1e-3)**(-1)
local_df['Proper Motion (RA)'] = local_results['pmra']
local_df['Proper Motion (Dec)'] = local_results['pmdec']
local_df['mag'] = local_results['phot_rp_mean_mag']
local_df['Color'] = local_results['bp_rp']
local_df['Absolute Magnitude'] = local_df['mag'] - \
5 * (np.log10(local_df['Distance']) - 1)
local_df['T Effective'] = local_results['teff_val']
local_df['Parallax'] = local_results['parallax']
local_df['Plx. Error'] = local_results['parallax_error']
local_df['Confidence'] = 1 - local_results['parallax_error'] / max(
local_results['parallax_error'])
pleiades_local = {
'Survey': 'local',
'Mean Distance': round(np.mean(local_df['Distance']), 1),
'Number of Stars': len(local_df['Distance']),
'text_loc1': (1.1, -2.2),
'text_loc2': (2, -0.2),
'Data': local_df,
'Full Results': local_results,
'Full Table': local_results_raw
}
return pleiades_local
def gaia_query(center, fov, *args, limit=10000, circular=True):
"""
https://gea.esac.esa.int/archive/documentation/GEDR3/Gaia_archive/chap_datamodel/sec_dm_main_tables/ssec_dm_gaia_source.html
"""
from astroquery.gaia import Gaia
if isinstance(center, SkyCoord):
ra = center.ra.to(u.deg).value
dec = center.dec.to(u.deg).value
if not isinstance(fov, u.Quantity):
fov = fov * u.deg
if fov.ndim == 1:
ra_fov, dec_fov = fov.to(u.deg).value
else:
ra_fov = dec_fov = fov.to(u.deg).value
radius = np.min([ra_fov, dec_fov]) / 2
fields = ','.join(args) if isinstance(args, (tuple, list)) else args
if circular:
job = Gaia.launch_job(
f"select top {limit} {fields} from gaiadr2.gaia_source where "
"1=CONTAINS("
f"POINT('ICRS', {ra}, {dec}), "
f"CIRCLE('ICRS',ra, dec, {radius}))"
"order by phot_g_mean_mag")
else:
job = Gaia.launch_job(
f"select top {limit} {fields} from gaiadr2.gaia_source where "
f"ra BETWEEN {ra-ra_fov/2} AND {ra+ra_fov/2} AND "
f"dec BETWEEN {dec-dec_fov/2} AND {dec+dec_fov/2} "
"order by phot_g_mean_mag")
return job.get_results()
def get_cepheids():
"""
Execute ADQL query for Gaia DR1 Cepheids using astroquery's TAP+ utils.
Returns
-------
table : `~astropy.table`
Astropy table of requested data.
"""
job = Gaia.launch_job("SELECT cep.*, gaia.ra, gaia.dec \
FROM gaiadr1.cepheid AS cep \
INNER JOIN gaiadr1.gaia_source as gaia \
on cep.source_id = gaia.source_id")
return job.get_results()
def failed_Kharchenko2013_match_Gaia():
#
# attempt:
# use the 2MASS object ID's, given in “pts_key” format from Kharchenko, to
# match against the tmass_best_neighbour table (tmass_oid column).
#
# result:
# fails, because I think these are different IDs. not sure if it ALWAYS
# fails, but it seems to fail enough of the time that using POSITIONS and
# MAGNITUDES directly is probably the safer algorithm.
csvpaths = glob('../data/cluster_data/MWSC_1sigma_members/????_*.csv')
for csvpath in np.sort(csvpaths):
outpath = csvpath.replace(
'MWSC_1sigma_members','MWSC_1sigma_members_Gaia_matched')
if os.path.exists(outpath):
print('found {}, skipping'.format(outpath))
continue
k13_tab = Table.read(csvpath, format='ascii')
print('{} -- {} stars'.format(csvpath, len(k13_tab)))
k13_tmass_oids = np.array(k13_tab['2MASS'])
gaia_source_ids = []
for k13_tmass_oid in k13_tmass_oids:
querystr = (
"select top 1 source_id, original_ext_source_id, "
"angular_distance, tmass_oid from gaiadr2.tmass_best_neighbour "
"where tmass_oid = {}".format(k13_tmass_oid)
)
job = Gaia.launch_job(
)
res = job.get_results()
if len(res)==0:
gaia_source_ids.append(-1)
else:
gaia_source_ids.append(res['source_id'])
gaia_source_ids = np.array(gaia_source_ids)
outdf = pd.DataFrame(
{'k13_tmass_oids':k13_tmass_oids,
'gaia_tmass_best_neighbour_DR2_ids':gaia_source_ids })
outdf.to_csv(outpath, index=False)
print('--> made {}'.format(outpath))
def query_gaiaedr3(input_table):
print('Gaia eDR3 query started')
gaia_query_string = "SELECT *,upload_table.db_names FROM gaiaedr3.gaia_source " \
"INNER JOIN tap_upload.upload_table ON " \
"gaiaedr3.gaia_source.source_id = tap_upload.upload_table.dr3_source_id "
job_gaia_query = Gaia.launch_job(gaia_query_string,
upload_resource=input_table,
upload_table_name="upload_table",
verbose=VERBOSE)
gaia_data = job_gaia_query.get_results()
print('Gaia eDR3 query complete')
return gaia_data
def gaia_query(self,
query_str,
query_type,
upload_resource=None,
upload_tablename=None):
"""
Queries Gaia archive with query_str and updates self with results
Arguments:
query_str: string: valid ADQL query in context of Gaia Archive
query_type: string, one of {'sync','async'} (use sync for debugging)
upload_resource: string, path to xml file containing list of gaia source ids
upload_tablename: string, name of table in upload_resource
Returns:
Nothing, self.objs updated in place with query results
self.tap_query_string updated with supplied query string
"""
#save the query string for posterity
self.tap_query_string = query_str
#fetch the data into a pandas data frame
#synchronous job gives better error message than async, use for debugging queries
#synchronous job limited to 2000 results or thereabouts.
if query_type == 'async':
job = Gaia.launch_job_async(query=query_str,
upload_resource=upload_resource,
upload_table_name=upload_tablename)
elif query_type == 'sync':
job = Gaia.launch_job(query=query_str,
upload_resource=upload_resource,
upload_table_name=upload_tablename)
else:
raise ValueError(
f'invalid query_type parameter: {query_type}; valid values are: \'async\' and \'sync\''
)
#get the results as pandas dataframe and index it
self.objs = job.get_results().to_pandas().set_index('source_id')
# hacks for edr3
if not ('r_est' in self.objs.columns):
if 'parallax' in self.objs.columns:
self.objs['r_est'] = 1000.0 / self.objs.parallax
self.objs.rename(columns={
'dr2_radial_velocity': 'radial_velocity',
'dr2_radial_velocity_error': 'radial_velocity_error'
},
inplace=True)
def gaia_query(ra, dec, radius=5):
'''Return an ADQL query string for Gaia DR2 + geometric distances
from Bailer-Jones et al. 2018 '''
radius = radius * u.arcsec
coords = SkyCoord(ra=ra, dec=dec, unit=(u.deg, u.deg), frame='icrs')
query_string = '''SELECT *, DISTANCE(POINT('ICRS',g.ra, g.dec),
POINT('ICRS', %s, %s)) as r
FROM gaiadr2.gaia_source AS g, external.gaiadr2_geometric_distance AS d
WHERE g.source_id = d.source_id AND CONTAINS(POINT('ICRS',g.ra, g.dec),
CIRCLE('ICRS',%15.10f, %15.10f,%15.10f))=1 ORDER BY r ASC''' % \
(ra, dec, ra, dec, radius.to(u.degree).value)
job = Gaia.launch_job(query_string, verbose=False)
#print(job.get_results())
return job.get_results()
def get_data(metric='distance', n_sources=1000):
if metric not in ['distance', 'brightness']:
return "Please choose 'distance' or 'brightness'."
# Translate english to SQL/Gaia and call.
param = 'parallax' if metric is 'distance' else 'lum_val'
if metric is 'distance':
# Want to remove sources closer than Alpha Cen, whose parallax angle is 0.768"
gaia_str = "SELECT top {} * FROM gaiadr2.gaia_source \
WHERE parallax > 700 \
AND phot_rp_mean_mag != 'Nan' \
AND bp_rp != 'Nan' \
AND lum_val != 'Nan' \
ORDER BY parallax DESC \
".format(n_sources)
else:
gaia_str = "SELECT top {} * FROM gaiadr2.gaia_source \
WHERE 'lum_val' > 0 \
AND phot_rp_mean_mag != 'Nan' \
AND bp_rp != 'Nan' \
AND lum_val != 'Nan' \
ORDER BY lum_val DESC \
".format(n_sources)
job = Gaia.launch_job(gaia_str) #, dump_to_file=True)
gaia_results_raw = job.get_results()
gaia_results = gaia_results_raw.to_pandas()
sort_feature = 'Distance' if metric is 'distance' else 'Absolute Magnitude'
df = pd.DataFrame()
df['Distance'] = (gaia_results['parallax'] * 1e-3)**(-1)
df['mag'] = gaia_results['phot_rp_mean_mag']
df['Color'] = gaia_results['bp_rp']
df['Absolute Magnitude'] = df['mag'] - \
5 * (np.log10(df['Distance']) - 1)
df['T Effective'] = gaia_results['teff_val']
df['Parallax'] = gaia_results['parallax']
df['Radius'] = gaia_results['radius_val']
df['Luminosity'] = gaia_results['lum_val']
df['Mass'] = df['Luminosity']**0.25
df['Sort Feature'] = sort_feature
df['Decimal Sort Feature'] = df[sort_feature] / np.nanmax(df[sort_feature])
df.to_csv('stars_by_{}-{}.csv'.format(metric, n_sources))
return df
def given_sourceid_get_radec(source_id):
jobstr = (
"select top 1 g.ra, g.dec, g.pmra, g.pmdec, g.phot_g_mean_mag from "
"gaiadr2.gaia_source as g where g.source_id = {:s}".
format(source_id)
)
job = Gaia.launch_job(jobstr)
gaia_r = job.get_results()
if len(gaia_r) != 1:
raise AssertionError('gaia match failed')
ra, dec = float(gaia_r['ra']), float(gaia_r['dec'])
return ra, dec
def given_sourceid_get_gaiarow(source_id, whichgaia='gaiadr2'):
"""
args:
source_id: from gaiadr2, gaiaedr3, etc.
"""
jobstr = (
"select top 1 g.ra, g.dec, g.pmra, g.pmdec, g.phot_g_mean_mag, g.phot_bp_mean_mag, g.phot_rp_mean_mag from "+
f"{whichgaia}.gaia_source as g where g.source_id = {source_id:s}"
)
job = Gaia.launch_job(jobstr)
gaia_r = job.get_results()
if len(gaia_r) != 1:
raise AssertionError('gaia match failed')
return gaia_r
def run_phot_crossmatch(self,
upload_table='dummy.vot',
verbose=True,
add_input_cols=""):
"""
Run a crossmatch between Gaia, 2MASS and WISE using Gaia internal cross-matched tables.
"""
# 1.- Search input table ===============================
upload_resource = glob.glob(upload_table)
upload_table_name = "input_table"
if len(upload_resource) == 0:
raise Exception('Upload Table not found')
if add_input_cols != "":
add_input_cols = "," + add_input_cols + " "
# 2.- Write ADQL query =================================
query = (
"SELECT " + self.gaia_cols_p + ", " + self.wise_cols + ", " +
self.tmass_cols + " " + add_input_cols +
"FROM tap_upload.input_table as input_table "
"LEFT OUTER JOIN gaiadr2.gaia_source as gaia ON input_table.source_id = gaia.source_id " # Inp Sample VS Gaia DR2
"LEFT OUTER JOIN gaiadr2.allwise_best_neighbour as xmatch ON gaia.source_id = xmatch.source_id " # Inp Sample VS WISE
"LEFT OUTER JOIN gaiadr1.allwise_original_valid as wise ON xmatch.allwise_oid = wise.allwise_oid " # Inp Sample VS WISE
"LEFT OUTER JOIN gaiadr2.tmass_best_neighbour as xmatch_2 ON gaia.source_id = xmatch_2.source_id " # Inp Sample VS 2MASS
"LEFT OUTER JOIN gaiadr1.tmass_original_valid as tmass ON xmatch_2.tmass_oid = tmass.tmass_oid " # Inp Sample VS 2MASS
f"WHERE cc_flags = '0000' "
f"AND ext_flag < 2 "
f"AND w3mpro_error IS NOT NULL "
f"AND w4mpro_error IS NOT NULL")
# 3.- Run ADQL query ===================================
print(f'RUNNING ADQL SYNCRHRONOUS QUERY ' + '=' * 57)
job = Gaia.launch_job(query=query,
upload_resource=upload_resource[0],
upload_table_name=upload_table_name,
verbose=True)
self.cat = job.get_results()
if verbose:
print('=' * 90)
print(f'SAMPLE OUTPUT N_els = {len(self.cat):3.0f}')
print('=' * 90)
def match_gaia (df, wcs, gaia_window, date):
"""
Input
df: Pandas DataFrame with 'x', 'y' columns for positions of possible stars
wcs: astropy.wcs object for unit transformation from fits image pixel coordinates
gaia_window: the size of the window (in arcsec) for the Gaia query
date: decimal year of the observation, used to propogate the Gaia epoch
Output
df: Pandas DataFrame with 'x', 'y', 'g', 'ra_prop', 'dec_prop', 'pmra', 'pmdec', 'parallax'
"""
from astroquery.gaia import Gaia
gaia_window = gaia_window / 3600.0
empty = np.zeros(len(df))*np.nan
df['ra_pix'], df['dec_pix'] = empty, empty
df['g'], df['ra_prop'], df['dec_prop'], df['pmra'], df['pmdec'], df['parallax'] = empty, empty, empty, empty, empty, empty
for i in range(len(df)):
x,y = df.loc[i,'x'],df.loc[i,'y']
ra,dec = wcs.pixel_to_world_values (x,y)
ra,dec = float(ra),float(dec)
df.loc[i,'ra_pix'], df.loc[i,'dec_pix'] = ra, dec
print ('Looking at RA:%.5f, Dec:%.5f'%(ra,dec))
query = """ SELECT TOP 1 designation, EPOCH_PROP(ASTROMETRIC_PARAMETERS(ra,dec,parallax,pmra,pmdec,radial_velocity), 2015.5, %f)
FROM gaiadr2.gaia_source
WHERE
CONTAINS(
POINT('ICRS', gaiadr2.gaia_source.ra, gaiadr2.gaia_source.dec),
CIRCLE('ICRS', %f, %f, %f)
)=1"""%(date,ra,dec,gaia_window)
job = Gaia.launch_job(query)
r = job.get_results()
try:
df.loc[i,'g'] = r['designation'][0].decode('utf-8')
df.loc[i,'ra_prop'] = r['epoch_prop'][0][0]
df.loc[i,'dec_prop'] = r['epoch_prop'][0][1]
df.loc[i,'parallax'] = r['epoch_prop'][0][2]
df.loc[i,'pmra'] = r['epoch_prop'][0][3]
df.loc[i,'pmdec'] = r['epoch_prop'][0][4]
except:
pass
del(job,r)
return df
def vector_match_tmassbestneighbor_to_gaiaids(stats,
homedir='/home/luke/',
outdir='../data/rms_vs_mag/',
projid=1301):
"""
if you do smart ADQL, it is like factor of >100x faster than doing
item-by-item crossmatching.
astroquery.Gaia is sick for this, because it lets you remotely upload
tables on the fly. (at least, somewhat small ones, of <~10^5 members)
"""
outfile = os.path.join(outdir, 'proj{}_xmatch.xml.gz'.format(projid))
xmlpath = '../data/rms_vs_mag/proj1301_gaiaids.xml'
if not os.path.exists(outfile):
gaiaids = stats['lcobj']
xmlpath = make_votable_given_ids(gaiaids)
Gaia.login(credentials_file=os.path.join(homedir, '.gaia_credentials'))
jobstr = ('SELECT top 100000 upl.source_id, tm.source_id, '
'tm.original_ext_source_id FROM '
'gaiadr2.tmass_best_neighbour AS tm '
'JOIN tap_upload.foobar as upl '
'using (source_id)')
if not os.path.exists(outfile):
# might do async if this times out. but it doesn't.
j = Gaia.launch_job(query=jobstr,
upload_resource=xmlpath,
upload_table_name="foobar",
verbose=True,
dump_to_file=True,
output_file=outfile)
Gaia.logout()
vot = parse(outfile)
tab = vot.get_first_table().to_table()
return tab
def load_gaia_stars(c, search_radius, bright_limit=-99, faint_limit=99):
inner_search_radius = 0
outer_search_radius = search_radius
c_icrs = c.transform_to('icrs')
radius = u.Quantity(outer_search_radius, u.deg)
query_string = "SELECT TOP 99999 source_id, ra,dec,phot_g_mean_mag FROM gaiaedr3.gaia_source WHERE phot_g_mean_mag <= " + str(
faint_limit
) + " AND 1=CONTAINS(POINT('ICRS',ra,dec), CIRCLE('ICRS'," + str(
c_icrs.ra.deg) + "," + str(c_icrs.dec.deg) + ", " + str(
radius.value) + "))"
print("Gaia query: ", query_string)
j = Gaia.launch_job(query_string, verbose=True)
cat = j.get_results()
#print("Gaia query: ",gaia_query)
print(f'{len(cat)} stars found within {search_radius} deg')
return cat
def fromGaia(self, name, **attributes):
self.attributes = attributes
self.name = name
assert ('DR2' in name)
source_id = name.split()[-1]
query = 'SELECT source_id, ra, dec, pmra, pmdec, parallax, parallax_error, phot_g_mean_mag, phot_bp_mean_mag, phot_rp_mean_mag FROM gaiadr2.gaia_source WHERE source_id={}'.format(
source_id)
self.speak("querying Gaia DR2 for {}".format(source_id))
result = Gaia.launch_job(query)
self.table = result.get_results()
ra = self.table['ra'].data[0]
dec = self.table['dec'].data[0]
gaiadr2_epoch = 2015.5
obstime = astropy.time.Time(gaiadr2_epoch, format='decimalyear')
self.icrs = astropy.coordinates.SkyCoord(ra,
dec,
unit=(u.deg, u.deg),
frame='icrs',
obstime=obstime)
self.pmra = self.table['pmra'].data[0] # in (projected) mas/yr
self.pmdec = self.table['pmdec'].data[0] # in mas/yr
self.attributes = {}
self.attributes['G_gaia'] = float(
self.table['phot_g_mean_mag'].data[0])
self.attributes['R_gaia'] = self.table['phot_rp_mean_mag'].data[0]
self.attributes['B_gaia'] = self.table['phot_bp_mean_mag'].data[0]
for k, v in attributes.items():
self.attributes[k] = v
self.speak('made {0} from Gaia DR2'.format(self))
# # * `SELECT` indicates that we are selecting data (as opposed to adding or modifying data). # # * `TOP` indicates that we only want the first 10 rows of the table, which is useful for testing a query before asking for all of the data. # # * `FROM` specifies which table we want data from. # # The third line is a list of column names, indicating which columns we want. # # In this example, the keywords are capitalized and the column names are lowercase. This is a common style, but it is not required. ADQL and SQL are not case-sensitive. # To run this query, we use the `Gaia` object, which represents our connection to the Gaia database, and invoke `launch_job`: # In[11]: job1 = Gaia.launch_job(query1) job1 # The result is an object that represents the job running on a Gaia server. # # If you print it, it displays metadata for the forthcoming table. # In[12]: print(job1) # Don't worry about `Results: None`. That does not actually mean there are no results. # # However, `Phase: COMPLETED` indicates that the job is complete, so we can get the results like this: # In[13]:
def get_gaia_data(self):
"""
Build a data structure for the Gaia data.
Paper on bands, colors, and equivalencies for GAIA:
arxiv.org/pdf/1008.0815.pdf'
Basically, since GAIA uses longer wavelength light, their colors don't map
to UBVRI bandpasses (p. 4, Table 1). This paper refers us to use B-R for
color, and since we used V magnitude when considering B-V color, I chose
to use the R band magnitude for my apparent magnitude values.
Args:
- ra, dec in degrees
"""
print("Building GAIA dataframe for {} sources...".format(
self.n_sources))
# AND parallax_error < 2 \
# try import gaia_results.csv except ImportError:
gaia_str = "SELECT top {} * FROM gaiadr2.gaia_source \
WHERE pmra between {} and {} \
AND pmdec between {} and {} \
AND ra between {} and {} \
AND dec between {} and {} \
AND parallax between {} and {} \
".format(self.n_sources, self.pm_ra_min, self.pm_ra_max,
self.pm_dec_min, self.pm_dec_max, self.ra_min,
self.ra_max, self.dec_min, self.dec_max,
self.plx_min, self.plx_max)
job = Gaia.launch_job(gaia_str) # , dump_to_file=True)
gaia_results_raw = job.get_results()
# gaia_results_raw['phot_rp_mean_mag'].description
gaia_results = gaia_results_raw.to_pandas()
# gaia_cols = sorted(gaia_results.columns)
print("Acquired data; now building dataframe...")
gaia_df = pd.DataFrame()
gaia_df['RA'] = gaia_results['ra']
gaia_df['Dec'] = gaia_results['dec']
gaia_df['Distance'] = (gaia_results['parallax'] * 1e-3)**(-1)
gaia_df['Proper Motion (RA)'] = gaia_results['pmra']
gaia_df['Proper Motion (Dec)'] = gaia_results['pmdec']
gaia_df['mag'] = gaia_results['phot_rp_mean_mag']
gaia_df['Color'] = gaia_results['bp_rp']
gaia_df['Absolute Magnitude'] = gaia_df['mag'] - \
5 * (np.log10(gaia_df['Distance']) - 1)
# gaia_df['T Effective'] = gaia_results['teff_val']
gaia_df['Parallax'] = gaia_results['parallax']
# gaia_df['Plx. Error'] = gaia_results['parallax_error']
# gaia_df['Confidence'] = 1 - gaia_results['parallax_error']/max(gaia_results['parallax_error'])
self.df = gaia_df.dropna()
self.mean_distance = round(np.mean(gaia_df['Distance']), 1)
self.n_stars = len(gaia_df['Distance'])
self.complete_sources_df = gaia_results
self.raw_gaia_results = gaia_results_raw
print(
'Finished getting data. Found {} sources (some may have been dropped due to NAs from Gaia or there were insufficient stars in the field).'
.format(len(self.df)))
return None
# Step 1: Gaia search.
input_coord = SkyCoord(ra=args.ra, dec=args.dec, unit=(u.degree, u.degree),
frame="icrs") # todo: allow user-specified frame
gaia_adql_constraints = f" AND {args.gaia_adql_constraints}" if args.gaia_adql_constraints else ""
logger.info(f"""Querying Gaia at {input_coord}
with radius {args.radius}
and constraints: {gaia_adql_constraints}""")
#j = Gaia.cone_search(input_coord, args.radius)
j = Gaia.launch_job(f"""SELECT DISTANCE(POINT('ICRS', ra, dec),
POINT('ICRS', {args.ra}, {args.dec})) AS dist, *
FROM gaiadr2.gaia_source
WHERE CONTAINS(POINT('ICRS', ra, dec),
CIRCLE('ICRS', {args.ra}, {args.dec}, {args.radius.to(u.deg).value})) = 1
{gaia_adql_constraints}
ORDER BY dist ASC""")
gaia_results = j.get_results()
G = len(gaia_results)
if G < 1:
raise NotImplementedError("no sources found in Gaia query")
if G > 1:
raise NotImplementedError("multiple sources found in Gaia query")
#python wobble_prepare.py 344.36658333333327 20.768833333333333
gaia_result = gaia_results[0]