def download_gaia_data(colname: str, xindex: Union[str, Table], outfile_path: Path) -> None:
"""Query and download Gaia data."""
if isinstance(xindex, str):
xindex_name = cast(str, xindex)
upload_resource = None
upload_table_name = None
else:
xindex_name = "TAP_UPLOAD.cel_xindex"
upload_resource = cast(Table, xindex)
upload_table_name = 'cel_xindex'
query = f"""SELECT
x.source_id, x.original_ext_source_id AS {colname},
g.ra, g.dec, g.parallax, g.parallax_error, g.pmra,
g.pmdec, g.phot_g_mean_mag, g.bp_rp, g.teff_val,
d.r_est, d.r_lo, d.r_hi
FROM
{xindex_name} x
JOIN gaiadr2.gaia_source g ON g.source_id = x.source_id
LEFT JOIN external.gaiadr2_geometric_distance d ON d.source_id = x.source_id"""
print(query)
job = Gaia.launch_job_async(
query,
upload_resource=upload_resource,
upload_table_name=upload_table_name,
dump_to_file=True,
output_file=outfile_path,
output_format='csv',
)
try:
job.save_results()
finally:
Gaia.remove_jobs(job.jobid)
def add_gaia(self):
"""Retrieve and add Gaia DR2 or DR1 parameters."""
if hasattr(self, 'IDS') is False:
raise RuntimeError('RUN set_simbad_fields() first!')
try:
self.simbad_identifiers = self.IDS.decode().split('|')
except AttributeError:
self.simbad_identifiers = self.IDS.split('|')
try:
gaia_dr2_id = [id for id in self.simbad_identifiers if 'Gaia DR2 ' in id][0].replace("'", "")
# print(gaia_dr2_id)
gacs_query = "SELECT * FROM gaiadr2.gaia_source WHERE source_id={}".format(gaia_dr2_id.split(' ')[-1])
job = Gaia.launch_job_async(gacs_query)
self.gaiadr2_table = job.get_results()
except IndexError:
try:
gaia_dr1_id = [id for id in self.simbad_identifiers if 'Gaia DR1 ' in id][0]
# return_gacs_query_as_table(query_string, output_file_seed, overwrite=False, verbose=True):
gacs_query = "SELECT * FROM gaiadr1.gaia_source WHERE source_id={}".format(gaia_dr1_id.split(' ')[-1])
job = Gaia.launch_job_async(gacs_query)
self.gaiadr1_table = job.get_results()
except IndexError:
print('No Gaia identifier listed in Simbad!')
def main():
# TODO: bad, hard-coded...
# base_path = '/Volumes/ProjectData/gaia-comoving-followup/'
base_path = '../data/'
db_path = path.join(base_path, 'db.sqlite')
engine = db_connect(db_path)
session = Session()
credentials = dict(user='******', password='******')
Gaia.login(**credentials)
for obs in session.query(Observation).all():
q = session.query(Photometry).join(Observation).filter(Observation.id == obs.id).count()
if q > 0:
logger.debug('Photometry already exists')
continue
if obs.tgas_source is None:
continue
tgas_source_id = obs.tgas_source.source_id
res = get_photometry(tgas_source_id)
phot_kw = dict()
for col in result_columns:
phot_kw[col] = res[col]
phot = Photometry(**phot_kw)
phot.observation = obs
session.add(phot)
session.commit()
def Gaia_adql(query, upload=None):
"""Run query on gaia archive and return results as a pandas dataframe.
If upload is a pandas dataframe then this is uploaded to the archive
and available as the table tap_upload.uploadedtable"""
# We take and return pandas DataFrames but use astropy tables to make a votable of coordinates to upload
with tempfile.NamedTemporaryFile(suffix='.xml',
mode='w') as file_to_upload:
if upload is not None:
table = Table.from_pandas(upload)
table.write(file_to_upload, format='votable')
# It seems Gaia DR2 source table produces many votable warnings that aren't important
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
with suppress_stdout():
if upload is not None:
job = Gaia.launch_job_async(
query=query,
upload_resource=file_to_upload.name,
upload_table_name="uploadedtable")
else:
job = Gaia.launch_job_async(query=query)
result = job.get_results()
result_df = result.to_pandas()
return result_df
def from_query(cls, query_str, login_info=None):
"""
Run the specified query and return a `GaiaData` instance with the
returned data.
This is meant only to be used for quick queries to the main Gaia science archive. For longer queries and more customized usage, use TAP access to any of the Gaia mirrors with, e.g., astroquery or pyvo.
This requires ``astroquery`` to be installed.
Parameters
----------
query_str : str
The string ADQL query to execute.
login_info : dict, optional
Username and password for the Gaia science archive as keys "user"
and "password". If not specified, will use anonymous access, subject
to the query limits.
Returns
-------
gaiadata : `GaiaData`
An instance of this object.
"""
from astroquery.gaia import Gaia
if login_info is not None:
Gaia.login(**login_info)
job = Gaia.launch_job_async(query_str)
tbl = job.get_results()
return cls(tbl)
def asyncQuery():
query2 = """SELECT TOP 3000
source_id, ref_epoch, ra, dec, parallax
FROM gaiadr2.gaia_source
WHERE parallax < 1
"""
job2 = Gaia.launch_job_async(query2)
print(job2)
results2 = job2.get_results()
results2.pprint_all()
Gaia.remove_jobs([job2.jobid])
def download_gaia_tyc(username: str) -> None:
"""Download TYC data from the Gaia archive."""
tyc_file = os.path.join('gaia', 'gaiadr2_tyc-result.csv')
if proceed_checkfile(tyc_file):
download_gaia_data('tyc2_id', 'gaiadr2.tycho2_best_neighbour', tyc_file)
# Use SIMBAD to fill in some of the missing entries
with contextlib.suppress(FileExistsError):
os.mkdir('simbad')
simbad_file = os.path.join('simbad', 'tyc-gaia.votable')
if proceed_checkfile(simbad_file):
ascc_file = os.path.join('vizier', 'ascc.tar.gz')
missing_ids = get_missing_tyc_ids(tyc_file, ascc_file)
print("Querying SIMBAD for Gaia DR2 identifiers")
simbad = Tap(url='http://simbad.u-strasbg.fr:80/simbad/sim-tap')
query = """SELECT
id1.id tyc_id, id2.id gaia_id
FROM
TAP_UPLOAD.missing_tyc src
JOIN IDENT id1 ON id1.id = src.id
JOIN IDENT id2 ON id2.oidref = id1.oidref
WHERE
id2.id LIKE 'Gaia DR2 %'"""
print(query)
job = simbad.launch_job_async(query,
upload_resource=missing_ids,
upload_table_name='missing_tyc',
output_file=simbad_file,
output_format='votable',
dump_to_file=True)
job.save_results()
tyc2_file = os.path.join('gaia', 'gaiadr2_tyc-result-extra.csv')
if proceed_checkfile(tyc2_file):
missing_ids = votable.parse(simbad_file).resources[0].tables[0].to_table()
missing_ids['tyc_id'] = [m[m.rfind(' ')+1:] for m in missing_ids['tyc_id'].astype('U')]
missing_ids.rename_column('tyc_id', 'original_ext_source_id')
missing_ids['gaia_id'] = [int(m[m.rfind(' ')+1:])
for m in missing_ids['gaia_id'].astype('U')]
missing_ids.rename_column('gaia_id', 'source_id')
Gaia.upload_table(upload_resource=missing_ids, table_name='tyc_missing')
try:
download_gaia_data('tyc2_id', 'user_'+username+'.tyc_missing', tyc2_file)
finally:
Gaia.delete_user_table('tyc_missing')
def _queryGaia(ID=None, coords=None, radius=2):
""" Query Gaia archive for bp-rp
Sends an ADQL query to the Gaia archive to look up a requested target ID or
set of coordinates.
If the query is based on coordinates a cone search will be performed and the
closest target is returned. Provided coordinates must be astropy.Skycoords.
Parameters
----------
ID : str
Gaia source ID to search for.
coord : astropy.Skycoords
An Astropy Skycoords object with the target coordinates. Must only
contain one target.
radius : float, optional
Radius in arcseconds to use for the sky cone search. Default is 20".
Returns
-------
bp_rp : float
Gaia bp-rp value of the requested target from the Gaia archive.
"""
from astroquery.gaia import Gaia
if ID is not None:
print('Querying Gaia archive for bp-rp values by target ID.')
adql_query = "select * from gaiadr2.gaia_source where source_id=%s" % (
ID)
try:
job = Gaia.launch_job(adql_query).get_results()
except:
return None
return float(job['bp_rp'][0])
elif coords is not None:
print('Querying Gaia archive for bp-rp values by target coordinates.')
ra = coords.ra.value
dec = coords.dec.value
adql_query = f"SELECT DISTANCE(POINT('ICRS', {ra}, {dec}), POINT('ICRS', ra, dec)) AS dist, * FROM gaiaedr3.gaia_source WHERE 1=CONTAINS( POINT('ICRS', {ra}, {dec}), CIRCLE('ICRS', ra, dec,{radius})) ORDER BY dist ASC"
try:
job = Gaia.launch_job(adql_query).get_results()
except:
return None
return float(job['bp_rp'][0])
else:
raise ValueError(
'No ID or coordinates provided when querying the Gaia archive.')
def get_hip_newreduction_data_file(path):
'''Get HIP data from gaia TAP server'''
from astroquery.gaia import Gaia
QUERY = '''
SELECT hip, ra, dec, hp_mag AS vmag
FROM public.hipparcos_newreduction
ORDER BY hip
'''
job = Gaia.launch_job_async(QUERY)
r = job.get_results()
r.write(path, overwrite=True, format='ascii.csv')
if not hasattr(job, 'get_jobid'):
print('Warning: cannot delete Gaia job')
return
Gaia.remove_jobs([job.get_jobid()])
def gaia_params(self):
"""Retrieve parallax, radius, teff and lum from Gaia."""
# If gaia DR2 id is provided, query by id
fields = np.array([
'parallax', 'parallax_error', 'teff_val',
'teff_percentile_lower', 'teff_percentile_upper',
'radius_val', 'radius_percentile_lower',
'radius_percentile_upper', 'lum_val',
'lum_percentile_lower', 'lum_percentile_upper'
])
query = 'select '
for f in fields[:-1]:
query += 'gaia.' + f + ', '
query += 'gaia.' + fields[-1]
query += ' from gaiadr2.gaia_source as gaia'
query += ' where gaia.source_id={0}'.format(self.g_id)
j = Gaia.launch_job_async(query)
res = j.get_results()
self.plx, self.plx_e = self._get_parallax(res)
self.temp, self.temp_e = self._get_teff(res)
self.rad, self.rad_e = self._get_radius(res)
self.lum, self.lum_e = self._get_lum(res)
self.dist, self.dist_e = self._get_distance(self.ra, self.dec,
self.radius, self.g_id)
pass
def query_gaia(ra_deg,
dec_deg,
radius_deg,
minmag=10,
maxmag=20,
maxsources=10000,
catalogname='gaiacatalog'):
job = Gaia.launch_job_async(
"SELECT * FROM gaiadr2.gaia_source AS g, gaiadr2.panstarrs1_best_neighbour \
AS pbest, gaiadr2.panstarrs1_original_valid AS ps1 WHERE g.source_id = pbest.source_id AND \
pbest.original_ext_source_id = ps1.obj_id AND CONTAINS(POINT('ICRS', g.ra, g.dec), \
CIRCLE('ICRS', %.4f, %.4f, %.4f))=1 AND ps1.r_mean_psf_mag > %.2f AND ps1.r_mean_psf_mag \
< %.2f AND pmra IS NOT NULL AND pmdec IS NOT NULL AND abs(pmdec) > 0 AND \
abs(pmdec) < 40 AND abs(pmra)>0 AND abs(pmra) < 40 AND ps1.n_detections > 6 \
AND pbest.number_of_mates=0 AND \ pbest.number_of_neighbours=1;" %
(ra_deg, dec_deg, radius_deg, 10, 20))
p = job.get_results()
p['ra_errdeg'] = p['ra_error'] / 3.6e6
p['dec_errdeg'] = p['dec_error'] / 3.6e6
p['FLAGS'] = 0
if os.path.exists(catalogname + '.ldac'):
os.remove(catalogname + '.ldac')
if os.path.exists('gaiacatalog.dat'):
os.remove('gaiacatalog.dat')
save_table_as_ldac(p, catalogname + '.ldac')
def query_survey(dataframe, idx, features, survey):
series = dataframe.loc[idx]
coord = SkyCoord(ra=series.RA, dec=series.Dec, unit=(u.deg, u.deg))
try:
if survey == '2MASS':
temp = Irsa.query_region(coord,
radius=search_radius,
catalog='fp_psc').to_pandas()
elif survey == 'GAIA':
temp = Gaia.query_object(coordinate=coord,
width=search_radius,
height=search_radius).to_pandas()
else:
print('Invalid Survey')
catalog = SkyCoord(ra=temp.ra, dec=temp.dec, unit=(u.deg, u.deg))
i, _, _ = match_coordinates_sky(coord, catalog)
for feature in features:
dataframe.at[idx, feature] = temp.at[int(i), feature]
except Exception as e:
print(e)
def jobGaiaDR2(ra0, dec0, limgmag, fov, yr):
ssl._create_default_https_context = ssl._create_unverified_context
job = Gaia.launch_job_async("SELECT * \
FROM gaiadr2.gaia_source \
WHERE CONTAINS(POINT('ICRS',gaiadr2.gaia_source.ra,gaiadr2.gaia_source.dec),CIRCLE('ICRS',%f,%f,%f))=1\
AND pmra IS NOT NULL AND abs(pmra)>0 \
AND pmdec IS NOT NULL AND abs(pmdec)>0\
AND phot_g_mean_mag<%f;" %(ra0,dec0,fov,limgmag) \
, dump_to_file=False)
gaiat = job.get_results()
# print(gaiat)
ra = np.array(gaiat['ra'])
dec = np.array(gaiat['dec'])
pmra = np.array(gaiat['pmra'])
pmdec = np.array(gaiat['pmdec'])
gmag = np.array(gaiat['phot_g_mean_mag'])
bpmag = np.array(gaiat['phot_bp_mean_mag'])
rpmag = np.array(gaiat['phot_rp_mean_mag'])
des = np.array(gaiat['designation'])
N = np.size(gmag)
for k in range(N):
alpha, delta = RADecFromTang((yr - 2015.5) * math.pi * pmra[k] / 648000000.0, \
(yr - 2015.5) * math.pi * pmdec[k] / 648000000.0, \
math.radians(ra[k]), math.radians(dec[k]))
print(ra[k], dec[k], math.degrees(alpha), math.degrees(delta))
ra[k], dec[k] = math.degrees(alpha), math.degrees(delta)
return ra, dec, gmag, bpmag, rpmag, des
def download_votable(tile,
offset=0.005,
output_dir=dirconfig.raw_gaia,
gaia_source='gaiadr2.gaia_source'):
"""
This function automatically download gaia-data from gaia-archive using AstroQuery module.
tile: a Tile objects.
Implemented query extract all the sources within the coordinates
offset: is a safe-margin (in degrees) added to the borders of each tile
output_dir : where vo-tables are saved
gaia_source: gaiaedr3.gaia_source or gaiadr2.gaia_source
"""
print('---------------------------------------------')
lmin = tile.lmin - offset
lmax = tile.lmax + offset
bmin = tile.bmin - offset
bmax = tile.bmax + offset
query = f'SELECT * FROM {gaia_source} WHERE l > {lmin:.4f} AND l < {lmax:.4f} AND b > {bmin:.4f} AND b < {bmax:.4f} '
print(query)
file_path = path.join(output_dir, tile.name + '_gaia.vot.gz')
t1 = time.time()
job = Gaia.launch_job_async(query,
dump_to_file=True,
output_file=file_path)
t2 = time.time()
print(f'Delta t: {t2 - t1:.4f} s')
def gaiadr2xdr3(source_idlist, nearest=True):
"""
returns the dr2 to dr3 cross matches for the given source_idlist
"""
upload_tablename, upload_resource, sidcol = source_id_to_xmlfile(
source_idlist)
query_str = ' '.join([
f'SELECT tu.{sidcol}, dr2.*',
f'from tap_upload.{upload_tablename} tu left join gaiaedr3.dr2_neighbourhood dr2',
f'on tu.{sidcol} = dr2.dr2_source_id'
])
try:
job = Gaia.launch_job_async(query=query_str,
upload_resource=upload_resource,
upload_table_name=upload_tablename)
df = job.get_results().to_pandas()
finally:
os.remove(upload_resource)
if nearest:
#just return the nearest dr3 source id based on angular distance
ret_df = df.sort_values(['dr2_source_id', 'angular_distance']).groupby(
'dr2_source_id', as_index=False).first().set_index('source_id')
else:
ret_df = df.set_index('source_id')
return ret_df
def query_GaiaEDR3(ra, dec, mag, filter, radius=60.):
G_guess = guess_Gaia_mag_from_color(mag, filter)
coord = SkyCoord(ra = ra, dec = dec, unit = (u.degree, u.degree), \
frame = 'icrs')
radius = u.Quantity(radius / 3600., u.deg)
result = Gaia.query_object(coordinate=coord, radius=radius)
Gmag0 = result["phot_g_mean_mag"]
filt = np.abs(Gmag0 - G_guess) < 1.0
parallax0 = result["parallax"][filt]
parallax_error0 = result["parallax_error"][filt]
gaiaid0 = result["source_id"][filt]
if len(parallax0) == 0:
gaiaid = ""
parallax = np.nan
parallax_error = np.nan
elif len(parallax0) > 1:
gaiaid = gaiaid0
parallax = parallax0
parallax_error = parallax_error0
else:
gaiaid = gaiaid0[0]
parallax = parallax0[0]
parallax_error = parallax_error0[0]
return (gaiaid, parallax, parallax_error)
def query_gaia_limEUCLIDSKY(self):
query = "select * from gaiadr2.gaia_source where abs(B)>{} and abs(ecl_lat)>{}".format(
self.galactic_latitude, self.celestial_latitude)
print("Start query_gaia_limEUCLIDSKY()")
if not os.path.exists("../../data/gaia_euclid.csv"):
filename = "gaia_euclid"
else:
print(
"PATH ../../data/gaia_euclid.csv already exist \n Overwrite existing file? [y/n]"
)
answer = input()
if answer == "y" or answer == "yes":
filename = "gaia_euclid"
elif answer == "n" or answer == "no":
print(
"Give another filename (no gaia_euclid) to save the data")
filename = input()
else:
print(
"Your answer: {} is not valid: Please return y, yes, n or no"
.format(answer))
return query_gaia_EUCLIDSKY()
job = Gaia.launch_job_async(query)
r = job.get_results()
ascii.write(r,
"../../data/gaia/{}.csv".format(filename),
delimiter=',')
return None
def get_gaia_data(self, index):
# get data about star
row = self.stars.iloc[index]
ls_id = row.ls_id
# get gaia id of star from decals
q = """SELECT
ra1, dec1, id1, ra2, dec2, id2, distance
FROM
ls_dr9.x1p5__tractor__gaia_edr3__gaia_source
WHERE
id1 = {} """.format(ls_id)
res = qc.query(sql=q)
decals_data = convert(res, 'pandas')
# get gaia id (or input id = 0 if no gaia data)
gaia_id = 0 if len(decals_data.id2) == 0 else decals_data.id2
# use gaia id to get info about it
query = """SELECT
source_id, ra, ra_error, dec, dec_error, parallax, pmra, pmra_error, pmdec, pmdec_error
FROM
gaiadr2.gaia_source
WHERE
source_id = {} """.format(int(gaia_id))
gaia_data = Gaia.launch_job(query).get_results().to_pandas()
return gaia_data
def query_gaia_KiDS_testpatch(
self, pointing): #pointing = [central_ra, central_dec]
patch_ra = [pointing[0] - 0.5, pointing[0] + 0.5]
patch_dec = [pointing[1] - 0.5, pointing[1] + 0.5]
query_KiDS_testpatch = "select * from gaiadr2.gaia_source where ra<{} and ra>{} and dec<{} and dec>{} and parallax IS NOT NULL".format(
patch_ra[1], patch_ra[0], patch_dec[1], patch_dec[0])
print("Start query_gaia_KiDS_testpatch()")
if not os.path.exists(
"../../data/gaia/gaia_KiDS_testpatch[{}_{}].csv".format(
pointing[0], pointing[1])):
filename = "gaia_KiDS_testpatch[{}_{}].csv".format(
pointing[0], pointing[1])
else:
print(
"PATH ../../data/gaia_KiDS_testpatch[{}_{}].csv already exist \n Overwrite existing file? [y/n]"
).format(pointing[0], pointing[1])
answer = input()
if answer == "y" or answer == "yes":
filename = "gaia_euclid"
elif answer == "n" or answer == "no":
print(
"Give another filename (no gaia_KiDS_testpatch) to save the data"
)
filename = input()
else:
print(
"Your answer: {} is not valid: Please return y, yes, n or no"
.format(answer))
return query_gaia_KiDS_testpatch()
job = Gaia.launch_job_async(query_KiDS_testpatch)
r = job.get_results()
ascii.write(r, "../../data/gaia/{}".format(filename), delimiter=',')
return None
def gaia_query(n, distance=200, **kwargs):
"""
Sends an archive query for d < 200 pc, with additional filters taken from
Gaia Data Release 2: Observational Hertzsprung-Russell diagrams (Sect. 2.1)
Gaia Collaboration, Babusiaux et al. (2018)
(https://doi.org/10.1051/0004-6361/201832843)
NOTE: 10000000 is a maximum query size (~76 MB / column)
Additional keyword arguments are passed to TapPlus.launch_job_async method.
"""
return Gaia.launch_job_async(
"select top {}".format(n) +
#" lum_val, teff_val,"
#" ra, dec, parallax,"
" bp_rp, phot_g_mean_mag+5*log10(parallax)-10 as mg"
" from gaiadr2.gaia_source"
" where parallax_over_error > 10"
" and visibility_periods_used > 8"
" and phot_g_mean_flux_over_error > 50"
" and phot_bp_mean_flux_over_error > 20"
" and phot_rp_mean_flux_over_error > 20"
" and phot_bp_rp_excess_factor <"
" 1.3+0.06*power(phot_bp_mean_mag-phot_rp_mean_mag,2)"
" and phot_bp_rp_excess_factor >"
" 1.0+0.015*power(phot_bp_mean_mag-phot_rp_mean_mag,2)"
" and astrometric_chi2_al/(astrometric_n_good_obs_al-5)<"
"1.44*greatest(1,exp(-0.4*(phot_g_mean_mag-19.5)))" +
" and 1000/parallax <= {}".format(distance),
**kwargs)
def get_bc_from_gaia(gaia_dr2_id, jd, rvabs=0):
"""
wrapper routine for using barycorrpy with Gaia DR2 coordinates
"""
# use 2015.5 as an epoch (Gaia DR2)
epoch = 2457206.375
# gaia_data = Gaia.query_object_async(coordinate=coord, width=width, height=height)
# q = Gaia.launch_job_async('SELECT * FROM gaiadr2.gaia_source WHERE source_id = ' + str(gaia_dr2_id))
q = Gaia.launch_job(
'SELECT * FROM gaiadr2.gaia_source WHERE source_id = ' +
str(gaia_dr2_id))
gaia_data = q.results
bc = barycorrpy.get_BC_vel(JDUTC=jd,
ra=gaia_data['ra'],
dec=gaia_data['dec'],
pmra=gaia_data['pmra'],
pmdec=gaia_data['pmdec'],
px=gaia_data['parallax'],
rv=rvabs * 1e3,
epoch=epoch,
obsname='AAO',
ephemeris='de430')
# bc = barycorrpy.get_BC_vel(JDUTC=utmjd, ra=ra, dec=dec, pmra=gaia_data['pmra'], pmdec=gaia_data['pmdec'],
# px=gaia_data['parallax'], rv=gaia_data['radial_velocity']*1e3, obsname='AAO', ephemeris='de430')
# bc = barycorrpy.get_BC_vel(JDUTC=utmjd, ra=ra, dec=dec, pmra=pmra, pmdec=pmdec,
# px=px, rv=rv, obsname='AAO', ephemeris='de430')
return bc[0][0]
def query_GaiaDR1(self, ID=None):
from astroquery.gaia import Gaia
key = 'Gaia DR1'
if ID is None:
ID = self.IDs[key]
tbl = Table(names = ('source_id',), dtype = (int,))
for i, gid in tqdm(enumerate(ID)):
if not isinstance(gid, str):
tbl.add_row(None)
tbl[-1][key] = int(re.sub(r"\D", "", gid))
elif len(gid) == 0:
tbl.add_row(None)
else:
gid = int(gid.replace(key+' ', ''))
adql_query = "select * from gaiadr1.gaia_source where source_id=%i" % (gid)
job = Gaia.launch_job(adql_query).get_results()
idx = np.where(job['source_id'].quantity == gid)[0]
if len(idx) > 0:
tbl = avstack([tbl, job[idx]])
else:
tbl.add_row(None)
tbl[-1][key] = int(re.sub(r"\D", "", gid))
self.GDR1 = tbl
return self.GDR1
def get_GAIA_data(coord, radius):
"""Query Gaia database.
Parameters
----------
coord : SkyCoord
Position to search.
radius : float
Radius of search cone.
Returns
-------
objects
table with the objects including the following info: ra (deg), ra_error (milliarcsec), dec (deg), dec_error (milliarcsec), mag
"""
j = Gaia.cone_search_async(coord, radius)
r = j.get_results()
#r.pprint()
#r.show_in_browser()
catalog_data = r.to_pandas()
catalog_data["mag"] = catalog_data["phot_g_mean_mag"]
catalog_data = catalog_data[["ra", "ra_error", "dec", "dec_error", "mag"]]
print("Found {} sources in GAIA within a radius of {}".format(
catalog_data.shape[0], radius))
#columns: ra, ra_error, dec, dec_error, mag
return catalog_data
def GaiaTable(gaia_ra, gaia_dec, starname):
global radsec
global info
global limmag
fov = radsec / 3600
try:
job = Gaia.launch_job_async("SELECT * FROM gaiadr2.gaia_source \
WHERE CONTAINS(POINT('ICRS',gaiadr2.gaia_source.ra,gaiadr2.gaia_source.dec), \
CIRCLE('ICRS',%f,%f,%f))=1 AND phot_g_mean_mag<%f \
AND pmra IS NOT NULL AND abs(pmra)>0 AND pmdec IS NOT NULL AND abs(pmdec)>0;" \
% (gaia_ra, gaia_dec, fov, limmag), dump_to_file=False)
gaiat = job.get_results()[
['source_id'] + list(info.columns)[:-2] +
['pmra', 'pmdec', 'pmra_error', 'pmdec_error']]
gaiat['phot_g_mean_mag'].name = 'mag'
df = gaiat.to_pandas()
df = df.set_index('source_id')
try:
df = df.drop(index=starname)
return df, True
except:
return df, True
except Exception as e:
print('Gaia \t' + ' ERROR:', e)
return ['NaN', False]
def download_gaia(tbl: Table) -> None:
"""Downloads Gaia data for stars not in catalog."""
if os.path.isfile(GAIA_PATH):
print('Gaia data already downloaded, skipping')
return
print("Querying Gaia")
source_ids = Table([
tbl[np.logical_not(np.logical_or(tbl['cel_exists'],
tbl['gaia'].mask))]['gaia']
])
query = r"""
SELECT
g.source_id, g.ra, g.dec, g.phot_g_mean_mag, g.bp_rp, d.r_est
FROM
TAP_UPLOAD.source_ids s
JOIN gaiadr2.gaia_source g ON g.source_id = s.gaia
LEFT JOIN external.gaiadr2_geometric_distance d ON d.source_id = g.source_id
"""
job = Gaia.launch_job_async(query,
output_file=GAIA_PATH,
output_format='votable',
dump_to_file=True,
upload_resource=source_ids,
upload_table_name='source_ids')
job.wait_for_job_end()
def get_data_subset(ra_deg,
dec_deg,
rad_deg,
dist,
dist_span=None,
rv_only=False):
if dist_span is not None:
max_parallax = 1e3 / (max(dist - dist_span, 1.))
min_parallax = 1e3 / (dist + dist_span)
else:
min_parallax = -1
max_parallax = 100
gaia_query = "SELECT source_id,ra,dec,parallax,parallax_error,pmra,pmra_error,pmdec,pmdec_error,phot_g_mean_mag,phot_bp_mean_mag,phot_rp_mean_mag,radial_velocity,radial_velocity_error,phot_variable_flag,a_g_val " +\
"FROM gaiadr2.gaia_source " +\
"WHERE parallax >= {:.4f} AND parallax <= {:.4f} ".format(min_parallax, max_parallax) +\
"AND CONTAINS(POINT('ICRS',gaiadr2.gaia_source.ra,gaiadr2.gaia_source.dec),CIRCLE('ICRS',{:.7f},{:.7f},{:.7f}))=1 ".format(ra_deg, dec_deg, rad_deg)
if rv_only:
gaia_query += 'AND (radial_velocity IS NOT NULL) '
# print ' QUERY:', gaia_quer
try:
gaia_job = Gaia.launch_job_async(gaia_query, dump_to_file=False)
gaia_data = gaia_job.get_results()
except:
print ' Problem querying data.'
return list([])
for g_c in gaia_data.colnames:
gaia_data[g_c].unit = ''
gaia_data['radial_velocity'].name = 'rv'
gaia_data['radial_velocity_error'].name = 'rv_error'
# print gaia_data
# print ' QUERY complete'
print ' Retireved lines:', len(gaia_data)
return gaia_data
def _get_catalog(self):
"""Returns GAIA catalog for current telescope coordinates."""
if self._catalog_coords is None or self._catalog_coords.separation(
self.telescope.real_pos) > 10. * u.arcmin:
self._catalog = Gaia.query_object_async(
coordinate=self.telescope.real_pos, radius=1. * u.deg)
return self._catalog
def getteff(c,cone=10):
G = nan
while G != G and cone < 180:
print "cone search radius",cone
#if True:
try:
width = u.Quantity(cone, u.arcsec)
height = u.Quantity(cone, u.arcsec)
result = Gaia.cone_search_async(c,width)
result = result.get_results()
#print result.columns
useindx = 0
print "len result",len(result)
if len(result) > 1:
Gmag = []
for i in range(len(result)):
Gmag.append(result[i]["phot_g_mean_mag"])
Gmag = array(Gmag)
print Gmag
useindx = argmin(Gmag)
G = result[useindx]["phot_g_mean_mag"]
teff = result[useindx]["teff_val"]
rstar = result[useindx]["radius_val"]
except IndexError:
G = nan
teff = nan
rstar = nan
cone += 20
return teff,rstar,G
def __init__(self, gaia_num, hiplogprob, dr='dr2'):
self.gaia_num = gaia_num
self.hiplogprob = hiplogprob
self.dr = dr
if self.dr == 'edr3':
self.gaia_epoch = 2016.0
elif self.dr == 'dr2':
self.gaia_epoch = 2015.5
else:
raise ValueError("`dr` must be either `dr2` or `edr3`")
self.hipparcos_epoch = 1991.25
query = """SELECT
TOP 1
ra, dec, ra_error, dec_error
FROM gaia{}.gaia_source
WHERE source_id = {}
""".format(self.dr, self.gaia_num)
job = Gaia.launch_job_async(query)
gaia_data = job.get_results()
self.ra = gaia_data['ra']
self.ra_err = gaia_data['ra_error']
self.dec = gaia_data['dec']
self.dec_err = gaia_data['dec_error']
# keep this number on hand for use in lnlike computation
self.mas2deg = (u.mas).to(u.degree)
def get_gaia_catalog(ra,
dec,
radius=4.,
limit=200,
catalog_fname='',
database='edr3'):
from astroquery.gaia import Gaia
query_args = {
'limit': limit,
'ra': ra,
'dec': dec,
'radius': radius / 60.,
'dr': database
}
query = """SELECT TOP {limit} ra, dec, phot_g_mean_mag FROM gaia{dr}.gaia_source
WHERE CONTAINS(POINT('ICRS', gaia{dr}.gaia_source.ra, gaia{dr}.gaia_source.dec),
CIRCLE('ICRS', {ra}, {dec}, {radius}))=1;""".format(
**query_args)
job = Gaia.launch_job_async(query,
dump_to_file=True,
output_format='csv',
verbose=False,
output_file=catalog_fname)
result = job.get_results()
return result
# Step 1: Gaia search.
input_coord = SkyCoord(ra=args.ra, dec=args.dec, unit=(u.degree, u.degree),
frame="icrs") # todo: allow user-specified frame
gaia_adql_constraints = f" AND {args.gaia_adql_constraints}" if args.gaia_adql_constraints else ""
logger.info(f"""Querying Gaia at {input_coord}
with radius {args.radius}
and constraints: {gaia_adql_constraints}""")
#j = Gaia.cone_search(input_coord, args.radius)
j = Gaia.launch_job(f"""SELECT DISTANCE(POINT('ICRS', ra, dec),
POINT('ICRS', {args.ra}, {args.dec})) AS dist, *
FROM gaiadr2.gaia_source
WHERE CONTAINS(POINT('ICRS', ra, dec),
CIRCLE('ICRS', {args.ra}, {args.dec}, {args.radius.to(u.deg).value})) = 1
{gaia_adql_constraints}
ORDER BY dist ASC""")
gaia_results = j.get_results()
G = len(gaia_results)
if G < 1:
raise NotImplementedError("no sources found in Gaia query")
if G > 1:
raise NotImplementedError("multiple sources found in Gaia query")
#python wobble_prepare.py 344.36658333333327 20.768833333333333
gaia_result = gaia_results[0]
def querycat_gaia(ralist=None, declist=None, cone_search=False,
width=10.0, height=10.0, radius=5.0, dr2=False,
test=False, debug=False):
"""
"""
import time
from astropy.table import Table, vstack
import astropy.units as u
from astropy.coordinates import SkyCoord
from astroquery.gaia import Gaia
if dr2:
help(Gaia)
gaiadr2_table = Gaia.load_table('gaiadr2.gaia_source')
for column in (gaiadr1_table.get_columns()):
print(column.get_name())
if debug:
help(Gaia)
width = u.Quantity(width/3600.0, u.degree)
height = u.Quantity(height/3600.0, u.degree)
radius = u.Quantity(radius/3600.0, u.degree)
if test is True:
ralist = [180.0]
declist = [0.0]
width = u.Quantity(30.0, u.arcsec)
height = u.Quantity(30.0, u.arcsec)
radius = u.Quantity(15.0, u.arcsec)
result_nrows = 0
for isource, (ra, dec) in enumerate(zip(ralist, declist)):
coord = SkyCoord(ra=ra, dec=dec,
unit=(u.degree, u.degree),
frame='icrs')
t0 = time.time()
if not cone_search:
result = Gaia.query_object_async(coordinate=coord,
width=width, height=height)
# help(result)
if debug:
result.pprint()
result.info('stats')
if cone_search:
job = Gaia.cone_search_async(coord, radius)
# help(job)
result = job.get_results()
print('Number of rows:', len(result))
print('Elapsed time(secs):',time.time() - t0)
if debug:
help(result)
result.pprint()
result.info('stats')
result_nrows = result_nrows + len(result)
if isource == 0:
result_all = result
if isource > 0:
result_all = vstack([result_all, result])
#def fix_vot_object():
table = result_all
print('icol, format, dtype')
# help(table)
# help(table.columns)
for (icol, column) in enumerate(table.columns):
print(icol, table.columns[icol].name,
table.columns[icol].format,
table.columns[icol].dtype)
# convert the columns for dtype = object which is not
# supported by FITs to bool
if table.columns[icol].dtype == 'object':
colname = table.columns[icol].name
NewColumn = Table.Column(table[colname].data, dtype='bool')
table.replace_column(colname, NewColumn)
print()
result_all = table
print('Number of Gaia sources returned:', result_nrows, len(result_all))
return result_all
#~ INNER JOIN gaiadr2.tmass_best_neighbour AS tmass
#~ ON gaia.source_id = tmass.source_id
#~ WHERE tmass.original_ext_source_id IN %s
#~ '''%str(tuple(tmass))
# Search with Gaia IDs
#gaiaid=[5283961585534643712, 5283965296387249920]
gaiaid = np.load('../data/bpmg_sourceids.npy')
query='''
SELECT gaia.*
FROM gaiadr2.gaia_source AS gaia
WHERE gaia.source_id IN %s
'''%str(tuple(gaiaid))
job = Gaia.launch_job_async(query, dump_to_file=True)
# Your astropy table with results
r = job.get_results()
#keys=['source_id', 'phot_bp_mean_flux','ra_pmdec_corr','ra_error','ra','pmra_error','ecl_lon','designation','l','phot_rp_mean_mag','parallax_pmdec_corr','ra_parallax_corr','pmdec_error','phot_g_mean_mag','pmra','parallax','radial_velocity','radial_velocity_error','ra_dec_corr','parallax_error','dec_pmdec_corr','dec_error','pmdec','parallax_over_error','b','ref_epoch','ra_pmra_corr','dec_parallax_corr','phot_bp_mean_mag','dec','dec_pmra_corr','pmra_pmdec_corr','parallax_pmra_corr','bp_rp','ecl_lat']
keys=['source_id','phot_bp_mean_flux','ra_error','ra','phot_rp_mean_mag',
'phot_g_mean_mag','phot_bp_mean_mag','bp_rp']
r2=r[keys]
d=dict(zip(r2['source_id'], r2)) # for easier crossmatch with source_id
print r2
# for source_id in gaiaid:
# try: