def Gaia_adql(query, upload=None):
"""Run query on gaia archive and return results as a pandas dataframe.
If upload is a pandas dataframe then this is uploaded to the archive
and available as the table tap_upload.uploadedtable"""
# We take and return pandas DataFrames but use astropy tables to make a votable of coordinates to upload
with tempfile.NamedTemporaryFile(suffix='.xml',
mode='w') as file_to_upload:
if upload is not None:
table = Table.from_pandas(upload)
table.write(file_to_upload, format='votable')
# It seems Gaia DR2 source table produces many votable warnings that aren't important
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
with suppress_stdout():
if upload is not None:
job = Gaia.launch_job_async(
query=query,
upload_resource=file_to_upload.name,
upload_table_name="uploadedtable")
else:
job = Gaia.launch_job_async(query=query)
result = job.get_results()
result_df = result.to_pandas()
return result_df
def add_gaia(self):
"""Retrieve and add Gaia DR2 or DR1 parameters."""
if hasattr(self, 'IDS') is False:
raise RuntimeError('RUN set_simbad_fields() first!')
try:
self.simbad_identifiers = self.IDS.decode().split('|')
except AttributeError:
self.simbad_identifiers = self.IDS.split('|')
try:
gaia_dr2_id = [id for id in self.simbad_identifiers if 'Gaia DR2 ' in id][0].replace("'", "")
# print(gaia_dr2_id)
gacs_query = "SELECT * FROM gaiadr2.gaia_source WHERE source_id={}".format(gaia_dr2_id.split(' ')[-1])
job = Gaia.launch_job_async(gacs_query)
self.gaiadr2_table = job.get_results()
except IndexError:
try:
gaia_dr1_id = [id for id in self.simbad_identifiers if 'Gaia DR1 ' in id][0]
# return_gacs_query_as_table(query_string, output_file_seed, overwrite=False, verbose=True):
gacs_query = "SELECT * FROM gaiadr1.gaia_source WHERE source_id={}".format(gaia_dr1_id.split(' ')[-1])
job = Gaia.launch_job_async(gacs_query)
self.gaiadr1_table = job.get_results()
except IndexError:
print('No Gaia identifier listed in Simbad!')
def getGaiaBox(ra, dec, width, height=None):
"""
Acquire table of all Gaia stars in a box centered at ra, dec
and having half-side-length equal to radius.
All three arguments should be in degrees.
Returns an astropy Table object.
"""
if height is None:
height = width
query ="SELECT s.source_id, s.ra, s.ra_error, s.dec, s.dec_error, s.parallax, s.parallax_error, " + \
"s.pmra, s.pmra_error, s.pmdec, s.pmdec_error, " + \
"s.ra_dec_corr, s.ra_parallax_corr, s.ra_pmra_corr, s.ra_pmdec_corr, " + \
"s.dec_parallax_corr, s.dec_pmra_corr, s.dec_pmdec_corr, " + \
"s.parallax_pmra_corr, s.parallax_pmdec_corr, " + \
"s.pmra_pmdec_corr, " + \
"s.phot_g_mean_mag as Gmag, s.phot_g_mean_flux_over_error as g_sn, " + \
"s.phot_bp_mean_mag as bpmag, s.phot_rp_mean_mag as rpmag " + \
"FROM gaiadr2.gaia_source s " + \
"INNER JOIN gaiadr2.ruwe r on s.source_id=r.source_id " + \
"WHERE s.astrometric_params_solved=31 " + \
" AND r.ruwe<1.4 AND s.visibility_periods_used>8 AND " + \
"CONTAINS(POINT('ICRS',s.ra,s.dec),BOX('ICRS',{:f},{:f},{:f},{:f}))=1;".format(\
ra, dec, \
width/np.cos(dec*np.pi/180.), height)
job = Gaia.launch_job_async(query, dump_to_file=False)
#print("Job launched")
#print(job)
return job.get_data()
def GaiaTable(gaia_ra, gaia_dec, starname):
global radsec
global info
global limmag
fov = radsec / 3600
try:
job = Gaia.launch_job_async("SELECT * FROM gaiadr2.gaia_source \
WHERE CONTAINS(POINT('ICRS',gaiadr2.gaia_source.ra,gaiadr2.gaia_source.dec), \
CIRCLE('ICRS',%f,%f,%f))=1 AND phot_g_mean_mag<%f \
AND pmra IS NOT NULL AND abs(pmra)>0 AND pmdec IS NOT NULL AND abs(pmdec)>0;" \
% (gaia_ra, gaia_dec, fov, limmag), dump_to_file=False)
gaiat = job.get_results()[
['source_id'] + list(info.columns)[:-2] +
['pmra', 'pmdec', 'pmra_error', 'pmdec_error']]
gaiat['phot_g_mean_mag'].name = 'mag'
df = gaiat.to_pandas()
df = df.set_index('source_id')
try:
df = df.drop(index=starname)
return df, True
except:
return df, True
except Exception as e:
print('Gaia \t' + ' ERROR:', e)
return ['NaN', False]
def download_gaia_data(colname: str, xindex: Union[str, Table], outfile_path: Path) -> None:
"""Query and download Gaia data."""
if isinstance(xindex, str):
xindex_name = cast(str, xindex)
upload_resource = None
upload_table_name = None
else:
xindex_name = "TAP_UPLOAD.cel_xindex"
upload_resource = cast(Table, xindex)
upload_table_name = 'cel_xindex'
query = f"""SELECT
x.source_id, x.original_ext_source_id AS {colname},
g.ra, g.dec, g.parallax, g.parallax_error, g.pmra,
g.pmdec, g.phot_g_mean_mag, g.bp_rp, g.teff_val,
d.r_est, d.r_lo, d.r_hi
FROM
{xindex_name} x
JOIN gaiadr2.gaia_source g ON g.source_id = x.source_id
LEFT JOIN external.gaiadr2_geometric_distance d ON d.source_id = x.source_id"""
print(query)
job = Gaia.launch_job_async(
query,
upload_resource=upload_resource,
upload_table_name=upload_table_name,
dump_to_file=True,
output_file=outfile_path,
output_format='csv',
)
try:
job.save_results()
finally:
Gaia.remove_jobs(job.jobid)
def get_gaia_catalog(ra,
dec,
radius=4.,
limit=200,
catalog_fname='',
database='edr3'):
from astroquery.gaia import Gaia
query_args = {
'limit': limit,
'ra': ra,
'dec': dec,
'radius': radius / 60.,
'dr': database
}
query = """SELECT TOP {limit} ra, dec, phot_g_mean_mag FROM gaia{dr}.gaia_source
WHERE CONTAINS(POINT('ICRS', gaia{dr}.gaia_source.ra, gaia{dr}.gaia_source.dec),
CIRCLE('ICRS', {ra}, {dec}, {radius}))=1;""".format(
**query_args)
job = Gaia.launch_job_async(query,
dump_to_file=True,
output_format='csv',
verbose=False,
output_file=catalog_fname)
result = job.get_results()
return result
def query_gaia_KiDS_testpatch(
self, pointing): #pointing = [central_ra, central_dec]
patch_ra = [pointing[0] - 0.5, pointing[0] + 0.5]
patch_dec = [pointing[1] - 0.5, pointing[1] + 0.5]
query_KiDS_testpatch = "select * from gaiadr2.gaia_source where ra<{} and ra>{} and dec<{} and dec>{} and parallax IS NOT NULL".format(
patch_ra[1], patch_ra[0], patch_dec[1], patch_dec[0])
print("Start query_gaia_KiDS_testpatch()")
if not os.path.exists(
"../../data/gaia/gaia_KiDS_testpatch[{}_{}].csv".format(
pointing[0], pointing[1])):
filename = "gaia_KiDS_testpatch[{}_{}].csv".format(
pointing[0], pointing[1])
else:
print(
"PATH ../../data/gaia_KiDS_testpatch[{}_{}].csv already exist \n Overwrite existing file? [y/n]"
).format(pointing[0], pointing[1])
answer = input()
if answer == "y" or answer == "yes":
filename = "gaia_euclid"
elif answer == "n" or answer == "no":
print(
"Give another filename (no gaia_KiDS_testpatch) to save the data"
)
filename = input()
else:
print(
"Your answer: {} is not valid: Please return y, yes, n or no"
.format(answer))
return query_gaia_KiDS_testpatch()
job = Gaia.launch_job_async(query_KiDS_testpatch)
r = job.get_results()
ascii.write(r, "../../data/gaia/{}".format(filename), delimiter=',')
return None
def from_query(cls, query_str, login_info=None):
"""
Run the specified query and return a `GaiaData` instance with the
returned data.
This is meant only to be used for quick queries to the main Gaia science archive. For longer queries and more customized usage, use TAP access to any of the Gaia mirrors with, e.g., astroquery or pyvo.
This requires ``astroquery`` to be installed.
Parameters
----------
query_str : str
The string ADQL query to execute.
login_info : dict, optional
Username and password for the Gaia science archive as keys "user"
and "password". If not specified, will use anonymous access, subject
to the query limits.
Returns
-------
gaiadata : `GaiaData`
An instance of this object.
"""
from astroquery.gaia import Gaia
if login_info is not None:
Gaia.login(**login_info)
job = Gaia.launch_job_async(query_str)
tbl = job.get_results()
return cls(tbl)
def gaia_params(self):
"""Retrieve parallax, radius, teff and lum from Gaia."""
# If gaia DR2 id is provided, query by id
fields = np.array([
'parallax', 'parallax_error', 'teff_val',
'teff_percentile_lower', 'teff_percentile_upper',
'radius_val', 'radius_percentile_lower',
'radius_percentile_upper', 'lum_val',
'lum_percentile_lower', 'lum_percentile_upper'
])
query = 'select '
for f in fields[:-1]:
query += 'gaia.' + f + ', '
query += 'gaia.' + fields[-1]
query += ' from gaiadr2.gaia_source as gaia'
query += ' where gaia.source_id={0}'.format(self.g_id)
j = Gaia.launch_job_async(query)
res = j.get_results()
self.plx, self.plx_e = self._get_parallax(res)
self.temp, self.temp_e = self._get_teff(res)
self.rad, self.rad_e = self._get_radius(res)
self.lum, self.lum_e = self._get_lum(res)
self.dist, self.dist_e = self._get_distance(self.ra, self.dec,
self.radius, self.g_id)
pass
def query_gaia_limEUCLIDSKY(self):
query = "select * from gaiadr2.gaia_source where abs(B)>{} and abs(ecl_lat)>{}".format(
self.galactic_latitude, self.celestial_latitude)
print("Start query_gaia_limEUCLIDSKY()")
if not os.path.exists("../../data/gaia_euclid.csv"):
filename = "gaia_euclid"
else:
print(
"PATH ../../data/gaia_euclid.csv already exist \n Overwrite existing file? [y/n]"
)
answer = input()
if answer == "y" or answer == "yes":
filename = "gaia_euclid"
elif answer == "n" or answer == "no":
print(
"Give another filename (no gaia_euclid) to save the data")
filename = input()
else:
print(
"Your answer: {} is not valid: Please return y, yes, n or no"
.format(answer))
return query_gaia_EUCLIDSKY()
job = Gaia.launch_job_async(query)
r = job.get_results()
ascii.write(r,
"../../data/gaia/{}.csv".format(filename),
delimiter=',')
return None
def get(self, **kwargs):
"""Build and performe query.
Parameters
----------
Parameters that are passed through **kwargs to
astroquery.gaia.Gaia.launch_job_async
For example:
dump_to_file : bool
If True, results will be stored in file
(default is False).
output_file : str
Name of the output file
Returns
-------
octable : opencluster.OCTable
Instance with query results,
None if dump_to_file is True
"""
query = self.build()
job = Gaia.launch_job_async(query=query, **kwargs)
if not kwargs.get("dump_to_file"):
table = job.get_results()
return table
def __init__(self, gaia_num, hiplogprob, dr='dr2'):
self.gaia_num = gaia_num
self.hiplogprob = hiplogprob
self.dr = dr
if self.dr == 'edr3':
self.gaia_epoch = 2016.0
elif self.dr == 'dr2':
self.gaia_epoch = 2015.5
else:
raise ValueError("`dr` must be either `dr2` or `edr3`")
self.hipparcos_epoch = 1991.25
query = """SELECT
TOP 1
ra, dec, ra_error, dec_error
FROM gaia{}.gaia_source
WHERE source_id = {}
""".format(self.dr, self.gaia_num)
job = Gaia.launch_job_async(query)
gaia_data = job.get_results()
self.ra = gaia_data['ra']
self.ra_err = gaia_data['ra_error']
self.dec = gaia_data['dec']
self.dec_err = gaia_data['dec_error']
# keep this number on hand for use in lnlike computation
self.mas2deg = (u.mas).to(u.degree)
def query_gaia(ra_deg,
dec_deg,
radius_deg,
minmag=10,
maxmag=20,
maxsources=10000,
catalogname='gaiacatalog'):
job = Gaia.launch_job_async(
"SELECT * FROM gaiadr2.gaia_source AS g, gaiadr2.panstarrs1_best_neighbour \
AS pbest, gaiadr2.panstarrs1_original_valid AS ps1 WHERE g.source_id = pbest.source_id AND \
pbest.original_ext_source_id = ps1.obj_id AND CONTAINS(POINT('ICRS', g.ra, g.dec), \
CIRCLE('ICRS', %.4f, %.4f, %.4f))=1 AND ps1.r_mean_psf_mag > %.2f AND ps1.r_mean_psf_mag \
< %.2f AND pmra IS NOT NULL AND pmdec IS NOT NULL AND abs(pmdec) > 0 AND \
abs(pmdec) < 40 AND abs(pmra)>0 AND abs(pmra) < 40 AND ps1.n_detections > 6 \
AND pbest.number_of_mates=0 AND \ pbest.number_of_neighbours=1;" %
(ra_deg, dec_deg, radius_deg, 10, 20))
p = job.get_results()
p['ra_errdeg'] = p['ra_error'] / 3.6e6
p['dec_errdeg'] = p['dec_error'] / 3.6e6
p['FLAGS'] = 0
if os.path.exists(catalogname + '.ldac'):
os.remove(catalogname + '.ldac')
if os.path.exists('gaiacatalog.dat'):
os.remove('gaiacatalog.dat')
save_table_as_ldac(p, catalogname + '.ldac')
def get_data_subset(ra_deg,
dec_deg,
rad_deg,
dist,
dist_span=None,
rv_only=False):
if dist_span is not None:
max_parallax = 1e3 / (max(dist - dist_span, 1.))
min_parallax = 1e3 / (dist + dist_span)
else:
min_parallax = -1
max_parallax = 100
gaia_query = "SELECT source_id,ra,dec,parallax,parallax_error,pmra,pmra_error,pmdec,pmdec_error,phot_g_mean_mag,phot_bp_mean_mag,phot_rp_mean_mag,radial_velocity,radial_velocity_error,phot_variable_flag,a_g_val " +\
"FROM gaiadr2.gaia_source " +\
"WHERE parallax >= {:.4f} AND parallax <= {:.4f} ".format(min_parallax, max_parallax) +\
"AND CONTAINS(POINT('ICRS',gaiadr2.gaia_source.ra,gaiadr2.gaia_source.dec),CIRCLE('ICRS',{:.7f},{:.7f},{:.7f}))=1 ".format(ra_deg, dec_deg, rad_deg)
if rv_only:
gaia_query += 'AND (radial_velocity IS NOT NULL) '
# print ' QUERY:', gaia_quer
try:
gaia_job = Gaia.launch_job_async(gaia_query, dump_to_file=False)
gaia_data = gaia_job.get_results()
except:
print ' Problem querying data.'
return list([])
for g_c in gaia_data.colnames:
gaia_data[g_c].unit = ''
gaia_data['radial_velocity'].name = 'rv'
gaia_data['radial_velocity_error'].name = 'rv_error'
# print gaia_data
# print ' QUERY complete'
print ' Retireved lines:', len(gaia_data)
return gaia_data
def jobGaiaDR2(ra0, dec0, limgmag, fov, yr):
ssl._create_default_https_context = ssl._create_unverified_context
job = Gaia.launch_job_async("SELECT * \
FROM gaiadr2.gaia_source \
WHERE CONTAINS(POINT('ICRS',gaiadr2.gaia_source.ra,gaiadr2.gaia_source.dec),CIRCLE('ICRS',%f,%f,%f))=1\
AND pmra IS NOT NULL AND abs(pmra)>0 \
AND pmdec IS NOT NULL AND abs(pmdec)>0\
AND phot_g_mean_mag<%f;" %(ra0,dec0,fov,limgmag) \
, dump_to_file=False)
gaiat = job.get_results()
# print(gaiat)
ra = np.array(gaiat['ra'])
dec = np.array(gaiat['dec'])
pmra = np.array(gaiat['pmra'])
pmdec = np.array(gaiat['pmdec'])
gmag = np.array(gaiat['phot_g_mean_mag'])
bpmag = np.array(gaiat['phot_bp_mean_mag'])
rpmag = np.array(gaiat['phot_rp_mean_mag'])
des = np.array(gaiat['designation'])
N = np.size(gmag)
for k in range(N):
alpha, delta = RADecFromTang((yr - 2015.5) * math.pi * pmra[k] / 648000000.0, \
(yr - 2015.5) * math.pi * pmdec[k] / 648000000.0, \
math.radians(ra[k]), math.radians(dec[k]))
print(ra[k], dec[k], math.degrees(alpha), math.degrees(delta))
ra[k], dec[k] = math.degrees(alpha), math.degrees(delta)
return ra, dec, gmag, bpmag, rpmag, des
def download_votable(tile,
offset=0.005,
output_dir=dirconfig.raw_gaia,
gaia_source='gaiadr2.gaia_source'):
"""
This function automatically download gaia-data from gaia-archive using AstroQuery module.
tile: a Tile objects.
Implemented query extract all the sources within the coordinates
offset: is a safe-margin (in degrees) added to the borders of each tile
output_dir : where vo-tables are saved
gaia_source: gaiaedr3.gaia_source or gaiadr2.gaia_source
"""
print('---------------------------------------------')
lmin = tile.lmin - offset
lmax = tile.lmax + offset
bmin = tile.bmin - offset
bmax = tile.bmax + offset
query = f'SELECT * FROM {gaia_source} WHERE l > {lmin:.4f} AND l < {lmax:.4f} AND b > {bmin:.4f} AND b < {bmax:.4f} '
print(query)
file_path = path.join(output_dir, tile.name + '_gaia.vot.gz')
t1 = time.time()
job = Gaia.launch_job_async(query,
dump_to_file=True,
output_file=file_path)
t2 = time.time()
print(f'Delta t: {t2 - t1:.4f} s')
def gaiadr2xdr3(source_idlist, nearest=True):
"""
returns the dr2 to dr3 cross matches for the given source_idlist
"""
upload_tablename, upload_resource, sidcol = source_id_to_xmlfile(
source_idlist)
query_str = ' '.join([
f'SELECT tu.{sidcol}, dr2.*',
f'from tap_upload.{upload_tablename} tu left join gaiaedr3.dr2_neighbourhood dr2',
f'on tu.{sidcol} = dr2.dr2_source_id'
])
try:
job = Gaia.launch_job_async(query=query_str,
upload_resource=upload_resource,
upload_table_name=upload_tablename)
df = job.get_results().to_pandas()
finally:
os.remove(upload_resource)
if nearest:
#just return the nearest dr3 source id based on angular distance
ret_df = df.sort_values(['dr2_source_id', 'angular_distance']).groupby(
'dr2_source_id', as_index=False).first().set_index('source_id')
else:
ret_df = df.set_index('source_id')
return ret_df
def gaia_query(n, distance=200, **kwargs):
"""
Sends an archive query for d < 200 pc, with additional filters taken from
Gaia Data Release 2: Observational Hertzsprung-Russell diagrams (Sect. 2.1)
Gaia Collaboration, Babusiaux et al. (2018)
(https://doi.org/10.1051/0004-6361/201832843)
NOTE: 10000000 is a maximum query size (~76 MB / column)
Additional keyword arguments are passed to TapPlus.launch_job_async method.
"""
return Gaia.launch_job_async(
"select top {}".format(n) +
#" lum_val, teff_val,"
#" ra, dec, parallax,"
" bp_rp, phot_g_mean_mag+5*log10(parallax)-10 as mg"
" from gaiadr2.gaia_source"
" where parallax_over_error > 10"
" and visibility_periods_used > 8"
" and phot_g_mean_flux_over_error > 50"
" and phot_bp_mean_flux_over_error > 20"
" and phot_rp_mean_flux_over_error > 20"
" and phot_bp_rp_excess_factor <"
" 1.3+0.06*power(phot_bp_mean_mag-phot_rp_mean_mag,2)"
" and phot_bp_rp_excess_factor >"
" 1.0+0.015*power(phot_bp_mean_mag-phot_rp_mean_mag,2)"
" and astrometric_chi2_al/(astrometric_n_good_obs_al-5)<"
"1.44*greatest(1,exp(-0.4*(phot_g_mean_mag-19.5)))" +
" and 1000/parallax <= {}".format(distance),
**kwargs)
def download_gaia(tbl: Table) -> None:
"""Downloads Gaia data for stars not in catalog."""
if os.path.isfile(GAIA_PATH):
print('Gaia data already downloaded, skipping')
return
print("Querying Gaia")
source_ids = Table([
tbl[np.logical_not(np.logical_or(tbl['cel_exists'],
tbl['gaia'].mask))]['gaia']
])
query = r"""
SELECT
g.source_id, g.ra, g.dec, g.phot_g_mean_mag, g.bp_rp, d.r_est
FROM
TAP_UPLOAD.source_ids s
JOIN gaiadr2.gaia_source g ON g.source_id = s.gaia
LEFT JOIN external.gaiadr2_geometric_distance d ON d.source_id = g.source_id
"""
job = Gaia.launch_job_async(query,
output_file=GAIA_PATH,
output_format='votable',
dump_to_file=True,
upload_resource=source_ids,
upload_table_name='source_ids')
job.wait_for_job_end()
def fetch_gaia(tmass_ids):
from astroquery.gaia import Gaia
# Can't prefix columns with the table name, so have to settle on using 'panda_start' and
# 'wise_start'
query = """
select
mine.tmass_id as tmass_id,
'' as gaia_start,
gaia.*,
'' as wise_start,
allwise.*
from gaiadr2.gaia_source as gaia
inner join gaiadr2.tmass_best_neighbour as tmass_xmatch
on gaia.source_id = tmass_xmatch.source_id
inner join tap_upload.mine as mine
on tmass_xmatch.original_ext_source_id = mine.tmass_id
inner join gaiadr2.allwise_best_neighbour as allwise_xmatch
on gaia.source_id = allwise_xmatch.source_id
inner join gaiadr1.allwise_original_valid as allwise
on allwise_xmatch.original_ext_source_id = allwise.designation"""
with tempfile.NamedTemporaryFile(suffix='.xml') as tmp:
os.remove(tmp.name) # astropy will complain if the file already exists
(astropy.table.Table(tmass_ids[:, None].astype(bytes),
names=['tmass_id']).write(tmp.name,
format='votable'))
log.info(f'Launching job for {len(tmass_ids)} 2MASS IDs')
job = Gaia.launch_job_async(query,
upload_resource=tmp.name,
upload_table_name='mine')
log.info(f'Job ID is {job.get_jobid()}')
while True:
time.sleep(5)
log.info(f'Job is {job.get_phase()}')
if job.get_phase() == 'COMPLETED':
table = job.get_results()
break
df = table.to_pandas().pipe(stringify)
gaia_start = list(df.columns).index('gaia_start')
wise_start = list(df.columns).index('wise_start')
indices = sp.arange(len(df.columns))
masks = {
'tmass': df.columns == 'tmass_id',
'gaia': (gaia_start < indices) & (indices < wise_start),
'wise': (wise_start < indices)
}
df = pd.concat({k: df.loc[:, m] for k, m in masks.items()}, 1)
# Some fields have a _2 suffixed because they're replicated in GAIA and WISE
df = df.rename(columns=lambda c: c.split('_2')[0])
return df
def gaia_query(ra_0, ra_1, dec_0, dec_1, threshold=20):
jobquery = (
"SELECT gaia_source.source_id,gaia_source.ra,gaia_source.dec,gaia_source.parallax,gaia_source.parallax_error,gaia_source.pmra,gaia_source.pmdec,gaia_source.phot_g_mean_mag,gaia_source.phot_bp_mean_mag,gaia_source.phot_rp_mean_mag,gaia_source.bp_rp,gaia_source.radial_velocity,gaia_source.phot_variable_flag,gaia_source.teff_val,gaia_source.lum_val FROM gaiadr2.gaia_source WHERE CONTAINS(POINT('ICRS',gaiadr2.gaia_source.ra,gaiadr2.gaia_source.dec),BOX('ICRS',"
+ str((ra_0 + ra_1) / 2) + "," + str((dec_0 + dec_1) / 2) + "," +
str(abs(ra_1 - ra_0)) + "," + str(abs(dec_1 - dec_0)) +
"))=1 AND (phot_bp_mean_mag<= " + str(threshold) + ")")
job = Gaia.launch_job_async(jobquery, dump_to_file=True)
return job.get_results()
def query_gaia_match(user_name,table_2_match,radius_arc):
'''
This function queries Gaia DR2 to find the match with table
table_2_match. This table needs to be uploaded to the Gaia
Archive in advance. The steps to upload the table can be found
in http://gea.esac.esa.int/archive-help/index.html.
Input:
user_name(str): User name in the Gaia Archive.
table_2_match(str): Name of the table uploaded to the Gaia
Archive.
radius_arc(float): Radius of search in arcsec.
Output:
file.vot: Output file from the Archive with all the columns
in table_2_match plus all Gaia DR2 columns.
'''
Gaia.login_gui()
#Ask for userName and userPassword to authenticated access mode
#This could be done with:
#Gaia.login(user='******', password='******')
#Cross-match user table and gaia source
job = Gaia.launch_job_async("""\
SELECT crossmatch_positional(\
'user_{}','{}',\
'gaiadr2','gaia_source',\
{},\
'xmatch')\
FROM dual;\
""".format(user_name,table_2_match,radius_arc))
#For the matches saved in test_xmatch, get the information from Gaia.
#The last line saves the information into a vot table
job2 = Gaia.launch_job_async("""\
SELECT c."dist", a.*, b.* \
FROM user_{}.{} AS a, \
gaiadr2.gaia_source AS b, \
user_{}.xmatch AS c \
WHERE (c.{}_{}_oid = a.{}_oid AND \
c.gaia_source_source_id = b.source_id)\
""".format(user_name,table_2_match,user_name,
table_2_match,table_2_match,table_2_match), dump_to_file=True)
Gaia.logout()
def get_gaia(df):
import time
import numpy as np
import pandas as pd
from astroquery.vizier import Vizier
from astropy.coordinates import SkyCoord
from astropy.coordinates import Angle
from astropy import units as u
from astropy.table import Column, Table, join
from astroquery.gaia import Gaia
Gaia.login(user='******',password='******')
qry = """
SELECT TOP 10 g.*, t.*
FROM gaiadr1.tmass_original_valid AS t
JOIN gaiadr2.tmass_neighbourhood AS xt ON xt.tmass_oid=t.tmass_oid
JOIN gaiadr2.gaia_source AS g ON g.source_id=xt.source_id
WHERE g.phot_g_mean_mag IS NOT NULL
"""
bkg=Gaia.launch_job_async(qry).get_results().to_pandas()
bkg['abs_g']=bkg.phot_g_mean_mag-5*np.log10(1000./bkg.parallax)+5.
columns=bkg.columns.tolist()
GAIAdf=pd.DataFrame(index=df.index,columns=columns)
for k,row in df.iterrows():
qry="""
SELECT g.*, t.*
FROM gaiadr1.tmass_original_valid AS t
LEFT OUTER JOIN gaiadr2.tmass_neighbourhood AS xt ON xt.tmass_oid = t.tmass_oid
LEFT OUTER JOIN gaiadr2.gaia_source AS g ON xt.source_id = g.source_id
where 1=CONTAINS(POINT('ICRS', t.ra, t.dec),CIRCLE('ICRS', {}, {}, 5./3600))
""".format(row['RA (deg)'],row['Dec (deg)'])
data=Gaia.launch_job_async(qry).get_results().to_pandas()
data['INDEX']=k
print(k,data.shape)
GAIAdf=GAIAdf.append(data).dropna(how='all')
Gaia.logout()
return GAIAdf
def query_gaia(self):
"""
Query Gaia EDR3 catalogue for bright stars (G < 12) and
store results in a FITS file.
"""
from astroquery.gaia import Gaia
gaia_dir = self.index_dir
# Query in two parts to overcome the 3-million-row limit
query = ('SELECT ra,dec,pmra,pmdec,phot_g_mean_mag,'
'phot_bp_mean_mag,phot_rp_mean_mag '
'FROM gaiaedr3.gaia_source '
'WHERE phot_g_mean_mag<11.5 '
'AND astrometric_params_solved=31')
fn_tab1 = os.path.join(gaia_dir, 'gaiaedr3_pyplate_1.fits')
job = Gaia.launch_job_async(query,
output_file=fn_tab1,
output_format='fits',
dump_to_file=True)
query = ('SELECT ra,dec,pmra,pmdec,phot_g_mean_mag,'
'phot_bp_mean_mag,phot_rp_mean_mag '
'FROM gaiaedr3.gaia_source '
'WHERE phot_g_mean_mag BETWEEN 11.5 AND 12 '
'AND astrometric_params_solved=31')
fn_tab2 = os.path.join(gaia_dir, 'gaiaedr3_pyplate_2.fits')
job = Gaia.launch_job_async(query,
output_file=fn_tab2,
output_format='fits',
dump_to_file=True)
# Read two tables and concatenate them
tab1 = Table.read(fn_tab1)
tab2 = Table.read(fn_tab2)
tab = vstack([tab1, tab2], join_type='exact')
fn_tab = os.path.join(gaia_dir, 'gaiaedr3_pyplate.fits')
tab.write(fn_tab, format='fits', overwrite=True)
# Remove partial tables
os.remove(fn_tab1)
os.remove(fn_tab2)
def getdata(RA,DEC,SR):
job = Gaia.launch_job_async("SELECT * \
FROM gaiadr2.gaia_source \
WHERE CONTAINS(POINT('ICRS',gaiadr2.gaia_source.ra,gaiadr2.gaia_source.dec),\
CIRCLE('ICRS'," + str(RA) + "," + str(DEC) + "," + str(SR) + "))=1 ORDER BY phot_g_mean_mag ASC;" \
, dump_to_file=True)
r = job.get_results()
return r
def target_contam_gaia(ra, dec, srad=15.5 * 20):
c = SkyCoord(ra=ra * u.degree, dec=dec * u.degree)
# Pgaia = Gaia.query_object_async(coordinate=c, radius=(5.0*u.arcsec))
sqltext = "SELECT * FROM gaiaedr3.gaia_source WHERE CONTAINS( \
POINT('ICRS',gaiaedr3.gaia_source.ra,gaiaedr3.gaia_source.dec), \
CIRCLE('ICRS'," + str(c.ra.value) + "," + str(
c.dec.value) + "," + str(srad / 3600.0) + "))=1;"
job = Gaia.launch_job_async(sqltext, dump_to_file=False)
obs_table_gaia = job.get_results()
return (obs_table_gaia)
def genSimulation(configpath, star, orders = [-1,0,1], seeing = 1):
""""Generates the arguments for plot_all to simulate the spectrogram of star through a slitless spectrogram which properties are contained in the configuration file.
Parameters
----------
configpath : string
path to the slitless spectrograph configutation file
star : string
the studied star identifier
orders : list of integers
the list of orders to represent.
seeing : float
the wanted seeing for the spectrogram in arcseconds, 1 arcsec by default
Returns
-------
stars : list
a list of star_class objects, containing the stars to take into account with their position on the captor,
img : 2d ndarray
a background image for the stars,
angle : float
the angle of dispersion used for the spectrogram,
config : configuration
the configuration used for the spectrogram,
LTh : list
a list of angles for Overlap_list,
Overlap_list : list
the list of the total overlap caused by surrounding stars on the studied star spectrum
"""
config = configuration(configpath)
imsize, pix2ars = config.ccd_imsize, config.pixel2arcsec
query = customSimbad.query_object(star)
ra0, dec0, mag0 = query["RA_d"][0], query["DEC_d"][0], query["FLUX_V"][0]
r = 2*imsize*config.pixel2arcsec/3600
maglim = mag0 + 7.5
job = Gaia.launch_job_async("SELECT ra, dec, DISTANCE(POINT('ICRS', ra, dec),POINT('ICRS', {0}, {1})) AS dist, phot_g_mean_mag AS flux FROM gaiadr2.gaia_source WHERE CONTAINS(POINT('ICRS', ra, dec),CIRCLE('ICRS', {0}, {1}, {2})) = 1 AND phot_g_mean_mag < {3} ORDER BY dist".format(ra0, dec0, r, maglim))
result = job.get_results()
ra0, dec0 = result['ra'][0], result['dec'][0]
config.wcs.wcs.crval = [ra0,dec0]
cra, cdec = config.wcs.wcs_pix2world([[imsize/2, imsize/2]], 0)[0]
stars = list_stars(result,config,maglim,seeing, orders)
angle, LTh, Overlap_list = best_angle(stars, config)
for star in stars :
star.rotate_orders(angle, use_radians = True)
LTh = LTh*180/np.pi
img = SkyView.get_images(position = '{}, {}'.format(cra, cdec), survey = 'DSS', width = imsize*pix2ars*u.arcsec, height = imsize*pix2ars*u.arcsec)[0][0].data
return (stars, img, angle, config, LTh, Overlap_list)
def make_gaia_catalog(ra, dec, radius_deg, catalog_min_mag, catalog_max_mag,
catname):
job = Gaia.launch_job_async(
"SELECT * FROM gaiadr2.gaia_source AS g, gaiadr2.panstarrs1_best_neighbour AS pbest, gaiadr2.panstarrs1_original_valid AS ps1 WHERE g.source_id = pbest.source_id AND pbest.original_ext_source_id = ps1.obj_id AND CONTAINS(POINT('ICRS', g.ra, g.dec), CIRCLE('ICRS', %.4f, %.4f, %.4f))=1 AND ps1.r_mean_psf_mag > %.2f AND ps1.r_mean_psf_mag < %.2f AND pmra IS NOT NULL AND pmdec IS NOT NULL AND abs(pmdec) > 0 AND abs(pmdec) < 40 AND abs(pmra)>0 AND abs(pmra) < 40 AND ps1.n_detections > 6 AND pbest.number_of_mates=0 AND pbest.number_of_neighbours=1;"
% (ra, dec, radius_deg, catalog_min_mag, catalog_max_mag),
dump_to_file=False)
p = job.get_results()
# convert RA and DEC errors from mas(milli arc second) to degrees
p['ra_errdeg'] = p['ra_error'] / 3.6e6
p['dec_errdeg'] = p['dec_error'] / 3.6e6
p['FLAGS'] = 0
p.remove_columns([
'astrometric_n_obs_al', 'astrometric_n_obs_ac',
'astrometric_n_good_obs_al', 'astrometric_n_bad_obs_al',
'astrometric_gof_al', 'astrometric_chi2_al',
'astrometric_excess_noise', 'astrometric_excess_noise_sig',
'astrometric_params_solved', 'astrometric_primary_flag',
'astrometric_weight_al', 'astrometric_pseudo_colour',
'astrometric_pseudo_colour_error', 'mean_varpi_factor_al',
'astrometric_matched_observations', 'visibility_periods_used',
'astrometric_sigma5d_max', 'frame_rotator_object_type',
'matched_observations', 'duplicated_source', 'phot_g_n_obs',
'phot_g_mean_flux', 'phot_g_mean_flux_error',
'phot_g_mean_flux_over_error', 'phot_g_mean_mag', 'phot_bp_n_obs',
'phot_bp_mean_flux', 'phot_bp_mean_flux_error',
'phot_bp_mean_flux_over_error', 'phot_bp_mean_mag', 'phot_rp_n_obs',
'phot_rp_mean_flux', 'phot_rp_mean_flux_error',
'phot_rp_mean_flux_over_error', 'phot_rp_mean_mag',
'phot_bp_rp_excess_factor', 'phot_proc_mode', 'bp_rp', 'bp_g', 'g_rp',
'radial_velocity', 'radial_velocity_error', 'rv_nb_transits',
'rv_template_teff', 'rv_template_logg', 'rv_template_fe_h', 'l', 'b',
'ecl_lon', 'ecl_lat', 'priam_flags', 'teff_val',
'teff_percentile_lower', 'teff_percentile_upper', 'a_g_val',
'a_g_percentile_lower', 'a_g_percentile_upper', 'e_bp_min_rp_val',
'e_bp_min_rp_percentile_lower', 'e_bp_min_rp_percentile_upper',
'flame_flags', 'radius_val', 'radius_percentile_lower',
'radius_percentile_upper', 'lum_val', 'lum_percentile_lower',
'lum_percentile_upper', 'gaia_astrometric_params', 'obj_name',
'obj_id', 'ra_2', 'dec_2', 'ra_error_2', 'dec_error_2', 'epoch_mean',
'zone_id', 'obj_info_flag', 'quality_flag', 'designation',
'phot_variable_flag', 'datalink_url', 'epoch_photometry_url',
'original_ext_source_id'
])
if os.path.exists(catname + '.txt'):
os.remove(catname + '.txt')
ascii.write(p, catname + '.txt')
if os.path.exists(catname + '.ldac'):
os.remove(catname + '.ldac')
save_table_as_ldac(p, catname + '.ldac')
def gaia_query(self):
"""
Performs a query on gaia DR2 using self.skycoord and self.radius.
Note: "Exception: 500" tends to happen when the coordinates used in search_string is not formatted correctly.
Arguments:
[none]
Returns:
CatalogTable with query result.
"""
catalog = ["gaia"]
ra = self.skycoord.ra.degree
dec = self.skycoord.dec.degree
#TODO: include code to write diagnostic info to log file incl. ra, dec, radius, search string
search_string = "SELECT {} FROM gaiadr2.gaia_source WHERE CONTAINS(POINT('ICRS',gaiadr2.gaia_source.ra,gaiadr2.gaia_source.dec),CIRCLE('ICRS',{},{},{}))=1;".format(
gaia_cols, ra, dec, self.radius)
#try:
out("Creating query...")
out(search_string)
job = Gaia.launch_job_async(search_string, dump_to_file=False)
#except gaierror as e:
# if str(e) != "[Errno 11001] getaddrinfo failed":
# raise
# else:
# out("This error is typically raised when there is no internet connection.")
# raise
out("Retrieving results...")
query_results = job.get_results()
out("Results retrieved.")
info_out(str(len(query_results['designation'])) + " sources detected.")
print(query_results.colnames)
# write Gaia query results to file
fname = tpath + "/gaia_query.dat"
query_results.write(fname, format='ascii.ecsv')
return (CatalogTable(catalog, query_results))
def run_cone_search(self,
qpar_SN=10,
qpar_vis=7,
qpar_ruwe=1.40,
verbose=True):
"""
Run a Cone Search ADQL query on the Gaia DR2 archive. By default the code applies a selection criteria: objets with
parallax S/N <10, visibility periods used < 7, and RUWE >1.40 are excluded.
"""
# 1.- Define ADQL query ================================
query = (
"SELECT " + self.gaia_cols + " "
",sqrt(gaia.astrometric_chi2_al/(gaia.astrometric_n_good_obs_al-5)) as unit_weight_e, g_ruwe.ruwe as ruwe "
"FROM gaiadr2.gaia_source as gaia "
"LEFT OUTER JOIN gaiadr2.ruwe AS g_ruwe ON gaia.source_id = g_ruwe.source_id "
"WHERE 1=CONTAINS( "
"POINT('ICRS',ra,dec), "
f"CIRCLE('ICRS',{self.ADQL['ra']:5.2F}, {self.ADQL['dec']:5.2F}, {self.ADQL['radii']:5.2F})) "
f"AND parallax >= {self.ADQL['para_m']:5.2F} AND parallax <= {self.ADQL['para_M']:5.2F} "
f"AND gaia.source_id IS NOT NULL AND gaia.parallax/gaia.parallax_error >{qpar_SN} "
f"AND gaia.visibility_periods_used >{qpar_vis} AND g_ruwe.ruwe <{qpar_ruwe}"
)
# 2.- Run ADQL query ===================================
warnings.simplefilter('ignore', category=AstropyWarning)
print(f'RUNNING ADQL ASYNCRHRONOUS QUERY ' + '=' * 57)
job = Gaia.launch_job_async(query=query, verbose=True)
self.cat = job.get_results()
flag_psn, flag_vis, flag_ruwe = '', '', ''
if qpar_SN == 10: flag_psn = '(Default)'
if qpar_vis == 7: flag_vis = '(Default)'
if qpar_ruwe == 1.40: flag_ruwe = '(Default)'
if qpar_SN < 10:
text = 'WARNING: Parallax->Distance conversion as d = 1/parallax is not realiable if parallax S/N < 10'
print('!' * len(text))
print(text)
print('!' * len(text))
if verbose:
print('=' * 90)
print(
f'Selection Criteria in Parallax S/N: Parallax S/N > {qpar_SN} {flag_psn}'
)
print(
f'Selection Criteria in Visibility Periods Used: Vis > {qpar_vis} {flag_vis}'
)
print(
f'Selection Criteria in RUWE: RUWE < {qpar_ruwe} {flag_ruwe}'
)
print()
print(f'SAMPLE OUTPUT N_els = {len(self.cat):3.0f}')
print('=' * 90)
print()
#~ INNER JOIN gaiadr2.tmass_best_neighbour AS tmass
#~ ON gaia.source_id = tmass.source_id
#~ WHERE tmass.original_ext_source_id IN %s
#~ '''%str(tuple(tmass))
# Search with Gaia IDs
#gaiaid=[5283961585534643712, 5283965296387249920]
gaiaid = np.load('../data/bpmg_sourceids.npy')
query='''
SELECT gaia.*
FROM gaiadr2.gaia_source AS gaia
WHERE gaia.source_id IN %s
'''%str(tuple(gaiaid))
job = Gaia.launch_job_async(query, dump_to_file=True)
# Your astropy table with results
r = job.get_results()
#keys=['source_id', 'phot_bp_mean_flux','ra_pmdec_corr','ra_error','ra','pmra_error','ecl_lon','designation','l','phot_rp_mean_mag','parallax_pmdec_corr','ra_parallax_corr','pmdec_error','phot_g_mean_mag','pmra','parallax','radial_velocity','radial_velocity_error','ra_dec_corr','parallax_error','dec_pmdec_corr','dec_error','pmdec','parallax_over_error','b','ref_epoch','ra_pmra_corr','dec_parallax_corr','phot_bp_mean_mag','dec','dec_pmra_corr','pmra_pmdec_corr','parallax_pmra_corr','bp_rp','ecl_lat']
keys=['source_id','phot_bp_mean_flux','ra_error','ra','phot_rp_mean_mag',
'phot_g_mean_mag','phot_bp_mean_mag','bp_rp']
r2=r[keys]
d=dict(zip(r2['source_id'], r2)) # for easier crossmatch with source_id
print r2
# for source_id in gaiaid:
# try:
