def test_list_async_jobs(self):
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
response = DummyResponse()
response.set_status_code(500)
response.set_message("ERROR")
jobDataFile = data_path('jobs_list.xml')
jobData = utils.read_file_content(jobDataFile)
response.set_data(method='GET',
context=None,
body=jobData,
headers=None)
req = "async"
connHandler.set_response(req, response)
with pytest.raises(Exception):
tap.list_async_jobs()
response.set_status_code(200)
response.set_message("OK")
jobs = tap.list_async_jobs()
assert len(jobs) == 2, \
"Wrong jobs number. Expected: %d, found %d" % \
(2, len(jobs))
assert jobs[0].get_jobid() == '12345', \
"Wrong job id. Expected: %s, found %s" % \
('12345', jobs[0].get_jobid())
assert jobs[0].get_phase() == 'COMPLETED', \
"Wrong job phase for job %s. Expected: %s, found %s" % \
(jobs[0].get_jobid(), 'COMPLETED', jobs[0].get_phase())
assert jobs[1].get_jobid() == '77777', \
"Wrong job id. Expected: %s, found %s" % \
('77777', jobs[1].get_jobid())
assert jobs[1].get_phase() == 'ERROR', \
"Wrong job phase for job %s. Expected: %s, found %s" % \
(jobs[1].get_jobid(), 'ERROR', jobs[1].get_phase())
def tap_query(url: str, query: str) -> Union[Table, None]:
"""
Query via a TapPlus query
:param url: str, the URL to the SQL database
:param query: str, the SQL query to run
:return: astropy.table.Table or None - the results of the gaia TAP query
"""
# ------------------------------------------------------------------
# try running gaia query
with warnings.catch_warnings(record=True) as _:
# construct TapPlus instance
tap = TapPlus(url=url, verbose=False)
# launch tap job
job = tap.launch_job(query=query)
# get tap table
table = job.get_results()
# ------------------------------------------------------------------
# if we have no entries we did not find object
if len(table) == 0:
# return None
return None
# else we return result
return table
def gaia_search(coords, radius):
"""
Gaia ADQL query cone searcher
Returns tables from both DR2 and EDR3
"""
radius_deg = (radius * u.arcsec).to(u.deg).value
gaia = TapPlus(url="https://gea.esac.esa.int/tap-server/tap",
verbose=False)
adql_query_dr2 = f"""SELECT *
FROM gaiadr2.gaia_source
WHERE
1=CONTAINS(POINT('ICRS', ra, dec),
CIRCLE('ICRS', {coords.ra.deg}, {coords.dec.deg}, {radius_deg}))
"""
job_dr2 = gaia.launch_job(adql_query_dr2)
#print(job_dr2)
gaia_dr2_result = job_dr2.get_results()
print(f'DR2: Found {len(gaia_dr2_result)} sources')
adql_query_edr3 = f"""SELECT *
FROM gaiaedr3.gaia_source
WHERE
1=CONTAINS(POINT('ICRS', ra, dec),
CIRCLE('ICRS', {coords.ra.deg}, {coords.dec.deg}, {radius_deg}))
"""
job_edr3 = gaia.launch_job(adql_query_edr3)
#print(job_edr3)
gaia_edr3_result = job_edr3.get_results()
print(f'DR3: Found {len(gaia_edr3_result)} sources')
return gaia_dr2_result, gaia_edr3_result
def _download_ps_table():
exoarch = TapPlus(url="https://exoplanetarchive.ipac.caltech.edu/TAP")
job = exoarch.launch_job_async("select * from ps")
# TODO: fix dtype conversion
df = job.get_results().to_pandas()
setattr(df, "_is_ps_table", True)
return df
def twomass_from_gaia(df):
"""
Get 2MASS IDs from Gaia DR2 ID and add in column of dataframe
Args:
df (pd.Dataframe): dataframe with object list
Returns:
df with updated 2mass IDs
"""
gaia_ids = df['GAIADR2ID'].astype(str).tolist()
# gaia_ids_end = df['GAIADR2ID'].astype(str).tolist()[100:]
# gaia_ids.extend(gaia_ids_end)
gaia_ids = '(' + ', '.join(gaia_ids) + ')'
query = ('SELECT source_id, original_ext_source_id '
'FROM gaiadr2.tmass_best_neighbour '
'WHERE source_id IN {}').format(gaia_ids)
gaia = TapPlus(url=GAIA_2MASS_URL)
job = gaia.launch_job(query=query)
tbl = job.get_results()
df_2mass = tbl.to_pandas()
df_2mass = df_2mass.astype(str).set_index('source_id', drop=False)
df_2mass = df_2mass.reindex(df['GAIADR2ID'].astype(str).tolist())
df_2mass = df_2mass['original_ext_source_id']
# Weird encoding workaround
df['TWOMASSID'] = df_2mass.str.strip("b\'\"").values
return df
def test_list_async_jobs():
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
response = DummyResponse()
response.set_status_code(500)
response.set_message("ERROR")
jobDataFile = data_path('jobs_list.xml')
jobData = utils.read_file_content(jobDataFile)
response.set_data(method='GET',
context=None,
body=jobData,
headers=None)
req = "async"
connHandler.set_response(req, response)
with pytest.raises(Exception):
tap.list_async_jobs()
response.set_status_code(200)
response.set_message("OK")
jobs = tap.list_async_jobs()
assert len(jobs) == 2
assert jobs[0].jobid == '12345'
assert jobs[0].get_phase() == 'COMPLETED'
assert jobs[1].jobid == '77777'
assert jobs[1].get_phase() == 'ERROR'
def getColumns(catalog):
from astroquery.utils.tap.core import TapPlus
tap = TapPlus(url="http://TAPVizieR.u-strasbg.fr/TAPVizieR/tap")
query_string = """SELECT Top 1 * FROM {}""".format(catalog)
job = tap.launch_job_async(query_string, dump_to_file=False)
res = job.get_results()
return res.columns
def __init__(self, tap_handler=None):
super(ISOClass, self).__init__()
if tap_handler is None:
self._tap = TapPlus(url=self.metadata_url)
else:
self._tap = tap_handler
def __init__(self, tap_handler=None):
super(ESAHubbleClass, self).__init__()
if tap_handler is None:
self._tap = TapPlus(url="http://hst.esac.esa.int"
"/tap-server/tap/")
else:
self._tap = tap_handler
def master_table_query(args):
import numpy as np
import pandas as pd
import time
import datetime
from astroquery.utils.tap.core import TapPlus
if args.verbose > 0:
print
print '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
print 'master_table_query'
print '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
print
print datetime.datetime.now()
# Extract info from argument list...
outputFile = args.outputFile
object_id_lo = args.object_id_lo
object_id_hi = args.object_id_hi
# Establish TAP connection to SkyMapper database...
skymapper = TapPlus(url="http://skymappertap.asvo.nci.org.au/ncitap/tap")
# Formulate the query...
query = """SELECT * FROM dr1.master where object_id between %d and %d""" \
% (object_id_lo, object_id_hi)
if args.verbose > 0:
print "Query: ", query
# Submit the query as an asynchronous job...
if args.verbose > 0:
print "Query start: ", datetime.datetime.now()
job = skymapper.launch_job_async(query)
if args.verbose > 0:
print "Query completed: ", datetime.datetime.now()
# Retrieve the results as an astropy Table...
if args.verbose > 0:
print "Retrieval start: ", datetime.datetime.now()
master_table = job.get_results()
if args.verbose > 0:
print "Retrieval completed: ", datetime.datetime.now()
# Sort table by image_id
master_table.sort('object_id')
# Save table to outputFile...
if args.verbose > 0:
print "File output start: ", datetime.datetime.now()
master_table.write(outputFile)
if args.verbose > 0:
print "File output completed: ", datetime.datetime.now()
if args.verbose > 0: print
return 0
def VizierTables():
from astroquery.utils.tap.core import TapPlus
tap = TapPlus(url="http://TAPVizieR.u-strasbg.fr/TAPVizieR/tap")
tables = tap.load_tables(only_names=True)
interestingNames = ("sdss", "2mass", "gaia", "tess", "usno")
for table in tables:
name = table.get_qualified_name().lower()
for iname in interestingNames:
if iname in name:
print(table.get_qualified_name())
def test_launch_sync_job(self):
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
responseLaunchJob = DummyResponse()
responseLaunchJob.set_status_code(500)
responseLaunchJob.set_message("ERROR")
jobDataFile = data_path('job_1.vot')
jobData = utils.read_file_content(jobDataFile)
responseLaunchJob.set_data(method='POST',
context=None,
body=jobData,
headers=None)
query = 'select top 5 * from table'
dTmp = {"q": query}
dTmpEncoded = connHandler.url_encode(dTmp)
p = dTmpEncoded.find("=")
q = dTmpEncoded[p + 1:]
dictTmp = {
"REQUEST": "doQuery",
"LANG": "ADQL",
"FORMAT": "votable",
"tapclient": str(TAP_CLIENT_ID),
"PHASE": "RUN",
"QUERY": str(q)
}
sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp)
jobRequest = "sync?" + sortedKey
connHandler.set_response(jobRequest, responseLaunchJob)
with pytest.raises(Exception):
tap.launch_job(query)
responseLaunchJob.set_status_code(200)
responseLaunchJob.set_message("OK")
job = tap.launch_job(query)
assert job is not None, "Expected a valid job"
assert job.async_ is False, "Expected a synchronous job"
assert job.get_phase() == 'COMPLETED', \
"Wrong job phase. Expected: %s, found %s" % \
('COMPLETED', job.get_phase())
assert job.failed is False, "Wrong job status (set Failed = True)"
# results
results = job.get_results()
assert len(results) == 3, \
"Wrong job results (num rows). Expected: %d, found %d" % \
(3, len(results))
self.__check_results_column(results, 'alpha', 'alpha', None,
np.float64)
self.__check_results_column(results, 'delta', 'delta', None,
np.float64)
self.__check_results_column(results, 'source_id', 'source_id', None,
np.object)
self.__check_results_column(results, 'table1_oid', 'table1_oid', None,
np.int32)
def connect(self, url=""):
if self.__tap_connector == None:
if url == "":
raise Exception("provide URL TAP Server")
else:
self.__tap_connector = TapPlus(url=url)
else:
if url != "":
self.__tap_connector = TapPlus(url=url)
else:
return self.__tap_connector
def test_load_tables(self):
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
responseLoadTable = DummyResponse()
responseLoadTable.set_status_code(500)
responseLoadTable.set_message("ERROR")
tableDataFile = data_path('test_tables.xml')
tableData = utils.read_file_content(tableDataFile)
responseLoadTable.set_data(method='GET',
context=None,
body=tableData,
headers=None)
tableRequest = "tables"
connHandler.set_response(tableRequest, responseLoadTable)
with pytest.raises(Exception):
tap.load_tables()
responseLoadTable.set_status_code(200)
responseLoadTable.set_message("OK")
res = tap.load_tables()
assert len(res) == 2, \
"Number of tables expected: %d, found: %d" % (2, len(res))
# Table 1
table = self.__find_table('public', 'table1', res)
assert table.description == 'Table1 desc', \
"Wrong description for table1. Expected: %s, found %s" % \
('Table1 desc', table.description)
columns = table.columns
assert len(columns) == 2, \
"Number of columns for table1. Expected: %d, found: %d" % \
(2, len(columns))
col = self.__find_column('table1_col1', columns)
self.__check_column(col, 'Table1 Column1 desc', '', 'VARCHAR',
'indexed')
col = self.__find_column('table1_col2', columns)
self.__check_column(col, 'Table1 Column2 desc', '', 'INTEGER', None)
# Table 2
table = self.__find_table('public', 'table2', res)
assert table.description == 'Table2 desc', \
"Wrong description for table2. Expected: %s, found %s" % \
('Table2 desc', table.description)
columns = table.columns
assert len(columns) == 3, \
"Number of columns for table2. Expected: %d, found: %d" % \
(3, len(columns))
col = self.__find_column('table2_col1', columns)
self.__check_column(col, 'Table2 Column1 desc', '', 'VARCHAR',
'indexed')
col = self.__find_column('table2_col2', columns)
self.__check_column(col, 'Table2 Column2 desc', '', 'INTEGER', None)
col = self.__find_column('table2_col3', columns)
self.__check_column(col, 'Table2 Column3 desc', '', 'INTEGER', None)
def test_abort_job(self):
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
jobid = '12345'
# Phase POST response
responsePhase = DummyResponse()
responsePhase.set_status_code(200)
responsePhase.set_message("OK")
responsePhase.set_data(method='POST',
context=None,
body=None,
headers=None)
req = "async/" + jobid + "/phase?PHASE=ABORT"
connHandler.set_response(req, responsePhase)
# Launch response
responseLaunchJob = DummyResponse()
responseLaunchJob.set_status_code(303)
responseLaunchJob.set_message("OK")
# list of list (httplib implementation for headers in response)
launchResponseHeaders = [[
'location', 'http://test:1111/tap/async/' + jobid
]]
responseLaunchJob.set_data(method='POST',
context=None,
body=None,
headers=launchResponseHeaders)
query = 'query'
dictTmp = {
"REQUEST": "doQuery",
"LANG": "ADQL",
"FORMAT": "votable",
"tapclient": str(TAP_CLIENT_ID),
"QUERY": str(query)
}
sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp)
req = "async?" + sortedKey
connHandler.set_response(req, responseLaunchJob)
job = tap.launch_job_async(query, autorun=False)
assert job is not None, "Expected a valid job"
assert job.get_phase() == 'PENDING', \
"Wrong job phase. Expected: %s, found %s" % \
('PENDING', job.get_phase())
# abort job
job.abort()
assert job.get_phase() == 'ABORT', \
"Wrong job phase. Expected: %s, found %s" % \
('ABORT', job.get_phase())
# try to abort again
with pytest.raises(Exception):
job.abort()
def test_load_tables(self):
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
responseLoadTable = DummyResponse()
responseLoadTable.set_status_code(500)
responseLoadTable.set_message("ERROR")
tableDataFile = data_path('test_tables.xml')
tableData = utils.read_file_content(tableDataFile)
responseLoadTable.set_data(method='GET',
context=None,
body=tableData,
headers=None)
tableRequest = "tables"
connHandler.set_response(tableRequest, responseLoadTable)
with pytest.raises(Exception):
tap.load_tables()
responseLoadTable.set_status_code(200)
responseLoadTable.set_message("OK")
res = tap.load_tables()
assert len(res) == 2, \
"Number of tables expected: %d, found: %d" % (2, len(res))
# Table 1
table = self.__find_table('public', 'table1', res)
assert table.get_description() == 'Table1 desc', \
"Wrong description for table1. Expected: %s, found %s" % \
('Table1 desc', table.get_description())
columns = table.get_columns()
assert len(columns) == 2, \
"Number of columns for table1. Expected: %d, found: %d" % \
(2, len(columns))
col = self.__find_column('table1_col1', columns)
self.__check_column(col, 'Table1 Column1 desc', '', 'VARCHAR', 'indexed')
col = self.__find_column('table1_col2', columns)
self.__check_column(col, 'Table1 Column2 desc', '', 'INTEGER', None)
# Table 2
table = self.__find_table('public', 'table2', res)
assert table.get_description() == 'Table2 desc', \
"Wrong description for table2. Expected: %s, found %s" % \
('Table2 desc', table.get_description())
columns = table.get_columns()
assert len(columns) == 3, \
"Number of columns for table2. Expected: %d, found: %d" % \
(3, len(columns))
col = self.__find_column('table2_col1', columns)
self.__check_column(col, 'Table2 Column1 desc', '', 'VARCHAR', 'indexed')
col = self.__find_column('table2_col2', columns)
self.__check_column(col, 'Table2 Column2 desc', '', 'INTEGER', None)
col = self.__find_column('table2_col3', columns)
self.__check_column(col, 'Table2 Column3 desc', '', 'INTEGER', None)
def bailerjones_new(src_id):
gaia = TapPlus(url="http://dc.zah.uni-heidelberg.de/tap", verbose=False)
adql_query_newbj = f"""SELECT
source_id,
r_med_geo, r_lo_geo, r_hi_geo,
r_med_photogeo, r_lo_photogeo, r_hi_photogeo
FROM gedr3dist.main
WHERE source_id = {src_id}
"""
job = gaia.launch_job(adql_query_newbj)
result = job.get_results()
return result
def doquery_from_table(table):
#Execute a TAP+ query using information from a single row of an astropy table
server = TapPlus(url = table['queryurl'], \
default_protocol_is_https = True, \
verbose = True)
if table['upload_table_name'] == '':
result = server.launch_job_async(query = table['query'], verbose = True, \
dump_to_file = True, output_file = table['outfile'])
else:
result = server.launch_job_async(query = table['query'], verbose = True, \
dump_to_file = True, output_file = table['outfile'], \
upload_table_name = table['upload_table_name'], \
upload_resource = table['upload_resource'])
return result
def get_nasa_exoplanet_archive_pscomppars(ver=TODAYSTR, N_max=int(1e4)):
"""
If newestpossible is True, will download the latest NEA pscomppars table,
from today. Otherwise, it'll take the most recent from those already
downloaded.
"""
from astroquery.utils.tap.core import TapPlus
savedir = os.path.join(LOCALDIR, 'catalogs')
if not os.path.exists(savedir):
try:
os.mkdir(savedir)
except:
raise NotImplementedError(f'Tried to make {savedir} and failed.')
nea_path = os.path.join(savedir,
f'nasaexoplanetarchive-pscomppars-{ver}.csv')
if not os.path.exists(nea_path):
tap = TapPlus(url="https://exoplanetarchive.ipac.caltech.edu/TAP/")
query = (
f'select top {N_max} ' +
'pl_name, hostname, pl_letter, gaia_id, tic_id, ra, dec, ' +
'discoverymethod, disc_year, pl_orbper, pl_orbsmax, pl_rade, ' +
'pl_radeerr1, pl_radeerr2, ' + 'pl_radjerr1, pl_radjerr2, ' +
'pl_radj, pl_bmasse, pl_bmasseerr1, pl_bmasseerr2, pl_bmassj, ' +
'pl_bmassjerr1, pl_bmassjerr2, pl_orbeccen, pl_imppar, ' +
'pl_insol, pl_insolerr1, pl_insolerr2, ' +
'pl_eqt, pl_eqt, pl_eqt, ' +
'st_teff, st_rad, st_mass, st_met, st_logg, st_rotp, sy_dist, ' +
'sy_disterr1, sy_disterr2, sy_plx, sy_plxerr1, sy_plxerr2, ' +
'sy_vmag, sy_tmag, ' + 'tran_flag, ' +
'st_age, st_ageerr1, st_ageerr2 from pscomppars')
print(query)
j = tap.launch_job(query=query)
r = j.get_results()
assert len(r) != N_max
df = r.to_pandas()
df.to_csv(nea_path, index=False)
else:
df = pd.read_csv(nea_path, sep=',')
return df
def test_launch_sync_job():
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
responseLaunchJob = DummyResponse()
responseLaunchJob.set_status_code(500)
responseLaunchJob.set_message("ERROR")
jobDataFile = data_path('job_1.vot')
jobData = utils.read_file_content(jobDataFile)
responseLaunchJob.set_data(method='POST',
context=None,
body=jobData,
headers=None)
query = 'select top 5 * from table'
dTmp = {"q": query}
dTmpEncoded = connHandler.url_encode(dTmp)
p = dTmpEncoded.find("=")
q = dTmpEncoded[p + 1:]
dictTmp = {
"REQUEST": "doQuery",
"LANG": "ADQL",
"FORMAT": "votable",
"tapclient": str(TAP_CLIENT_ID),
"PHASE": "RUN",
"QUERY": str(q)
}
sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp)
jobRequest = f"sync?{sortedKey}"
connHandler.set_response(jobRequest, responseLaunchJob)
with pytest.raises(Exception):
tap.launch_job(query)
responseLaunchJob.set_status_code(200)
responseLaunchJob.set_message("OK")
job = tap.launch_job(query)
assert job is not None
assert job.async_ is False
assert job.get_phase() == 'COMPLETED'
assert job.failed is False
# results
results = job.get_results()
assert len(results) == 3
__check_results_column(results, 'ra', 'ra', None, np.float64)
__check_results_column(results, 'dec', 'dec', None, np.float64)
__check_results_column(results, 'source_id', 'source_id', None, object)
__check_results_column(results, 'table1_oid', 'table1_oid', None, np.int32)
def test_get_epic_metadata(self):
tap_url = "http://nxsadev.esac.esa.int/tap-server/tap/"
target_name = "4XMM J122934.7+015657"
radius = 0.01
epic_source_table = "xsa.v_epic_source"
epic_source_column = "epic_source_equatorial_spoint"
cat_4xmm_table = "xsa.v_epic_source_cat"
cat_4xmm_column = "epic_source_cat_equatorial_spoint"
stack_4xmm_table = "xsa.v_epic_xmm_stack_cat"
stack_4xmm_column = "epic_stack_cat_equatorial_spoint"
slew_source_table = "xsa.v_slew_source_cat"
slew_source_column = "slew_source_cat_equatorial_spoint"
xsa = XMMNewtonClass(TapPlus(url=tap_url))
epic_source, cat_4xmm, stack_4xmm, slew_source = xsa.get_epic_metadata(
target_name=target_name, radius=radius)
c = SkyCoord.from_name(target_name, parse=True)
query = ("select * from {} "
"where 1=contains({}, circle('ICRS', {}, {}, {}));")
table = xsa.query_xsa_tap(
query.format(epic_source_table, epic_source_column, c.ra.degree,
c.dec.degree, radius))
assert report_diff_values(epic_source, table)
table = xsa.query_xsa_tap(
query.format(cat_4xmm_table, cat_4xmm_column, c.ra.degree,
c.dec.degree, radius))
assert report_diff_values(cat_4xmm, table)
table = xsa.query_xsa_tap(
query.format(stack_4xmm_table, stack_4xmm_column, c.ra.degree,
c.dec.degree, radius))
assert report_diff_values(stack_4xmm, table)
table = xsa.query_xsa_tap(
query.format(slew_source_table, slew_source_column, c.ra.degree,
c.dec.degree, radius))
assert report_diff_values(slew_source, table)
def get_exoplanetarchive_planetarysystems(tabletype="ps", overwrite=1,
n_max=int(3e5), verbose=1):
"""
Args:
overwrite: if true, will download the LATEST tables from the NASA
exoplanet archive. otherwise, uses the cache.
tabletype: "ps" or "pscomppars"
Returns:
dataframe with the Planetary Systems, "ps" table. (Nb. there is a
composite parameters table too, "pscomppars").
https://exoplanetarchive.ipac.caltech.edu/docs/API_PS_columns.html
"""
dlpath = os.path.join(
CACHEDIR, f'{tabletype}_exoplanetarchive.xml.gz'
)
if os.path.exists(dlpath) and overwrite:
os.remove(dlpath)
if not os.path.exists(dlpath):
tap = TapPlus(url="https://exoplanetarchive.ipac.caltech.edu/TAP/")
jobstr = (
f'''
SELECT top {n_max:d} *
FROM {tabletype}
'''
)
query = jobstr
if verbose:
print(20*'.')
print('Executing:')
print(query)
print(20*'.')
j = tap.launch_job(query=query, verbose=True, dump_to_file=True,
output_file=dlpath)
df = given_votable_get_df(dlpath, assert_equal=None)
return df
def test_datalink():
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap",
datalink_context="datalink",
connhandler=connHandler)
responseResultsJob = DummyResponse()
responseResultsJob.set_status_code(200)
responseResultsJob.set_message("OK")
jobDataFile = data_path('job_1.vot')
jobData = utils.read_file_content(jobDataFile)
responseResultsJob.set_data(method='GET',
context=None,
body=jobData,
headers=None)
req = "links?ID=1,2"
connHandler.set_response(req, responseResultsJob)
# error
responseResultsJob.set_status_code(500)
responseResultsJob.set_message("ERROR")
with pytest.raises(Exception):
# missing IDS parameter
tap.get_datalinks(ids=None)
# OK
responseResultsJob.set_status_code(200)
responseResultsJob.set_message("OK")
# results
results = tap.get_datalinks("1,2")
assert len(results) == 3
results = tap.get_datalinks([1, 2])
assert len(results) == 3
results = tap.get_datalinks(['1', '2'])
assert len(results) == 3
def test_logout(self, mock_logout):
conn_handler = DummyConnHandler()
tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler)
tap = GaiaClass(conn_handler, tapplus, show_server_messages=False)
tap.logout()
assert (mock_logout.call_count == 2)
mock_logout.side_effect = HTTPError("Login error")
tap.logout()
assert (mock_logout.call_count == 3)
def test_login_gui(self, mock_login_gui, mock_login):
conn_handler = DummyConnHandler()
tapplus = TapPlus("http://test:1111/tap", connhandler=conn_handler)
tap = GaiaClass(conn_handler, tapplus, show_server_messages=False)
tap.login_gui()
assert (mock_login_gui.call_count == 1)
mock_login_gui.side_effect = HTTPError("Login error")
tap.login("user", "password")
assert (mock_login.call_count == 1)
def download_extended_catalog(self, merge=True):
""" Dowloads smaller catalog source from PS:
https://outerspace.stsci.edu/display/PANSTARRS/PS1+Source+extraction+and+catalogs
Here merging:
StackModelFitDeV:
Contains the de Vaucouleurs fit parameters for stack
detections brighter than some limit (is this a S/N cut or a mag limit?)
outside the galactic plane.
All filters are matched into a single row.
Given are mag, radius, axial ratio, position angle, RA, Dec, chisq of fit.
StackModelFitSer:
Contains the Sersic fit parameters for stack
detections brighter than magnitude 21.5 outside the galactic plane.
All filters are matched into a single row.
Given are mag, radius, axial ratio, Sersic index, position angle, RA, Dec, chisq of fit.
"""
if self.catdata is None:
self.download_catalog(update=True)
objid = ",".join(["%s" % s for s in self.catdata["objID"].values])
from astroquery.utils.tap.core import TapPlus
tap_service = TapPlus(
url="http://vao.stsci.edu/PS1DR2/tapservice.aspx")
job = tap_service.launch_job_async("""
SELECT *
FROM dbo.StackModelFitDeV AS dev
LEFT JOIN dbo.StackModelFitSer AS ser
ON dev.objID = ser.objID
WHERE
dev.objID IN ({})
""".format(objid))
results = job.get_results().to_pandas()
if merge:
self._catdata = pandas.merge(self.catdata, results, on="objID")
self._is_extended_cat_set = True
else:
self._is_extended_cat_set = False
return results
def test_query_object(self):
connHandler = DummyConnHandler()
tapplus = TapPlus("http://test:1111/tap", connhandler=connHandler)
tap = GaiaClass(connHandler, tapplus)
# Launch response: we use default response because the query contains
# decimals
responseLaunchJob = DummyResponse()
responseLaunchJob.set_status_code(200)
responseLaunchJob.set_message("OK")
jobDataFile = data_path('job_1.vot')
jobData = utils.read_file_content(jobDataFile)
responseLaunchJob.set_data(method='POST',
context=None,
body=jobData,
headers=None)
# The query contains decimals: force default response
connHandler.set_default_response(responseLaunchJob)
sc = SkyCoord(ra=29.0,
dec=15.0,
unit=(u.degree, u.degree),
frame='icrs')
with pytest.raises(ValueError) as err:
tap.query_object(sc)
assert "Missing required argument: 'width'" in err.value.args[0]
width = Quantity(12, u.deg)
with pytest.raises(ValueError) as err:
tap.query_object(sc, width=width)
assert "Missing required argument: 'height'" in err.value.args[0]
height = Quantity(10, u.deg)
table = tap.query_object(sc, width=width, height=height)
assert len(table) == 3, \
"Wrong job results (num rows). Expected: %d, found %d" % \
(3, len(table))
self.__check_results_column(table, 'alpha', 'alpha', None, np.float64)
self.__check_results_column(table, 'delta', 'delta', None, np.float64)
self.__check_results_column(table, 'source_id', 'source_id', None,
np.object)
self.__check_results_column(table, 'table1_oid', 'table1_oid', None,
np.int32)
# by radius
radius = Quantity(1, u.deg)
table = tap.query_object(sc, radius=radius)
assert len(table) == 3, \
"Wrong job results (num rows). Expected: %d, found %d" % \
(3, len(table))
self.__check_results_column(table, 'alpha', 'alpha', None, np.float64)
self.__check_results_column(table, 'delta', 'delta', None, np.float64)
self.__check_results_column(table, 'source_id', 'source_id', None,
np.object)
self.__check_results_column(table, 'table1_oid', 'table1_oid', None,
np.int32)
def __init__(self, service_url, schema, master_table, natural_join=None):
self.service_url = service_url
self.schema = schema
self.natural_join = natural_join
self.service = TapPlus(url=self.service_url)
self.master_table = GraphControl.get_table(self.service,
schema,
master_table,
None,
None,
natural_join=natural_join)
def test_rename_table():
tableName = 'user_test.table_test_rename'
newTableName = 'user_test.table_test_rename_new'
newColumnNames = {'ra': 'alpha', 'dec': 'delta'}
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
dummyResponse = DummyResponse()
dummyResponse.set_status_code(200)
dummyResponse.set_message("OK")
tableDataFile = data_path('test_table_rename.xml')
tableData = utils.read_file_content(tableDataFile)
dummyResponse.set_data(method='GET',
context=None,
body=tableData,
headers=None)
with pytest.raises(Exception):
tap.rename_table()
with pytest.raises(Exception):
tap.rename_table(table_name=tableName)
with pytest.raises(Exception):
tap.rename_table(table_name=tableName,
new_table_name=None,
new_column_names_dict=None)
# Test OK.
responseRenameTable = DummyResponse()
responseRenameTable.set_status_code(200)
responseRenameTable.set_message("OK")
dictArgs = {
"action": "rename",
"new_column_names": "ra:alpha,dec:delta",
"new_table_name": newTableName,
"table_name": tableName,
}
data = connHandler.url_encode(dictArgs)
req = f"TableTool?{data}"
connHandler.set_response(req, responseRenameTable)
tap.rename_table(table_name=tableName,
new_table_name=newTableName,
new_column_names_dict=newColumnNames)
def query_gaia(url, query) -> Union[Table, None]:
"""
Query Gaia via a TapPlus query
:param url: str, the URL to the SQL database
:param query: str, the SQL query to run
:return: astropy.table.Table or None - the results of the gaia TAP query
"""
# set fucntion name
func_name = __NAME__ + '.query_gaia()'
# check for astroquery and return a fail and warning if not installed
try:
from astroquery.utils.tap.core import TapPlus
except Exception as e:
eargs = [type(e), str(e), func_name]
WLOG(params, 'warning', TextEntry('10-016-00009', args=eargs))
return None
# ------------------------------------------------------------------
# try running gaia query
try:
with warnings.catch_warnings(record=True) as _:
# construct gaia TapPlus instance
gaia = TapPlus(url=url)
# launch gaia job
job = gaia.launch_job(query=query)
# get gaia table
table = job.get_results()
except Exception as e:
wargs = [url, query, type(e), e, func_name]
WLOG(params, 'warning', TextEntry('10-016-00008', args=wargs))
# return No row and True to fail
return None
# ------------------------------------------------------------------
# if we have no entries we did not find object
if len(table) == 0:
# return None
return None
# else we return result
return table
def query_gaia():
gaia = TapPlus(url='http://gea.esac.esa.int/tap-server/tap')
job = gaia.launch_job_async('select top 100000 ra, dec, phot_g_mean_mag \
from gaiadr1.gaia_source order by source_id')
results = job.get_results()
results = np.array(
[result.as_void().view((float, 3)) for result in results])
x = results[:, 2] * np.cos(results[:, 0]) * np.cos(results[:, 1])
y = results[:, 2] * np.sin(results[:, 0]) * np.cos(results[:, 1])
z = results[:, 2] * np.sin(results[:, 1])
sample_size = 1000
random_idns = np.random.choice(results.shape[0], sample_size)
ax.scatter(x[random_idns], y[random_idns], z[random_idns], s=3)
plt.show()
def test_load_table(self):
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
responseLoadTable = DummyResponse()
responseLoadTable.set_status_code(500)
responseLoadTable.set_message("ERROR")
tableDataFile = data_path('test_table1.xml')
tableData = utils.read_file_content(tableDataFile)
responseLoadTable.set_data(method='GET',
context=None,
body=tableData,
headers=None)
tableSchema = "public"
tableName = "table1"
fullQualifiedTableName = tableSchema + "." + tableName
tableRequest = "tables?tables=" + fullQualifiedTableName
connHandler.set_response(tableRequest, responseLoadTable)
with pytest.raises(Exception):
tap.load_table(fullQualifiedTableName)
responseLoadTable.set_status_code(200)
responseLoadTable.set_message("OK")
table = tap.load_table(fullQualifiedTableName)
assert table is not None, \
"Table '%s' not found" % (fullQualifiedTableName)
assert table.get_description() == 'Table1 desc', \
"Wrong description for table1. Expected: %s, found %s" % \
('Table1 desc', table.get_description())
columns = table.get_columns()
assert len(columns) == 2, \
"Number of columns for table1. Expected: %d, found: %d" % \
(2, len(columns))
col = self.__find_column('table1_col1', columns)
self.__check_column(col, 'Table1 Column1 desc', '', 'VARCHAR', 'indexed')
col = self.__find_column('table1_col2', columns)
self.__check_column(col, 'Table1 Column2 desc', '', 'INTEGER', None)
def test_load_tables_parameters(self):
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
responseLoadTable = DummyResponse()
responseLoadTable.set_status_code(200)
responseLoadTable.set_message("OK")
tableDataFile = data_path('test_tables.xml')
tableData = utils.read_file_content(tableDataFile)
responseLoadTable.set_data(method='GET',
context=None,
body=tableData,
headers=None)
tableRequest = "tables"
connHandler.set_response(tableRequest, responseLoadTable)
# empty request
tap.load_tables()
request = connHandler.get_last_request()
assert request == tableRequest, \
"Empty request. Expected: '%s', found: '%s'" % \
(tableRequest, request)
# flag only_names=false & share_accessible=false: equals to empty request
tap.load_tables(only_names=False, include_shared_tables=False)
request = connHandler.get_last_request()
assert request == tableRequest, \
"Empty request. Expected: '%s', found: '%s'" % \
(tableRequest, request)
# flag only_names
tableRequest = "tables?only_tables=true"
connHandler.set_response(tableRequest, responseLoadTable)
tap.load_tables(only_names=True)
request = connHandler.get_last_request()
assert request == tableRequest, \
"Flag only_names. Expected: '%s', found: '%s'" % \
(tableRequest, request)
# flag share_accessible=true
tableRequest = "tables?share_accessible=true"
connHandler.set_response(tableRequest, responseLoadTable)
tap.load_tables(include_shared_tables=True)
request = connHandler.get_last_request()
assert request == tableRequest, \
"Flag share_accessigle. Expected: '%s', found: '%s'" % \
(tableRequest, request)
# flag only_names=true & share_accessible=true
tableRequest = "tables?only_tables=true&share_accessible=true"
connHandler.set_response(tableRequest, responseLoadTable)
tap.load_tables(only_names=True, include_shared_tables=True)
request = connHandler.get_last_request()
assert request == tableRequest, \
"Flags only_names and share_accessible. Expected: '%s', found: '%s'" % \
(tableRequest, request)
def test_launc_async_job(self):
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
jobid = '12345'
# Launch response
responseLaunchJob = DummyResponse()
responseLaunchJob.set_status_code(500)
responseLaunchJob.set_message("ERROR")
# list of list (httplib implementation for headers in response)
launchResponseHeaders = [
['location', 'http://test:1111/tap/async/' + jobid]
]
responseLaunchJob.set_data(method='POST',
context=None,
body=None,
headers=launchResponseHeaders)
query = 'query'
dictTmp = {
"REQUEST": "doQuery",
"LANG": "ADQL",
"FORMAT": "votable",
"tapclient": str(TAP_CLIENT_ID),
"PHASE": "RUN",
"QUERY": str(query)}
sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp)
req = "async?" + sortedKey
connHandler.set_response(req, responseLaunchJob)
# Phase response
responsePhase = DummyResponse()
responsePhase.set_status_code(500)
responsePhase.set_message("ERROR")
responsePhase.set_data(method='GET',
context=None,
body="COMPLETED",
headers=None)
req = "async/" + jobid + "/phase"
connHandler.set_response(req, responsePhase)
# Results response
responseResultsJob = DummyResponse()
responseResultsJob.set_status_code(500)
responseResultsJob.set_message("ERROR")
jobDataFile = data_path('job_1.vot')
jobData = utils.read_file_content(jobDataFile)
responseResultsJob.set_data(method='GET',
context=None,
body=jobData,
headers=None)
req = "async/" + jobid + "/results/result"
connHandler.set_response(req, responseResultsJob)
with pytest.raises(Exception):
tap.launch_job_async(query)
responseLaunchJob.set_status_code(303)
responseLaunchJob.set_message("OK")
with pytest.raises(Exception):
tap.launch_job_async(query)
responsePhase.set_status_code(200)
responsePhase.set_message("OK")
with pytest.raises(Exception):
tap.launch_job_async(query)
responseResultsJob.set_status_code(200)
responseResultsJob.set_message("OK")
job = tap.launch_job_async(query)
assert job is not None, "Expected a valid job"
assert job.is_sync() is False, "Expected an asynchronous job"
assert job.get_phase() == 'COMPLETED', \
"Wrong job phase. Expected: %s, found %s" % \
('COMPLETED', job.get_phase())
assert job.is_failed() is False, "Wrong job status (set Failed = True)"
# results
results = job.get_results()
assert len(results) == 3, \
"Wrong job results (num rows). Expected: %d, found %d" % \
(3, len(results))
self.__check_results_column(results,
'alpha',
'alpha',
None,
np.float64)
self.__check_results_column(results,
'delta',
'delta',
None,
np.float64)
self.__check_results_column(results,
'source_id',
'source_id',
None,
np.object)
self.__check_results_column(results,
'table1_oid',
'table1_oid',
None,
np.int32)
def test_launch_sync_job(self):
connHandler = DummyConnHandler()
tap = TapPlus("http://test:1111/tap", connhandler=connHandler)
responseLaunchJob = DummyResponse()
responseLaunchJob.set_status_code(500)
responseLaunchJob.set_message("ERROR")
jobDataFile = data_path('job_1.vot')
jobData = utils.read_file_content(jobDataFile)
responseLaunchJob.set_data(method='POST',
context=None,
body=jobData,
headers=None)
query = 'select top 5 * from table'
dTmp = {"q": query}
dTmpEncoded = connHandler.url_encode(dTmp)
p = dTmpEncoded.find("=")
q = dTmpEncoded[p+1:]
dictTmp = {
"REQUEST": "doQuery",
"LANG": "ADQL",
"FORMAT": "votable",
"tapclient": str(TAP_CLIENT_ID),
"PHASE": "RUN",
"QUERY": str(q)}
sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp)
jobRequest = "sync?" + sortedKey
connHandler.set_response(jobRequest, responseLaunchJob)
with pytest.raises(Exception):
tap.launch_job(query)
responseLaunchJob.set_status_code(200)
responseLaunchJob.set_message("OK")
job = tap.launch_job(query)
assert job is not None, "Expected a valid job"
assert job.is_sync(), "Expected a synchronous job"
assert job.get_phase() == 'COMPLETED', \
"Wrong job phase. Expected: %s, found %s" % \
('COMPLETED', job.get_phase())
assert job.is_failed() is False, "Wrong job status (set Failed = True)"
# results
results = job.get_results()
assert len(results) == 3, \
"Wrong job results (num rows). Expected: %d, found %d" % \
(3, len(results))
self.__check_results_column(results,
'alpha',
'alpha',
None,
np.float64)
self.__check_results_column(results,
'delta',
'delta',
None,
np.float64)
self.__check_results_column(results,
'source_id',
'source_id',
None,
np.object)
self.__check_results_column(results,
'table1_oid',
'table1_oid',
None,
np.int32)
