def test_login_with_user_password(self):
for auth_session in [
None, authsession.AuthSession(),
requests.Session()
]:
cadc = Cadc(auth_session=auth_session)
now = datetime.utcnow()
query = \
"select top 1 * from caom2.Plane where metaRelease>'{}'".\
format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
result = cadc.exec_sync(query)
assert len(result) == 0
cadc.login(os.environ['CADC_USER'], os.environ['CADC_PASSWD'])
query = "select top 1 * from caom2.Plane where metaRelease>'{}'".\
format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
result = cadc.exec_sync(query)
assert len(result) == 1
# repeat after logout
cadc.logout()
query = \
"select top 1 * from caom2.Plane where metaRelease>'{}'".\
format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
result = cadc.exec_sync(query)
assert len(result) == 0
# login in again
cadc.login(os.environ['CADC_USER'], os.environ['CADC_PASSWD'])
query = "select top 1 * from caom2.Plane where metaRelease>'{}'". \
format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
result = cadc.exec_sync(query)
assert len(result) == 1
def test_exec_sync():
# save results in a file
# create the VOTable result
# example from http://docs.astropy.org/en/stable/io/votable/
votable = VOTableFile()
resource = Resource()
votable.resources.append(resource)
table = Table(votable)
resource.tables.append(table)
table.fields.extend([
Field(votable, name="filename", datatype="char", arraysize="*"),
Field(votable, name="matrix", datatype="double", arraysize="2x2")])
table.create_arrays(2)
table.array[0] = ('test1.xml', [[1, 0], [0, 1]])
table.array[1] = ('test2.xml', [[0.5, 0.3], [0.2, 0.1]])
buffer = BytesIO()
votable.to_xml(buffer)
cadc = Cadc(auth_session=requests.Session())
response = Mock()
response.to_table.return_value = buffer.getvalue()
cadc.cadctap.search = Mock(return_value=response)
output_file = '{}/test_vooutput.xml'.format(tempfile.tempdir)
cadc.exec_sync('some query', output_file=output_file)
actual = parse(output_file)
assert len(votable.resources) == len(actual.resources) == 1
assert len(votable.resources[0].tables) ==\
len(actual.resources[0].tables) == 1
actual_table = actual.resources[0].tables[0]
try:
# TODO remove when astropy LTS upgraded
from astropy.utils.diff import report_diff_values
assert report_diff_values(table, actual_table, fileobj=sys.stdout)
except ImportError:
pass
def test_query(self):
cadc = Cadc()
result = cadc.exec_sync(
"select count(*) from caom2.Observation where target_name='M31'")
assert 1000 < result[0][0]
# test that no proprietary results are returned when not logged in
now = datetime.utcnow()
query = "select top 1 * from caom2.Plane where " \
"metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
result = cadc.exec_sync(query)
assert len(result) == 0
def test_login_with_cert(self):
# repeat previous test
cadc = Cadc()
cadc.logout()
now = datetime.utcnow()
# following query is to test login with certificates when an
# anonymous query is executed first.
cadc.exec_sync('select top 1 * from caom2.Observation')
cadc.login(certificate_file=os.environ['CADC_CERT'])
query = "select top 1 * from caom2.Plane where " \
"metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
result = cadc.exec_sync(query)
assert len(result) == 1
cadc.logout()
def test_async(self):
# test async calls
cadc = Cadc()
# run the query in sync mode first
expected = cadc.exec_sync(
"select top 3 observationID from caom2.Observation where "
"collection='IRIS' order by observationID")
# now run the query in async mode
job = cadc.create_async(
"select top 3 observationID from caom2.Observation where "
"collection='IRIS' order by observationID")
job = job.run().wait()
job.raise_if_error()
result = job.fetch_result().to_table()
assert len(expected) == len(result)
for ii in range(0, 2):
assert expected['observationID'][ii] == result['observationID'][ii]
# load job again
loaded_job = cadc.load_async_job(job.job_id)
result = loaded_job.fetch_result().to_table()
assert len(expected) == len(result)
for ii in range(0, 2):
assert expected['observationID'][ii] == result['observationID'][ii]
def test_authsession(self):
# repeat previous test
auth_session = requests.Session()
auth_session.cert = os.environ['CADC_CERT']
cadc = Cadc(auth_session=auth_session)
now = datetime.utcnow()
query = "select top 1 * from caom2.Plane where " \
"metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
result = cadc.exec_sync(query)
assert len(result) == 1
annon_session = requests.Session()
cadc = Cadc(auth_session=annon_session)
now = datetime.utcnow()
query = "select top 1 * from caom2.Plane where " \
"metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
result = cadc.exec_sync(query)
assert len(result) == 0
def test_login_with_user_password(self):
cadc = Cadc()
cadc.logout()
now = datetime.utcnow()
cadc.login(os.environ['CADC_USER'], os.environ['CADC_PASSWD'])
query = "select top 1 * from caom2.Plane where " \
"metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
result = cadc.exec_sync(query)
cadc.logout()
assert len(result) == 1
def test_login_with_cert(self):
for auth_session in [requests.Session()]:
cadc = Cadc(auth_session=auth_session)
now = datetime.utcnow()
query = \
"select top 1 * from caom2.Plane where metaRelease>'{}'".\
format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
result = cadc.exec_sync(query)
assert len(result) == 0
# following query is to test login with certificates when an
# anonymous query is executed first.
cadc.login(certificate_file=os.environ['CADC_CERT'])
query = \
"select top 1 * from caom2.Plane where metaRelease>'{}'".\
format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
result = cadc.exec_sync(query)
assert len(result) == 1
cadc.logout()
query = \
"select top 1 * from caom2.Plane where metaRelease>'{}'".\
format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
result = cadc.exec_sync(query)
assert len(result) == 0
def test_get_collections():
cadc = Cadc()
def mock_run_query(query, output_format=None, maxrec=None,
output_file=None):
assert query == \
'select distinct collection, energy_emBand from caom2.EnumField'
assert output_format is None
assert maxrec is None
assert output_file is None
table = AstroTable(rows=[('CFHT', 'Optical'), ('CFHT', 'Infrared'),
('JCMT', 'Millimeter'), ('DAO', 'Optical'),
('DAO', 'Infrared')],
names=('collection', 'energy_emBand'))
return table
cadc.exec_sync = mock_run_query
result = cadc.get_collections()
assert len(result) == 3
assert 'CFHT' in result
assert 'JCMT' in result
assert 'DAO' in result
def test_get_images_against_AS(self):
cadc = Cadc()
coords = '08h45m07.5s +54d18m00s'
radius = 0.05 * u.deg
# Compare results from cadc advanced search to get_images
query = cadc._args_to_payload(**{
'coordinates': coords,
'radius': radius,
'data_product_type': 'image'
})['query']
result = cadc.exec_sync(query)
uri_list = [uri.decode('ascii') for uri in result['publisherID']]
access_url = 'https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/download'
icrs_coords = parse_coordinates(coords).icrs
data = {
'uris':
' '.join(uri_list),
'params':
'cutout=Circle ICRS {} {} {}'.format(icrs_coords.ra.degree,
icrs_coords.dec.degree,
radius.value),
'method':
'URL List'
}
resp_urls = requests.post(access_url, data).text.split('\r\n')
# Filter out the errors and empty strings
filtered_resp_urls = list(
filter(lambda url: not url.startswith('ERROR') and url != '',
resp_urls))
# This function should return nearly the same urls
image_urls = cadc.get_images(coords, radius, get_url_list=True)
assert len(filtered_resp_urls) == len(image_urls)
def test_query_format(self):
cadc = Cadc()
query = "select top 1 observationID, collection from caom2.Observation"
result = cadc.exec_sync(query, output_format='csv')
print(result)
class QueryBuilder:
def __init__(self, table):
print("ADQL Builder")
self.cadc = Cadc()
try:
self.table = self._is_valid_table(table)
except TableNotExistError:
print("catch TableNotExistError ")
return
self.column_type_dictionary = {}
self.dropdown_list = self._get_columns()
self.combobox_list = ['a', 'b', 'c']
self.method_list = ['like', '>', '=', '<', '<=', '>=']
self.condition_list = []
self.query_output = widgets.Output()
self.out = widgets.Output(
layout=widgets.Layout(flex='1 1 auto', width='auto'))
self.column_name = widgets.Dropdown(
options=self.dropdown_list,
description='column',
layout=widgets.Layout(flex='1 1 auto', width='auto'),
style={'description_width': 'initial'})
self.method = widgets.Dropdown(options=self.method_list,
description='',
layout=widgets.Layout(flex='1 1 auto',
width='auto'),
style={'description_width': 'initial'})
"""self.column_value = widgets.Combobox(
value='',
placeholder='value',
options=self.combobox_list,
description='',
layout=widgets.Layout(flex='1 1 auto',
width='auto'),
style={'description_width': 'initial'})"""
self.column_value = widgets.Text(value='',
placeholder='value',
description='',
layout=widgets.Layout(flex='1 1 auto',
width='auto'))
self.add_button = widgets.Button(
description="",
icon='plus',
style=widgets.ButtonStyle(button_color='#E58975'),
layout=widgets.Layout(width='5%'))
self.add_button.on_click(self._button_clicked)
self.delete_button = widgets.Button(
description="Delete",
icon='',
style=widgets.ButtonStyle(button_color='#E58975'),
layout=widgets.Layout(height="25px", width='70px'))
self.delete_button.on_click(self._button_clicked)
self.query_button = widgets.Button(
description="Query",
icon='',
style=widgets.ButtonStyle(button_color='#E58975'),
layout=widgets.Layout(height="25px", width='70PX'))
self.query_button.on_click(self._query_clicked)
self.clear_button = widgets.Button(
description="Clear",
icon='',
style=widgets.ButtonStyle(button_color='#E58975'),
layout=widgets.Layout(height="25px", width='70px'))
self.clear_button.on_click(self._button_clicked)
self.clear_button.click()
self.ui = widgets.HBox([
self.column_name, self.method, self.column_value, self.add_button
])
self.buttons_ui = widgets.VBox(
[self.delete_button, self.clear_button, self.query_button],
layout=widgets.Layout(top='8px'))
self.output_ui = widgets.HBox([self.out, self.buttons_ui])
def adql_builder(self):
display(self.ui, self.output_ui, self.query_output)
def _button_clicked(self, b):
with self.out:
clear_output()
if b.description == "Clear":
self.condition_list = []
elif b.description == "Delete":
if len(self.condition_list) > 0:
self.condition_list.remove(self.conditions.value)
else:
self.condition_list.append(
f"{self.column_name.value} {self.method.value} {self.column_value.value}"
)
self.conditions = widgets.Select(
options=self.condition_list,
layout=widgets.Layout(flex='1 1 auto', width='auto'),
description='')
ui_condition = widgets.HBox([self.conditions])
display(ui_condition)
def _is_valid_table(self, table_name):
output_q = self.cadc.exec_sync(
f"SELECT * From tap_schema.tables WHERE table_name='{table_name}'")
if len(output_q) == 0:
raise TableNotExistError
else:
return table_name
def _query_clicked(self, b):
with self.query_output:
clear_output()
Query = self.get_query()
output = self.cadc.exec_sync(Query)
display(output)
#print(self.condition_list)
#output = self.cadc.exec_sync(f"SELECT * FROM {self.table} WHERE { }"
def get_query(self):
Query = f"SELECT * FROM {self.table}"
if len(self.condition_list) > 0:
Query += " WHERE ("
for item in self.condition_list:
item = item.split(" ")
if item[1] == 'like' and self.column_type_dictionary[
item[0]] == 'char':
Query += f" AND {item[0]} " + f"{item[1]} " + f"'%{str(item[2])}%'"
elif item[1] == '=' and self.column_type_dictionary[
item[0]] == 'char':
Query += f" AND {item[0]}" + f"{item[1]}" + f"'{str(item[2])}'"
elif self.column_type_dictionary[item[0]] == 'int':
if item[1] == '=' or item[1] == '>' or item[1] == '<' or item[
1] == '<=' or item[1] == '>=':
Query += f" AND {item[0]}" + f"{item[1]}" + f"{int(item[2])}"
elif self.column_type_dictionary[item[0]] == 'double':
if item[1] == '=' or item[1] == '>' or item[1] == '<' or item[
1] == '<=' or item[1] == '>=':
Query += f" AND {item[0]}" + f"{item[1]}" + f"{float(item[2])}"
Query += ")"
Query = Query.replace("WHERE ( AND", "WHERE (")
print(f"Query: {Query}")
return Query
def _get_columns(self):
output = self.cadc.exec_sync(
f"SELECT column_name, datatype from tap_schema.columns WHERE table_name = '{self.table}' "
)
column_lst = list(output['column_name'])
type_lst = list(output['datatype'])
for i in range(0, len(column_lst)):
self.column_type_dictionary[column_lst[i]] = type_lst[i]
return column_lst
def text(self):
space_object = widgets.Text(
value="",
description="Target Object/Target Coordinates",
continuous_update=True,
layout=widgets.Layout(flex='1 1 auto', width='auto'),
style={'description_width': '31.5%'})
display(space_object)
