def define_subject(config):
"""Common code to figure out which credentials to use based on the
content of a Config instance."""
subject = None
if config.proxy_fqn is not None and os.path.exists(config.proxy_fqn):
logging.debug(
f'Using proxy certificate {config.proxy_fqn} for credentials.'
)
subject = net.Subject(username=None, certificate=config.proxy_fqn)
elif config.netrc_file is not None:
netrc_fqn = os.path.join(config.working_directory, config.netrc_file)
if os.path.exists(netrc_fqn):
logging.debug(f'Using netrc file {netrc_fqn} for credentials.')
subject = net.Subject(
username=None, certificate=None, netrc=netrc_fqn
)
else:
logging.warning(f'Cannot find netrc file {netrc_fqn}')
else:
logging.warning(
f'Proxy certificate is {config.proxy_fqn}, netrc file is '
f'{config.netrc_file}.'
)
raise mc.CadcException(
'No credentials provided (proxy certificate or netrc file). '
'Cannot create an anonymous subject.'
)
return subject
def test_create_index(caps_get_mock, base_get_mock, base_post_mock):
caps_get_mock.return_value = BASE_URL
client = CadcTapClient(net.Subject())
response1 = Mock()
response1.status_code = 303
job_location = 'http://go.here'
response1.headers = {'Location': job_location}
base_post_mock.return_value = response1
response2 = Mock()
response2.status_code = 200
response2.text = "EXECUTING"
base_get_mock.side_effect = [response2]
response3 = Mock()
response3.status_code = 200
response3.text = "COMPLETED"
base_get_mock.side_effect = [response2, response3]
client.create_index('schema.sometable', 'col1', unique=True)
# expected post calls
post_calls = [
call((TABLE_UPDATE_CAPABILITY_ID, None),
allow_redirects=False,
data={
'table': 'schema.sometable',
'unique': 'true',
'index': 'col1'
}),
call('{}/phase'.format(job_location), data={'PHASE': 'RUN'})
]
base_post_mock.assert_has_calls(post_calls)
# expected get calls
get_calls = [
call('{}/phase'.format(job_location), data={'WAIT': 1}),
call('{}/phase'.format(job_location), data={'WAIT': 1})
]
base_get_mock.assert_has_calls(get_calls)
# error cases
with pytest.raises(AttributeError):
client.create_index(None, 'col1')
with pytest.raises(AttributeError):
client.create_index('sometable', None)
response4 = Mock()
response4.status_code = 200
response4.text = 'ABORTED'
base_get_mock.side_effect = [response4, response4]
client = CadcTapClient(net.Subject())
with pytest.raises(RuntimeError):
client.create_index('sometable', 'col1')
response5 = Mock()
response5.status_code = 500
base_get_mock.side_effect = [response1, response4, response4]
client = CadcTapClient(net.Subject())
with pytest.raises(RuntimeError):
client.create_index('sometable', 'col1')
def read_file_list_from_archive(config):
ad_resource_id = 'ivo://cadc.nrc.ca/ad'
agent = 'vlass2caom2/1.0'
subject = net.Subject(certificate=config.proxy_fqn)
client = net.BaseWsClient(
resource_id=ad_resource_id,
subject=subject,
agent=agent,
retry=True,
)
query_meta = (f"SELECT fileName FROM archive_files WHERE archiveName = "
f"'{config.archive}'")
data = {'QUERY': query_meta, 'LANG': 'ADQL', 'FORMAT': 'csv'}
logging.debug(f'Query is {query_meta}')
try:
response = client.get(
f'https://{client.host}/ad/sync?{parse.urlencode((data))}',
cert=config.proxy_fqn,
)
if response.status_code == 200:
# ignore the column name as the first part of the response
artifact_files_list = response.text.split()[1:]
return artifact_files_list
else:
raise mc.CadcException(f'Query failure {response}')
except Exception as e:
raise mc.CadcException(f'Failed ad content query: {e}')
def update(self, observation, **kwargs):
"""
Processes an observation and updates it
"""
assert isinstance(observation, Observation), (
"observation {} is not an Observation".format(observation))
for plane in observation.planes.values():
for artifact in plane.artifacts.values():
url = urlparse(artifact.uri)
if url.scheme != 'ad':
raise ValueError('Unsupported schema in uri: {}'.format(
artifact.uri))
[archive, file] = url.path.split('/')
# create cadc data web service client
if 'subject' in kwargs:
client = CadcDataClient(kwargs['subject'])
else:
client = CadcDataClient(net.Subject())
metadata = client.get_file_info(archive, file)
uri = artifact.uri.replace('/{}'.format(file),
'/{}'.format(metadata['name']))
checksum = ChecksumURI('md5:{}'.format(metadata['md5sum']))
print("old - uri({}), encoding({}), size({}), type({})".format(
artifact.uri, artifact.content_checksum,
artifact.content_length, artifact.content_type))
artifact.uri = uri
artifact.content_checksum = checksum
artifact.content_length = int(metadata['size'])
artifact.content_type = str(metadata['type'])
print("updated - uri({}), encoding({}), size({}), type({})".
format(artifact.uri, artifact.content_checksum,
artifact.content_length, artifact.content_type))
def test_create_table(caps_get_mock, base_put_mock):
caps_get_mock.return_value = BASE_URL
client = CadcTapClient(net.Subject())
# default format
def_table = os.path.join(TESTDATA_DIR, 'createTable.vosi')
def_table_content = open(def_table, 'rb').read()
client.create_table('sometable', def_table, 'VOSITable')
base_put_mock.assert_called_with(
(TABLES_CAPABILITY_ID, 'sometable'), data=def_table_content,
headers={'Content-Type': '{}'.format(
ALLOWED_TB_DEF_TYPES['VOSITable'])})
# VOTable format
base_put_mock.reset_mock()
client.create_table('sometable', def_table, 'VOTable')
base_put_mock.assert_called_with(
(TABLES_CAPABILITY_ID, 'sometable'), data=def_table_content,
headers={'Content-Type': '{}'.format(
ALLOWED_TB_DEF_TYPES['VOTable'])})
# default is VOTable format
base_put_mock.reset_mock()
client.create_table('sometable', def_table)
base_put_mock.assert_called_with(
(TABLES_CAPABILITY_ID, 'sometable'), data=def_table_content,
headers={'Content-Type': '{}'.format(
ALLOWED_TB_DEF_TYPES['VOSITable'])})
# error cases
with pytest.raises(AttributeError):
client.create_table(None, def_table)
with pytest.raises(AttributeError):
client.create_table('sometable', None)
def test_cadc_client_basicaa(base_mock):
service_realm = 'some.domain.com'
service_url = 'https://{}'.format(service_realm)
base_mock.return_value._get_url.return_value = service_url
cookie_response = Mock()
cookie_response.status_code = 200
cookie_response.text = 'cookie val'
subject = net.Subject()
subject._netrc = True # avoid checking for the real .netrc file
base_mock.return_value.post.return_value = cookie_response
# mock the get_auth of the subject to return user/passwd from netrc file
get_auth = Mock(return_value=('user', 'pswd'))
subject.get_auth = get_auth
# after calling CadcTapClient the subject is augmented with cookie info
CadcTapClient(subject)
get_auth.assert_called_with(service_realm)
assert len(subject.cookies) == len(cadctap.core.CADC_REALMS)
for cookie in subject.cookies:
# note "" around the value required by the cookie
assert cookie.value == '"cookie val"'
base_mock.return_value.post.assert_called_with(
('ivo://ivoa.net/std/UMS#login-0.1', None),
data='username=user&password=pswd',
headers={'Content-type': 'application/x-www-form-urlencoded',
'Accept': 'text/plain'})
def get_file(archive, file_name, cutout=None, destination=None):
anonSubject = net.Subject()
data_client = CadcDataClient(anonSubject)
return data_client.get_file(archive,
file_name,
cutout=cutout,
destination=destination)
def test_load_table(caps_get_mock, base_put_mock):
caps_get_mock.return_value = BASE_URL
client = CadcTapClient(net.Subject())
test_load_tb = os.path.join(TESTDATA_DIR, 'loadTable.txt')
# default format (tsv) using stdin
with open(test_load_tb, 'rb') as fh:
sys.stdin = fh
with patch('cadctap.core.open') as open_mock:
open_mock.return_value = fh
client.load('schema.sometable', '-')
base_put_mock.assert_called_with(
(TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['tsv'])})
# default format (tsv)
with open(test_load_tb, 'rb') as fh:
with patch('cadctap.core.open') as open_mock:
open_mock.return_value = fh
client.load('schema.sometable', [test_load_tb])
base_put_mock.assert_called_with(
(TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['tsv'])})
# tsv format
with open(test_load_tb, 'rb') as fh:
with patch('cadctap.core.open') as open_mock:
open_mock.return_value = fh
client.load('schema.sometable', [test_load_tb], fformat='tsv')
base_put_mock.assert_called_with(
(TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['tsv'])})
# csv format
with open(test_load_tb, 'rb') as fh:
with patch('cadctap.core.open') as open_mock:
open_mock.return_value = fh
client.load('schema.sometable', [test_load_tb], fformat='csv')
base_put_mock.assert_called_with(
(TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['csv'])})
# FITS table format
with open(test_load_tb, 'rb') as fh:
with patch('cadctap.core.open') as open_mock:
open_mock.return_value = fh
client.load('schema.sometable', [test_load_tb],
fformat='FITSTable')
base_put_mock.assert_called_with(
(TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['FITSTable'])})
# error cases
with pytest.raises(AttributeError):
client.load(None, [test_load_table])
with pytest.raises(AttributeError):
client.load('sometable', None)
with pytest.raises(AttributeError):
client.load('sometable', [])
def test_delete_table(base_delete_mock):
client = CadcTapClient(net.Subject())
client.delete_table('sometable')
base_delete_mock.assert_called_with((TABLES_CAPABILITY_ID,
'sometable'))
# error case
with pytest.raises(AttributeError):
client.delete(None)
def get_obs_id_from_vos(self):
logging.debug(f'Begin get_obs_id_from_vos for {self._vos_uri}.')
headers = fits2caom2.get_vos_headers(
self._vos_uri,
subject=net.Subject(certificate='/usr/src/app/cadcproxy.pem'))
self._obs_id = headers[0].get('DATALAB')
if self._obs_id is None:
raise mc.CadcException(
f'Could not get obs id from {self._vos_uri}')
logging.debug('End get_obs_id_from_vos.')
def test_error_cases():
def get_my_access_url(service):
if service == cadctap.core.PERMISSIONS_CAPABILITY_ID:
raise Exception(cadctap.core.PERMISSIONS_CAPABILITY_ID)
else:
return "http://some.org/some/path"
with patch('cadcutils.net.ws.WsCapabilities.get_access_url') as amock:
amock.side_effect = get_my_access_url
client = CadcTapClient(net.Subject())
assert not client.permissions_support
def read_list_from_caom(config):
query = "SELECT A.uri FROM caom2.Observation AS O " \
"JOIN caom2.Plane AS P ON O.obsID = P.obsID " \
"JOIN caom2.Artifact AS A ON P.planeID = A.planeID " \
"WHERE O.collection='{}'".format(config.archive)
subject = net.Subject(certificate=config.proxy_fqn)
tap_client = CadcTapClient(subject, resource_id=config.tap_id)
buffer = io.BytesIO()
tap_client.query(query, output_file=buffer)
temp = parse_single_table(buffer).to_table()
return [ii.decode().replace('ad:{}/'.format(config.archive), '').strip()
for ii in temp['uri']]
def test_query(base_post_mock):
client = youcat.YoucatClient(net.Subject())
# default format
def_name = 'tmptable'
def_table = os.path.join(TESTDATA_DIR, 'votable.xml')
fields = {'LANG': 'ADQL', 'QUERY': 'query', 'FORMAT': 'VOTable'}
tablefile = os.path.basename(def_table)
fields['UPLOAD'] = '{},param:{}'.format(def_name, tablefile)
fields[tablefile] = (def_table, open(def_table, 'rb'))
client.query('query', tmptable='tmptable:' + def_table)
print(base_post_mock.call_args_list[0][0][0])
assert base_post_mock.call_args_list[0][0][0] == \
(youcat.QUERY_CAPABILITY_ID, None)
def test_query(caps_get_mock, base_post_mock):
caps_get_mock.return_value = BASE_URL
response = Mock()
response.status_code = 200
response.raw.read.return_value = b'<VOTable format>'
response.text = 'Header 1\nVal1\nVal2\n'
# NOTE: post mock returns a context manager with the responose, hence
# the __enter__
base_post_mock.return_value.__enter__.return_value = response
client = CadcTapClient(net.Subject())
# default format
def_name = 'tmptable'
def_table = os.path.join(TESTDATA_DIR, 'votable.xml')
fields = {'LANG': 'ADQL',
'QUERY': 'query',
'FORMAT': 'VOTable'}
tablefile = os.path.basename(def_table)
fields['UPLOAD'] = '{},param:{}'.format(def_name, tablefile)
fields[tablefile] = (def_table, open(def_table, 'rb'))
with patch('cadctap.core.sys.stdout', new_callable=BytesIO):
client.query('query', tmptable='tmptable:'+def_table)
assert base_post_mock.call_args_list[0][0][0] == \
'{}/{}'.format(BASE_URL, 'sync')
base_post_mock.reset_mock()
with patch('sys.stdout', new_callable=StringIO) as stdout_mock:
client.query('query', data_only=True, response_format='tsv')
assert stdout_mock.getvalue() == 'Val1\nVal2\n'
# save in a file
tf = tempfile.NamedTemporaryFile()
base_post_mock.reset_mock()
base_post_mock.return_value.__enter__.return_value = response
client.query('query', output_file=tf.name, response_format='tsv')
actual = open(tf.name).read()
assert actual == 'Header 1\n-----------------------\n' \
'Val1\nVal2\n\n(2 rows affected)\n'
# different format => result from server not altered
base_post_mock.reset_mock()
base_post_mock.return_value.__enter__.return_value = response
with patch('sys.stdout', new_callable=BytesIO) as stdout_mock:
client.query('query')
assert stdout_mock.getvalue() == response.raw.read()
def test_load_table(base_put_mock):
client = youcat.YoucatClient(net.Subject())
test_load_tb = os.path.join(TESTDATA_DIR, 'loadTable.txt')
# default format (tsv)
with open(test_load_tb, 'rb') as fh:
with patch('cadctap.youcat.open') as open_mock:
open_mock.return_value = fh
client.load('schema.sometable', [test_load_tb])
base_put_mock.assert_called_with(
(youcat.TABLES_CAPABILITY_ID, 'schema.sometable'),
data=fh,
headers={'Content-Type': str(youcat.ALLOWED_CONTENT_TYPES['tsv'])})
# tsv format
with open(test_load_tb, 'rb') as fh:
with patch('cadctap.youcat.open') as open_mock:
open_mock.return_value = fh
client.load('schema.sometable', [test_load_tb], fformat='tsv')
base_put_mock.assert_called_with(
(youcat.TABLES_CAPABILITY_ID, 'schema.sometable'),
data=fh,
headers={'Content-Type': str(youcat.ALLOWED_CONTENT_TYPES['tsv'])})
# csv format
with open(test_load_tb, 'rb') as fh:
with patch('cadctap.youcat.open') as open_mock:
open_mock.return_value = fh
client.load('schema.sometable', [test_load_tb], fformat='csv')
base_put_mock.assert_called_with(
(youcat.TABLES_CAPABILITY_ID, 'schema.sometable'),
data=fh,
headers={'Content-Type': str(youcat.ALLOWED_CONTENT_TYPES['csv'])})
# error cases
with pytest.raises(AttributeError):
client.load(None, [test_load_table])
with pytest.raises(AttributeError):
client.load('sometable', None)
with pytest.raises(AttributeError):
client.load('sometable', [])
def read_file_list_from_archive(config):
ad_resource_id = 'ivo://cadc.nrc.ca/ad'
agent = '{}/{}'.format(APPLICATION, '1.0')
subject = net.Subject(certificate=config.proxy_fqn)
client = net.BaseWsClient(resource_id=ad_resource_id,
subject=subject, agent=agent, retry=True)
query_meta = "SELECT fileName FROM archive_files WHERE " \
"archiveName = '{}'".format(config.archive)
data = {'QUERY': query_meta, 'LANG': 'ADQL', 'FORMAT': 'csv'}
logging.debug('Query is {}'.format(query_meta))
try:
response = client.get('https://{}/ad/sync?{}'.format(
client.host, parse.urlencode(data)), cert=config.proxy_fqn)
if response.status_code == 200:
# ignore the column name as the first part of the response
artifact_files_list = response.text.split()[1:]
return artifact_files_list
else:
raise mc.CadcException('Query failure {!r}'.format(response))
except Exception as e:
raise mc.CadcException('Failed ad content query: {}'.format(e))
def testCalls(self, time_mock):
client = ws.BaseWsClient('https://httpbin.org', net.Subject(), 'FOO')
response = client.get('https://httpbin.org')
self.assertEqual(response.status_code, requests.codes.ok)
with self.assertRaises(exceptions.InternalServerException):
client.get('https://httpbin.org/status/500')
time_mock.reset_mock()
with self.assertRaises(exceptions.HttpException):
client.get('https://httpbin.org/status/503')
calls = [
call(DEFAULT_RETRY_DELAY),
call(min(DEFAULT_RETRY_DELAY * 2, MAX_RETRY_DELAY)),
call(min(DEFAULT_RETRY_DELAY * 4, MAX_RETRY_DELAY)),
call(min(DEFAULT_RETRY_DELAY * 8, MAX_RETRY_DELAY)),
call(min(DEFAULT_RETRY_DELAY * 16, MAX_RETRY_DELAY)),
call(min(DEFAULT_RETRY_DELAY * 32, MAX_RETRY_DELAY))
]
time_mock.assert_has_calls(calls)
def test_schema(base_get_mock):
client = CadcTapClient(net.Subject())
# default format
client.schema()
base_get_mock.assert_called_with(
(TABLES_CAPABILITY_ID, None))
import numpy
from cadcutils import net
from caom2 import *
from astropy import time
from astropy import units, constants
import logging
import os
from caom2repo import CAOM2RepoClient
import math
import pickle
from astroquery.alma import Alma
# Create a CAOM2RepoClient object.
certificate = os.path.join(os.getenv('HOME'), '.ssl/cadcproxy.pem')
resource_id = 'ivo://cadc.nrc.ca/sc2repo'
repo_client = CAOM2RepoClient(net.Subject(certificate=certificate), resource_id=resource_id)
def nan2None(x):
return not numpy.isnan(x) and x or None
def build_position(ra, dec, radius=24*units.arcsec):
points = []
vertices = []
segment_type = SegmentType['MOVE']
x1 = y1 = None
for theta in range(360, 0, -5):
x = radius.to('degree').value*math.cos(math.radians(theta)) + ra
y = radius.to('degree').value*math.sin(math.radians(theta)) + dec
from cadcdata import CadcDataClient
from cadcutils import net
fname = "cadcUrlList.txt"
with open(fname) as f:
txt = f.readlines()
txt = [x.strip() for x in txt]
print(len(txt))
txt = list(map(lambda x: x[73:81], txt))
for pid in txt:
if "." in pid:
pid = pid[:-1]
else:
pid = pid
try:
client = CadcDataClient(net.Subject())
client.get_file('CFHT', pid + '.fits.fz')
print(pid)
except Exception as e:
print(e)
continue
def test_get_subject(from_cmd_line_mock, netrc_mock, client_mock):
args = Mock()
args.service = 'tap'
args.anon = None
# authentication option specified
netrc_subject = net.Subject(netrc=True)
from_cmd_line_mock.return_value = netrc_subject
ret_subject = _get_subject(args)
assert(ret_subject is not None)
assert(ret_subject == netrc_subject)
# no authentication options, pick -n option
anon_subject = net.Subject()
netrc_subject = net.Subject(netrc=True)
from_cmd_line_mock.return_value = anon_subject
netrc_instance = netrc_mock.return_value
netrc_instance.hosts = {'netrc.host': 'netrc.host.ca'}
client_instance = client_mock.return_value
client_instance._tap_client._host = 'netrc.host'
ret_subject = _get_subject(args)
assert(ret_subject is not None)
assert(ret_subject.anon is False)
assert(ret_subject.certificate is None)
assert(ret_subject.netrc is not None)
# no authentication options, pick --cert option
orig_home = os.environ['HOME']
try:
# has certificate
os.environ['HOME'] = TESTDATA_DIR
anon_subject = net.Subject()
netrc_subject = net.Subject(netrc=True)
from_cmd_line_mock.return_value = anon_subject
netrc_instance = netrc_mock.return_value
netrc_instance.hosts = {'netrc.host': 'netrc.host.ca'}
client_instance = client_mock.return_value
client_instance._tap_client._host = 'no.such.host'
ret_subject = _get_subject(args)
assert(ret_subject is not None)
assert(ret_subject.anon is False)
assert(ret_subject.certificate is not None)
assert(ret_subject.netrc is False)
finally:
os.environ['HOME'] = orig_home
# no authentication options, pick --anon option
orig_home = os.environ['HOME']
try:
# has no certificate
os.environ['HOME'] = 'tmp'
anon_subject = net.Subject()
netrc_subject = net.Subject(netrc=True)
from_cmd_line_mock.return_value = anon_subject
netrc_instance = netrc_mock.return_value
netrc_instance.hosts = {'netrc.host': 'netrc.host.ca'}
client_instance = client_mock.return_value
client_instance._tap_client._host = 'no.such.host'
ret_subject = _get_subject(args)
assert(ret_subject is not None)
assert(ret_subject == anon_subject)
finally:
os.environ['HOME'] = orig_home
def test_set_permissions(caps_get_mock, post_mock):
caps_get_mock.return_value = BASE_URL
client = CadcTapClient(net.Subject())
resource = 'mytable'
client.set_permissions(resource=resource, read_anon=True)
post_mock.assert_called_with((PERMISSIONS_CAPABILITY_ID, resource),
data='public=true\n',
headers={'Content-Type': 'text/plain'})
post_mock.reset_mock()
client.set_permissions(resource=resource, read_anon=False,
read_only='ivo://cadc.nrc.ca/groups?ABC')
post_mock.assert_called_with(
(PERMISSIONS_CAPABILITY_ID, resource),
data='public=false\nr-group=ivo://cadc.nrc.ca/groups?ABC\n',
headers={'Content-Type': 'text/plain'})
post_mock.reset_mock()
client.set_permissions(resource=resource,
read_write='ivo://cadc.nrc.ca/groups?DEF',
read_only='ivo://cadc.nrc.ca/groups?ABC')
post_mock.assert_called_with(
(PERMISSIONS_CAPABILITY_ID, resource),
data='r-group=ivo://cadc.nrc.ca/groups?ABC\n'
'rw-group=ivo://cadc.nrc.ca/groups?DEF\n',
headers={'Content-Type': 'text/plain'})
post_mock.reset_mock()
client.set_permissions(resource=resource,
read_write='',
read_only='')
post_mock.assert_called_with(
(PERMISSIONS_CAPABILITY_ID, resource),
data='r-group=\nrw-group=\n',
headers={'Content-Type': 'text/plain'})
with pytest.raises(AttributeError, match='No resource'):
client.set_permissions(None)
with pytest.raises(
AttributeError, match='Expected URI for read group: ABC'):
client.set_permissions(resource, read_only='ABC')
with pytest.raises(
AttributeError, match='Expected URI for write group: ABC'):
client.set_permissions(resource, read_write='ABC')
# test dummy call
client.set_permissions(resource=resource)
# errors in permission modes
# extra group
with pytest.raises(argparse.ArgumentError):
opt = Mock
opt.groups = 'A B'
opt.mode = {'who': 'g', 'op': '+', 'what': 'r'}
_get_permission_modes(opt)
with pytest.raises(argparse.ArgumentError):
opt = Mock
opt.groups = 'A'
opt.mode = {'who': 'g', 'op': '-', 'what': 'r'}
_get_permission_modes(opt)
with pytest.raises(argparse.ArgumentError):
opt = Mock
opt.groups = 'A'
opt.mode = {'who': 'o', 'op': '+', 'what': 'r'}
_get_permission_modes(opt)
def test_schema(caps_get_mock, base_get_mock):
caps_get_mock.return_value = BASE_URL
base_get_mock.return_value.text = \
open(os.path.join(TESTDATA_DIR, 'db_schema.xml'), 'r').read()
client = CadcTapClient(net.Subject())
# default schema
db_schema = client.get_schema()
assert 3 == len(db_schema)
for schema in db_schema:
assert 'DB schema' == schema.description
assert ['Table', 'Description'] == schema.columns
if 'ivoa' == schema.name:
assert 3 == len(schema.rows)
elif 'caom2' in schema.name:
assert 14 == len(schema.rows)
elif 'tap_schema' == schema.name:
assert 5 == len(schema.rows)
else:
assert False, 'Unexpected schema'
base_get_mock.assert_called_with((TABLES_CAPABILITY_ID, None),
params={'detail': 'min'})
# table schema
table_schema_mock = Mock()
base_get_mock.reset_mock()
table_schema_mock.text = open(os.path.join(TESTDATA_DIR,
'table_schema.xml'), 'r').read()
permission_mock = Mock()
permission_mock.text = 'owner=someone\npublic=true\nr-group=\n' \
'rw-group=ivo://cadc.nrc.ca/gms?CADC'
base_get_mock.side_effect = [table_schema_mock, permission_mock]
tb_schema = client.get_table_schema('caom2.Observation')
assert 3 == len(tb_schema)
assert 'caom2.Observation' == tb_schema[0].name
assert 'the main CAOM Observation table' == tb_schema[0].description
assert ['Name', 'Type', 'Index', 'Description'] == tb_schema[0].columns
assert 45 == len(tb_schema[0].rows)
assert 'Foreign Keys' == tb_schema[1].name
assert 'Foreign Keys for table' == tb_schema[1].description
assert ['Target Table', 'Target Col', 'From Column', 'Description'] == \
tb_schema[1].columns
assert 1 == len(tb_schema[1].rows)
assert 'caom2.ObservationMember' == tb_schema[1].rows[0][0]
assert 1 == len(tb_schema[2].rows)
assert ['Owner', 'Others Read', 'Group Read', 'Group Write'] == \
tb_schema[2].columns
assert 'Permissions' == tb_schema[2].name
assert 'Permissions for caom2.Observation' == \
tb_schema[2].description
assert 'someone' == tb_schema[2].rows[0][0]
assert 'true' == tb_schema[2].rows[0][1]
assert '-' == tb_schema[2].rows[0][2]
assert 'CADC' == tb_schema[2].rows[0][3]
calls = [call(('ivo://ivoa.net/std/VOSI#tables-1.1', 'caom2.Observation'),
params={'detail': 'min'}),
call(('ivo://ivoa.net/std/VOSI#table-permissions-1.x',
'caom2.Observation'))]
base_get_mock.assert_has_calls(calls)
# check displays are also working although the visual part not tested
client.get_schema = Mock(return_value=db_schema)
client.schema('foo')
client.get_table_schema = Mock(return_value=tb_schema)
client.schema('caom2.Observation')
# test get_schema now
client = CadcTapClient(net.Subject())
client._db_schemas = {'tap': 'Schema goes in here'}
client.get_permissions = Mock(return_value='Permissions go in here')
assert not client.get_schema('foo')
assert ['Schema goes in here', 'Permissions go in here'] == \
client.get_schema('tap')
collection = sys.argv[1].upper()
if collection == 'NEOSSAT':
service = 'shared'
archive = 'NEOSSAT'
collection = 'NEOSS'
elif collection == 'GEM':
service = 'gemini'
archive = 'GEMINI'
elif collection == 'VLASS':
service = 'cirada'
archive = 'VLASS'
else:
service = collection.lower()
archive = collection
proxy_fqn = '/usr/src/app/cadcproxy.pem'
subject = net.Subject(certificate=proxy_fqn)
ad_client = CadcTapClient(subject, resource_id='ivo://cadc.nrc.ca/ad')
ops_client = CadcTapClient(subject, resource_id=f'ivo://cadc.nrc.ca/ams/{service}')
caom_client = CAOM2RepoClient(subject, resource_id='ivo://cadc.nrc.ca/ams')
print(':::1 - Find the name of a file to test with.')
ops_query = f"SELECT O.observationID, A.uri " \
f"FROM caom2.Observation as O " \
f"JOIN caom2.Plane as P on O.obsID = P.obsID " \
f"JOIN caom2.Artifact as A on P.planeID = A.planeID " \
f"WHERE O.collection = '{archive}' " \
f"AND A.uri like '%.fits%' " \
f"LIMIT 1"
ops_buffer = io.StringIO()
ops_client.query(ops_query, output_file=ops_buffer, data_only=True, response_format='csv')
