def read_list_from_caom(config):
query = "SELECT A.uri FROM caom2.Observation AS O " \
"JOIN caom2.Plane AS P ON O.obsID = P.obsID " \
"JOIN caom2.Artifact AS A ON P.planeID = A.planeID " \
"WHERE O.collection='{}'".format(config.archive)
subject = net.Subject(certificate=config.proxy_fqn)
tap_client = CadcTapClient(subject, resource_id=config.tap_id)
buffer = io.BytesIO()
tap_client.query(query, output_file=buffer)
temp = parse_single_table(buffer).to_table()
return [ii.decode().replace('ad:{}/'.format(config.archive), '').strip()
for ii in temp['uri']]
def __init__(self, config):
super().__init__()
self._config = config
self._subject = clc.define_subject(self._config)
self._sc2_client = CadcTapClient(
subject=self._subject, resource_id='ivo://cadc.nrc.ca/sc2tap'
)
self._prod_client = CadcTapClient(
subject=self._subject, resource_id='ivo://cadc.nrc.ca/ams/gemini'
)
self._connected = mc.TaskType.SCRAPE not in config.task_types
self._logger = logging.getLogger(self.__class__.__name__)
def test_query(base_post_mock):
client = CadcTapClient(net.Subject())
# default format
def_name = 'tmptable'
def_table = os.path.join(TESTDATA_DIR, 'votable.xml')
fields = {'LANG': 'ADQL',
'QUERY': 'query',
'FORMAT': 'VOTable'}
tablefile = os.path.basename(def_table)
fields['UPLOAD'] = '{},param:{}'.format(def_name, tablefile)
fields[tablefile] = (def_table, open(def_table, 'rb'))
client.query('query', tmptable='tmptable:'+def_table)
print(base_post_mock.call_args_list[0][0][0])
assert base_post_mock.call_args_list[0][0][0] == \
(QUERY_CAPABILITY_ID, None, 'uws:Sync')
def test_cadc_client_basicaa(base_mock):
service_realm = 'some.domain.com'
service_url = 'https://{}'.format(service_realm)
base_mock.return_value._get_url.return_value = service_url
cookie_response = Mock()
cookie_response.status_code = 200
cookie_response.text = 'cookie val'
subject = net.Subject()
subject._netrc = True # avoid checking for the real .netrc file
base_mock.return_value.post.return_value = cookie_response
# mock the get_auth of the subject to return user/passwd from netrc file
get_auth = Mock(return_value=('user', 'pswd'))
subject.get_auth = get_auth
# after calling CadcTapClient the subject is augmented with cookie info
CadcTapClient(subject)
get_auth.assert_called_with(service_realm)
assert len(subject.cookies) == len(cadctap.core.CADC_REALMS)
for cookie in subject.cookies:
# note "" around the value required by the cookie
assert cookie.value == '"cookie val"'
base_mock.return_value.post.assert_called_with(
('ivo://ivoa.net/std/UMS#login-0.1', None),
data='username=user&password=pswd',
headers={'Content-type': 'application/x-www-form-urlencoded',
'Accept': 'text/plain'})
def test_query(caps_get_mock, base_post_mock):
caps_get_mock.return_value = BASE_URL
response = Mock()
response.status_code = 200
response.raw.read.return_value = b'<VOTable format>'
response.text = 'Header 1\nVal1\nVal2\n'
# NOTE: post mock returns a context manager with the responose, hence
# the __enter__
base_post_mock.return_value.__enter__.return_value = response
client = CadcTapClient(net.Subject())
# default format
def_name = 'tmptable'
def_table = os.path.join(TESTDATA_DIR, 'votable.xml')
fields = {'LANG': 'ADQL',
'QUERY': 'query',
'FORMAT': 'VOTable'}
tablefile = os.path.basename(def_table)
fields['UPLOAD'] = '{},param:{}'.format(def_name, tablefile)
fields[tablefile] = (def_table, open(def_table, 'rb'))
with patch('cadctap.core.sys.stdout', new_callable=BytesIO):
client.query('query', tmptable='tmptable:'+def_table)
assert base_post_mock.call_args_list[0][0][0] == \
'{}/{}'.format(BASE_URL, 'sync')
base_post_mock.reset_mock()
with patch('sys.stdout', new_callable=StringIO) as stdout_mock:
client.query('query', data_only=True, response_format='tsv')
assert stdout_mock.getvalue() == 'Val1\nVal2\n'
# save in a file
tf = tempfile.NamedTemporaryFile()
base_post_mock.reset_mock()
base_post_mock.return_value.__enter__.return_value = response
client.query('query', output_file=tf.name, response_format='tsv')
actual = open(tf.name).read()
assert actual == 'Header 1\n-----------------------\n' \
'Val1\nVal2\n\n(2 rows affected)\n'
# different format => result from server not altered
base_post_mock.reset_mock()
base_post_mock.return_value.__enter__.return_value = response
with patch('sys.stdout', new_callable=BytesIO) as stdout_mock:
client.query('query')
assert stdout_mock.getvalue() == response.raw.read()
def init_global(config):
global om
om = gom.GeminiObsMetadata()
get_gofr(config)
global gofr
if gofr.tap_client is None and config.is_connected:
subject = mc.define_subject(config)
tap_client = CadcTapClient(subject=subject, resource_id=config.tap_id)
gofr.tap_client = tap_client
def visit(observation, **kwargs):
mc.check_param(observation, Observation)
working_directory = kwargs.get('working_directory', './')
storage_name = kwargs.get('storage_name')
if storage_name is None:
raise mc.CadcException(
f'Must have a storage_name parameter for provenance_augmentation '
f'for {observation.observation_id}')
config = mc.Config()
config.get_executors()
if mc.TaskType.SCRAPE in config.task_types:
logging.warning(f'Provenance augmentation does not work for SCRAPE.')
return {'provenance': 0}
subject = clc.define_subject(config)
tap_client = CadcTapClient(subject, config.tap_id)
count = 0
obs_members = TypedSet(ObservationURI, )
for plane in observation.planes.values():
plane_inputs = TypedSet(PlaneURI, )
for artifact in plane.artifacts.values():
if storage_name.file_uri == artifact.uri:
count = _do_provenance(
working_directory,
storage_name.file_name,
observation,
tap_client,
plane_inputs,
obs_members,
config,
)
if plane.provenance is not None:
plane.provenance.inputs.update(plane_inputs)
if isinstance(observation, DerivedObservation):
observation.members.update(obs_members)
if len(observation.members) > 0:
observable = kwargs.get('observable')
caom_repo_client = kwargs.get('caom_repo_client')
if caom_repo_client is None:
logging.warning(f'Warning: Must have a caom_repo_client for '
f'members metadata for '
f'{observation.observation_id}.')
_do_members_metadata(
observation,
caom_repo_client,
observation.members,
observable.metrics,
)
logging.info(
f'Done provenance_augmentation for {observation.observation_id}')
return {'provenance': count}
def test_error_cases():
def get_my_access_url(service):
if service == cadctap.core.PERMISSIONS_CAPABILITY_ID:
raise Exception(cadctap.core.PERMISSIONS_CAPABILITY_ID)
else:
return "http://some.org/some/path"
with patch('cadcutils.net.ws.WsCapabilities.get_access_url') as amock:
amock.side_effect = get_my_access_url
client = CadcTapClient(net.Subject())
assert not client.permissions_support
def test_delete_table(base_delete_mock):
client = CadcTapClient(net.Subject())
client.delete_table('sometable')
base_delete_mock.assert_called_with((TABLES_CAPABILITY_ID,
'sometable'))
# error case
with pytest.raises(AttributeError):
client.delete(None)
def test_create_table(base_put_mock):
client = CadcTapClient(net.Subject())
# default format
def_table = os.path.join(TESTDATA_DIR, 'createTable.vosi')
def_table_content = open(def_table, 'rb').read()
client.create_table('sometable', def_table)
base_put_mock.assert_called_with(
(TABLES_CAPABILITY_ID, 'sometable'), data=def_table_content,
headers={'Content-Type': '{}'.format(
ALLOWED_TB_DEF_TYPES['VOSITable'])})
# VOTable format
base_put_mock.reset_mock()
client.create_table('sometable', def_table, 'VOTable')
base_put_mock.assert_called_with(
(TABLES_CAPABILITY_ID, 'sometable'), data=def_table_content,
headers={'Content-Type': '{}'.format(
ALLOWED_TB_DEF_TYPES['VOTable'])})
# error cases
with pytest.raises(AttributeError):
client.create_table(None, def_table)
with pytest.raises(AttributeError):
client.create_table('sometable', None)
def _init(self, config):
if mc.TaskType.SCRAPE in config.task_types:
self._logger.info(
f'SCRAPE\'ing data - no clients will be initialized.'
)
else:
subject = define_subject(config)
self._metadata_client = CAOM2RepoClient(
subject, config.logging_level, config.resource_id
)
self._data_client = declare_client(config)
if config.tap_id is not None:
self._query_client = CadcTapClient(
subject=subject, resource_id=config.tap_id
)
self._metrics = mc.Metrics(config)
def get_obs_id(self):
obs_id = external_metadata.get_obs_id_from_headers(self._file_id)
if obs_id is None:
config = mc.Config()
config.get_executors()
subject = mc.define_subject(config)
tap_client = CadcTapClient(subject=subject,
resource_id=config.tap_id)
obs_id = external_metadata.get_obs_id_from_cadc(
self._file_id, tap_client, COLLECTION)
if obs_id is None:
headers = fits2caom2.get_cadc_headers(
f'ad:GEMINI/{self._file_name}', subject)
obs_id = headers[0].get('DATALAB')
if obs_id is None:
raise mc.CadcException(f'No obs id for {self._file_name}')
return obs_id
def test_set_permissions(caps_get_mock, post_mock):
caps_get_mock.return_value = BASE_URL
client = CadcTapClient(net.Subject())
resource = 'mytable'
client.set_permissions(resource=resource, read_anon=True)
post_mock.assert_called_with((PERMISSIONS_CAPABILITY_ID, resource),
data='public=true\n',
headers={'Content-Type': 'text/plain'})
post_mock.reset_mock()
client.set_permissions(resource=resource, read_anon=False,
read_only='ivo://cadc.nrc.ca/groups?ABC')
post_mock.assert_called_with(
(PERMISSIONS_CAPABILITY_ID, resource),
data='public=false\nr-group=ivo://cadc.nrc.ca/groups?ABC\n',
headers={'Content-Type': 'text/plain'})
post_mock.reset_mock()
client.set_permissions(resource=resource,
read_write='ivo://cadc.nrc.ca/groups?DEF',
read_only='ivo://cadc.nrc.ca/groups?ABC')
post_mock.assert_called_with(
(PERMISSIONS_CAPABILITY_ID, resource),
data='r-group=ivo://cadc.nrc.ca/groups?ABC\n'
'rw-group=ivo://cadc.nrc.ca/groups?DEF\n',
headers={'Content-Type': 'text/plain'})
post_mock.reset_mock()
client.set_permissions(resource=resource,
read_write='',
read_only='')
post_mock.assert_called_with(
(PERMISSIONS_CAPABILITY_ID, resource),
data='r-group=\nrw-group=\n',
headers={'Content-Type': 'text/plain'})
with pytest.raises(AttributeError, match='No resource'):
client.set_permissions(None)
with pytest.raises(
AttributeError, match='Expected URI for read group: ABC'):
client.set_permissions(resource, read_only='ABC')
with pytest.raises(
AttributeError, match='Expected URI for write group: ABC'):
client.set_permissions(resource, read_write='ABC')
# test dummy call
client.set_permissions(resource=resource)
# errors in permission modes
# extra group
with pytest.raises(argparse.ArgumentError):
opt = Mock
opt.groups = 'A B'
opt.mode = {'who': 'g', 'op': '+', 'what': 'r'}
_get_permission_modes(opt)
with pytest.raises(argparse.ArgumentError):
opt = Mock
opt.groups = 'A'
opt.mode = {'who': 'g', 'op': '-', 'what': 'r'}
_get_permission_modes(opt)
with pytest.raises(argparse.ArgumentError):
opt = Mock
opt.groups = 'A'
opt.mode = {'who': 'o', 'op': '+', 'what': 'r'}
_get_permission_modes(opt)
def test_schema(base_get_mock):
client = CadcTapClient(net.Subject())
# default format
client.schema()
base_get_mock.assert_called_with(
(TABLES_CAPABILITY_ID, None))
def test_create_index(base_get_mock, base_post_mock):
client = CadcTapClient(net.Subject())
response1 = Mock()
response1.status_code = 303
job_location = 'http://go.here'
response1.headers = {'Location': job_location}
base_post_mock.return_value = response1
response2 = Mock()
response2.status_code = 200
response2.text = "EXECUTING"
base_get_mock.side_effect = [response2]
response3 = Mock()
response3.status_code = 200
response3.text = "COMPLETED"
base_get_mock.side_effect = [response2, response3]
client.create_index('schema.sometable', 'col1', unique=True)
# expected post calls
post_calls = [call((TABLE_UPDATE_CAPABILITY_ID, None),
allow_redirects=False,
data={'table': 'schema.sometable',
'uniquer': True,
'index': 'col1'}),
call('{}/phase'.format(job_location),
data={'PHASE': 'RUN'})]
base_post_mock.assert_has_calls(post_calls)
# expected get calls
get_calls = [call('{}/phase'.format(job_location), data={'WAIT': 1}),
call('{}/phase'.format(job_location), data={'WAIT': 1})]
base_get_mock.assert_has_calls(get_calls)
# error cases
with pytest.raises(AttributeError):
client.create_index(None, 'col1')
with pytest.raises(AttributeError):
client.create_index('sometable', None)
response4 = Mock()
response4.status_code = 200
response4.text = 'ABORTED'
base_get_mock.side_effect = [response4, response4]
client = CadcTapClient(net.Subject())
with pytest.raises(RuntimeError):
client.create_index('sometable', 'col1')
response5 = Mock()
response5.status_code = 500
base_get_mock.side_effect = [response1, response4, response4]
client = CadcTapClient(net.Subject())
with pytest.raises(RuntimeError):
client.create_index('sometable', 'col1')
def test_load_table(base_put_mock):
client = CadcTapClient(net.Subject())
test_load_tb = os.path.join(TESTDATA_DIR, 'loadTable.txt')
# default format (tsv)
with open(test_load_tb, 'rb') as fh:
with patch('cadctap.core.open') as open_mock:
open_mock.return_value = fh
client.load('schema.sometable', [test_load_tb])
base_put_mock.assert_called_with(
(TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['tsv'])})
# tsv format
with open(test_load_tb, 'rb') as fh:
with patch('cadctap.core.open') as open_mock:
open_mock.return_value = fh
client.load('schema.sometable', [test_load_tb], fformat='tsv')
base_put_mock.assert_called_with(
(TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['tsv'])})
# csv format
with open(test_load_tb, 'rb') as fh:
with patch('cadctap.core.open') as open_mock:
open_mock.return_value = fh
client.load('schema.sometable', [test_load_tb], fformat='csv')
base_put_mock.assert_called_with(
(TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['csv'])})
# FITS table format
with open(test_load_tb, 'rb') as fh:
with patch('cadctap.core.open') as open_mock:
open_mock.return_value = fh
client.load('schema.sometable', [test_load_tb],
fformat='FITSTable')
base_put_mock.assert_called_with(
(TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['FITSTable'])})
# error cases
with pytest.raises(AttributeError):
client.load(None, [test_load_table])
with pytest.raises(AttributeError):
client.load('sometable', None)
with pytest.raises(AttributeError):
client.load('sometable', [])
def test_schema(caps_get_mock, base_get_mock):
caps_get_mock.return_value = BASE_URL
base_get_mock.return_value.text = \
open(os.path.join(TESTDATA_DIR, 'db_schema.xml'), 'r').read()
client = CadcTapClient(net.Subject())
# default schema
db_schema = client.get_schema()
assert 3 == len(db_schema)
for schema in db_schema:
assert 'DB schema' == schema.description
assert ['Table', 'Description'] == schema.columns
if 'ivoa' == schema.name:
assert 3 == len(schema.rows)
elif 'caom2' in schema.name:
assert 14 == len(schema.rows)
elif 'tap_schema' == schema.name:
assert 5 == len(schema.rows)
else:
assert False, 'Unexpected schema'
base_get_mock.assert_called_with((TABLES_CAPABILITY_ID, None),
params={'detail': 'min'})
# table schema
table_schema_mock = Mock()
base_get_mock.reset_mock()
table_schema_mock.text = open(os.path.join(TESTDATA_DIR,
'table_schema.xml'), 'r').read()
permission_mock = Mock()
permission_mock.text = 'owner=someone\npublic=true\nr-group=\n' \
'rw-group=ivo://cadc.nrc.ca/gms?CADC'
base_get_mock.side_effect = [table_schema_mock, permission_mock]
tb_schema = client.get_table_schema('caom2.Observation')
assert 3 == len(tb_schema)
assert 'caom2.Observation' == tb_schema[0].name
assert 'the main CAOM Observation table' == tb_schema[0].description
assert ['Name', 'Type', 'Index', 'Description'] == tb_schema[0].columns
assert 45 == len(tb_schema[0].rows)
assert 'Foreign Keys' == tb_schema[1].name
assert 'Foreign Keys for table' == tb_schema[1].description
assert ['Target Table', 'Target Col', 'From Column', 'Description'] == \
tb_schema[1].columns
assert 1 == len(tb_schema[1].rows)
assert 'caom2.ObservationMember' == tb_schema[1].rows[0][0]
assert 1 == len(tb_schema[2].rows)
assert ['Owner', 'Others Read', 'Group Read', 'Group Write'] == \
tb_schema[2].columns
assert 'Permissions' == tb_schema[2].name
assert 'Permissions for caom2.Observation' == \
tb_schema[2].description
assert 'someone' == tb_schema[2].rows[0][0]
assert 'true' == tb_schema[2].rows[0][1]
assert '-' == tb_schema[2].rows[0][2]
assert 'CADC' == tb_schema[2].rows[0][3]
calls = [call(('ivo://ivoa.net/std/VOSI#tables-1.1', 'caom2.Observation'),
params={'detail': 'min'}),
call(('ivo://ivoa.net/std/VOSI#table-permissions-1.x',
'caom2.Observation'))]
base_get_mock.assert_has_calls(calls)
# check displays are also working although the visual part not tested
client.get_schema = Mock(return_value=db_schema)
client.schema('foo')
client.get_table_schema = Mock(return_value=tb_schema)
client.schema('caom2.Observation')
# test get_schema now
client = CadcTapClient(net.Subject())
client._db_schemas = {'tap': 'Schema goes in here'}
client.get_permissions = Mock(return_value='Permissions go in here')
assert not client.get_schema('foo')
assert ['Schema goes in here', 'Permissions go in here'] == \
client.get_schema('tap')
def __init__(self, config, preview_suffix='jpg'):
super(QueryTimeBoxDataSourceTS, self).__init__(config)
self._preview_suffix = preview_suffix
subject = mc.define_subject(config)
self._client = CadcTapClient(subject, resource_id=self._config.tap_id)
self._logger = logging.getLogger(self.__class__.__name__)
if collection == 'NEOSSAT':
service = 'shared'
archive = 'NEOSSAT'
collection = 'NEOSS'
elif collection == 'GEM':
service = 'gemini'
archive = 'GEMINI'
elif collection == 'VLASS':
service = 'cirada'
archive = 'VLASS'
else:
service = collection.lower()
archive = collection
proxy_fqn = '/usr/src/app/cadcproxy.pem'
subject = net.Subject(certificate=proxy_fqn)
ad_client = CadcTapClient(subject, resource_id='ivo://cadc.nrc.ca/ad')
ops_client = CadcTapClient(subject, resource_id=f'ivo://cadc.nrc.ca/ams/{service}')
caom_client = CAOM2RepoClient(subject, resource_id='ivo://cadc.nrc.ca/ams')
print(':::1 - Find the name of a file to test with.')
ops_query = f"SELECT O.observationID, A.uri " \
f"FROM caom2.Observation as O " \
f"JOIN caom2.Plane as P on O.obsID = P.obsID " \
f"JOIN caom2.Artifact as A on P.planeID = A.planeID " \
f"WHERE O.collection = '{archive}' " \
f"AND A.uri like '%.fits%' " \
f"LIMIT 1"
ops_buffer = io.StringIO()
ops_client.query(ops_query, output_file=ops_buffer, data_only=True, response_format='csv')
ops_table = Table.read(ops_buffer.getvalue().split('\n'), format='csv')