示例#1
0
def read_list_from_caom(config):
    query = "SELECT A.uri FROM caom2.Observation AS O " \
            "JOIN caom2.Plane AS P ON O.obsID = P.obsID " \
            "JOIN caom2.Artifact AS A ON P.planeID = A.planeID " \
            "WHERE O.collection='{}'".format(config.archive)
    subject = net.Subject(certificate=config.proxy_fqn)
    tap_client = CadcTapClient(subject, resource_id=config.tap_id)
    buffer = io.BytesIO()
    tap_client.query(query, output_file=buffer)
    temp = parse_single_table(buffer).to_table()
    return [ii.decode().replace('ad:{}/'.format(config.archive), '').strip()
            for ii in temp['uri']]
示例#2
0
 def __init__(self, config):
     super().__init__()
     self._config = config
     self._subject = clc.define_subject(self._config)
     self._sc2_client = CadcTapClient(
         subject=self._subject, resource_id='ivo://cadc.nrc.ca/sc2tap'
     )
     self._prod_client = CadcTapClient(
         subject=self._subject, resource_id='ivo://cadc.nrc.ca/ams/gemini'
     )
     self._connected = mc.TaskType.SCRAPE not in config.task_types
     self._logger = logging.getLogger(self.__class__.__name__)
示例#3
0
def test_query(base_post_mock):
    client = CadcTapClient(net.Subject())
    # default format
    def_name = 'tmptable'
    def_table = os.path.join(TESTDATA_DIR, 'votable.xml')

    fields = {'LANG': 'ADQL',
              'QUERY': 'query',
              'FORMAT': 'VOTable'}
    tablefile = os.path.basename(def_table)
    fields['UPLOAD'] = '{},param:{}'.format(def_name, tablefile)
    fields[tablefile] = (def_table, open(def_table, 'rb'))
    client.query('query', tmptable='tmptable:'+def_table)
    print(base_post_mock.call_args_list[0][0][0])
    assert base_post_mock.call_args_list[0][0][0] == \
        (QUERY_CAPABILITY_ID, None, 'uws:Sync')
示例#4
0
def test_cadc_client_basicaa(base_mock):
    service_realm = 'some.domain.com'
    service_url = 'https://{}'.format(service_realm)
    base_mock.return_value._get_url.return_value = service_url
    cookie_response = Mock()
    cookie_response.status_code = 200
    cookie_response.text = 'cookie val'
    subject = net.Subject()
    subject._netrc = True  # avoid checking for the real .netrc file
    base_mock.return_value.post.return_value = cookie_response
    # mock the get_auth of the subject to return user/passwd from netrc file
    get_auth = Mock(return_value=('user', 'pswd'))
    subject.get_auth = get_auth
    # after calling CadcTapClient the subject is augmented with cookie info
    CadcTapClient(subject)
    get_auth.assert_called_with(service_realm)
    assert len(subject.cookies) == len(cadctap.core.CADC_REALMS)
    for cookie in subject.cookies:
        # note "" around the value required by the cookie
        assert cookie.value == '"cookie val"'
    base_mock.return_value.post.assert_called_with(
        ('ivo://ivoa.net/std/UMS#login-0.1', None),
        data='username=user&password=pswd',
        headers={'Content-type': 'application/x-www-form-urlencoded',
                 'Accept': 'text/plain'})
示例#5
0
def test_query(caps_get_mock, base_post_mock):
    caps_get_mock.return_value = BASE_URL
    response = Mock()
    response.status_code = 200
    response.raw.read.return_value = b'<VOTable format>'
    response.text = 'Header 1\nVal1\nVal2\n'
    # NOTE: post mock returns a context manager with the responose, hence
    # the __enter__
    base_post_mock.return_value.__enter__.return_value = response
    client = CadcTapClient(net.Subject())
    # default format
    def_name = 'tmptable'
    def_table = os.path.join(TESTDATA_DIR, 'votable.xml')

    fields = {'LANG': 'ADQL',
              'QUERY': 'query',
              'FORMAT': 'VOTable'}
    tablefile = os.path.basename(def_table)
    fields['UPLOAD'] = '{},param:{}'.format(def_name, tablefile)
    fields[tablefile] = (def_table, open(def_table, 'rb'))
    with patch('cadctap.core.sys.stdout', new_callable=BytesIO):
        client.query('query', tmptable='tmptable:'+def_table)
    assert base_post_mock.call_args_list[0][0][0] == \
        '{}/{}'.format(BASE_URL, 'sync')

    base_post_mock.reset_mock()
    with patch('sys.stdout', new_callable=StringIO) as stdout_mock:
        client.query('query', data_only=True, response_format='tsv')
    assert stdout_mock.getvalue() == 'Val1\nVal2\n'

    # save in a file
    tf = tempfile.NamedTemporaryFile()
    base_post_mock.reset_mock()
    base_post_mock.return_value.__enter__.return_value = response
    client.query('query', output_file=tf.name, response_format='tsv')
    actual = open(tf.name).read()
    assert actual == 'Header 1\n-----------------------\n' \
                     'Val1\nVal2\n\n(2 rows affected)\n'

    # different format => result from server not altered
    base_post_mock.reset_mock()
    base_post_mock.return_value.__enter__.return_value = response
    with patch('sys.stdout', new_callable=BytesIO) as stdout_mock:
        client.query('query')
    assert stdout_mock.getvalue() == response.raw.read()
def init_global(config):
    global om
    om = gom.GeminiObsMetadata()
    get_gofr(config)
    global gofr
    if gofr.tap_client is None and config.is_connected:
        subject = mc.define_subject(config)
        tap_client = CadcTapClient(subject=subject, resource_id=config.tap_id)
        gofr.tap_client = tap_client
def visit(observation, **kwargs):
    mc.check_param(observation, Observation)

    working_directory = kwargs.get('working_directory', './')
    storage_name = kwargs.get('storage_name')
    if storage_name is None:
        raise mc.CadcException(
            f'Must have a storage_name parameter for provenance_augmentation '
            f'for {observation.observation_id}')
    config = mc.Config()
    config.get_executors()
    if mc.TaskType.SCRAPE in config.task_types:
        logging.warning(f'Provenance augmentation does not work for SCRAPE.')
        return {'provenance': 0}

    subject = clc.define_subject(config)
    tap_client = CadcTapClient(subject, config.tap_id)

    count = 0
    obs_members = TypedSet(ObservationURI, )

    for plane in observation.planes.values():
        plane_inputs = TypedSet(PlaneURI, )
        for artifact in plane.artifacts.values():
            if storage_name.file_uri == artifact.uri:
                count = _do_provenance(
                    working_directory,
                    storage_name.file_name,
                    observation,
                    tap_client,
                    plane_inputs,
                    obs_members,
                    config,
                )

        if plane.provenance is not None:
            plane.provenance.inputs.update(plane_inputs)

    if isinstance(observation, DerivedObservation):
        observation.members.update(obs_members)
        if len(observation.members) > 0:
            observable = kwargs.get('observable')
            caom_repo_client = kwargs.get('caom_repo_client')
            if caom_repo_client is None:
                logging.warning(f'Warning: Must have a caom_repo_client for '
                                f'members metadata for '
                                f'{observation.observation_id}.')
            _do_members_metadata(
                observation,
                caom_repo_client,
                observation.members,
                observable.metrics,
            )

    logging.info(
        f'Done provenance_augmentation for {observation.observation_id}')
    return {'provenance': count}
示例#8
0
def test_error_cases():
    def get_my_access_url(service):
        if service == cadctap.core.PERMISSIONS_CAPABILITY_ID:
            raise Exception(cadctap.core.PERMISSIONS_CAPABILITY_ID)
        else:
            return "http://some.org/some/path"

    with patch('cadcutils.net.ws.WsCapabilities.get_access_url') as amock:
        amock.side_effect = get_my_access_url
        client = CadcTapClient(net.Subject())
        assert not client.permissions_support
示例#9
0
def test_delete_table(base_delete_mock):
    client = CadcTapClient(net.Subject())
    client.delete_table('sometable')
    base_delete_mock.assert_called_with((TABLES_CAPABILITY_ID,
                                        'sometable'))

    # error case
    with pytest.raises(AttributeError):
        client.delete(None)
示例#10
0
def test_create_table(base_put_mock):
    client = CadcTapClient(net.Subject())
    # default format
    def_table = os.path.join(TESTDATA_DIR, 'createTable.vosi')
    def_table_content = open(def_table, 'rb').read()
    client.create_table('sometable', def_table)
    base_put_mock.assert_called_with(
        (TABLES_CAPABILITY_ID, 'sometable'), data=def_table_content,
        headers={'Content-Type': '{}'.format(
            ALLOWED_TB_DEF_TYPES['VOSITable'])})

    # VOTable format
    base_put_mock.reset_mock()
    client.create_table('sometable', def_table, 'VOTable')
    base_put_mock.assert_called_with(
        (TABLES_CAPABILITY_ID, 'sometable'), data=def_table_content,
        headers={'Content-Type': '{}'.format(
            ALLOWED_TB_DEF_TYPES['VOTable'])})

    # error cases
    with pytest.raises(AttributeError):
        client.create_table(None, def_table)
    with pytest.raises(AttributeError):
        client.create_table('sometable', None)
 def _init(self, config):
     if mc.TaskType.SCRAPE in config.task_types:
         self._logger.info(
             f'SCRAPE\'ing data - no clients will be initialized.'
         )
     else:
         subject = define_subject(config)
         self._metadata_client = CAOM2RepoClient(
             subject, config.logging_level, config.resource_id
         )
         self._data_client = declare_client(config)
         if config.tap_id is not None:
             self._query_client = CadcTapClient(
                 subject=subject, resource_id=config.tap_id
             )
     self._metrics = mc.Metrics(config)
示例#12
0
 def get_obs_id(self):
     obs_id = external_metadata.get_obs_id_from_headers(self._file_id)
     if obs_id is None:
         config = mc.Config()
         config.get_executors()
         subject = mc.define_subject(config)
         tap_client = CadcTapClient(subject=subject,
                                    resource_id=config.tap_id)
         obs_id = external_metadata.get_obs_id_from_cadc(
             self._file_id, tap_client, COLLECTION)
         if obs_id is None:
             headers = fits2caom2.get_cadc_headers(
                 f'ad:GEMINI/{self._file_name}', subject)
             obs_id = headers[0].get('DATALAB')
             if obs_id is None:
                 raise mc.CadcException(f'No obs id for {self._file_name}')
     return obs_id
示例#13
0
def test_set_permissions(caps_get_mock, post_mock):
    caps_get_mock.return_value = BASE_URL
    client = CadcTapClient(net.Subject())
    resource = 'mytable'
    client.set_permissions(resource=resource, read_anon=True)
    post_mock.assert_called_with((PERMISSIONS_CAPABILITY_ID, resource),
                                 data='public=true\n',
                                 headers={'Content-Type': 'text/plain'})

    post_mock.reset_mock()
    client.set_permissions(resource=resource, read_anon=False,
                           read_only='ivo://cadc.nrc.ca/groups?ABC')
    post_mock.assert_called_with(
        (PERMISSIONS_CAPABILITY_ID, resource),
        data='public=false\nr-group=ivo://cadc.nrc.ca/groups?ABC\n',
        headers={'Content-Type': 'text/plain'})

    post_mock.reset_mock()
    client.set_permissions(resource=resource,
                           read_write='ivo://cadc.nrc.ca/groups?DEF',
                           read_only='ivo://cadc.nrc.ca/groups?ABC')
    post_mock.assert_called_with(
        (PERMISSIONS_CAPABILITY_ID, resource),
        data='r-group=ivo://cadc.nrc.ca/groups?ABC\n'
             'rw-group=ivo://cadc.nrc.ca/groups?DEF\n',
        headers={'Content-Type': 'text/plain'})

    post_mock.reset_mock()
    client.set_permissions(resource=resource,
                           read_write='',
                           read_only='')
    post_mock.assert_called_with(
        (PERMISSIONS_CAPABILITY_ID, resource),
        data='r-group=\nrw-group=\n',
        headers={'Content-Type': 'text/plain'})

    with pytest.raises(AttributeError, match='No resource'):
        client.set_permissions(None)

    with pytest.raises(
            AttributeError, match='Expected URI for read group: ABC'):
        client.set_permissions(resource, read_only='ABC')

    with pytest.raises(
            AttributeError, match='Expected URI for write group: ABC'):
        client.set_permissions(resource, read_write='ABC')
    # test dummy call
    client.set_permissions(resource=resource)

    # errors in permission modes
    # extra group
    with pytest.raises(argparse.ArgumentError):
        opt = Mock
        opt.groups = 'A B'
        opt.mode = {'who': 'g', 'op': '+', 'what': 'r'}
        _get_permission_modes(opt)

    with pytest.raises(argparse.ArgumentError):
        opt = Mock
        opt.groups = 'A'
        opt.mode = {'who': 'g', 'op': '-', 'what': 'r'}
        _get_permission_modes(opt)

    with pytest.raises(argparse.ArgumentError):
        opt = Mock
        opt.groups = 'A'
        opt.mode = {'who': 'o', 'op': '+', 'what': 'r'}
        _get_permission_modes(opt)
示例#14
0
def test_schema(base_get_mock):
    client = CadcTapClient(net.Subject())
    # default format
    client.schema()
    base_get_mock.assert_called_with(
        (TABLES_CAPABILITY_ID, None))
示例#15
0
def test_create_index(base_get_mock, base_post_mock):
    client = CadcTapClient(net.Subject())
    response1 = Mock()
    response1.status_code = 303
    job_location = 'http://go.here'
    response1.headers = {'Location': job_location}
    base_post_mock.return_value = response1
    response2 = Mock()
    response2.status_code = 200
    response2.text = "EXECUTING"
    base_get_mock.side_effect = [response2]
    response3 = Mock()
    response3.status_code = 200
    response3.text = "COMPLETED"
    base_get_mock.side_effect = [response2, response3]
    client.create_index('schema.sometable', 'col1', unique=True)

    # expected post calls
    post_calls = [call((TABLE_UPDATE_CAPABILITY_ID, None),
                  allow_redirects=False,
                  data={'table': 'schema.sometable',
                        'uniquer': True,
                        'index': 'col1'}),
                  call('{}/phase'.format(job_location),
                  data={'PHASE': 'RUN'})]
    base_post_mock.assert_has_calls(post_calls)

    # expected get calls
    get_calls = [call('{}/phase'.format(job_location), data={'WAIT': 1}),
                 call('{}/phase'.format(job_location), data={'WAIT': 1})]
    base_get_mock.assert_has_calls(get_calls)

    # error cases
    with pytest.raises(AttributeError):
        client.create_index(None, 'col1')
    with pytest.raises(AttributeError):
        client.create_index('sometable', None)
    response4 = Mock()
    response4.status_code = 200
    response4.text = 'ABORTED'
    base_get_mock.side_effect = [response4, response4]
    client = CadcTapClient(net.Subject())
    with pytest.raises(RuntimeError):
        client.create_index('sometable', 'col1')

    response5 = Mock()
    response5.status_code = 500
    base_get_mock.side_effect = [response1, response4, response4]
    client = CadcTapClient(net.Subject())
    with pytest.raises(RuntimeError):
        client.create_index('sometable', 'col1')
示例#16
0
def test_load_table(base_put_mock):
    client = CadcTapClient(net.Subject())
    test_load_tb = os.path.join(TESTDATA_DIR, 'loadTable.txt')

    # default format (tsv)
    with open(test_load_tb, 'rb') as fh:
        with patch('cadctap.core.open') as open_mock:
            open_mock.return_value = fh
            client.load('schema.sometable', [test_load_tb])
    base_put_mock.assert_called_with(
        (TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
        headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['tsv'])})

    # tsv format
    with open(test_load_tb, 'rb') as fh:
        with patch('cadctap.core.open') as open_mock:
            open_mock.return_value = fh
            client.load('schema.sometable', [test_load_tb], fformat='tsv')
    base_put_mock.assert_called_with(
        (TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
        headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['tsv'])})

    # csv format
    with open(test_load_tb, 'rb') as fh:
        with patch('cadctap.core.open') as open_mock:
            open_mock.return_value = fh
            client.load('schema.sometable', [test_load_tb], fformat='csv')
    base_put_mock.assert_called_with(
        (TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
        headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['csv'])})

    # FITS table format
    with open(test_load_tb, 'rb') as fh:
        with patch('cadctap.core.open') as open_mock:
            open_mock.return_value = fh
            client.load('schema.sometable', [test_load_tb],
                        fformat='FITSTable')
    base_put_mock.assert_called_with(
        (TABLE_LOAD_CAPABILITY_ID, 'schema.sometable'), data=fh,
        headers={'Content-Type': str(ALLOWED_CONTENT_TYPES['FITSTable'])})

    # error cases
    with pytest.raises(AttributeError):
        client.load(None, [test_load_table])
    with pytest.raises(AttributeError):
        client.load('sometable', None)
    with pytest.raises(AttributeError):
        client.load('sometable', [])
示例#17
0
def test_schema(caps_get_mock, base_get_mock):
    caps_get_mock.return_value = BASE_URL
    base_get_mock.return_value.text = \
        open(os.path.join(TESTDATA_DIR, 'db_schema.xml'), 'r').read()
    client = CadcTapClient(net.Subject())
    # default schema
    db_schema = client.get_schema()
    assert 3 == len(db_schema)
    for schema in db_schema:
        assert 'DB schema' == schema.description
        assert ['Table', 'Description'] == schema.columns
        if 'ivoa' == schema.name:
            assert 3 == len(schema.rows)
        elif 'caom2' in schema.name:
            assert 14 == len(schema.rows)
        elif 'tap_schema' == schema.name:
            assert 5 == len(schema.rows)
        else:
            assert False, 'Unexpected schema'
    base_get_mock.assert_called_with((TABLES_CAPABILITY_ID, None),
                                     params={'detail': 'min'})
    # table schema
    table_schema_mock = Mock()
    base_get_mock.reset_mock()
    table_schema_mock.text = open(os.path.join(TESTDATA_DIR,
                                               'table_schema.xml'), 'r').read()
    permission_mock = Mock()
    permission_mock.text = 'owner=someone\npublic=true\nr-group=\n' \
                           'rw-group=ivo://cadc.nrc.ca/gms?CADC'
    base_get_mock.side_effect = [table_schema_mock, permission_mock]
    tb_schema = client.get_table_schema('caom2.Observation')
    assert 3 == len(tb_schema)
    assert 'caom2.Observation' == tb_schema[0].name
    assert 'the main CAOM Observation table' == tb_schema[0].description
    assert ['Name', 'Type', 'Index', 'Description'] == tb_schema[0].columns
    assert 45 == len(tb_schema[0].rows)
    assert 'Foreign Keys' == tb_schema[1].name
    assert 'Foreign Keys for table' == tb_schema[1].description
    assert ['Target Table', 'Target Col', 'From Column', 'Description'] == \
        tb_schema[1].columns
    assert 1 == len(tb_schema[1].rows)
    assert 'caom2.ObservationMember' == tb_schema[1].rows[0][0]
    assert 1 == len(tb_schema[2].rows)
    assert ['Owner', 'Others Read', 'Group Read', 'Group Write'] == \
        tb_schema[2].columns
    assert 'Permissions' == tb_schema[2].name
    assert 'Permissions for caom2.Observation' == \
           tb_schema[2].description
    assert 'someone' == tb_schema[2].rows[0][0]
    assert 'true' == tb_schema[2].rows[0][1]
    assert '-' == tb_schema[2].rows[0][2]
    assert 'CADC' == tb_schema[2].rows[0][3]
    calls = [call(('ivo://ivoa.net/std/VOSI#tables-1.1', 'caom2.Observation'),
                  params={'detail': 'min'}),
             call(('ivo://ivoa.net/std/VOSI#table-permissions-1.x',
                   'caom2.Observation'))]
    base_get_mock.assert_has_calls(calls)
    # check displays are also working although the visual part not tested
    client.get_schema = Mock(return_value=db_schema)
    client.schema('foo')
    client.get_table_schema = Mock(return_value=tb_schema)
    client.schema('caom2.Observation')

    # test get_schema now
    client = CadcTapClient(net.Subject())
    client._db_schemas = {'tap': 'Schema goes in here'}
    client.get_permissions = Mock(return_value='Permissions go in here')
    assert not client.get_schema('foo')
    assert ['Schema goes in here', 'Permissions go in here'] == \
        client.get_schema('tap')
示例#18
0
 def __init__(self, config, preview_suffix='jpg'):
     super(QueryTimeBoxDataSourceTS, self).__init__(config)
     self._preview_suffix = preview_suffix
     subject = mc.define_subject(config)
     self._client = CadcTapClient(subject, resource_id=self._config.tap_id)
     self._logger = logging.getLogger(self.__class__.__name__)
示例#19
0
if collection == 'NEOSSAT':
    service = 'shared'
    archive = 'NEOSSAT'
    collection = 'NEOSS'
elif collection == 'GEM':
    service = 'gemini'
    archive = 'GEMINI'
elif collection == 'VLASS':
    service = 'cirada'
    archive = 'VLASS'
else:
    service = collection.lower()
    archive = collection
proxy_fqn = '/usr/src/app/cadcproxy.pem'
subject = net.Subject(certificate=proxy_fqn)
ad_client = CadcTapClient(subject, resource_id='ivo://cadc.nrc.ca/ad')
ops_client = CadcTapClient(subject, resource_id=f'ivo://cadc.nrc.ca/ams/{service}')
caom_client = CAOM2RepoClient(subject, resource_id='ivo://cadc.nrc.ca/ams')

print(':::1 - Find the name of a file to test with.')
ops_query = f"SELECT O.observationID, A.uri " \
f"FROM caom2.Observation as O " \
f"JOIN caom2.Plane as P on O.obsID = P.obsID " \
f"JOIN caom2.Artifact as A on P.planeID = A.planeID " \
f"WHERE O.collection = '{archive}' " \
f"AND A.uri like '%.fits%' " \
f"LIMIT 1"

ops_buffer = io.StringIO()
ops_client.query(ops_query, output_file=ops_buffer, data_only=True, response_format='csv')
ops_table = Table.read(ops_buffer.getvalue().split('\n'), format='csv')