Esempio n. 1
0
    def test_plugin_class(self):
        # plugin class does not change the observation
        collection = 'cfht'
        observation_id = '7000000o'
        level = logging.DEBUG
        visitor = CAOM2RepoClient(auth.Subject(), level)
        obs = SimpleObservation(collection, observation_id)
        expect_obs = copy.deepcopy(obs)
        visitor._load_plugin_class(os.path.join(THIS_DIR, 'passplugin.py'))
        visitor.plugin.update(obs)
        self.assertEqual(expect_obs, obs)

        # plugin class adds a plane to the observation
        visitor = CAOM2RepoClient(auth.Subject(), level)
        obs = SimpleObservation('cfht', '7000000o')
        expect_obs = copy.deepcopy(obs)
        visitor._load_plugin_class(os.path.join(THIS_DIR, 'addplaneplugin.py'))
        visitor.plugin.update(obs)
        self.assertNotEqual(expect_obs, obs)
        self.assertEqual(len(expect_obs.planes) + 1, len(obs.planes))

        # non-existent the plugin file
        with self.assertRaises(Exception):
            visitor._load_plugin_class(os.path.join(THIS_DIR, 'blah.py'))

        # non-existent ObservationUpdater class in the plugin file
        with self.assertRaises(Exception):
            visitor._load_plugin_class(
                os.path.join(THIS_DIR, 'test_visitor.py'))

        # non-existent update method in ObservationUpdater class
        with self.assertRaises(Exception):
            visitor._load_plugin_class(
                os.path.join(THIS_DIR, 'noupdateplugin.py'))
Esempio n. 2
0
    def test_create_and_visit(self):

        #logger.setLevel(logging.DEBUG)
        logger.info('-- START: test_create_and_visit --')

        start = datetime.now()
        name = obs_name = 'caom2pyinttest{}'.format(start.microsecond)

        try:
            env_a = os.environ['A']
            cert_file = env_a + '/test-certificates/x509_CADCAuthtest1.pem'
            subject = auth.Subject(certificate=cert_file)
            client = CAOM2RepoClient(subject)
            
            # create one observation for today
            algorithm = observation.SimpleObservation._DEFAULT_ALGORITHM_NAME

            logger.debug("test obs name {}".format(name))
            obs = observation.SimpleObservation("TEST", obs_name)
            obs.algorithm = algorithm
            client.put_observation(obs)

            plugin = os.path.join(THIS_DIR, 'visitor-plugin.py')
            (visited, updated, skipped, failed) = client.visit(plugin, 'TEST', start=start, halt_on_error=True)
            logger.debug("observations visited: {}".format(len(visited)))
            
            self.assertGreater(len(visited), 0, msg="No Observations Visited")
            
        finally:
            try:
                client.delete_observation("TEST", name)
            except:
                logger.warning('Failed to delete test observation, continuing')
            logger.info('-- END :test_create_and_visit --')
Esempio n. 3
0
    def test_visit_retry_on_412(self):
        # observation changed on server while visited
        core.BATCH_SIZE = 3  # size of the batch is 3
        obs = [['a'], []]
        level = logging.DEBUG
        visitor = CAOM2RepoClient(auth.Subject(), level)
        observation = SimpleObservation('cfht', 'a')
        observation.acc_meta_checksum = ChecksumURI('md5:abc')
        visitor.get_observation = MagicMock(
            side_effect=[observation, observation])

        exception_412 = exceptions.UnexpectedException()
        exception_412.orig_exception = Mock()
        exception_412.orig_exception.response = Mock(status_code=412)
        visitor.post_observation = MagicMock(side_effect=[exception_412, None])
        visitor._get_observations = MagicMock(side_effect=obs)

        (visited, updated, skipped,
         failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
                                 'cfht')
        self.assertEqual(1, len(visited))
        self.assertEqual(1, len(updated))
        self.assertEqual(0, len(skipped))
        self.assertEqual(0, len(failed))
        # get and post called twice to recover from error HTTP status 412 -
        # precondition
        self.assertEqual(2, visitor.get_observation.call_count)
        self.assertEqual(2, visitor.post_observation.call_count)
        visitor.post_observation.assert_called_with(
            observation, observation.acc_meta_checksum.uri)
Esempio n. 4
0
    def test_multiprocess_with_unexpected_type_error(self, get_mock):
        core.BATCH_SIZE = 3  # size of the batch is 3
        obs_ids = [['a', 'b', 'c'], ['d'], []]
        get_mock.side_effect = \
            self.mock_get_observation_with_unexpected_type_error
        level = logging.DEBUG
        visitor = CAOM2RepoClient(auth.Subject(), level)
        visitor.get_observation = PickableMagicMock(
            return_value=PickableMagicMock(spec=SimpleObservation))
        visitor.post_observation = PickableMagicMock()
        visitor._get_observations = PickableMagicMock(side_effect=obs_ids)

        try:
            (visited, updated, skipped,
             failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
                                     'cfht',
                                     start=None,
                                     end=None,
                                     obs_file=None,
                                     nthreads=3,
                                     halt_on_error=True)
        except TypeError as e:
            self.assertTrue("unexpected TypeError" in str(e))
        finally:
            logging.info("DONE")
Esempio n. 5
0
    def test_multiprocess_with_more_different_statuses(self, get_mock):
        core.BATCH_SIZE = 3  # size of the batch is 3
        # make it return different status. errorplugin returns according to the
        # id of the observation: True for 'UPDATE', False for 'SKIP' and
        # raises exception for 'ERROR'
        obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']]
        get_mock.side_effect = self.mock_get_observation
        level = logging.DEBUG
        visitor = CAOM2RepoClient(auth.Subject(), level)
        visitor.get_observation = PickableMagicMock(
            return_value=PickableMagicMock(spec=SimpleObservation))
        visitor.post_observation = PickableMagicMock()
        visitor._get_observations = PickableMagicMock(side_effect=obs_ids)

        (visited, updated, skipped,
         failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
                                 'cfht',
                                 start=None,
                                 end=None,
                                 obs_file=None,
                                 nthreads=3)

        try:
            self.assertEqual(5, len(visited))
            self.assertEqual(2, len(updated))
            self.assertEqual(2, len(skipped))
            self.assertEqual(1, len(failed))
        finally:
            # lp.join()
            logging.info("DONE")
Esempio n. 6
0
    def test_post_observation(self, mock_conn, caps_mock):
        caps_mock.get_service_host.return_value = 'some.host.com'
        caps_mock.return_value.get_access_url.return_value =\
            'http://serviceurl/caom2repo/auth'
        collection = 'cfht'
        observation_id = '7000000o'
        service = 'caom2repo'
        service_url = 'www.cadc.nrc.ca'

        obs = SimpleObservation(collection, observation_id)
        level = logging.DEBUG
        visitor = CAOM2RepoClient(auth.Subject(netrc='somenetrc'),
                                  level,
                                  host=service_url)
        response = MagicMock()
        response.status = 200
        mock_conn.return_value = response
        iobuffer = BytesIO()
        ObservationWriter().write(obs, iobuffer)
        obsxml = iobuffer.getvalue()
        response.content = obsxml

        visitor.post_observation(obs)
        self.assertEqual('POST', mock_conn.call_args[0][0].method)
        self.assertEqual(
            '/{}/auth/{}/{}'.format(service, collection, observation_id),
            mock_conn.call_args[0][0].path_url)
        self.assertEqual('application/xml',
                         mock_conn.call_args[0][0].headers['Content-Type'])
        self.assertEqual(obsxml, mock_conn.call_args[0][0].body)

        # signal problems
        http_error = requests.HTTPError()
        response.status_code = 500
        http_error.response = response
        response.raise_for_status.side_effect = [http_error]
        with self.assertRaises(exceptions.InternalServerException):
            visitor.update(obs)

        # temporary transient errors
        http_error = requests.HTTPError()
        response.status_code = 503
        http_error.response = response
        response.raise_for_status.side_effect = [http_error, None]
        visitor.post_observation(obs)

        # permanent transient errors
        http_error = requests.HTTPError()
        response.status_code = 503
        http_error.response = response

        def raise_error():
            raise http_error

        response.raise_for_status.side_effect = raise_error
        with self.assertRaises(exceptions.HttpException):
            visitor.post_observation(obs)
Esempio n. 7
0
    def test_get_observations(self, mock_get, caps_mock):
        # This is almost similar to the previous test except that it gets
        # observations matching a collection and start/end criteria
        # Also, patch the CAOM2RepoClient now.
        caps_mock.get_service_host.return_value = 'some.host.com'
        caps_mock.return_value.get_access_url.return_value =\
            'http://serviceurl/caom2repo/pub'
        response = MagicMock()
        response.status_code = 200
        last_datetime = '2000-10-10T12:30:00.333'
        response.text = \
            ('CFHT\t700000o\t2000-10-10T12:20:11.123\t'
             '3e00ca6129dc8358315015204ab9fe15\nCFHT\t700001o\t' +
             last_datetime + '\t3e00ca6129dc8358315015204ab9fe15')
        mock_get.return_value = response

        level = logging.DEBUG
        visitor = CAOM2RepoClient(auth.Subject(), level)
        end_date = util.utils.str2ivoa(last_datetime)

        expect_observations = ['700000o', '700001o']
        self.assertEqual(expect_observations,
                         visitor._get_observations('cfht'))
        self.assertEqual(end_date, visitor._start)
        mock_get.assert_called_once_with(
            ('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2',
             'cfht'),
            params={'MAXREC': core.BATCH_SIZE})

        mock_get.reset_mock()
        visitor._get_observations('cfht',
                                  end=datetime.strptime(
                                      '2000-11-11', '%Y-%m-%d'))
        mock_get.assert_called_once_with(
            ('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2',
             'cfht'),
            params={
                'END': '2000-11-11T00:00:00.000',
                'MAXREC': core.BATCH_SIZE
            })

        mock_get.reset_mock()
        visitor._get_observations(
            'cfht',
            start=datetime.strptime('2000-11-11', '%Y-%m-%d'),
            end=datetime.strptime('2000-11-12', '%Y-%m-%d'))
        mock_get.assert_called_once_with(
            ('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2',
             'cfht'),
            params={
                'START': '2000-11-11T00:00:00.000',
                'END': '2000-11-12T00:00:00.000',
                'MAXREC': core.BATCH_SIZE
            })
Esempio n. 8
0
    def test_get_observation(self, mock_get, caps_mock):
        caps_mock.get_service_host.return_value = 'some.host.com'
        caps_mock.return_value.get_access_url.return_value =\
            'http://serviceurl/caom2repo/pub'
        collection = 'cfht'
        observation_id = '7000000o'
        service_url = 'www.cadc.nrc.ca/caom2repo'
        obs = SimpleObservation(collection, observation_id)
        writer = ObservationWriter()
        ibuffer = BytesIO()
        writer.write(obs, ibuffer)
        response = MagicMock()
        response.status_code = 200
        response.content = ibuffer.getvalue()
        mock_get.return_value = response
        ibuffer.seek(0)  # reposition the buffer for reading
        level = logging.DEBUG
        visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url)
        self.assertEqual(obs,
                         visitor.get_observation(collection, observation_id))

        # signal problems
        http_error = requests.HTTPError()
        response.status_code = 500
        http_error.response = response
        response.raise_for_status.side_effect = [http_error]
        with self.assertRaises(exceptions.InternalServerException):
            visitor.get_observation(collection, observation_id)

        # temporary transient errors
        http_error = requests.HTTPError()
        response.status_code = 503
        http_error.response = response
        response.raise_for_status.side_effect = [http_error, None]
        visitor.read(collection, observation_id)

        # permanent transient errors
        http_error = requests.HTTPError()
        response.status_code = 503
        http_error.response = response

        def raise_error():
            raise http_error

        response.raise_for_status.side_effect = raise_error
        with self.assertRaises(exceptions.HttpException):
            visitor.get_observation(collection, observation_id)
Esempio n. 9
0
    def test_delete_observation(self, mock_conn, caps_mock):
        caps_mock.get_service_host.return_value = 'some.host.com'
        caps_mock.return_value.get_access_url.return_value =\
            'http://serviceurl/caom2repo/pub'
        collection = 'cfht'
        observation_id = '7000000o'
        service_url = 'www.cadc.nrc.ca'
        level = logging.DEBUG

        visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url)
        response = MagicMock()
        response.status = 200
        mock_conn.return_value = response

        visitor.delete_observation(collection, observation_id)
        self.assertEqual('DELETE', mock_conn.call_args[0][0].method)

        # signal problems
        http_error = requests.HTTPError()
        response.status_code = 500
        http_error.response = response
        response.raise_for_status.side_effect = [http_error]
        with self.assertRaises(exceptions.InternalServerException):
            visitor.delete(collection, observation_id)

        # temporary transient errors
        http_error = requests.HTTPError()
        response.status_code = 503
        http_error.response = response
        response.raise_for_status.side_effect = [http_error, None]
        visitor.delete_observation(collection, observation_id)

        # permanent transient errors
        http_error = requests.HTTPError()
        response.status_code = 503
        http_error.response = response

        def raise_error():
            raise http_error

        response.raise_for_status.side_effect = raise_error
        with self.assertRaises(exceptions.HttpException):
            visitor.delete_observation(collection, observation_id)
Esempio n. 10
0
    def test_get_obs_from_file(self):
        level = logging.DEBUG
        visitor = CAOM2RepoClient(auth.Subject(), level)

        # no start or end
        with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file:
            obs_id_list = visitor._get_obs_from_file(obs_file, None, None,
                                                     False)
            self.assertEqual('obs_id_1', obs_id_list[0])
            self.assertEqual('obs_id_2', obs_id_list[1])
            self.assertEqual('obs_id_3', obs_id_list[2])

        # last_modified_date is earlier than start
        with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file:
            obs_id_list = visitor._get_obs_from_file(
                obs_file, util.str2ivoa('2000-10-11T12:30:00.333'), None,
                False)
            self.assertEqual('obs_id_1', obs_id_list[0])

        # last_modified_date is between start and end
        with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file:
            obs_id_list = visitor._get_obs_from_file(
                obs_file, util.str2ivoa('2000-10-9T12:30:00.333'),
                util.str2ivoa('2016-10-11T12:30:00.333'), False)
            self.assertEqual('obs_id_1', obs_id_list[0])
            self.assertEqual('obs_id_2', obs_id_list[1])

        # last_modified_date is after end
        with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file:
            obs_id_list = visitor._get_obs_from_file(
                obs_file, util.str2ivoa('2000-10-9T12:30:00.333'),
                util.str2ivoa('2017-10-11T12:30:00.333'), False)
            self.assertEqual('obs_id_1', obs_id_list[0])
            self.assertEqual('obs_id_2', obs_id_list[1])
            self.assertEqual('obs_id_3', obs_id_list[2])

        # error in file
        with open(os.path.join(THIS_DIR, 'data/obs_id_error.txt')) as obs_file:
            with self.assertRaises(Exception):
                obs_id_list = visitor._get_obs_from_file(
                    obs_file, util.str2ivoa('2000-10-9T12:30:00.333'),
                    util.str2ivoa('2016-10-11T12:30:00.333'), True)
Esempio n. 11
0
    def test_multiprocess_with_more_obs_id(self, client_mock):
        core.BATCH_SIZE = 3  # size of the batch is 3
        obs_ids = [['a', 'b', 'c'], ['d', 'e', 'f'], []]
        client_mock.return_value.get_observation.side_effect = \
            self.mock_get_observation
        level = logging.DEBUG
        visitor = CAOM2RepoClient(auth.Subject(), level)
        visitor.get_observation = PickableMagicMock(
            return_value=PickableMagicMock(spec=SimpleObservation))
        visitor.post_observation = PickableMagicMock()
        visitor._get_observations = PickableMagicMock(side_effect=obs_ids)

        (visited, updated, skipped,
         failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
                                 'cfht',
                                 start=None,
                                 end=None,
                                 obs_file=None,
                                 nthreads=3)

        try:
            self.assertEqual(6, len(visited))
            self.assertEqual(6, len(updated))
            self.assertEqual(0, len(skipped))
            self.assertEqual(0, len(failed))
            self.assertTrue('a' in visited)
            self.assertTrue('b' in visited)
            self.assertTrue('c' in visited)
            self.assertTrue('d' in visited)
            self.assertTrue('e' in visited)
            self.assertTrue('f' in visited)
            self.assertFalse('g' in visited)
            self.assertTrue('a' in updated)
            self.assertTrue('b' in updated)
            self.assertTrue('c' in updated)
            self.assertTrue('d' in updated)
            self.assertTrue('e' in updated)
            self.assertTrue('f' in updated)
            self.assertFalse('g' in updated)
        finally:
            # lp.join()
            logging.info("DONE")
Esempio n. 12
0
    def test_shortcuts(self):
        level = logging.DEBUG
        target = CAOM2RepoClient(auth.Subject(), level)
        obs = SimpleObservation('CFHT', 'abc')

        target.put_observation = Mock()
        target.create(obs)
        target.put_observation.assert_called_with(obs)

        target.get_observation = Mock()
        target.read('CFHT', 'abc')
        target.get_observation.assert_called_with('CFHT', 'abc')

        target.post_observation = Mock()
        target.update(obs)
        target.post_observation.assert_called_with(obs)

        target.delete_observation = Mock()
        target.delete('CFHT', 'abc')
        target.delete_observation.assert_called_with('CFHT', 'abc')
Esempio n. 13
0
def test_info_file(basews_mock):
    client = CadcDataClient(auth.Subject())
    # test an info
    file_name = 'myfile'
    file_name = 'myfile.txt'
    archive = 'TEST'
    size = '123'
    md5sum = '0x123'
    type = 'txt'
    encoding = 'gzip'
    lastmod = '11/11/11T11:11:11.000'
    usize = '1234'
    umd5sum = '0x1234'

    h = {}
    h['Content-Disposition'] = 'inline; filename={}'.format(file_name)
    h['Content-Length'] = size
    h['Content-MD5'] = md5sum
    h['Content-Type'] = type
    h['Content-Encoding'] = encoding
    h['Last-Modified'] = lastmod
    h['X-Uncompressed-Length'] = usize
    h['X-Uncompressed-MD5'] = umd5sum
    response = Mock()
    response.headers = h
    basews_mock.return_value.head.return_value = response
    info = client.get_file_info('TEST', 'myfile')
    assert archive == info['archive']
    assert file_name == info['name']
    assert size == info['size']
    assert md5sum == info['md5sum']
    assert type == info['type']
    assert encoding == info['encoding']
    assert lastmod == info['lastmod']
    assert usize == info['usize']
    assert umd5sum == info['umd5sum']
Esempio n. 14
0
def test_put_file(basews_mock):
    client = CadcDataClient(auth.Subject())
    # test a put
    file_name = '/tmp/putfile.txt'
    file_content = 'ABCDEFGH12345'
    hash_md5 = hashlib.md5()
    hash_md5.update(file_content.encode())
    hash_md5 = hash_md5.hexdigest()
    # write the file
    with open(file_name, 'w') as f:
        f.write(file_content)
    put_mock = Mock()
    basews_mock.return_value.put = put_mock
    with pytest.raises(exceptions.UnauthorizedException):
        client.put_file('TEST', 'putfile', file_name)
    client._data_client.subject.anon = False  # authenticate the user
    transf_end_point = 'http://test.ca/endpoint'

    def mock_get_trans_protocols(archive, file_name, is_get, headers):
        protocol = Mock()
        protocol.endpoint = '{}/{}'.format(transf_end_point, file_name)
        return [protocol]

    client._get_transfer_protocols = mock_get_trans_protocols
    client.put_file('TEST', file_name)
    # Note Content* headers automatically created by cadc-data except when
    # MAGIC_WANT -- libmagic not present
    put_mock.assert_called_with('{}/{}'.format(transf_end_point,
                                               os.path.basename(file_name)),
                                data=ANY,
                                headers={
                                    'Content-Type': 'text/plain',
                                    'Content-Encoding': 'us-ascii',
                                    'Content-MD5': '{}'.format(hash_md5)
                                })

    # mimic libmagic missing
    cadcdata.core.MAGIC_WARN = 'Some warning'
    put_mock.reset_mock()
    client.put_file('TEST', file_name)
    put_mock.assert_called_with(
        '{}/{}'.format(transf_end_point, os.path.basename(file_name)),
        data=ANY,
        headers={'Content-MD5': '835e7e6cd54e18ae21d50af963b0c32b'})
    cadcdata.core.MAGIC_WARN = None

    # specify an archive stream and override the name of the file
    input_name = 'abc'
    client.put_file('TEST',
                    file_name,
                    archive_stream='default',
                    input_name=input_name)
    put_mock.assert_called_with('{}/{}'.format(transf_end_point, input_name),
                                data=ANY,
                                headers={
                                    'Content-Encoding': 'us-ascii',
                                    'X-CADC-Stream': 'default',
                                    'Content-Type': 'text/plain',
                                    'Content-MD5': '{}'.format(hash_md5)
                                })
    # specify the mime types
    client.put_file('TEST',
                    file_name,
                    archive_stream='default',
                    mime_type='ASCII',
                    mime_encoding='GZIP')
    put_mock.assert_called_with('{}/{}'.format(transf_end_point,
                                               os.path.basename(file_name)),
                                data=ANY,
                                headers={
                                    'Content-Encoding': 'GZIP',
                                    'X-CADC-Stream': 'default',
                                    'Content-Type': 'ASCII',
                                    'Content-MD5': '{}'.format(hash_md5)
                                })
    os.remove(file_name)
Esempio n. 15
0
def test_get_file(trans_reader_mock, basews_mock):
    # test a simple get - no decompress
    file_name = '/tmp/afile.txt'
    file_chunks = ['aaaa'.encode(), 'bbbb'.encode(), ''.encode()]
    response = Mock()
    hash_md5 = hashlib.md5()
    for i in file_chunks:
        hash_md5.update(i)
    response.headers.get.return_value = \
        'filename={}'.format('orig_file_name')
    response.raw.read.side_effect = file_chunks  # returns multiple blocks
    basews_mock.return_value.get.return_value = response
    client = CadcDataClient(auth.Subject())
    with pytest.raises(exceptions.HttpException):
        # no URLs returned in the transfer negotiations
        client.get_file('TEST', 'afile', destination=file_name)
    t = transfer.Transfer('ad:TEST/afile', 'pullFromVoSpace')
    p = transfer.Protocol
    p.endpoint = Mock()
    t.protocols = [p]
    trans_reader_mock.return_value.read.return_value = t
    client.get_file('TEST', 'afile', destination=file_name, md5_check=False)
    expected_content = \
        (''.join([c.decode() for c in file_chunks])).encode()
    with open(file_name, 'rb') as f:
        assert expected_content == f.read()
    os.remove(file_name)
    # do it again with the file now open
    response = Mock()
    response.headers = {
        'filename': 'orig_file_name',
        'content-MD5': hash_md5.hexdigest()
    }
    response.raw.read.side_effect = file_chunks
    basews_mock.return_value.get.return_value = response
    with open(file_name, 'wb') as f:
        client.get_file('TEST', 'afile', destination=f)
    with open(file_name, 'rb') as f:
        assert expected_content == f.read()
    os.remove(file_name)

    # test a get with decompress and md5 check enabled
    file_name = 'bfile.txt'
    file_content = 'aaaabbbb'
    hash_md5 = hashlib.md5()
    hash_md5.update(file_content.encode())
    file_chunks = [file_content.encode(), ''.encode()]
    decoded_file_content = 'MNOPRST6789'
    decoded_file_chunks = [decoded_file_content.encode(), ''.encode()]
    response = Mock()
    response.headers = \
        {'content-MD5': '{}'.format(hash_md5.hexdigest()),
         'filename': file_name}
    response.raw.read.side_effect = file_chunks
    response.raw._decode.side_effect = decoded_file_chunks
    basews_mock.return_value.get.return_value = response
    client = CadcDataClient(auth.Subject())
    client.get_file('TEST',
                    file_name=file_name,
                    decompress=True,
                    md5_check=True)
    with open(file_name, 'r') as f:
        # note the check against the decoded content
        assert decoded_file_content == f.read()
    os.remove(file_name)

    # repeat test with a bad md5
    file_name = 'bfile.txt'
    file_content = 'ABCDEFGH12345'
    file_chunks = [file_content.encode(), ''.encode()]
    decoded_file_content = 'MNOPRST6789'
    decoded_file_chunks = [decoded_file_content.encode(), ''.encode()]
    response = Mock()
    response.headers = {'content-MD5': 'abc', 'filename': file_name}
    response.raw.read.side_effect = file_chunks
    response.raw._decode.side_effect = decoded_file_chunks
    basews_mock.return_value.get.return_value = response
    client = CadcDataClient(auth.Subject())
    with pytest.raises(exceptions.HttpException):
        client.get_file('TEST',
                        file_name=file_name,
                        decompress=True,
                        md5_check=True)

    # test process_bytes and send the content to /dev/null after.
    # Use no decompress
    def concatenate_chunks(chunk):
        global mycontent
        mycontent = '{}{}'.format(mycontent, chunk.decode())

    file_name = 'bfile.txt'
    file_content = 'ABCDEFGH12345'
    file_chunks = [
        file_content[i:i + 5].encode() for i in xrange(0, len(file_content), 5)
    ]
    file_chunks.append('')  # last chunk is empty
    response = Mock()
    response.headers = {'filename': '{}.gz'.format(file_name)}
    response.raw.read.side_effect = file_chunks
    basews_mock.return_value.get.return_value = response
    client = CadcDataClient(auth.Subject())
    client.logger.setLevel(logging.INFO)
    # md5_check does not take place because no content-MD5 received
    # from server
    client.get_file('TEST',
                    'afile',
                    destination='/dev/null',
                    process_bytes=concatenate_chunks)
    assert file_content == mycontent

    # failed md5 checksum
    response = Mock()
    response.headers = {
        'filename': '{}.gz'.format(file_name),
        'content-MD5': '33'
    }
    response.raw.read.side_effect = file_chunks
    basews_mock.return_value.get.return_value = response
    client = CadcDataClient(auth.Subject())
    client.logger.setLevel(logging.INFO)
    # md5_check does not take place because no content-MD5 received
    # from server
    with pytest.raises(exceptions.HttpException):
        client.get_file('TEST',
                        'afile',
                        destination='/dev/null',
                        process_bytes=concatenate_chunks)

    # test get fhead
    response = Mock()
    response.headers.get.return_value = 'filename={}.gz'.format(file_name)
    response.raw.read.side_effect = file_chunks
    response.history = []
    response.status_code = 200
    response.url = 'someurl'
    post_mock = Mock(return_value=response)
    basews_mock.return_value.post = post_mock
    file_name = 'getfile'
    archive = 'TEST'
    p.endpoint = 'http://someurl/transfer/{}/{}'.format(archive, file_name)
    client.get_file('TEST',
                    'getfile',
                    decompress=True,
                    wcs=True,
                    md5_check=False)
    trans_doc = \
        ('<vos:transfer xmlns:'
         'vos="http://www.ivoa.net/xml/VOSpace/v2.0">\n  '
         '<vos:target>ad:TEST/getfile</vos:target>\n  '
         '<vos:direction>pullFromVoSpace</vos:direction>\n  '
         '<vos:protocol uri="ivo://ivoa.net/vospace/core#httpget"/>\n'
         '  <vos:protocol uri="ivo://ivoa.net/vospace/core#httpsget"/>\n'
         '</vos:transfer>\n').encode()
    post_mock.assert_called_with(resource=(TRANSFER_RESOURCE_ID, None),
                                 params={'wcs': True},
                                 data=trans_doc,
                                 headers={'Content-Type': 'text/xml'})
    response.raw.read.side_effect = file_chunks
    post_mock.reset_mock()
    client.get_file('TEST',
                    'getfile',
                    decompress=True,
                    fhead=True,
                    md5_check=False)
    post_mock.assert_called_with(resource=(TRANSFER_RESOURCE_ID, None),
                                 params={'fhead': True},
                                 data=trans_doc,
                                 headers={'Content-Type': 'text/xml'})
    response.raw.read.side_effect = file_chunks
    post_mock.reset_mock()
    client.get_file('TEST',
                    'getfile',
                    decompress=True,
                    cutout='[1:1]',
                    md5_check=False)
    post_mock.assert_called_with(resource=(TRANSFER_RESOURCE_ID, None),
                                 params={'cutout': '[1:1]'},
                                 data=trans_doc,
                                 headers={'Content-Type': 'text/xml'})
    response.raw.read.side_effect = file_chunks
    post_mock.reset_mock()
    client.get_file('TEST',
                    'getfile',
                    decompress=True,
                    cutout='[[1:1], 2]',
                    md5_check=False)
    post_mock.assert_called_with(resource=(TRANSFER_RESOURCE_ID, None),
                                 params={'cutout': '[[1:1], 2]'},
                                 data=trans_doc,
                                 headers={'Content-Type': 'text/xml'})
Esempio n. 16
0
    def testSubject(self, os_mock):
        # anon subject
        subject = auth.Subject()
        self.assertTrue(subject.anon)
        self.assertEqual(None, subject.certificate)
        self.assertEqual({}, subject._hosts_auth)
        self.assertEqual(None, subject.get_auth('realm1'))

        # cert subject
        cert = 'somecert'
        subject = auth.Subject(certificate=cert)
        self.assertFalse(subject.anon)
        self.assertEqual(cert, subject.certificate)
        self.assertEqual({}, subject._hosts_auth)
        self.assertEqual(None, subject.get_auth('realm1'))

        # empty netrc subject
        m = mock_open()
        with patch('six.moves.builtins.open', m, create=True):
            subject = auth.Subject(netrc='somefile')
        self.assertFalse(subject.anon)
        self.assertEqual(None, subject.certificate)
        self.assertEqual({}, subject._hosts_auth)
        self.assertEqual(None, subject.get_auth('realm1'))

        # netrc with content
        netrc_content = {'realm1': ('user1', None, 'pass1'),
                         'realm2': ('user1', None, 'pass2')}
        expected_host_auth = {'realm1': ('user1', 'pass1'),
                              'realm2': ('user1', 'pass2')}
        os_mock.path.join.return_value = '/home/myhome/.netrc'
        with patch('cadcutils.net.auth.netrclib') as netrclib_mock:
            netrclib_mock.netrc.return_value.hosts = netrc_content
            subject = auth.Subject(netrc=True)
        self.assertFalse(subject.anon)
        self.assertEqual(None, subject.certificate)
        self.assertEqual('/home/myhome/.netrc', subject.netrc)
        self.assertEqual(expected_host_auth, subject._hosts_auth)
        self.assertEqual(('user1', 'pass1'), subject.get_auth('realm1'))
        self.assertEqual(('user1', 'pass2'), subject.get_auth('realm2'))
        self.assertEqual(None, subject.get_auth('realm3'))

        # subject with username
        username = '******'
        passwd = 'passwd1'
        subject = auth.Subject(username=username)
        self.assertFalse(subject.anon)
        self.assertEqual(None, subject.certificate)
        self.assertEqual({}, subject._hosts_auth)
        with patch('cadcutils.net.auth.getpass') as getpass_mock:
            getpass_mock.getpass.return_value = passwd
            self.assertEqual((username, passwd), subject.get_auth('realm1'))

        parser = get_base_parser(subparsers=False)
        args = parser.parse_args(['--resource-id', 'blah'])
        subject = auth.Subject.from_cmd_line_args(args)
        self.assertTrue(subject.anon)

        sys.argv = ['cadc-client', '--resource-id', 'blah', '--cert',
                    'mycert.pem']
        args = parser.parse_args()
        subject = auth.Subject.from_cmd_line_args(args)
        self.assertFalse(subject.anon)
        self.assertEqual('mycert.pem', subject.certificate)
Esempio n. 17
0
    def test_ops(self, post_mock, get_mock, delete_mock, head_mock, put_mock,
                 netrclib_mock, caps_mock):
        anon_subject = auth.Subject()
        with self.assertRaises(ValueError):
            ws.BaseWsClient(None, anon_subject, "TestApp")
        resource = 'aresource'
        service = 'myservice'
        resource_id = 'ivo://www.canfar.phys.uvic.ca/{}'.format(service)
        # test anonymous access
        cm = Mock()
        cm.get_access_url.return_value = "http://host/availability"
        caps_mock.return_value = cm
        client = ws.BaseWsClient(resource_id, anon_subject, 'TestApp')
        resource_uri = urlparse(resource_id)
        base_url = 'http://{}{}/pub'.format(resource_uri.netloc,
                                            resource_uri.path)
        resource_url = 'http://{}{}/{}'.format(resource_uri.netloc, base_url,
                                               resource)
        self.assertEqual(anon_subject, client.subject)
        self.assertTrue(client.retry)
        self.assertEqual('TestApp', client.agent)
        self.assertTrue(client.retry)
        self.assertEqual(None, client._session)  # lazy initialization
        client.get(resource_url)
        get_mock.assert_called_with(resource_url, params=None)
        params = {'arg1': 'abc', 'arg2': 123, 'arg3': True}
        client.post(resource_url, **params)
        post_mock.assert_called_with(resource_url, **params)
        client.delete(resource_url)
        delete_mock.assert_called_with(resource_url)
        client.head(resource_url)
        head_mock.assert_called_with(resource_url)
        client.put(resource_url, **params)
        put_mock.assert_called_with(resource_url, **params)
        self.assertTrue(isinstance(client._session, ws.RetrySession))

        # test basic authentication access
        post_mock.reset_mock()
        get_mock.reset_mock()
        put_mock.reset_mock()
        delete_mock.reset_mock()
        head_mock.reset_mock()
        host = 'www.different.org'
        subject = auth.Subject(netrc='somecert')
        client = ws.BaseWsClient(resource_id,
                                 subject,
                                 'TestApp',
                                 retry=False,
                                 host=host)
        base_url = 'http://{}{}/auth'.format(host, resource_uri.path)
        resource_url = '{}/{}'.format(base_url, resource)
        self.assertEqual('TestApp', client.agent)
        self.assertFalse(client.retry)
        client.get(resource_url)
        get_mock.assert_called_with(resource_url, params=None)
        params = {'arg1': 'abc', 'arg2': 123, 'arg3': True}
        client.post(resource_url, **params)
        post_mock.assert_called_with(resource_url, **params)
        client.delete(resource_url)
        delete_mock.assert_called_with(resource_url)
        client.head(resource_url)
        head_mock.assert_called_with(resource_url)
        client.put(resource_url, **params)
        put_mock.assert_called_with(resource_url, **params)
        self.assertTrue(isinstance(client._session, ws.RetrySession))

        # test cert authentication
        post_mock.reset_mock()
        get_mock.reset_mock()
        put_mock.reset_mock()
        delete_mock.reset_mock()
        head_mock.reset_mock()
        certfile = 'some/certfile.pem'
        subject = auth.Subject(certificate=certfile)
        client = ws.BaseWsClient(resource_id, subject, 'TestApp')
        base_url = 'https://{}{}/pub'.format(resource_uri.netloc,
                                             resource_uri.path)
        resource_url = '{}/{}'.format(base_url, resource)
        self.assertEqual('TestApp', client.agent)
        self.assertTrue(client.retry)
        client.get(resource_url)
        get_mock.assert_called_with(resource_url, params=None)
        params = {'arg1': 'abc', 'arg2': 123, 'arg3': True}
        client.post(resource_url, **params)
        post_mock.assert_called_with(resource_url, **params)
        client.delete(resource_url)
        delete_mock.assert_called_with(resource_url)
        client.head(resource_url)
        head_mock.assert_called_with(resource_url)
        client.put(resource_url, **params)
        put_mock.assert_called_with(resource_url, **params)
        self.assertTrue(isinstance(client._session, ws.RetrySession))
        self.assertEqual((certfile, certfile), client._session.cert)

        # repeat above tests test with the temporary sc2repo test
        resource = 'aresource'
        service = 'sc2repo'
        resource_id = 'ivo://www.canfar.phys.uvic.ca/{}'.format(service)
        # test anonymous access
        client = ws.BaseWsClient(resource_id, auth.Subject(), 'TestApp')
        resource_uri = urlparse(resource_id)
        base_url = 'http://{}{}/observations'.format(resource_uri.netloc,
                                                     resource_uri.path)
        resource_url = '{}/{}'.format(base_url, resource)
        self.assertTrue(client.retry)
        self.assertEqual('TestApp', client.agent)
        self.assertTrue(client.retry)
        self.assertEqual(None, client._session)  # lazy initialization
        client.get(resource_url)
        get_mock.assert_called_with(resource_url, params=None)
        params = {'arg1': 'abc', 'arg2': 123, 'arg3': True}
        client.post(resource_url, **params)
        post_mock.assert_called_with(resource_url, **params)
        client.delete(resource_url)
        delete_mock.assert_called_with(resource_url)
        client.head(resource_url)
        head_mock.assert_called_with(resource_url)
        client.put(resource_url, **params)
        put_mock.assert_called_with(resource_url, **params)
        self.assertTrue(isinstance(client._session, ws.RetrySession))

        # test basic authentication access
        post_mock.reset_mock()
        get_mock.reset_mock()
        put_mock.reset_mock()
        delete_mock.reset_mock()
        head_mock.reset_mock()
        host = 'caom2workshop.canfar.net'
        subject = auth.Subject(netrc=True)
        subject._hosts_auth[host] = ('auser', 'apasswd')
        client = ws.BaseWsClient(resource_id,
                                 subject,
                                 'TestApp',
                                 retry=False,
                                 host=host)
        base_url = 'http://{}{}/auth-observations'.format(
            host, resource_uri.path)
        resource_url = '{}/{}'.format(base_url, resource)
        self.assertEqual('TestApp', client.agent)
        self.assertFalse(client.retry)
        client.get(resource_url)
        get_mock.assert_called_with(resource_url, params=None)
        params = {'arg1': 'abc', 'arg2': 123, 'arg3': True}
        client.post(resource_url, **params)
        post_mock.assert_called_with(resource_url, **params)
        client.delete(resource_url)
        delete_mock.assert_called_with(resource_url)
        client.head(resource_url)
        head_mock.assert_called_with(resource_url)
        client.put(resource_url, **params)
        put_mock.assert_called_with(resource_url, **params)
        self.assertTrue(isinstance(client._session, ws.RetrySession))

        # test cert authentication
        post_mock.reset_mock()
        get_mock.reset_mock()
        put_mock.reset_mock()
        delete_mock.reset_mock()
        head_mock.reset_mock()
        certfile = 'some/certfile.pem'
        client = ws.BaseWsClient(resource_id,
                                 auth.Subject(certificate=certfile), 'TestApp')
        base_url = 'https://{}{}/observations'.format(resource_uri.netloc,
                                                      resource_uri.path)
        resource_url = '{}/{}'.format(base_url, resource)
        self.assertEqual('TestApp', client.agent)
        self.assertTrue(client.retry)
        client.get(resource_url)
        get_mock.assert_called_with(resource_url, params=None)
        params = {'arg1': 'abc', 'arg2': 123, 'arg3': True}
        client.post(resource_url, **params)
        post_mock.assert_called_with(resource_url, **params)
        client.delete(resource_url)
        delete_mock.assert_called_with(resource_url)
        client.head(resource_url)
        head_mock.assert_called_with(resource_url)
        client.put(resource_url, **params)
        put_mock.assert_called_with(resource_url, **params)
        self.assertTrue(isinstance(client._session, ws.RetrySession))
        self.assertEqual((certfile, certfile), client._session.cert)
Esempio n. 18
0
    def test_misc(self):
        """
        Tests miscellaneous functions
        """
        service_url = 'http://somehost.com/service'
        with patch('cadcutils.net.ws.WsCapabilities') as caps_mock:
            caps_mock.return_value.get_service_host.return_value =\
                'somehost.com'
            caps_mock.return_value.get_access_url.return_value = service_url
            client = ws.BaseWsClient("someresourceID", auth.Subject(),
                                     'TestApp')
            self.assertEqual('{}'.format(service_url),
                             client._get_url(('myfeature', None)))
            caps_mock.return_value.get_access_url.assert_called_with(
                'myfeature')
            self.assertEqual('{}'.format(service_url),
                             client._get_url(('myfeature', '')))

        test_host = 'testhost.com'
        with patch('cadcutils.net.ws.os.makedirs'):
            client = ws.BaseWsClient("someresourceID",
                                     auth.Subject(),
                                     'TestApp',
                                     host=test_host)
        self.assertEqual(test_host, client.host)

        # test with resource as url
        with patch('cadcutils.net.ws.WsCapabilities') as caps_mock:
            caps_mock.return_value.get_service_host.return_value =\
                'somehost.com'
            cm = Mock()
            cm.get_access_url.return_value = "http://host/availability"
            caps_mock.return_value = cm
            client = ws.BaseWsClient("someresourceID", auth.Subject(),
                                     'TestApp')
            resource_url = 'http://someurl.com/path/'
            self.assertEqual(resource_url, client._get_url(resource_url))
            # repeat with overriden host name
            client = ws.BaseWsClient("someresourceID",
                                     auth.Subject(),
                                     'TestApp',
                                     host=test_host)
            # same result
            self.assertEqual(resource_url, client._get_url(resource_url))

            # test exceptions with different status in the response
            session = ws.RetrySession()
            response = requests.Response()
            response.status_code = requests.codes.not_found
            with self.assertRaises(exceptions.NotFoundException):
                session.check_status(response)
            response.status_code = requests.codes.unauthorized
            with self.assertRaises(exceptions.UnauthorizedException):
                session.check_status(response)
            response.status_code = requests.codes.forbidden
            with self.assertRaises(exceptions.ForbiddenException):
                session.check_status(response)
            response.status_code = requests.codes.bad_request
            with self.assertRaises(exceptions.BadRequestException):
                session.check_status(response)
            response.status_code = requests.codes.conflict
            with self.assertRaises(exceptions.AlreadyExistsException):
                session.check_status(response)
            response.status_code = requests.codes.internal_server_error
            with self.assertRaises(exceptions.InternalServerException):
                session.check_status(response)
            response.status_code = requests.codes.unavailable
            with self.assertRaises(requests.HTTPError):
                session.check_status(response)
            response.status_code = requests.codes.not_extended
            with self.assertRaises(exceptions.UnexpectedException):
                session.check_status(response)
Esempio n. 19
0
    def test_get_caps(self, get_mock, file_modtime_mock):
        """
        Tests the capabilities part of WsCapabilities
        """
        # test when registry information is read from the server
        # (cache is outdated)
        service = 'myservice'
        resource_id = 'ivo://canfar.phys.uvic.ca/{}'.format(service)
        resource_cap_url = 'www.canfar.net/myservice'
        # set the modified time of the cache file to 0 to make sure the
        # info is retrieved from server
        file_modtime_mock.return_value = 0
        # test anonymous access
        expected_content = capabilities__content.replace(
            'WS_URL', resource_cap_url)
        response = Mock(text=expected_content)
        get_mock.return_value = response
        caps = ws.WsCapabilities(
            Mock(resource_id=resource_id, subject=auth.Subject()))

        # mock _get_capability_url to return some url without attempting
        # to access the server
        def get_url():
            return 'http://some.url/capabilities'

        caps._get_capability_url = get_url
        # remove the cached file if exists
        if os.path.isfile(caps.caps_file):
            os.remove(caps.caps_file)
        caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url)
        self.assertEqual(
            'http://{}/capabilities'.format(resource_cap_url),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
        self.assertEqual(
            'http://{}/availability'.format(resource_cap_url),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
        self.assertEqual(
            'http://{}/pub'.format(resource_cap_url),
            caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))
        actual_content = open(caps.caps_file, 'r').read()
        self.assertEqual(expected_content, actual_content)

        # mock _get_capability_url to return a subservice
        service = 'myservice/mysubservice'
        resource_id = 'ivo://canfar.phys.uvic.ca/{}'.format(service)
        resource_cap_url = 'www.canfar.net/myservice/mysubservice'
        # set the modified time of the cache file to 0 to make sure the
        # info is retrieved from server
        file_modtime_mock.return_value = 0
        expected_content = capabilities__content.replace(
            'WS_URL', resource_cap_url)
        # test anonymous access
        response = Mock(text=expected_content)
        get_mock.return_value = response
        caps = ws.WsCapabilities(
            Mock(resource_id=resource_id, subject=auth.Subject()))
        # remove the cached file if exists
        if os.path.isfile(caps.caps_file):
            os.remove(caps.caps_file)
        caps._get_capability_url = get_url
        caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url)
        self.assertEqual(
            'http://{}/capabilities'.format(resource_cap_url),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
        self.assertEqual(
            'http://{}/availability'.format(resource_cap_url),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
        self.assertEqual(
            'http://{}/pub'.format(resource_cap_url),
            caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))
        actual_content = open(caps.caps_file, 'r').read()
        self.assertEqual(expected_content, actual_content)

        # repeat for basic auth subject. Mock the netrc library to
        # prevent a lookup for $HOME/.netrc
        with patch('cadcutils.net.auth.netrclib'):
            client = Mock(resource_id=resource_id,
                          subject=auth.Subject(netrc=True))
        client.get.return_value = response
        caps = ws.WsCapabilities(client)
        caps._get_capability_url = get_url
        # capabilities works even if it has only one anonymous interface
        self.assertEqual(
            'http://{}/capabilities'.format(resource_cap_url),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
        # same for availability
        self.assertEqual(
            'http://{}/availability'.format(resource_cap_url),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))

        # repeat for https
        with patch('os.path.isfile'):
            client = Mock(resource_id=resource_id,
                          subject=auth.Subject(certificate='somecert.pem'))
        client.get.return_value = response
        caps = ws.WsCapabilities(client)
        caps._get_capability_url = get_url
        # capabilities works even if it has only one anonymous interface
        self.assertEqual(
            'http://{}/capabilities'.format(resource_cap_url),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
        # same for availability
        self.assertEqual(
            'http://{}/availability'.format(resource_cap_url),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
        self.assertEqual(
            'https://{}'.format(resource_cap_url),
            caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))

        # test when capabilities information is retrieved from the cache file
        get_mock.reset_mock()
        get_mock.return_value = None
        file_modtime_mock.reset_mock()
        service = 'myservice2'
        resource_id = 'ivo://canfar.phys.uvic.ca/{}'.format(service)
        resource_cap_url2 = 'canfar.phys.uvic.ca/myservice2'
        expected_content = capabilities__content.replace(
            'WS_URL', resource_cap_url2)
        file_modtime_mock.return_value = time.time()
        client = Mock(resource_id=resource_cap_url2, subject=auth.Subject())
        caps = ws.WsCapabilities(client)
        caps._get_capability_url = get_url
        caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url2)
        # manually write the content
        with open(caps.caps_file, 'w') as f:
            f.write(expected_content)
        self.assertEqual(
            'http://{}/capabilities'.format(resource_cap_url2),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
        self.assertEqual(
            'http://{}/availability'.format(resource_cap_url2),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
        self.assertEqual(
            'http://{}/pub'.format(resource_cap_url2),
            caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))
        actual_content = open(caps.caps_file, 'r').read()
        self.assertEqual(expected_content, actual_content)

        # repeat for basic auth subject. Mock the netrc library to prevent a
        # lookup for $HOME/.netrc
        with patch('cadcutils.net.auth.netrclib'):
            client = Mock(resource_id=resource_id,
                          subject=auth.Subject(netrc=True))
        caps = ws.WsCapabilities(client)
        caps._get_capability_url = get_url
        # does not work with user-password of the subject set above
        self.assertEqual(
            'http://{}/capabilities'.format(resource_cap_url2),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
        self.assertEqual(
            'http://{}/availability'.format(resource_cap_url2),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
        self.assertEqual(
            'http://{}/auth'.format(resource_cap_url2),
            caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))

        # repeat for https
        with patch('os.path.isfile'):
            client = Mock(resource_id=resource_id,
                          subject=auth.Subject(certificate='somecert.pem'))
        caps = ws.WsCapabilities(client)
        caps._get_capability_url = get_url
        # does not work with user-password of the subject set above
        self.assertEqual(
            'http://{}/capabilities'.format(resource_cap_url2),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
        self.assertEqual(
            'http://{}/availability'.format(resource_cap_url2),
            caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
        self.assertEqual(
            'https://{}'.format(resource_cap_url2),
            caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))
Esempio n. 20
0
    def test_process(self):
        core.BATCH_SIZE = 3  # size of the batch is 3
        obs = [['a', 'b', 'c'], ['d'], []]
        level = logging.DEBUG
        visitor = CAOM2RepoClient(auth.Subject(), level)
        visitor.get_observation = MagicMock(return_value=MagicMock(
            spec=SimpleObservation))
        visitor.post_observation = MagicMock()
        visitor._get_observations = MagicMock(side_effect=obs)

        (visited, updated, skipped,
         failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
                                 'cfht')
        self.assertEqual(4, len(visited))
        self.assertEqual(4, len(updated))
        self.assertEqual(0, len(skipped))
        self.assertEqual(0, len(failed))

        obs = [['a', 'b', 'c'], ['d', 'e', 'f'], []]
        visitor._get_observations = MagicMock(side_effect=obs)
        (visited, updated, skipped,
         failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
                                 'cfht')
        self.assertEqual(6, len(visited))
        self.assertEqual(6, len(updated))
        self.assertEqual(0, len(skipped))
        self.assertEqual(0, len(failed))

        # make it return different status. errorplugin returns according to the
        # id of the observation: True for 'UPDATE', False for 'SKIP' and
        # raises exception for 'ERROR'
        obs_ids = [['UPDATE', 'SKIP', 'ERROR'], []]
        obs = [
            SimpleObservation(collection='TEST', observation_id='UPDATE'),
            SimpleObservation(collection='TEST', observation_id='SKIP'),
            SimpleObservation(collection='TEST', observation_id='ERROR')
        ]
        visitor._get_observations = MagicMock(side_effect=obs_ids)
        visitor.get_observation = MagicMock(side_effect=obs)
        (visited, updated, skipped,
         failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
                                 'cfht')
        self.assertEqual(3, len(visited))
        self.assertEqual(1, len(updated))
        self.assertEqual(1, len(skipped))
        self.assertEqual(1, len(failed))

        # repeat with other obs
        obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']]
        obs = [
            SimpleObservation(collection='TEST', observation_id='UPDATE'),
            SimpleObservation(collection='TEST', observation_id='SKIP'),
            SimpleObservation(collection='TEST', observation_id='ERROR'),
            SimpleObservation(collection='TEST', observation_id='UPDATE'),
            SimpleObservation(collection='TEST', observation_id='SKIP')
        ]
        visitor._get_observations = MagicMock(side_effect=obs_ids)
        visitor.get_observation = MagicMock(side_effect=obs)
        (visited, updated, skipped,
         failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
                                 'cfht')
        self.assertEqual(5, len(visited))
        self.assertEqual(2, len(updated))
        self.assertEqual(2, len(skipped))
        self.assertEqual(1, len(failed))

        # repeat but halt on first ERROR -> process only 3 observations
        obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']]
        obs = [
            SimpleObservation(collection='TEST', observation_id='UPDATE'),
            SimpleObservation(collection='TEST', observation_id='SKIP'),
            SimpleObservation(collection='TEST', observation_id='ERROR'),
            SimpleObservation(collection='TEST', observation_id='UPDATE'),
            SimpleObservation(collection='TEST', observation_id='SKIP')
        ]
        visitor._get_observations = MagicMock(side_effect=obs_ids)
        visitor.get_observation = MagicMock(side_effect=obs)
        with self.assertRaises(SystemError):
            visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
                          'cfht',
                          halt_on_error=True)

        # test with time boundaries
        core.BATCH_SIZE = 3  # size of the batch is 3
        response = MagicMock()
        response.text = """ARCHIVE\ta\t2011-01-01T11:00:00.000
                           ARCHIVE\tb\t211-01-01T11:00:10.000
                           ARCHIVE\tc\t2011-01-01T12:00:00.000"""
        response2 = MagicMock()
        response2.text = """ARCHIVE\td\t2011-02-02T11:00:00.000"""
        level = logging.DEBUG
        visitor = CAOM2RepoClient(auth.Subject(), level)
        visitor.get_observation = MagicMock(return_value=MagicMock(
            spec=SimpleObservation))
        visitor.post_observation = MagicMock()
        visitor._repo_client.get = MagicMock(side_effect=[response, response2])

        start = '2010-10-10T12:00:00.000'
        end = '2012-12-12T11:11:11.000'
        (visited, updated, skipped,
         failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
                                 'cfht',
                                 start=util.str2ivoa(start),
                                 end=util.str2ivoa(end))

        self.assertEqual(4, len(visited))
        self.assertEqual(4, len(updated))
        self.assertEqual(0, len(skipped))
        self.assertEqual(0, len(failed))
        calls = [
            call((core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'),
                 params={
                     'START': start,
                     'END': end,
                     'MAXREC': 3
                 }),
            call(
                (core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'),
                params={
                    'START': '2011-01-01T12:00:00.000',
                    # datetime of the last record in the batch
                    'END': end,
                    'MAXREC': 3
                })
        ]
        visitor._repo_client.get.assert_has_calls(calls)
Esempio n. 21
0
    def test_get_caps(self, get_mock, file_mock, file_modtime_mock):
        """
        Tests the capabilities part of WsCapabilities
        """
        # test when registry information is read from the server
        # (cache is outdated)
        service = 'myservice'
        resource_id = 'ivo://canfar.phys.uvic.ca/{}'.format(service)
        resource_cap_url = 'www.canfar.net/myservice'
        # set the modified time of the cache file to 0 to make sure the
        # info is retrieved from server
        file_modtime_mock.return_value = 0
        # test anonymous access
        fh_mock = Mock()
        file_mock.write = fh_mock
        response = Mock(
            text=capabilities__content.replace('WS_URL', resource_cap_url))
        get_mock.return_value = response
        caps = ws.WsCapabilities(Mock(resource_id=resource_id,
                                      subject=auth.Subject()))

        # mock _get_capability_url to return some url without attempting
        # to access the server
        def get_url():
            return 'http://some.url/capabilities'

        caps._get_capability_url = get_url
        caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url)
        self.assertEqual('http://{}/capabilities'.format(resource_cap_url),
                         caps.get_access_url(
                             'ivo://ivoa.net/std/VOSI#capabilities'))
        self.assertEqual('http://{}/availability'.format(resource_cap_url),
                         caps.get_access_url(
                             'ivo://ivoa.net/std/VOSI#availability'))
        self.assertEqual('http://{}/pub'.format(resource_cap_url),
                         caps.get_access_url(
                             'vos://cadc.nrc.ca~service/CADC/mystnd01'))
        resource_url = urlparse(resource_id)
        file_mock.assert_called_once_with(os.path.join(ws.CACHE_LOCATION,
                                                       resource_url.netloc,
                                                       resource_url.path.strip(
                                                           '/')), 'w')
        # TODO not sure why need to access write this way
        file_mock().__enter__.return_value.write.assert_called_once_with(
            capabilities__content.replace('WS_URL', resource_cap_url))

        # repeat for basic auth subject. Mock the netrc library to
        # prevent a lookup for $HOME/.netrc
        with patch('cadcutils.net.auth.netrclib'):
            client = Mock(resource_id=resource_id,
                          subject=auth.Subject(netrc=True))
        client.get.return_value = response
        caps = ws.WsCapabilities(client)
        caps._get_capability_url = get_url
        # capabilities works even if it has only one anonymous interface
        self.assertEqual('http://{}/capabilities'.format(resource_cap_url),
                         caps.get_access_url(
                             'ivo://ivoa.net/std/VOSI#capabilities'))
        # same for availability
        self.assertEqual('http://{}/availability'.format(resource_cap_url),
                         caps.get_access_url(
                             'ivo://ivoa.net/std/VOSI#availability'))

        # repeat for https
        with patch('os.path.isfile'):
            client = Mock(resource_id=resource_id,
                          subject=auth.Subject(certificate='somecert.pem'))
        client.get.return_value = response
        caps = ws.WsCapabilities(client)
        caps._get_capability_url = get_url
        # capabilities works even if it has only one anonymous interface
        self.assertEqual('http://{}/capabilities'.format(resource_cap_url),
                         caps.get_access_url(
                             'ivo://ivoa.net/std/VOSI#capabilities'))
        # same for availability
        self.assertEqual('http://{}/availability'.format(resource_cap_url),
                         caps.get_access_url(
                             'ivo://ivoa.net/std/VOSI#availability'))
        self.assertEqual('https://{}'.format(resource_cap_url),
                         caps.get_access_url(
                             'vos://cadc.nrc.ca~service/CADC/mystnd01'))

        # test when capabilities information is retrieved from the cache file
        get_mock.reset_mock()
        get_mock.return_value = None
        file_modtime_mock.reset_mock()
        file_mock.reset_mock()
        resource_cap_url2 = 'http://www.canfar.net/myservice2'
        cache_content2 = capabilities__content.replace('WS_URL',
                                                       resource_cap_url2)
        file_modtime_mock.return_value = time.time()
        file_mock().__enter__.return_value.read.return_value = cache_content2
        client = Mock(resource_id=resource_id, subject=auth.Subject())
        caps = ws.WsCapabilities(client)
        caps._get_capability_url = get_url
        caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url2)
        self.assertEqual('http://{}/capabilities'.format(resource_cap_url2),
                         caps.get_access_url(
                             'ivo://ivoa.net/std/VOSI#capabilities'))
        self.assertEqual('http://{}/availability'.format(resource_cap_url2),
                         caps.get_access_url(
                             'ivo://ivoa.net/std/VOSI#availability'))
        self.assertEqual('http://{}/pub'.format(resource_cap_url2),
                         caps.get_access_url(
                             'vos://cadc.nrc.ca~service/CADC/mystnd01'))
        resource_url = urlparse(resource_id)
        file_mock().__enter__.return_value.read.return_value = \
            capabilities__content.replace('WS_URL', resource_cap_url2)

        # repeat for basic auth subject. Mock the netrc library to prevent a
        # lookup for $HOME/.netrc
        with patch('cadcutils.net.auth.netrclib'):
            client = Mock(resource_id=resource_id,
                          subject=auth.Subject(netrc=True))
        caps = ws.WsCapabilities(client)
        caps._get_capability_url = get_url
        # does not work with user-password of the subject set above
        self.assertEqual('http://{}/capabilities'.format(resource_cap_url2),
                         caps.get_access_url(
                             'ivo://ivoa.net/std/VOSI#capabilities'))
        self.assertEqual('http://{}/availability'.format(resource_cap_url2),
                         caps.get_access_url(
                             'ivo://ivoa.net/std/VOSI#availability'))
        self.assertEqual('http://{}/auth'.format(resource_cap_url2),
                         caps.get_access_url(
                             'vos://cadc.nrc.ca~service/CADC/mystnd01'))

        # repeat for https
        with patch('os.path.isfile'):
            client = Mock(resource_id=resource_id,
                          subject=auth.Subject(certificate='somecert.pem'))
        caps = ws.WsCapabilities(client)
        caps._get_capability_url = get_url
        # does not work with user-password of the subject set above
        self.assertEqual('http://{}/capabilities'.format(resource_cap_url2),
                         caps.get_access_url(
                             'ivo://ivoa.net/std/VOSI#capabilities'))
        self.assertEqual('http://{}/availability'.format(resource_cap_url2),
                         caps.get_access_url(
                             'ivo://ivoa.net/std/VOSI#availability'))
        self.assertEqual('https://{}'.format(resource_cap_url2),
                         caps.get_access_url(
                             'vos://cadc.nrc.ca~service/CADC/mystnd01'))