def test_plugin_class(self): # plugin class does not change the observation collection = 'cfht' observation_id = '7000000o' level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) obs = SimpleObservation(collection, observation_id) expect_obs = copy.deepcopy(obs) visitor._load_plugin_class(os.path.join(THIS_DIR, 'passplugin.py')) visitor.plugin.update(obs) self.assertEqual(expect_obs, obs) # plugin class adds a plane to the observation visitor = CAOM2RepoClient(auth.Subject(), level) obs = SimpleObservation('cfht', '7000000o') expect_obs = copy.deepcopy(obs) visitor._load_plugin_class(os.path.join(THIS_DIR, 'addplaneplugin.py')) visitor.plugin.update(obs) self.assertNotEqual(expect_obs, obs) self.assertEqual(len(expect_obs.planes) + 1, len(obs.planes)) # non-existent the plugin file with self.assertRaises(Exception): visitor._load_plugin_class(os.path.join(THIS_DIR, 'blah.py')) # non-existent ObservationUpdater class in the plugin file with self.assertRaises(Exception): visitor._load_plugin_class( os.path.join(THIS_DIR, 'test_visitor.py')) # non-existent update method in ObservationUpdater class with self.assertRaises(Exception): visitor._load_plugin_class( os.path.join(THIS_DIR, 'noupdateplugin.py'))
def test_create_and_visit(self): #logger.setLevel(logging.DEBUG) logger.info('-- START: test_create_and_visit --') start = datetime.now() name = obs_name = 'caom2pyinttest{}'.format(start.microsecond) try: env_a = os.environ['A'] cert_file = env_a + '/test-certificates/x509_CADCAuthtest1.pem' subject = auth.Subject(certificate=cert_file) client = CAOM2RepoClient(subject) # create one observation for today algorithm = observation.SimpleObservation._DEFAULT_ALGORITHM_NAME logger.debug("test obs name {}".format(name)) obs = observation.SimpleObservation("TEST", obs_name) obs.algorithm = algorithm client.put_observation(obs) plugin = os.path.join(THIS_DIR, 'visitor-plugin.py') (visited, updated, skipped, failed) = client.visit(plugin, 'TEST', start=start, halt_on_error=True) logger.debug("observations visited: {}".format(len(visited))) self.assertGreater(len(visited), 0, msg="No Observations Visited") finally: try: client.delete_observation("TEST", name) except: logger.warning('Failed to delete test observation, continuing') logger.info('-- END :test_create_and_visit --')
def test_visit_retry_on_412(self): # observation changed on server while visited core.BATCH_SIZE = 3 # size of the batch is 3 obs = [['a'], []] level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) observation = SimpleObservation('cfht', 'a') observation.acc_meta_checksum = ChecksumURI('md5:abc') visitor.get_observation = MagicMock( side_effect=[observation, observation]) exception_412 = exceptions.UnexpectedException() exception_412.orig_exception = Mock() exception_412.orig_exception.response = Mock(status_code=412) visitor.post_observation = MagicMock(side_effect=[exception_412, None]) visitor._get_observations = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') self.assertEqual(1, len(visited)) self.assertEqual(1, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) # get and post called twice to recover from error HTTP status 412 - # precondition self.assertEqual(2, visitor.get_observation.call_count) self.assertEqual(2, visitor.post_observation.call_count) visitor.post_observation.assert_called_with( observation, observation.acc_meta_checksum.uri)
def test_multiprocess_with_unexpected_type_error(self, get_mock): core.BATCH_SIZE = 3 # size of the batch is 3 obs_ids = [['a', 'b', 'c'], ['d'], []] get_mock.side_effect = \ self.mock_get_observation_with_unexpected_type_error level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) visitor.get_observation = PickableMagicMock( return_value=PickableMagicMock(spec=SimpleObservation)) visitor.post_observation = PickableMagicMock() visitor._get_observations = PickableMagicMock(side_effect=obs_ids) try: (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht', start=None, end=None, obs_file=None, nthreads=3, halt_on_error=True) except TypeError as e: self.assertTrue("unexpected TypeError" in str(e)) finally: logging.info("DONE")
def test_multiprocess_with_more_different_statuses(self, get_mock): core.BATCH_SIZE = 3 # size of the batch is 3 # make it return different status. errorplugin returns according to the # id of the observation: True for 'UPDATE', False for 'SKIP' and # raises exception for 'ERROR' obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']] get_mock.side_effect = self.mock_get_observation level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) visitor.get_observation = PickableMagicMock( return_value=PickableMagicMock(spec=SimpleObservation)) visitor.post_observation = PickableMagicMock() visitor._get_observations = PickableMagicMock(side_effect=obs_ids) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht', start=None, end=None, obs_file=None, nthreads=3) try: self.assertEqual(5, len(visited)) self.assertEqual(2, len(updated)) self.assertEqual(2, len(skipped)) self.assertEqual(1, len(failed)) finally: # lp.join() logging.info("DONE")
def test_post_observation(self, mock_conn, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/auth' collection = 'cfht' observation_id = '7000000o' service = 'caom2repo' service_url = 'www.cadc.nrc.ca' obs = SimpleObservation(collection, observation_id) level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(netrc='somenetrc'), level, host=service_url) response = MagicMock() response.status = 200 mock_conn.return_value = response iobuffer = BytesIO() ObservationWriter().write(obs, iobuffer) obsxml = iobuffer.getvalue() response.content = obsxml visitor.post_observation(obs) self.assertEqual('POST', mock_conn.call_args[0][0].method) self.assertEqual( '/{}/auth/{}/{}'.format(service, collection, observation_id), mock_conn.call_args[0][0].path_url) self.assertEqual('application/xml', mock_conn.call_args[0][0].headers['Content-Type']) self.assertEqual(obsxml, mock_conn.call_args[0][0].body) # signal problems http_error = requests.HTTPError() response.status_code = 500 http_error.response = response response.raise_for_status.side_effect = [http_error] with self.assertRaises(exceptions.InternalServerException): visitor.update(obs) # temporary transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response response.raise_for_status.side_effect = [http_error, None] visitor.post_observation(obs) # permanent transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response def raise_error(): raise http_error response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.post_observation(obs)
def test_get_observations(self, mock_get, caps_mock): # This is almost similar to the previous test except that it gets # observations matching a collection and start/end criteria # Also, patch the CAOM2RepoClient now. caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/pub' response = MagicMock() response.status_code = 200 last_datetime = '2000-10-10T12:30:00.333' response.text = \ ('CFHT\t700000o\t2000-10-10T12:20:11.123\t' '3e00ca6129dc8358315015204ab9fe15\nCFHT\t700001o\t' + last_datetime + '\t3e00ca6129dc8358315015204ab9fe15') mock_get.return_value = response level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) end_date = util.utils.str2ivoa(last_datetime) expect_observations = ['700000o', '700001o'] self.assertEqual(expect_observations, visitor._get_observations('cfht')) self.assertEqual(end_date, visitor._start) mock_get.assert_called_once_with( ('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2', 'cfht'), params={'MAXREC': core.BATCH_SIZE}) mock_get.reset_mock() visitor._get_observations('cfht', end=datetime.strptime( '2000-11-11', '%Y-%m-%d')) mock_get.assert_called_once_with( ('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2', 'cfht'), params={ 'END': '2000-11-11T00:00:00.000', 'MAXREC': core.BATCH_SIZE }) mock_get.reset_mock() visitor._get_observations( 'cfht', start=datetime.strptime('2000-11-11', '%Y-%m-%d'), end=datetime.strptime('2000-11-12', '%Y-%m-%d')) mock_get.assert_called_once_with( ('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2', 'cfht'), params={ 'START': '2000-11-11T00:00:00.000', 'END': '2000-11-12T00:00:00.000', 'MAXREC': core.BATCH_SIZE })
def test_get_observation(self, mock_get, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/pub' collection = 'cfht' observation_id = '7000000o' service_url = 'www.cadc.nrc.ca/caom2repo' obs = SimpleObservation(collection, observation_id) writer = ObservationWriter() ibuffer = BytesIO() writer.write(obs, ibuffer) response = MagicMock() response.status_code = 200 response.content = ibuffer.getvalue() mock_get.return_value = response ibuffer.seek(0) # reposition the buffer for reading level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url) self.assertEqual(obs, visitor.get_observation(collection, observation_id)) # signal problems http_error = requests.HTTPError() response.status_code = 500 http_error.response = response response.raise_for_status.side_effect = [http_error] with self.assertRaises(exceptions.InternalServerException): visitor.get_observation(collection, observation_id) # temporary transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response response.raise_for_status.side_effect = [http_error, None] visitor.read(collection, observation_id) # permanent transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response def raise_error(): raise http_error response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.get_observation(collection, observation_id)
def test_delete_observation(self, mock_conn, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/pub' collection = 'cfht' observation_id = '7000000o' service_url = 'www.cadc.nrc.ca' level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url) response = MagicMock() response.status = 200 mock_conn.return_value = response visitor.delete_observation(collection, observation_id) self.assertEqual('DELETE', mock_conn.call_args[0][0].method) # signal problems http_error = requests.HTTPError() response.status_code = 500 http_error.response = response response.raise_for_status.side_effect = [http_error] with self.assertRaises(exceptions.InternalServerException): visitor.delete(collection, observation_id) # temporary transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response response.raise_for_status.side_effect = [http_error, None] visitor.delete_observation(collection, observation_id) # permanent transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response def raise_error(): raise http_error response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.delete_observation(collection, observation_id)
def test_get_obs_from_file(self): level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) # no start or end with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file: obs_id_list = visitor._get_obs_from_file(obs_file, None, None, False) self.assertEqual('obs_id_1', obs_id_list[0]) self.assertEqual('obs_id_2', obs_id_list[1]) self.assertEqual('obs_id_3', obs_id_list[2]) # last_modified_date is earlier than start with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file: obs_id_list = visitor._get_obs_from_file( obs_file, util.str2ivoa('2000-10-11T12:30:00.333'), None, False) self.assertEqual('obs_id_1', obs_id_list[0]) # last_modified_date is between start and end with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file: obs_id_list = visitor._get_obs_from_file( obs_file, util.str2ivoa('2000-10-9T12:30:00.333'), util.str2ivoa('2016-10-11T12:30:00.333'), False) self.assertEqual('obs_id_1', obs_id_list[0]) self.assertEqual('obs_id_2', obs_id_list[1]) # last_modified_date is after end with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file: obs_id_list = visitor._get_obs_from_file( obs_file, util.str2ivoa('2000-10-9T12:30:00.333'), util.str2ivoa('2017-10-11T12:30:00.333'), False) self.assertEqual('obs_id_1', obs_id_list[0]) self.assertEqual('obs_id_2', obs_id_list[1]) self.assertEqual('obs_id_3', obs_id_list[2]) # error in file with open(os.path.join(THIS_DIR, 'data/obs_id_error.txt')) as obs_file: with self.assertRaises(Exception): obs_id_list = visitor._get_obs_from_file( obs_file, util.str2ivoa('2000-10-9T12:30:00.333'), util.str2ivoa('2016-10-11T12:30:00.333'), True)
def test_multiprocess_with_more_obs_id(self, client_mock): core.BATCH_SIZE = 3 # size of the batch is 3 obs_ids = [['a', 'b', 'c'], ['d', 'e', 'f'], []] client_mock.return_value.get_observation.side_effect = \ self.mock_get_observation level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) visitor.get_observation = PickableMagicMock( return_value=PickableMagicMock(spec=SimpleObservation)) visitor.post_observation = PickableMagicMock() visitor._get_observations = PickableMagicMock(side_effect=obs_ids) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht', start=None, end=None, obs_file=None, nthreads=3) try: self.assertEqual(6, len(visited)) self.assertEqual(6, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) self.assertTrue('a' in visited) self.assertTrue('b' in visited) self.assertTrue('c' in visited) self.assertTrue('d' in visited) self.assertTrue('e' in visited) self.assertTrue('f' in visited) self.assertFalse('g' in visited) self.assertTrue('a' in updated) self.assertTrue('b' in updated) self.assertTrue('c' in updated) self.assertTrue('d' in updated) self.assertTrue('e' in updated) self.assertTrue('f' in updated) self.assertFalse('g' in updated) finally: # lp.join() logging.info("DONE")
def test_shortcuts(self): level = logging.DEBUG target = CAOM2RepoClient(auth.Subject(), level) obs = SimpleObservation('CFHT', 'abc') target.put_observation = Mock() target.create(obs) target.put_observation.assert_called_with(obs) target.get_observation = Mock() target.read('CFHT', 'abc') target.get_observation.assert_called_with('CFHT', 'abc') target.post_observation = Mock() target.update(obs) target.post_observation.assert_called_with(obs) target.delete_observation = Mock() target.delete('CFHT', 'abc') target.delete_observation.assert_called_with('CFHT', 'abc')
def test_info_file(basews_mock): client = CadcDataClient(auth.Subject()) # test an info file_name = 'myfile' file_name = 'myfile.txt' archive = 'TEST' size = '123' md5sum = '0x123' type = 'txt' encoding = 'gzip' lastmod = '11/11/11T11:11:11.000' usize = '1234' umd5sum = '0x1234' h = {} h['Content-Disposition'] = 'inline; filename={}'.format(file_name) h['Content-Length'] = size h['Content-MD5'] = md5sum h['Content-Type'] = type h['Content-Encoding'] = encoding h['Last-Modified'] = lastmod h['X-Uncompressed-Length'] = usize h['X-Uncompressed-MD5'] = umd5sum response = Mock() response.headers = h basews_mock.return_value.head.return_value = response info = client.get_file_info('TEST', 'myfile') assert archive == info['archive'] assert file_name == info['name'] assert size == info['size'] assert md5sum == info['md5sum'] assert type == info['type'] assert encoding == info['encoding'] assert lastmod == info['lastmod'] assert usize == info['usize'] assert umd5sum == info['umd5sum']
def test_put_file(basews_mock): client = CadcDataClient(auth.Subject()) # test a put file_name = '/tmp/putfile.txt' file_content = 'ABCDEFGH12345' hash_md5 = hashlib.md5() hash_md5.update(file_content.encode()) hash_md5 = hash_md5.hexdigest() # write the file with open(file_name, 'w') as f: f.write(file_content) put_mock = Mock() basews_mock.return_value.put = put_mock with pytest.raises(exceptions.UnauthorizedException): client.put_file('TEST', 'putfile', file_name) client._data_client.subject.anon = False # authenticate the user transf_end_point = 'http://test.ca/endpoint' def mock_get_trans_protocols(archive, file_name, is_get, headers): protocol = Mock() protocol.endpoint = '{}/{}'.format(transf_end_point, file_name) return [protocol] client._get_transfer_protocols = mock_get_trans_protocols client.put_file('TEST', file_name) # Note Content* headers automatically created by cadc-data except when # MAGIC_WANT -- libmagic not present put_mock.assert_called_with('{}/{}'.format(transf_end_point, os.path.basename(file_name)), data=ANY, headers={ 'Content-Type': 'text/plain', 'Content-Encoding': 'us-ascii', 'Content-MD5': '{}'.format(hash_md5) }) # mimic libmagic missing cadcdata.core.MAGIC_WARN = 'Some warning' put_mock.reset_mock() client.put_file('TEST', file_name) put_mock.assert_called_with( '{}/{}'.format(transf_end_point, os.path.basename(file_name)), data=ANY, headers={'Content-MD5': '835e7e6cd54e18ae21d50af963b0c32b'}) cadcdata.core.MAGIC_WARN = None # specify an archive stream and override the name of the file input_name = 'abc' client.put_file('TEST', file_name, archive_stream='default', input_name=input_name) put_mock.assert_called_with('{}/{}'.format(transf_end_point, input_name), data=ANY, headers={ 'Content-Encoding': 'us-ascii', 'X-CADC-Stream': 'default', 'Content-Type': 'text/plain', 'Content-MD5': '{}'.format(hash_md5) }) # specify the mime types client.put_file('TEST', file_name, archive_stream='default', mime_type='ASCII', mime_encoding='GZIP') put_mock.assert_called_with('{}/{}'.format(transf_end_point, os.path.basename(file_name)), data=ANY, headers={ 'Content-Encoding': 'GZIP', 'X-CADC-Stream': 'default', 'Content-Type': 'ASCII', 'Content-MD5': '{}'.format(hash_md5) }) os.remove(file_name)
def test_get_file(trans_reader_mock, basews_mock): # test a simple get - no decompress file_name = '/tmp/afile.txt' file_chunks = ['aaaa'.encode(), 'bbbb'.encode(), ''.encode()] response = Mock() hash_md5 = hashlib.md5() for i in file_chunks: hash_md5.update(i) response.headers.get.return_value = \ 'filename={}'.format('orig_file_name') response.raw.read.side_effect = file_chunks # returns multiple blocks basews_mock.return_value.get.return_value = response client = CadcDataClient(auth.Subject()) with pytest.raises(exceptions.HttpException): # no URLs returned in the transfer negotiations client.get_file('TEST', 'afile', destination=file_name) t = transfer.Transfer('ad:TEST/afile', 'pullFromVoSpace') p = transfer.Protocol p.endpoint = Mock() t.protocols = [p] trans_reader_mock.return_value.read.return_value = t client.get_file('TEST', 'afile', destination=file_name, md5_check=False) expected_content = \ (''.join([c.decode() for c in file_chunks])).encode() with open(file_name, 'rb') as f: assert expected_content == f.read() os.remove(file_name) # do it again with the file now open response = Mock() response.headers = { 'filename': 'orig_file_name', 'content-MD5': hash_md5.hexdigest() } response.raw.read.side_effect = file_chunks basews_mock.return_value.get.return_value = response with open(file_name, 'wb') as f: client.get_file('TEST', 'afile', destination=f) with open(file_name, 'rb') as f: assert expected_content == f.read() os.remove(file_name) # test a get with decompress and md5 check enabled file_name = 'bfile.txt' file_content = 'aaaabbbb' hash_md5 = hashlib.md5() hash_md5.update(file_content.encode()) file_chunks = [file_content.encode(), ''.encode()] decoded_file_content = 'MNOPRST6789' decoded_file_chunks = [decoded_file_content.encode(), ''.encode()] response = Mock() response.headers = \ {'content-MD5': '{}'.format(hash_md5.hexdigest()), 'filename': file_name} response.raw.read.side_effect = file_chunks response.raw._decode.side_effect = decoded_file_chunks basews_mock.return_value.get.return_value = response client = CadcDataClient(auth.Subject()) client.get_file('TEST', file_name=file_name, decompress=True, md5_check=True) with open(file_name, 'r') as f: # note the check against the decoded content assert decoded_file_content == f.read() os.remove(file_name) # repeat test with a bad md5 file_name = 'bfile.txt' file_content = 'ABCDEFGH12345' file_chunks = [file_content.encode(), ''.encode()] decoded_file_content = 'MNOPRST6789' decoded_file_chunks = [decoded_file_content.encode(), ''.encode()] response = Mock() response.headers = {'content-MD5': 'abc', 'filename': file_name} response.raw.read.side_effect = file_chunks response.raw._decode.side_effect = decoded_file_chunks basews_mock.return_value.get.return_value = response client = CadcDataClient(auth.Subject()) with pytest.raises(exceptions.HttpException): client.get_file('TEST', file_name=file_name, decompress=True, md5_check=True) # test process_bytes and send the content to /dev/null after. # Use no decompress def concatenate_chunks(chunk): global mycontent mycontent = '{}{}'.format(mycontent, chunk.decode()) file_name = 'bfile.txt' file_content = 'ABCDEFGH12345' file_chunks = [ file_content[i:i + 5].encode() for i in xrange(0, len(file_content), 5) ] file_chunks.append('') # last chunk is empty response = Mock() response.headers = {'filename': '{}.gz'.format(file_name)} response.raw.read.side_effect = file_chunks basews_mock.return_value.get.return_value = response client = CadcDataClient(auth.Subject()) client.logger.setLevel(logging.INFO) # md5_check does not take place because no content-MD5 received # from server client.get_file('TEST', 'afile', destination='/dev/null', process_bytes=concatenate_chunks) assert file_content == mycontent # failed md5 checksum response = Mock() response.headers = { 'filename': '{}.gz'.format(file_name), 'content-MD5': '33' } response.raw.read.side_effect = file_chunks basews_mock.return_value.get.return_value = response client = CadcDataClient(auth.Subject()) client.logger.setLevel(logging.INFO) # md5_check does not take place because no content-MD5 received # from server with pytest.raises(exceptions.HttpException): client.get_file('TEST', 'afile', destination='/dev/null', process_bytes=concatenate_chunks) # test get fhead response = Mock() response.headers.get.return_value = 'filename={}.gz'.format(file_name) response.raw.read.side_effect = file_chunks response.history = [] response.status_code = 200 response.url = 'someurl' post_mock = Mock(return_value=response) basews_mock.return_value.post = post_mock file_name = 'getfile' archive = 'TEST' p.endpoint = 'http://someurl/transfer/{}/{}'.format(archive, file_name) client.get_file('TEST', 'getfile', decompress=True, wcs=True, md5_check=False) trans_doc = \ ('<vos:transfer xmlns:' 'vos="http://www.ivoa.net/xml/VOSpace/v2.0">\n ' '<vos:target>ad:TEST/getfile</vos:target>\n ' '<vos:direction>pullFromVoSpace</vos:direction>\n ' '<vos:protocol uri="ivo://ivoa.net/vospace/core#httpget"/>\n' ' <vos:protocol uri="ivo://ivoa.net/vospace/core#httpsget"/>\n' '</vos:transfer>\n').encode() post_mock.assert_called_with(resource=(TRANSFER_RESOURCE_ID, None), params={'wcs': True}, data=trans_doc, headers={'Content-Type': 'text/xml'}) response.raw.read.side_effect = file_chunks post_mock.reset_mock() client.get_file('TEST', 'getfile', decompress=True, fhead=True, md5_check=False) post_mock.assert_called_with(resource=(TRANSFER_RESOURCE_ID, None), params={'fhead': True}, data=trans_doc, headers={'Content-Type': 'text/xml'}) response.raw.read.side_effect = file_chunks post_mock.reset_mock() client.get_file('TEST', 'getfile', decompress=True, cutout='[1:1]', md5_check=False) post_mock.assert_called_with(resource=(TRANSFER_RESOURCE_ID, None), params={'cutout': '[1:1]'}, data=trans_doc, headers={'Content-Type': 'text/xml'}) response.raw.read.side_effect = file_chunks post_mock.reset_mock() client.get_file('TEST', 'getfile', decompress=True, cutout='[[1:1], 2]', md5_check=False) post_mock.assert_called_with(resource=(TRANSFER_RESOURCE_ID, None), params={'cutout': '[[1:1], 2]'}, data=trans_doc, headers={'Content-Type': 'text/xml'})
def testSubject(self, os_mock): # anon subject subject = auth.Subject() self.assertTrue(subject.anon) self.assertEqual(None, subject.certificate) self.assertEqual({}, subject._hosts_auth) self.assertEqual(None, subject.get_auth('realm1')) # cert subject cert = 'somecert' subject = auth.Subject(certificate=cert) self.assertFalse(subject.anon) self.assertEqual(cert, subject.certificate) self.assertEqual({}, subject._hosts_auth) self.assertEqual(None, subject.get_auth('realm1')) # empty netrc subject m = mock_open() with patch('six.moves.builtins.open', m, create=True): subject = auth.Subject(netrc='somefile') self.assertFalse(subject.anon) self.assertEqual(None, subject.certificate) self.assertEqual({}, subject._hosts_auth) self.assertEqual(None, subject.get_auth('realm1')) # netrc with content netrc_content = {'realm1': ('user1', None, 'pass1'), 'realm2': ('user1', None, 'pass2')} expected_host_auth = {'realm1': ('user1', 'pass1'), 'realm2': ('user1', 'pass2')} os_mock.path.join.return_value = '/home/myhome/.netrc' with patch('cadcutils.net.auth.netrclib') as netrclib_mock: netrclib_mock.netrc.return_value.hosts = netrc_content subject = auth.Subject(netrc=True) self.assertFalse(subject.anon) self.assertEqual(None, subject.certificate) self.assertEqual('/home/myhome/.netrc', subject.netrc) self.assertEqual(expected_host_auth, subject._hosts_auth) self.assertEqual(('user1', 'pass1'), subject.get_auth('realm1')) self.assertEqual(('user1', 'pass2'), subject.get_auth('realm2')) self.assertEqual(None, subject.get_auth('realm3')) # subject with username username = '******' passwd = 'passwd1' subject = auth.Subject(username=username) self.assertFalse(subject.anon) self.assertEqual(None, subject.certificate) self.assertEqual({}, subject._hosts_auth) with patch('cadcutils.net.auth.getpass') as getpass_mock: getpass_mock.getpass.return_value = passwd self.assertEqual((username, passwd), subject.get_auth('realm1')) parser = get_base_parser(subparsers=False) args = parser.parse_args(['--resource-id', 'blah']) subject = auth.Subject.from_cmd_line_args(args) self.assertTrue(subject.anon) sys.argv = ['cadc-client', '--resource-id', 'blah', '--cert', 'mycert.pem'] args = parser.parse_args() subject = auth.Subject.from_cmd_line_args(args) self.assertFalse(subject.anon) self.assertEqual('mycert.pem', subject.certificate)
def test_ops(self, post_mock, get_mock, delete_mock, head_mock, put_mock, netrclib_mock, caps_mock): anon_subject = auth.Subject() with self.assertRaises(ValueError): ws.BaseWsClient(None, anon_subject, "TestApp") resource = 'aresource' service = 'myservice' resource_id = 'ivo://www.canfar.phys.uvic.ca/{}'.format(service) # test anonymous access cm = Mock() cm.get_access_url.return_value = "http://host/availability" caps_mock.return_value = cm client = ws.BaseWsClient(resource_id, anon_subject, 'TestApp') resource_uri = urlparse(resource_id) base_url = 'http://{}{}/pub'.format(resource_uri.netloc, resource_uri.path) resource_url = 'http://{}{}/{}'.format(resource_uri.netloc, base_url, resource) self.assertEqual(anon_subject, client.subject) self.assertTrue(client.retry) self.assertEqual('TestApp', client.agent) self.assertTrue(client.retry) self.assertEqual(None, client._session) # lazy initialization client.get(resource_url) get_mock.assert_called_with(resource_url, params=None) params = {'arg1': 'abc', 'arg2': 123, 'arg3': True} client.post(resource_url, **params) post_mock.assert_called_with(resource_url, **params) client.delete(resource_url) delete_mock.assert_called_with(resource_url) client.head(resource_url) head_mock.assert_called_with(resource_url) client.put(resource_url, **params) put_mock.assert_called_with(resource_url, **params) self.assertTrue(isinstance(client._session, ws.RetrySession)) # test basic authentication access post_mock.reset_mock() get_mock.reset_mock() put_mock.reset_mock() delete_mock.reset_mock() head_mock.reset_mock() host = 'www.different.org' subject = auth.Subject(netrc='somecert') client = ws.BaseWsClient(resource_id, subject, 'TestApp', retry=False, host=host) base_url = 'http://{}{}/auth'.format(host, resource_uri.path) resource_url = '{}/{}'.format(base_url, resource) self.assertEqual('TestApp', client.agent) self.assertFalse(client.retry) client.get(resource_url) get_mock.assert_called_with(resource_url, params=None) params = {'arg1': 'abc', 'arg2': 123, 'arg3': True} client.post(resource_url, **params) post_mock.assert_called_with(resource_url, **params) client.delete(resource_url) delete_mock.assert_called_with(resource_url) client.head(resource_url) head_mock.assert_called_with(resource_url) client.put(resource_url, **params) put_mock.assert_called_with(resource_url, **params) self.assertTrue(isinstance(client._session, ws.RetrySession)) # test cert authentication post_mock.reset_mock() get_mock.reset_mock() put_mock.reset_mock() delete_mock.reset_mock() head_mock.reset_mock() certfile = 'some/certfile.pem' subject = auth.Subject(certificate=certfile) client = ws.BaseWsClient(resource_id, subject, 'TestApp') base_url = 'https://{}{}/pub'.format(resource_uri.netloc, resource_uri.path) resource_url = '{}/{}'.format(base_url, resource) self.assertEqual('TestApp', client.agent) self.assertTrue(client.retry) client.get(resource_url) get_mock.assert_called_with(resource_url, params=None) params = {'arg1': 'abc', 'arg2': 123, 'arg3': True} client.post(resource_url, **params) post_mock.assert_called_with(resource_url, **params) client.delete(resource_url) delete_mock.assert_called_with(resource_url) client.head(resource_url) head_mock.assert_called_with(resource_url) client.put(resource_url, **params) put_mock.assert_called_with(resource_url, **params) self.assertTrue(isinstance(client._session, ws.RetrySession)) self.assertEqual((certfile, certfile), client._session.cert) # repeat above tests test with the temporary sc2repo test resource = 'aresource' service = 'sc2repo' resource_id = 'ivo://www.canfar.phys.uvic.ca/{}'.format(service) # test anonymous access client = ws.BaseWsClient(resource_id, auth.Subject(), 'TestApp') resource_uri = urlparse(resource_id) base_url = 'http://{}{}/observations'.format(resource_uri.netloc, resource_uri.path) resource_url = '{}/{}'.format(base_url, resource) self.assertTrue(client.retry) self.assertEqual('TestApp', client.agent) self.assertTrue(client.retry) self.assertEqual(None, client._session) # lazy initialization client.get(resource_url) get_mock.assert_called_with(resource_url, params=None) params = {'arg1': 'abc', 'arg2': 123, 'arg3': True} client.post(resource_url, **params) post_mock.assert_called_with(resource_url, **params) client.delete(resource_url) delete_mock.assert_called_with(resource_url) client.head(resource_url) head_mock.assert_called_with(resource_url) client.put(resource_url, **params) put_mock.assert_called_with(resource_url, **params) self.assertTrue(isinstance(client._session, ws.RetrySession)) # test basic authentication access post_mock.reset_mock() get_mock.reset_mock() put_mock.reset_mock() delete_mock.reset_mock() head_mock.reset_mock() host = 'caom2workshop.canfar.net' subject = auth.Subject(netrc=True) subject._hosts_auth[host] = ('auser', 'apasswd') client = ws.BaseWsClient(resource_id, subject, 'TestApp', retry=False, host=host) base_url = 'http://{}{}/auth-observations'.format( host, resource_uri.path) resource_url = '{}/{}'.format(base_url, resource) self.assertEqual('TestApp', client.agent) self.assertFalse(client.retry) client.get(resource_url) get_mock.assert_called_with(resource_url, params=None) params = {'arg1': 'abc', 'arg2': 123, 'arg3': True} client.post(resource_url, **params) post_mock.assert_called_with(resource_url, **params) client.delete(resource_url) delete_mock.assert_called_with(resource_url) client.head(resource_url) head_mock.assert_called_with(resource_url) client.put(resource_url, **params) put_mock.assert_called_with(resource_url, **params) self.assertTrue(isinstance(client._session, ws.RetrySession)) # test cert authentication post_mock.reset_mock() get_mock.reset_mock() put_mock.reset_mock() delete_mock.reset_mock() head_mock.reset_mock() certfile = 'some/certfile.pem' client = ws.BaseWsClient(resource_id, auth.Subject(certificate=certfile), 'TestApp') base_url = 'https://{}{}/observations'.format(resource_uri.netloc, resource_uri.path) resource_url = '{}/{}'.format(base_url, resource) self.assertEqual('TestApp', client.agent) self.assertTrue(client.retry) client.get(resource_url) get_mock.assert_called_with(resource_url, params=None) params = {'arg1': 'abc', 'arg2': 123, 'arg3': True} client.post(resource_url, **params) post_mock.assert_called_with(resource_url, **params) client.delete(resource_url) delete_mock.assert_called_with(resource_url) client.head(resource_url) head_mock.assert_called_with(resource_url) client.put(resource_url, **params) put_mock.assert_called_with(resource_url, **params) self.assertTrue(isinstance(client._session, ws.RetrySession)) self.assertEqual((certfile, certfile), client._session.cert)
def test_misc(self): """ Tests miscellaneous functions """ service_url = 'http://somehost.com/service' with patch('cadcutils.net.ws.WsCapabilities') as caps_mock: caps_mock.return_value.get_service_host.return_value =\ 'somehost.com' caps_mock.return_value.get_access_url.return_value = service_url client = ws.BaseWsClient("someresourceID", auth.Subject(), 'TestApp') self.assertEqual('{}'.format(service_url), client._get_url(('myfeature', None))) caps_mock.return_value.get_access_url.assert_called_with( 'myfeature') self.assertEqual('{}'.format(service_url), client._get_url(('myfeature', ''))) test_host = 'testhost.com' with patch('cadcutils.net.ws.os.makedirs'): client = ws.BaseWsClient("someresourceID", auth.Subject(), 'TestApp', host=test_host) self.assertEqual(test_host, client.host) # test with resource as url with patch('cadcutils.net.ws.WsCapabilities') as caps_mock: caps_mock.return_value.get_service_host.return_value =\ 'somehost.com' cm = Mock() cm.get_access_url.return_value = "http://host/availability" caps_mock.return_value = cm client = ws.BaseWsClient("someresourceID", auth.Subject(), 'TestApp') resource_url = 'http://someurl.com/path/' self.assertEqual(resource_url, client._get_url(resource_url)) # repeat with overriden host name client = ws.BaseWsClient("someresourceID", auth.Subject(), 'TestApp', host=test_host) # same result self.assertEqual(resource_url, client._get_url(resource_url)) # test exceptions with different status in the response session = ws.RetrySession() response = requests.Response() response.status_code = requests.codes.not_found with self.assertRaises(exceptions.NotFoundException): session.check_status(response) response.status_code = requests.codes.unauthorized with self.assertRaises(exceptions.UnauthorizedException): session.check_status(response) response.status_code = requests.codes.forbidden with self.assertRaises(exceptions.ForbiddenException): session.check_status(response) response.status_code = requests.codes.bad_request with self.assertRaises(exceptions.BadRequestException): session.check_status(response) response.status_code = requests.codes.conflict with self.assertRaises(exceptions.AlreadyExistsException): session.check_status(response) response.status_code = requests.codes.internal_server_error with self.assertRaises(exceptions.InternalServerException): session.check_status(response) response.status_code = requests.codes.unavailable with self.assertRaises(requests.HTTPError): session.check_status(response) response.status_code = requests.codes.not_extended with self.assertRaises(exceptions.UnexpectedException): session.check_status(response)
def test_get_caps(self, get_mock, file_modtime_mock): """ Tests the capabilities part of WsCapabilities """ # test when registry information is read from the server # (cache is outdated) service = 'myservice' resource_id = 'ivo://canfar.phys.uvic.ca/{}'.format(service) resource_cap_url = 'www.canfar.net/myservice' # set the modified time of the cache file to 0 to make sure the # info is retrieved from server file_modtime_mock.return_value = 0 # test anonymous access expected_content = capabilities__content.replace( 'WS_URL', resource_cap_url) response = Mock(text=expected_content) get_mock.return_value = response caps = ws.WsCapabilities( Mock(resource_id=resource_id, subject=auth.Subject())) # mock _get_capability_url to return some url without attempting # to access the server def get_url(): return 'http://some.url/capabilities' caps._get_capability_url = get_url # remove the cached file if exists if os.path.isfile(caps.caps_file): os.remove(caps.caps_file) caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url) self.assertEqual( 'http://{}/capabilities'.format(resource_cap_url), caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities')) self.assertEqual( 'http://{}/availability'.format(resource_cap_url), caps.get_access_url('ivo://ivoa.net/std/VOSI#availability')) self.assertEqual( 'http://{}/pub'.format(resource_cap_url), caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01')) actual_content = open(caps.caps_file, 'r').read() self.assertEqual(expected_content, actual_content) # mock _get_capability_url to return a subservice service = 'myservice/mysubservice' resource_id = 'ivo://canfar.phys.uvic.ca/{}'.format(service) resource_cap_url = 'www.canfar.net/myservice/mysubservice' # set the modified time of the cache file to 0 to make sure the # info is retrieved from server file_modtime_mock.return_value = 0 expected_content = capabilities__content.replace( 'WS_URL', resource_cap_url) # test anonymous access response = Mock(text=expected_content) get_mock.return_value = response caps = ws.WsCapabilities( Mock(resource_id=resource_id, subject=auth.Subject())) # remove the cached file if exists if os.path.isfile(caps.caps_file): os.remove(caps.caps_file) caps._get_capability_url = get_url caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url) self.assertEqual( 'http://{}/capabilities'.format(resource_cap_url), caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities')) self.assertEqual( 'http://{}/availability'.format(resource_cap_url), caps.get_access_url('ivo://ivoa.net/std/VOSI#availability')) self.assertEqual( 'http://{}/pub'.format(resource_cap_url), caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01')) actual_content = open(caps.caps_file, 'r').read() self.assertEqual(expected_content, actual_content) # repeat for basic auth subject. Mock the netrc library to # prevent a lookup for $HOME/.netrc with patch('cadcutils.net.auth.netrclib'): client = Mock(resource_id=resource_id, subject=auth.Subject(netrc=True)) client.get.return_value = response caps = ws.WsCapabilities(client) caps._get_capability_url = get_url # capabilities works even if it has only one anonymous interface self.assertEqual( 'http://{}/capabilities'.format(resource_cap_url), caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities')) # same for availability self.assertEqual( 'http://{}/availability'.format(resource_cap_url), caps.get_access_url('ivo://ivoa.net/std/VOSI#availability')) # repeat for https with patch('os.path.isfile'): client = Mock(resource_id=resource_id, subject=auth.Subject(certificate='somecert.pem')) client.get.return_value = response caps = ws.WsCapabilities(client) caps._get_capability_url = get_url # capabilities works even if it has only one anonymous interface self.assertEqual( 'http://{}/capabilities'.format(resource_cap_url), caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities')) # same for availability self.assertEqual( 'http://{}/availability'.format(resource_cap_url), caps.get_access_url('ivo://ivoa.net/std/VOSI#availability')) self.assertEqual( 'https://{}'.format(resource_cap_url), caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01')) # test when capabilities information is retrieved from the cache file get_mock.reset_mock() get_mock.return_value = None file_modtime_mock.reset_mock() service = 'myservice2' resource_id = 'ivo://canfar.phys.uvic.ca/{}'.format(service) resource_cap_url2 = 'canfar.phys.uvic.ca/myservice2' expected_content = capabilities__content.replace( 'WS_URL', resource_cap_url2) file_modtime_mock.return_value = time.time() client = Mock(resource_id=resource_cap_url2, subject=auth.Subject()) caps = ws.WsCapabilities(client) caps._get_capability_url = get_url caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url2) # manually write the content with open(caps.caps_file, 'w') as f: f.write(expected_content) self.assertEqual( 'http://{}/capabilities'.format(resource_cap_url2), caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities')) self.assertEqual( 'http://{}/availability'.format(resource_cap_url2), caps.get_access_url('ivo://ivoa.net/std/VOSI#availability')) self.assertEqual( 'http://{}/pub'.format(resource_cap_url2), caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01')) actual_content = open(caps.caps_file, 'r').read() self.assertEqual(expected_content, actual_content) # repeat for basic auth subject. Mock the netrc library to prevent a # lookup for $HOME/.netrc with patch('cadcutils.net.auth.netrclib'): client = Mock(resource_id=resource_id, subject=auth.Subject(netrc=True)) caps = ws.WsCapabilities(client) caps._get_capability_url = get_url # does not work with user-password of the subject set above self.assertEqual( 'http://{}/capabilities'.format(resource_cap_url2), caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities')) self.assertEqual( 'http://{}/availability'.format(resource_cap_url2), caps.get_access_url('ivo://ivoa.net/std/VOSI#availability')) self.assertEqual( 'http://{}/auth'.format(resource_cap_url2), caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01')) # repeat for https with patch('os.path.isfile'): client = Mock(resource_id=resource_id, subject=auth.Subject(certificate='somecert.pem')) caps = ws.WsCapabilities(client) caps._get_capability_url = get_url # does not work with user-password of the subject set above self.assertEqual( 'http://{}/capabilities'.format(resource_cap_url2), caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities')) self.assertEqual( 'http://{}/availability'.format(resource_cap_url2), caps.get_access_url('ivo://ivoa.net/std/VOSI#availability')) self.assertEqual( 'https://{}'.format(resource_cap_url2), caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))
def test_process(self): core.BATCH_SIZE = 3 # size of the batch is 3 obs = [['a', 'b', 'c'], ['d'], []] level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) visitor.get_observation = MagicMock(return_value=MagicMock( spec=SimpleObservation)) visitor.post_observation = MagicMock() visitor._get_observations = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') self.assertEqual(4, len(visited)) self.assertEqual(4, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) obs = [['a', 'b', 'c'], ['d', 'e', 'f'], []] visitor._get_observations = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') self.assertEqual(6, len(visited)) self.assertEqual(6, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) # make it return different status. errorplugin returns according to the # id of the observation: True for 'UPDATE', False for 'SKIP' and # raises exception for 'ERROR' obs_ids = [['UPDATE', 'SKIP', 'ERROR'], []] obs = [ SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP'), SimpleObservation(collection='TEST', observation_id='ERROR') ] visitor._get_observations = MagicMock(side_effect=obs_ids) visitor.get_observation = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht') self.assertEqual(3, len(visited)) self.assertEqual(1, len(updated)) self.assertEqual(1, len(skipped)) self.assertEqual(1, len(failed)) # repeat with other obs obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']] obs = [ SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP'), SimpleObservation(collection='TEST', observation_id='ERROR'), SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP') ] visitor._get_observations = MagicMock(side_effect=obs_ids) visitor.get_observation = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht') self.assertEqual(5, len(visited)) self.assertEqual(2, len(updated)) self.assertEqual(2, len(skipped)) self.assertEqual(1, len(failed)) # repeat but halt on first ERROR -> process only 3 observations obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']] obs = [ SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP'), SimpleObservation(collection='TEST', observation_id='ERROR'), SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP') ] visitor._get_observations = MagicMock(side_effect=obs_ids) visitor.get_observation = MagicMock(side_effect=obs) with self.assertRaises(SystemError): visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht', halt_on_error=True) # test with time boundaries core.BATCH_SIZE = 3 # size of the batch is 3 response = MagicMock() response.text = """ARCHIVE\ta\t2011-01-01T11:00:00.000 ARCHIVE\tb\t211-01-01T11:00:10.000 ARCHIVE\tc\t2011-01-01T12:00:00.000""" response2 = MagicMock() response2.text = """ARCHIVE\td\t2011-02-02T11:00:00.000""" level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) visitor.get_observation = MagicMock(return_value=MagicMock( spec=SimpleObservation)) visitor.post_observation = MagicMock() visitor._repo_client.get = MagicMock(side_effect=[response, response2]) start = '2010-10-10T12:00:00.000' end = '2012-12-12T11:11:11.000' (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht', start=util.str2ivoa(start), end=util.str2ivoa(end)) self.assertEqual(4, len(visited)) self.assertEqual(4, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) calls = [ call((core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'), params={ 'START': start, 'END': end, 'MAXREC': 3 }), call( (core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'), params={ 'START': '2011-01-01T12:00:00.000', # datetime of the last record in the batch 'END': end, 'MAXREC': 3 }) ] visitor._repo_client.get.assert_has_calls(calls)
def test_get_caps(self, get_mock, file_mock, file_modtime_mock): """ Tests the capabilities part of WsCapabilities """ # test when registry information is read from the server # (cache is outdated) service = 'myservice' resource_id = 'ivo://canfar.phys.uvic.ca/{}'.format(service) resource_cap_url = 'www.canfar.net/myservice' # set the modified time of the cache file to 0 to make sure the # info is retrieved from server file_modtime_mock.return_value = 0 # test anonymous access fh_mock = Mock() file_mock.write = fh_mock response = Mock( text=capabilities__content.replace('WS_URL', resource_cap_url)) get_mock.return_value = response caps = ws.WsCapabilities(Mock(resource_id=resource_id, subject=auth.Subject())) # mock _get_capability_url to return some url without attempting # to access the server def get_url(): return 'http://some.url/capabilities' caps._get_capability_url = get_url caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url) self.assertEqual('http://{}/capabilities'.format(resource_cap_url), caps.get_access_url( 'ivo://ivoa.net/std/VOSI#capabilities')) self.assertEqual('http://{}/availability'.format(resource_cap_url), caps.get_access_url( 'ivo://ivoa.net/std/VOSI#availability')) self.assertEqual('http://{}/pub'.format(resource_cap_url), caps.get_access_url( 'vos://cadc.nrc.ca~service/CADC/mystnd01')) resource_url = urlparse(resource_id) file_mock.assert_called_once_with(os.path.join(ws.CACHE_LOCATION, resource_url.netloc, resource_url.path.strip( '/')), 'w') # TODO not sure why need to access write this way file_mock().__enter__.return_value.write.assert_called_once_with( capabilities__content.replace('WS_URL', resource_cap_url)) # repeat for basic auth subject. Mock the netrc library to # prevent a lookup for $HOME/.netrc with patch('cadcutils.net.auth.netrclib'): client = Mock(resource_id=resource_id, subject=auth.Subject(netrc=True)) client.get.return_value = response caps = ws.WsCapabilities(client) caps._get_capability_url = get_url # capabilities works even if it has only one anonymous interface self.assertEqual('http://{}/capabilities'.format(resource_cap_url), caps.get_access_url( 'ivo://ivoa.net/std/VOSI#capabilities')) # same for availability self.assertEqual('http://{}/availability'.format(resource_cap_url), caps.get_access_url( 'ivo://ivoa.net/std/VOSI#availability')) # repeat for https with patch('os.path.isfile'): client = Mock(resource_id=resource_id, subject=auth.Subject(certificate='somecert.pem')) client.get.return_value = response caps = ws.WsCapabilities(client) caps._get_capability_url = get_url # capabilities works even if it has only one anonymous interface self.assertEqual('http://{}/capabilities'.format(resource_cap_url), caps.get_access_url( 'ivo://ivoa.net/std/VOSI#capabilities')) # same for availability self.assertEqual('http://{}/availability'.format(resource_cap_url), caps.get_access_url( 'ivo://ivoa.net/std/VOSI#availability')) self.assertEqual('https://{}'.format(resource_cap_url), caps.get_access_url( 'vos://cadc.nrc.ca~service/CADC/mystnd01')) # test when capabilities information is retrieved from the cache file get_mock.reset_mock() get_mock.return_value = None file_modtime_mock.reset_mock() file_mock.reset_mock() resource_cap_url2 = 'http://www.canfar.net/myservice2' cache_content2 = capabilities__content.replace('WS_URL', resource_cap_url2) file_modtime_mock.return_value = time.time() file_mock().__enter__.return_value.read.return_value = cache_content2 client = Mock(resource_id=resource_id, subject=auth.Subject()) caps = ws.WsCapabilities(client) caps._get_capability_url = get_url caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url2) self.assertEqual('http://{}/capabilities'.format(resource_cap_url2), caps.get_access_url( 'ivo://ivoa.net/std/VOSI#capabilities')) self.assertEqual('http://{}/availability'.format(resource_cap_url2), caps.get_access_url( 'ivo://ivoa.net/std/VOSI#availability')) self.assertEqual('http://{}/pub'.format(resource_cap_url2), caps.get_access_url( 'vos://cadc.nrc.ca~service/CADC/mystnd01')) resource_url = urlparse(resource_id) file_mock().__enter__.return_value.read.return_value = \ capabilities__content.replace('WS_URL', resource_cap_url2) # repeat for basic auth subject. Mock the netrc library to prevent a # lookup for $HOME/.netrc with patch('cadcutils.net.auth.netrclib'): client = Mock(resource_id=resource_id, subject=auth.Subject(netrc=True)) caps = ws.WsCapabilities(client) caps._get_capability_url = get_url # does not work with user-password of the subject set above self.assertEqual('http://{}/capabilities'.format(resource_cap_url2), caps.get_access_url( 'ivo://ivoa.net/std/VOSI#capabilities')) self.assertEqual('http://{}/availability'.format(resource_cap_url2), caps.get_access_url( 'ivo://ivoa.net/std/VOSI#availability')) self.assertEqual('http://{}/auth'.format(resource_cap_url2), caps.get_access_url( 'vos://cadc.nrc.ca~service/CADC/mystnd01')) # repeat for https with patch('os.path.isfile'): client = Mock(resource_id=resource_id, subject=auth.Subject(certificate='somecert.pem')) caps = ws.WsCapabilities(client) caps._get_capability_url = get_url # does not work with user-password of the subject set above self.assertEqual('http://{}/capabilities'.format(resource_cap_url2), caps.get_access_url( 'ivo://ivoa.net/std/VOSI#capabilities')) self.assertEqual('http://{}/availability'.format(resource_cap_url2), caps.get_access_url( 'ivo://ivoa.net/std/VOSI#availability')) self.assertEqual('https://{}'.format(resource_cap_url2), caps.get_access_url( 'vos://cadc.nrc.ca~service/CADC/mystnd01'))