def test_plugin_class(self):
# plugin class does not change the observation
collection = 'cfht'
observation_id = '7000000o'
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
obs = SimpleObservation(collection, observation_id)
expect_obs = copy.deepcopy(obs)
visitor._load_plugin_class(os.path.join(THIS_DIR, 'passplugin.py'))
visitor.plugin.update(obs)
self.assertEqual(expect_obs, obs)
# plugin class adds a plane to the observation
visitor = CAOM2RepoClient(auth.Subject(), level)
obs = SimpleObservation('cfht', '7000000o')
expect_obs = copy.deepcopy(obs)
visitor._load_plugin_class(os.path.join(THIS_DIR, 'addplaneplugin.py'))
visitor.plugin.update(obs)
self.assertNotEqual(expect_obs, obs)
self.assertEqual(len(expect_obs.planes) + 1, len(obs.planes))
# non-existent the plugin file
with self.assertRaises(Exception):
visitor._load_plugin_class(os.path.join(THIS_DIR, 'blah.py'))
# non-existent ObservationUpdater class in the plugin file
with self.assertRaises(Exception):
visitor._load_plugin_class(
os.path.join(THIS_DIR, 'test_visitor.py'))
# non-existent update method in ObservationUpdater class
with self.assertRaises(Exception):
visitor._load_plugin_class(
os.path.join(THIS_DIR, 'noupdateplugin.py'))
def test_create_and_visit(self):
#logger.setLevel(logging.DEBUG)
logger.info('-- START: test_create_and_visit --')
start = datetime.now()
name = obs_name = 'caom2pyinttest{}'.format(start.microsecond)
try:
env_a = os.environ['A']
cert_file = env_a + '/test-certificates/x509_CADCAuthtest1.pem'
subject = auth.Subject(certificate=cert_file)
client = CAOM2RepoClient(subject)
# create one observation for today
algorithm = observation.SimpleObservation._DEFAULT_ALGORITHM_NAME
logger.debug("test obs name {}".format(name))
obs = observation.SimpleObservation("TEST", obs_name)
obs.algorithm = algorithm
client.put_observation(obs)
plugin = os.path.join(THIS_DIR, 'visitor-plugin.py')
(visited, updated, skipped, failed) = client.visit(plugin, 'TEST', start=start, halt_on_error=True)
logger.debug("observations visited: {}".format(len(visited)))
self.assertGreater(len(visited), 0, msg="No Observations Visited")
finally:
try:
client.delete_observation("TEST", name)
except:
logger.warning('Failed to delete test observation, continuing')
logger.info('-- END :test_create_and_visit --')
def test_visit_retry_on_412(self):
# observation changed on server while visited
core.BATCH_SIZE = 3 # size of the batch is 3
obs = [['a'], []]
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
observation = SimpleObservation('cfht', 'a')
observation.acc_meta_checksum = ChecksumURI('md5:abc')
visitor.get_observation = MagicMock(
side_effect=[observation, observation])
exception_412 = exceptions.UnexpectedException()
exception_412.orig_exception = Mock()
exception_412.orig_exception.response = Mock(status_code=412)
visitor.post_observation = MagicMock(side_effect=[exception_412, None])
visitor._get_observations = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht')
self.assertEqual(1, len(visited))
self.assertEqual(1, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
# get and post called twice to recover from error HTTP status 412 -
# precondition
self.assertEqual(2, visitor.get_observation.call_count)
self.assertEqual(2, visitor.post_observation.call_count)
visitor.post_observation.assert_called_with(
observation, observation.acc_meta_checksum.uri)
def test_multiprocess_with_unexpected_type_error(self, get_mock):
core.BATCH_SIZE = 3 # size of the batch is 3
obs_ids = [['a', 'b', 'c'], ['d'], []]
get_mock.side_effect = \
self.mock_get_observation_with_unexpected_type_error
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
visitor.get_observation = PickableMagicMock(
return_value=PickableMagicMock(spec=SimpleObservation))
visitor.post_observation = PickableMagicMock()
visitor._get_observations = PickableMagicMock(side_effect=obs_ids)
try:
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht',
start=None,
end=None,
obs_file=None,
nthreads=3,
halt_on_error=True)
except TypeError as e:
self.assertTrue("unexpected TypeError" in str(e))
finally:
logging.info("DONE")
def test_multiprocess_with_more_different_statuses(self, get_mock):
core.BATCH_SIZE = 3 # size of the batch is 3
# make it return different status. errorplugin returns according to the
# id of the observation: True for 'UPDATE', False for 'SKIP' and
# raises exception for 'ERROR'
obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']]
get_mock.side_effect = self.mock_get_observation
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
visitor.get_observation = PickableMagicMock(
return_value=PickableMagicMock(spec=SimpleObservation))
visitor.post_observation = PickableMagicMock()
visitor._get_observations = PickableMagicMock(side_effect=obs_ids)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
'cfht',
start=None,
end=None,
obs_file=None,
nthreads=3)
try:
self.assertEqual(5, len(visited))
self.assertEqual(2, len(updated))
self.assertEqual(2, len(skipped))
self.assertEqual(1, len(failed))
finally:
# lp.join()
logging.info("DONE")
def test_post_observation(self, mock_conn, caps_mock):
caps_mock.get_service_host.return_value = 'some.host.com'
caps_mock.return_value.get_access_url.return_value =\
'http://serviceurl/caom2repo/auth'
collection = 'cfht'
observation_id = '7000000o'
service = 'caom2repo'
service_url = 'www.cadc.nrc.ca'
obs = SimpleObservation(collection, observation_id)
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(netrc='somenetrc'),
level,
host=service_url)
response = MagicMock()
response.status = 200
mock_conn.return_value = response
iobuffer = BytesIO()
ObservationWriter().write(obs, iobuffer)
obsxml = iobuffer.getvalue()
response.content = obsxml
visitor.post_observation(obs)
self.assertEqual('POST', mock_conn.call_args[0][0].method)
self.assertEqual(
'/{}/auth/{}/{}'.format(service, collection, observation_id),
mock_conn.call_args[0][0].path_url)
self.assertEqual('application/xml',
mock_conn.call_args[0][0].headers['Content-Type'])
self.assertEqual(obsxml, mock_conn.call_args[0][0].body)
# signal problems
http_error = requests.HTTPError()
response.status_code = 500
http_error.response = response
response.raise_for_status.side_effect = [http_error]
with self.assertRaises(exceptions.InternalServerException):
visitor.update(obs)
# temporary transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
response.raise_for_status.side_effect = [http_error, None]
visitor.post_observation(obs)
# permanent transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
def raise_error():
raise http_error
response.raise_for_status.side_effect = raise_error
with self.assertRaises(exceptions.HttpException):
visitor.post_observation(obs)
def test_get_observations(self, mock_get, caps_mock):
# This is almost similar to the previous test except that it gets
# observations matching a collection and start/end criteria
# Also, patch the CAOM2RepoClient now.
caps_mock.get_service_host.return_value = 'some.host.com'
caps_mock.return_value.get_access_url.return_value =\
'http://serviceurl/caom2repo/pub'
response = MagicMock()
response.status_code = 200
last_datetime = '2000-10-10T12:30:00.333'
response.text = \
('CFHT\t700000o\t2000-10-10T12:20:11.123\t'
'3e00ca6129dc8358315015204ab9fe15\nCFHT\t700001o\t' +
last_datetime + '\t3e00ca6129dc8358315015204ab9fe15')
mock_get.return_value = response
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
end_date = util.utils.str2ivoa(last_datetime)
expect_observations = ['700000o', '700001o']
self.assertEqual(expect_observations,
visitor._get_observations('cfht'))
self.assertEqual(end_date, visitor._start)
mock_get.assert_called_once_with(
('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2',
'cfht'),
params={'MAXREC': core.BATCH_SIZE})
mock_get.reset_mock()
visitor._get_observations('cfht',
end=datetime.strptime(
'2000-11-11', '%Y-%m-%d'))
mock_get.assert_called_once_with(
('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2',
'cfht'),
params={
'END': '2000-11-11T00:00:00.000',
'MAXREC': core.BATCH_SIZE
})
mock_get.reset_mock()
visitor._get_observations(
'cfht',
start=datetime.strptime('2000-11-11', '%Y-%m-%d'),
end=datetime.strptime('2000-11-12', '%Y-%m-%d'))
mock_get.assert_called_once_with(
('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2',
'cfht'),
params={
'START': '2000-11-11T00:00:00.000',
'END': '2000-11-12T00:00:00.000',
'MAXREC': core.BATCH_SIZE
})
def test_get_observation(self, mock_get, caps_mock):
caps_mock.get_service_host.return_value = 'some.host.com'
caps_mock.return_value.get_access_url.return_value =\
'http://serviceurl/caom2repo/pub'
collection = 'cfht'
observation_id = '7000000o'
service_url = 'www.cadc.nrc.ca/caom2repo'
obs = SimpleObservation(collection, observation_id)
writer = ObservationWriter()
ibuffer = BytesIO()
writer.write(obs, ibuffer)
response = MagicMock()
response.status_code = 200
response.content = ibuffer.getvalue()
mock_get.return_value = response
ibuffer.seek(0) # reposition the buffer for reading
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url)
self.assertEqual(obs,
visitor.get_observation(collection, observation_id))
# signal problems
http_error = requests.HTTPError()
response.status_code = 500
http_error.response = response
response.raise_for_status.side_effect = [http_error]
with self.assertRaises(exceptions.InternalServerException):
visitor.get_observation(collection, observation_id)
# temporary transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
response.raise_for_status.side_effect = [http_error, None]
visitor.read(collection, observation_id)
# permanent transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
def raise_error():
raise http_error
response.raise_for_status.side_effect = raise_error
with self.assertRaises(exceptions.HttpException):
visitor.get_observation(collection, observation_id)
def test_delete_observation(self, mock_conn, caps_mock):
caps_mock.get_service_host.return_value = 'some.host.com'
caps_mock.return_value.get_access_url.return_value =\
'http://serviceurl/caom2repo/pub'
collection = 'cfht'
observation_id = '7000000o'
service_url = 'www.cadc.nrc.ca'
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url)
response = MagicMock()
response.status = 200
mock_conn.return_value = response
visitor.delete_observation(collection, observation_id)
self.assertEqual('DELETE', mock_conn.call_args[0][0].method)
# signal problems
http_error = requests.HTTPError()
response.status_code = 500
http_error.response = response
response.raise_for_status.side_effect = [http_error]
with self.assertRaises(exceptions.InternalServerException):
visitor.delete(collection, observation_id)
# temporary transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
response.raise_for_status.side_effect = [http_error, None]
visitor.delete_observation(collection, observation_id)
# permanent transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
def raise_error():
raise http_error
response.raise_for_status.side_effect = raise_error
with self.assertRaises(exceptions.HttpException):
visitor.delete_observation(collection, observation_id)
def test_get_obs_from_file(self):
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
# no start or end
with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file:
obs_id_list = visitor._get_obs_from_file(obs_file, None, None,
False)
self.assertEqual('obs_id_1', obs_id_list[0])
self.assertEqual('obs_id_2', obs_id_list[1])
self.assertEqual('obs_id_3', obs_id_list[2])
# last_modified_date is earlier than start
with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file:
obs_id_list = visitor._get_obs_from_file(
obs_file, util.str2ivoa('2000-10-11T12:30:00.333'), None,
False)
self.assertEqual('obs_id_1', obs_id_list[0])
# last_modified_date is between start and end
with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file:
obs_id_list = visitor._get_obs_from_file(
obs_file, util.str2ivoa('2000-10-9T12:30:00.333'),
util.str2ivoa('2016-10-11T12:30:00.333'), False)
self.assertEqual('obs_id_1', obs_id_list[0])
self.assertEqual('obs_id_2', obs_id_list[1])
# last_modified_date is after end
with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file:
obs_id_list = visitor._get_obs_from_file(
obs_file, util.str2ivoa('2000-10-9T12:30:00.333'),
util.str2ivoa('2017-10-11T12:30:00.333'), False)
self.assertEqual('obs_id_1', obs_id_list[0])
self.assertEqual('obs_id_2', obs_id_list[1])
self.assertEqual('obs_id_3', obs_id_list[2])
# error in file
with open(os.path.join(THIS_DIR, 'data/obs_id_error.txt')) as obs_file:
with self.assertRaises(Exception):
obs_id_list = visitor._get_obs_from_file(
obs_file, util.str2ivoa('2000-10-9T12:30:00.333'),
util.str2ivoa('2016-10-11T12:30:00.333'), True)
def test_multiprocess_with_more_obs_id(self, client_mock):
core.BATCH_SIZE = 3 # size of the batch is 3
obs_ids = [['a', 'b', 'c'], ['d', 'e', 'f'], []]
client_mock.return_value.get_observation.side_effect = \
self.mock_get_observation
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
visitor.get_observation = PickableMagicMock(
return_value=PickableMagicMock(spec=SimpleObservation))
visitor.post_observation = PickableMagicMock()
visitor._get_observations = PickableMagicMock(side_effect=obs_ids)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht',
start=None,
end=None,
obs_file=None,
nthreads=3)
try:
self.assertEqual(6, len(visited))
self.assertEqual(6, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
self.assertTrue('a' in visited)
self.assertTrue('b' in visited)
self.assertTrue('c' in visited)
self.assertTrue('d' in visited)
self.assertTrue('e' in visited)
self.assertTrue('f' in visited)
self.assertFalse('g' in visited)
self.assertTrue('a' in updated)
self.assertTrue('b' in updated)
self.assertTrue('c' in updated)
self.assertTrue('d' in updated)
self.assertTrue('e' in updated)
self.assertTrue('f' in updated)
self.assertFalse('g' in updated)
finally:
# lp.join()
logging.info("DONE")
def test_shortcuts(self):
level = logging.DEBUG
target = CAOM2RepoClient(auth.Subject(), level)
obs = SimpleObservation('CFHT', 'abc')
target.put_observation = Mock()
target.create(obs)
target.put_observation.assert_called_with(obs)
target.get_observation = Mock()
target.read('CFHT', 'abc')
target.get_observation.assert_called_with('CFHT', 'abc')
target.post_observation = Mock()
target.update(obs)
target.post_observation.assert_called_with(obs)
target.delete_observation = Mock()
target.delete('CFHT', 'abc')
target.delete_observation.assert_called_with('CFHT', 'abc')
def test_info_file(basews_mock):
client = CadcDataClient(auth.Subject())
# test an info
file_name = 'myfile'
file_name = 'myfile.txt'
archive = 'TEST'
size = '123'
md5sum = '0x123'
type = 'txt'
encoding = 'gzip'
lastmod = '11/11/11T11:11:11.000'
usize = '1234'
umd5sum = '0x1234'
h = {}
h['Content-Disposition'] = 'inline; filename={}'.format(file_name)
h['Content-Length'] = size
h['Content-MD5'] = md5sum
h['Content-Type'] = type
h['Content-Encoding'] = encoding
h['Last-Modified'] = lastmod
h['X-Uncompressed-Length'] = usize
h['X-Uncompressed-MD5'] = umd5sum
response = Mock()
response.headers = h
basews_mock.return_value.head.return_value = response
info = client.get_file_info('TEST', 'myfile')
assert archive == info['archive']
assert file_name == info['name']
assert size == info['size']
assert md5sum == info['md5sum']
assert type == info['type']
assert encoding == info['encoding']
assert lastmod == info['lastmod']
assert usize == info['usize']
assert umd5sum == info['umd5sum']
def test_put_file(basews_mock):
client = CadcDataClient(auth.Subject())
# test a put
file_name = '/tmp/putfile.txt'
file_content = 'ABCDEFGH12345'
hash_md5 = hashlib.md5()
hash_md5.update(file_content.encode())
hash_md5 = hash_md5.hexdigest()
# write the file
with open(file_name, 'w') as f:
f.write(file_content)
put_mock = Mock()
basews_mock.return_value.put = put_mock
with pytest.raises(exceptions.UnauthorizedException):
client.put_file('TEST', 'putfile', file_name)
client._data_client.subject.anon = False # authenticate the user
transf_end_point = 'http://test.ca/endpoint'
def mock_get_trans_protocols(archive, file_name, is_get, headers):
protocol = Mock()
protocol.endpoint = '{}/{}'.format(transf_end_point, file_name)
return [protocol]
client._get_transfer_protocols = mock_get_trans_protocols
client.put_file('TEST', file_name)
# Note Content* headers automatically created by cadc-data except when
# MAGIC_WANT -- libmagic not present
put_mock.assert_called_with('{}/{}'.format(transf_end_point,
os.path.basename(file_name)),
data=ANY,
headers={
'Content-Type': 'text/plain',
'Content-Encoding': 'us-ascii',
'Content-MD5': '{}'.format(hash_md5)
})
# mimic libmagic missing
cadcdata.core.MAGIC_WARN = 'Some warning'
put_mock.reset_mock()
client.put_file('TEST', file_name)
put_mock.assert_called_with(
'{}/{}'.format(transf_end_point, os.path.basename(file_name)),
data=ANY,
headers={'Content-MD5': '835e7e6cd54e18ae21d50af963b0c32b'})
cadcdata.core.MAGIC_WARN = None
# specify an archive stream and override the name of the file
input_name = 'abc'
client.put_file('TEST',
file_name,
archive_stream='default',
input_name=input_name)
put_mock.assert_called_with('{}/{}'.format(transf_end_point, input_name),
data=ANY,
headers={
'Content-Encoding': 'us-ascii',
'X-CADC-Stream': 'default',
'Content-Type': 'text/plain',
'Content-MD5': '{}'.format(hash_md5)
})
# specify the mime types
client.put_file('TEST',
file_name,
archive_stream='default',
mime_type='ASCII',
mime_encoding='GZIP')
put_mock.assert_called_with('{}/{}'.format(transf_end_point,
os.path.basename(file_name)),
data=ANY,
headers={
'Content-Encoding': 'GZIP',
'X-CADC-Stream': 'default',
'Content-Type': 'ASCII',
'Content-MD5': '{}'.format(hash_md5)
})
os.remove(file_name)
def test_get_file(trans_reader_mock, basews_mock):
# test a simple get - no decompress
file_name = '/tmp/afile.txt'
file_chunks = ['aaaa'.encode(), 'bbbb'.encode(), ''.encode()]
response = Mock()
hash_md5 = hashlib.md5()
for i in file_chunks:
hash_md5.update(i)
response.headers.get.return_value = \
'filename={}'.format('orig_file_name')
response.raw.read.side_effect = file_chunks # returns multiple blocks
basews_mock.return_value.get.return_value = response
client = CadcDataClient(auth.Subject())
with pytest.raises(exceptions.HttpException):
# no URLs returned in the transfer negotiations
client.get_file('TEST', 'afile', destination=file_name)
t = transfer.Transfer('ad:TEST/afile', 'pullFromVoSpace')
p = transfer.Protocol
p.endpoint = Mock()
t.protocols = [p]
trans_reader_mock.return_value.read.return_value = t
client.get_file('TEST', 'afile', destination=file_name, md5_check=False)
expected_content = \
(''.join([c.decode() for c in file_chunks])).encode()
with open(file_name, 'rb') as f:
assert expected_content == f.read()
os.remove(file_name)
# do it again with the file now open
response = Mock()
response.headers = {
'filename': 'orig_file_name',
'content-MD5': hash_md5.hexdigest()
}
response.raw.read.side_effect = file_chunks
basews_mock.return_value.get.return_value = response
with open(file_name, 'wb') as f:
client.get_file('TEST', 'afile', destination=f)
with open(file_name, 'rb') as f:
assert expected_content == f.read()
os.remove(file_name)
# test a get with decompress and md5 check enabled
file_name = 'bfile.txt'
file_content = 'aaaabbbb'
hash_md5 = hashlib.md5()
hash_md5.update(file_content.encode())
file_chunks = [file_content.encode(), ''.encode()]
decoded_file_content = 'MNOPRST6789'
decoded_file_chunks = [decoded_file_content.encode(), ''.encode()]
response = Mock()
response.headers = \
{'content-MD5': '{}'.format(hash_md5.hexdigest()),
'filename': file_name}
response.raw.read.side_effect = file_chunks
response.raw._decode.side_effect = decoded_file_chunks
basews_mock.return_value.get.return_value = response
client = CadcDataClient(auth.Subject())
client.get_file('TEST',
file_name=file_name,
decompress=True,
md5_check=True)
with open(file_name, 'r') as f:
# note the check against the decoded content
assert decoded_file_content == f.read()
os.remove(file_name)
# repeat test with a bad md5
file_name = 'bfile.txt'
file_content = 'ABCDEFGH12345'
file_chunks = [file_content.encode(), ''.encode()]
decoded_file_content = 'MNOPRST6789'
decoded_file_chunks = [decoded_file_content.encode(), ''.encode()]
response = Mock()
response.headers = {'content-MD5': 'abc', 'filename': file_name}
response.raw.read.side_effect = file_chunks
response.raw._decode.side_effect = decoded_file_chunks
basews_mock.return_value.get.return_value = response
client = CadcDataClient(auth.Subject())
with pytest.raises(exceptions.HttpException):
client.get_file('TEST',
file_name=file_name,
decompress=True,
md5_check=True)
# test process_bytes and send the content to /dev/null after.
# Use no decompress
def concatenate_chunks(chunk):
global mycontent
mycontent = '{}{}'.format(mycontent, chunk.decode())
file_name = 'bfile.txt'
file_content = 'ABCDEFGH12345'
file_chunks = [
file_content[i:i + 5].encode() for i in xrange(0, len(file_content), 5)
]
file_chunks.append('') # last chunk is empty
response = Mock()
response.headers = {'filename': '{}.gz'.format(file_name)}
response.raw.read.side_effect = file_chunks
basews_mock.return_value.get.return_value = response
client = CadcDataClient(auth.Subject())
client.logger.setLevel(logging.INFO)
# md5_check does not take place because no content-MD5 received
# from server
client.get_file('TEST',
'afile',
destination='/dev/null',
process_bytes=concatenate_chunks)
assert file_content == mycontent
# failed md5 checksum
response = Mock()
response.headers = {
'filename': '{}.gz'.format(file_name),
'content-MD5': '33'
}
response.raw.read.side_effect = file_chunks
basews_mock.return_value.get.return_value = response
client = CadcDataClient(auth.Subject())
client.logger.setLevel(logging.INFO)
# md5_check does not take place because no content-MD5 received
# from server
with pytest.raises(exceptions.HttpException):
client.get_file('TEST',
'afile',
destination='/dev/null',
process_bytes=concatenate_chunks)
# test get fhead
response = Mock()
response.headers.get.return_value = 'filename={}.gz'.format(file_name)
response.raw.read.side_effect = file_chunks
response.history = []
response.status_code = 200
response.url = 'someurl'
post_mock = Mock(return_value=response)
basews_mock.return_value.post = post_mock
file_name = 'getfile'
archive = 'TEST'
p.endpoint = 'http://someurl/transfer/{}/{}'.format(archive, file_name)
client.get_file('TEST',
'getfile',
decompress=True,
wcs=True,
md5_check=False)
trans_doc = \
('<vos:transfer xmlns:'
'vos="http://www.ivoa.net/xml/VOSpace/v2.0">\n '
'<vos:target>ad:TEST/getfile</vos:target>\n '
'<vos:direction>pullFromVoSpace</vos:direction>\n '
'<vos:protocol uri="ivo://ivoa.net/vospace/core#httpget"/>\n'
' <vos:protocol uri="ivo://ivoa.net/vospace/core#httpsget"/>\n'
'</vos:transfer>\n').encode()
post_mock.assert_called_with(resource=(TRANSFER_RESOURCE_ID, None),
params={'wcs': True},
data=trans_doc,
headers={'Content-Type': 'text/xml'})
response.raw.read.side_effect = file_chunks
post_mock.reset_mock()
client.get_file('TEST',
'getfile',
decompress=True,
fhead=True,
md5_check=False)
post_mock.assert_called_with(resource=(TRANSFER_RESOURCE_ID, None),
params={'fhead': True},
data=trans_doc,
headers={'Content-Type': 'text/xml'})
response.raw.read.side_effect = file_chunks
post_mock.reset_mock()
client.get_file('TEST',
'getfile',
decompress=True,
cutout='[1:1]',
md5_check=False)
post_mock.assert_called_with(resource=(TRANSFER_RESOURCE_ID, None),
params={'cutout': '[1:1]'},
data=trans_doc,
headers={'Content-Type': 'text/xml'})
response.raw.read.side_effect = file_chunks
post_mock.reset_mock()
client.get_file('TEST',
'getfile',
decompress=True,
cutout='[[1:1], 2]',
md5_check=False)
post_mock.assert_called_with(resource=(TRANSFER_RESOURCE_ID, None),
params={'cutout': '[[1:1], 2]'},
data=trans_doc,
headers={'Content-Type': 'text/xml'})
def testSubject(self, os_mock):
# anon subject
subject = auth.Subject()
self.assertTrue(subject.anon)
self.assertEqual(None, subject.certificate)
self.assertEqual({}, subject._hosts_auth)
self.assertEqual(None, subject.get_auth('realm1'))
# cert subject
cert = 'somecert'
subject = auth.Subject(certificate=cert)
self.assertFalse(subject.anon)
self.assertEqual(cert, subject.certificate)
self.assertEqual({}, subject._hosts_auth)
self.assertEqual(None, subject.get_auth('realm1'))
# empty netrc subject
m = mock_open()
with patch('six.moves.builtins.open', m, create=True):
subject = auth.Subject(netrc='somefile')
self.assertFalse(subject.anon)
self.assertEqual(None, subject.certificate)
self.assertEqual({}, subject._hosts_auth)
self.assertEqual(None, subject.get_auth('realm1'))
# netrc with content
netrc_content = {'realm1': ('user1', None, 'pass1'),
'realm2': ('user1', None, 'pass2')}
expected_host_auth = {'realm1': ('user1', 'pass1'),
'realm2': ('user1', 'pass2')}
os_mock.path.join.return_value = '/home/myhome/.netrc'
with patch('cadcutils.net.auth.netrclib') as netrclib_mock:
netrclib_mock.netrc.return_value.hosts = netrc_content
subject = auth.Subject(netrc=True)
self.assertFalse(subject.anon)
self.assertEqual(None, subject.certificate)
self.assertEqual('/home/myhome/.netrc', subject.netrc)
self.assertEqual(expected_host_auth, subject._hosts_auth)
self.assertEqual(('user1', 'pass1'), subject.get_auth('realm1'))
self.assertEqual(('user1', 'pass2'), subject.get_auth('realm2'))
self.assertEqual(None, subject.get_auth('realm3'))
# subject with username
username = '******'
passwd = 'passwd1'
subject = auth.Subject(username=username)
self.assertFalse(subject.anon)
self.assertEqual(None, subject.certificate)
self.assertEqual({}, subject._hosts_auth)
with patch('cadcutils.net.auth.getpass') as getpass_mock:
getpass_mock.getpass.return_value = passwd
self.assertEqual((username, passwd), subject.get_auth('realm1'))
parser = get_base_parser(subparsers=False)
args = parser.parse_args(['--resource-id', 'blah'])
subject = auth.Subject.from_cmd_line_args(args)
self.assertTrue(subject.anon)
sys.argv = ['cadc-client', '--resource-id', 'blah', '--cert',
'mycert.pem']
args = parser.parse_args()
subject = auth.Subject.from_cmd_line_args(args)
self.assertFalse(subject.anon)
self.assertEqual('mycert.pem', subject.certificate)
def test_ops(self, post_mock, get_mock, delete_mock, head_mock, put_mock,
netrclib_mock, caps_mock):
anon_subject = auth.Subject()
with self.assertRaises(ValueError):
ws.BaseWsClient(None, anon_subject, "TestApp")
resource = 'aresource'
service = 'myservice'
resource_id = 'ivo://www.canfar.phys.uvic.ca/{}'.format(service)
# test anonymous access
cm = Mock()
cm.get_access_url.return_value = "http://host/availability"
caps_mock.return_value = cm
client = ws.BaseWsClient(resource_id, anon_subject, 'TestApp')
resource_uri = urlparse(resource_id)
base_url = 'http://{}{}/pub'.format(resource_uri.netloc,
resource_uri.path)
resource_url = 'http://{}{}/{}'.format(resource_uri.netloc, base_url,
resource)
self.assertEqual(anon_subject, client.subject)
self.assertTrue(client.retry)
self.assertEqual('TestApp', client.agent)
self.assertTrue(client.retry)
self.assertEqual(None, client._session) # lazy initialization
client.get(resource_url)
get_mock.assert_called_with(resource_url, params=None)
params = {'arg1': 'abc', 'arg2': 123, 'arg3': True}
client.post(resource_url, **params)
post_mock.assert_called_with(resource_url, **params)
client.delete(resource_url)
delete_mock.assert_called_with(resource_url)
client.head(resource_url)
head_mock.assert_called_with(resource_url)
client.put(resource_url, **params)
put_mock.assert_called_with(resource_url, **params)
self.assertTrue(isinstance(client._session, ws.RetrySession))
# test basic authentication access
post_mock.reset_mock()
get_mock.reset_mock()
put_mock.reset_mock()
delete_mock.reset_mock()
head_mock.reset_mock()
host = 'www.different.org'
subject = auth.Subject(netrc='somecert')
client = ws.BaseWsClient(resource_id,
subject,
'TestApp',
retry=False,
host=host)
base_url = 'http://{}{}/auth'.format(host, resource_uri.path)
resource_url = '{}/{}'.format(base_url, resource)
self.assertEqual('TestApp', client.agent)
self.assertFalse(client.retry)
client.get(resource_url)
get_mock.assert_called_with(resource_url, params=None)
params = {'arg1': 'abc', 'arg2': 123, 'arg3': True}
client.post(resource_url, **params)
post_mock.assert_called_with(resource_url, **params)
client.delete(resource_url)
delete_mock.assert_called_with(resource_url)
client.head(resource_url)
head_mock.assert_called_with(resource_url)
client.put(resource_url, **params)
put_mock.assert_called_with(resource_url, **params)
self.assertTrue(isinstance(client._session, ws.RetrySession))
# test cert authentication
post_mock.reset_mock()
get_mock.reset_mock()
put_mock.reset_mock()
delete_mock.reset_mock()
head_mock.reset_mock()
certfile = 'some/certfile.pem'
subject = auth.Subject(certificate=certfile)
client = ws.BaseWsClient(resource_id, subject, 'TestApp')
base_url = 'https://{}{}/pub'.format(resource_uri.netloc,
resource_uri.path)
resource_url = '{}/{}'.format(base_url, resource)
self.assertEqual('TestApp', client.agent)
self.assertTrue(client.retry)
client.get(resource_url)
get_mock.assert_called_with(resource_url, params=None)
params = {'arg1': 'abc', 'arg2': 123, 'arg3': True}
client.post(resource_url, **params)
post_mock.assert_called_with(resource_url, **params)
client.delete(resource_url)
delete_mock.assert_called_with(resource_url)
client.head(resource_url)
head_mock.assert_called_with(resource_url)
client.put(resource_url, **params)
put_mock.assert_called_with(resource_url, **params)
self.assertTrue(isinstance(client._session, ws.RetrySession))
self.assertEqual((certfile, certfile), client._session.cert)
# repeat above tests test with the temporary sc2repo test
resource = 'aresource'
service = 'sc2repo'
resource_id = 'ivo://www.canfar.phys.uvic.ca/{}'.format(service)
# test anonymous access
client = ws.BaseWsClient(resource_id, auth.Subject(), 'TestApp')
resource_uri = urlparse(resource_id)
base_url = 'http://{}{}/observations'.format(resource_uri.netloc,
resource_uri.path)
resource_url = '{}/{}'.format(base_url, resource)
self.assertTrue(client.retry)
self.assertEqual('TestApp', client.agent)
self.assertTrue(client.retry)
self.assertEqual(None, client._session) # lazy initialization
client.get(resource_url)
get_mock.assert_called_with(resource_url, params=None)
params = {'arg1': 'abc', 'arg2': 123, 'arg3': True}
client.post(resource_url, **params)
post_mock.assert_called_with(resource_url, **params)
client.delete(resource_url)
delete_mock.assert_called_with(resource_url)
client.head(resource_url)
head_mock.assert_called_with(resource_url)
client.put(resource_url, **params)
put_mock.assert_called_with(resource_url, **params)
self.assertTrue(isinstance(client._session, ws.RetrySession))
# test basic authentication access
post_mock.reset_mock()
get_mock.reset_mock()
put_mock.reset_mock()
delete_mock.reset_mock()
head_mock.reset_mock()
host = 'caom2workshop.canfar.net'
subject = auth.Subject(netrc=True)
subject._hosts_auth[host] = ('auser', 'apasswd')
client = ws.BaseWsClient(resource_id,
subject,
'TestApp',
retry=False,
host=host)
base_url = 'http://{}{}/auth-observations'.format(
host, resource_uri.path)
resource_url = '{}/{}'.format(base_url, resource)
self.assertEqual('TestApp', client.agent)
self.assertFalse(client.retry)
client.get(resource_url)
get_mock.assert_called_with(resource_url, params=None)
params = {'arg1': 'abc', 'arg2': 123, 'arg3': True}
client.post(resource_url, **params)
post_mock.assert_called_with(resource_url, **params)
client.delete(resource_url)
delete_mock.assert_called_with(resource_url)
client.head(resource_url)
head_mock.assert_called_with(resource_url)
client.put(resource_url, **params)
put_mock.assert_called_with(resource_url, **params)
self.assertTrue(isinstance(client._session, ws.RetrySession))
# test cert authentication
post_mock.reset_mock()
get_mock.reset_mock()
put_mock.reset_mock()
delete_mock.reset_mock()
head_mock.reset_mock()
certfile = 'some/certfile.pem'
client = ws.BaseWsClient(resource_id,
auth.Subject(certificate=certfile), 'TestApp')
base_url = 'https://{}{}/observations'.format(resource_uri.netloc,
resource_uri.path)
resource_url = '{}/{}'.format(base_url, resource)
self.assertEqual('TestApp', client.agent)
self.assertTrue(client.retry)
client.get(resource_url)
get_mock.assert_called_with(resource_url, params=None)
params = {'arg1': 'abc', 'arg2': 123, 'arg3': True}
client.post(resource_url, **params)
post_mock.assert_called_with(resource_url, **params)
client.delete(resource_url)
delete_mock.assert_called_with(resource_url)
client.head(resource_url)
head_mock.assert_called_with(resource_url)
client.put(resource_url, **params)
put_mock.assert_called_with(resource_url, **params)
self.assertTrue(isinstance(client._session, ws.RetrySession))
self.assertEqual((certfile, certfile), client._session.cert)
def test_misc(self):
"""
Tests miscellaneous functions
"""
service_url = 'http://somehost.com/service'
with patch('cadcutils.net.ws.WsCapabilities') as caps_mock:
caps_mock.return_value.get_service_host.return_value =\
'somehost.com'
caps_mock.return_value.get_access_url.return_value = service_url
client = ws.BaseWsClient("someresourceID", auth.Subject(),
'TestApp')
self.assertEqual('{}'.format(service_url),
client._get_url(('myfeature', None)))
caps_mock.return_value.get_access_url.assert_called_with(
'myfeature')
self.assertEqual('{}'.format(service_url),
client._get_url(('myfeature', '')))
test_host = 'testhost.com'
with patch('cadcutils.net.ws.os.makedirs'):
client = ws.BaseWsClient("someresourceID",
auth.Subject(),
'TestApp',
host=test_host)
self.assertEqual(test_host, client.host)
# test with resource as url
with patch('cadcutils.net.ws.WsCapabilities') as caps_mock:
caps_mock.return_value.get_service_host.return_value =\
'somehost.com'
cm = Mock()
cm.get_access_url.return_value = "http://host/availability"
caps_mock.return_value = cm
client = ws.BaseWsClient("someresourceID", auth.Subject(),
'TestApp')
resource_url = 'http://someurl.com/path/'
self.assertEqual(resource_url, client._get_url(resource_url))
# repeat with overriden host name
client = ws.BaseWsClient("someresourceID",
auth.Subject(),
'TestApp',
host=test_host)
# same result
self.assertEqual(resource_url, client._get_url(resource_url))
# test exceptions with different status in the response
session = ws.RetrySession()
response = requests.Response()
response.status_code = requests.codes.not_found
with self.assertRaises(exceptions.NotFoundException):
session.check_status(response)
response.status_code = requests.codes.unauthorized
with self.assertRaises(exceptions.UnauthorizedException):
session.check_status(response)
response.status_code = requests.codes.forbidden
with self.assertRaises(exceptions.ForbiddenException):
session.check_status(response)
response.status_code = requests.codes.bad_request
with self.assertRaises(exceptions.BadRequestException):
session.check_status(response)
response.status_code = requests.codes.conflict
with self.assertRaises(exceptions.AlreadyExistsException):
session.check_status(response)
response.status_code = requests.codes.internal_server_error
with self.assertRaises(exceptions.InternalServerException):
session.check_status(response)
response.status_code = requests.codes.unavailable
with self.assertRaises(requests.HTTPError):
session.check_status(response)
response.status_code = requests.codes.not_extended
with self.assertRaises(exceptions.UnexpectedException):
session.check_status(response)
def test_get_caps(self, get_mock, file_modtime_mock):
"""
Tests the capabilities part of WsCapabilities
"""
# test when registry information is read from the server
# (cache is outdated)
service = 'myservice'
resource_id = 'ivo://canfar.phys.uvic.ca/{}'.format(service)
resource_cap_url = 'www.canfar.net/myservice'
# set the modified time of the cache file to 0 to make sure the
# info is retrieved from server
file_modtime_mock.return_value = 0
# test anonymous access
expected_content = capabilities__content.replace(
'WS_URL', resource_cap_url)
response = Mock(text=expected_content)
get_mock.return_value = response
caps = ws.WsCapabilities(
Mock(resource_id=resource_id, subject=auth.Subject()))
# mock _get_capability_url to return some url without attempting
# to access the server
def get_url():
return 'http://some.url/capabilities'
caps._get_capability_url = get_url
# remove the cached file if exists
if os.path.isfile(caps.caps_file):
os.remove(caps.caps_file)
caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url)
self.assertEqual(
'http://{}/capabilities'.format(resource_cap_url),
caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
self.assertEqual(
'http://{}/availability'.format(resource_cap_url),
caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
self.assertEqual(
'http://{}/pub'.format(resource_cap_url),
caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))
actual_content = open(caps.caps_file, 'r').read()
self.assertEqual(expected_content, actual_content)
# mock _get_capability_url to return a subservice
service = 'myservice/mysubservice'
resource_id = 'ivo://canfar.phys.uvic.ca/{}'.format(service)
resource_cap_url = 'www.canfar.net/myservice/mysubservice'
# set the modified time of the cache file to 0 to make sure the
# info is retrieved from server
file_modtime_mock.return_value = 0
expected_content = capabilities__content.replace(
'WS_URL', resource_cap_url)
# test anonymous access
response = Mock(text=expected_content)
get_mock.return_value = response
caps = ws.WsCapabilities(
Mock(resource_id=resource_id, subject=auth.Subject()))
# remove the cached file if exists
if os.path.isfile(caps.caps_file):
os.remove(caps.caps_file)
caps._get_capability_url = get_url
caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url)
self.assertEqual(
'http://{}/capabilities'.format(resource_cap_url),
caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
self.assertEqual(
'http://{}/availability'.format(resource_cap_url),
caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
self.assertEqual(
'http://{}/pub'.format(resource_cap_url),
caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))
actual_content = open(caps.caps_file, 'r').read()
self.assertEqual(expected_content, actual_content)
# repeat for basic auth subject. Mock the netrc library to
# prevent a lookup for $HOME/.netrc
with patch('cadcutils.net.auth.netrclib'):
client = Mock(resource_id=resource_id,
subject=auth.Subject(netrc=True))
client.get.return_value = response
caps = ws.WsCapabilities(client)
caps._get_capability_url = get_url
# capabilities works even if it has only one anonymous interface
self.assertEqual(
'http://{}/capabilities'.format(resource_cap_url),
caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
# same for availability
self.assertEqual(
'http://{}/availability'.format(resource_cap_url),
caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
# repeat for https
with patch('os.path.isfile'):
client = Mock(resource_id=resource_id,
subject=auth.Subject(certificate='somecert.pem'))
client.get.return_value = response
caps = ws.WsCapabilities(client)
caps._get_capability_url = get_url
# capabilities works even if it has only one anonymous interface
self.assertEqual(
'http://{}/capabilities'.format(resource_cap_url),
caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
# same for availability
self.assertEqual(
'http://{}/availability'.format(resource_cap_url),
caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
self.assertEqual(
'https://{}'.format(resource_cap_url),
caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))
# test when capabilities information is retrieved from the cache file
get_mock.reset_mock()
get_mock.return_value = None
file_modtime_mock.reset_mock()
service = 'myservice2'
resource_id = 'ivo://canfar.phys.uvic.ca/{}'.format(service)
resource_cap_url2 = 'canfar.phys.uvic.ca/myservice2'
expected_content = capabilities__content.replace(
'WS_URL', resource_cap_url2)
file_modtime_mock.return_value = time.time()
client = Mock(resource_id=resource_cap_url2, subject=auth.Subject())
caps = ws.WsCapabilities(client)
caps._get_capability_url = get_url
caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url2)
# manually write the content
with open(caps.caps_file, 'w') as f:
f.write(expected_content)
self.assertEqual(
'http://{}/capabilities'.format(resource_cap_url2),
caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
self.assertEqual(
'http://{}/availability'.format(resource_cap_url2),
caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
self.assertEqual(
'http://{}/pub'.format(resource_cap_url2),
caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))
actual_content = open(caps.caps_file, 'r').read()
self.assertEqual(expected_content, actual_content)
# repeat for basic auth subject. Mock the netrc library to prevent a
# lookup for $HOME/.netrc
with patch('cadcutils.net.auth.netrclib'):
client = Mock(resource_id=resource_id,
subject=auth.Subject(netrc=True))
caps = ws.WsCapabilities(client)
caps._get_capability_url = get_url
# does not work with user-password of the subject set above
self.assertEqual(
'http://{}/capabilities'.format(resource_cap_url2),
caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
self.assertEqual(
'http://{}/availability'.format(resource_cap_url2),
caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
self.assertEqual(
'http://{}/auth'.format(resource_cap_url2),
caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))
# repeat for https
with patch('os.path.isfile'):
client = Mock(resource_id=resource_id,
subject=auth.Subject(certificate='somecert.pem'))
caps = ws.WsCapabilities(client)
caps._get_capability_url = get_url
# does not work with user-password of the subject set above
self.assertEqual(
'http://{}/capabilities'.format(resource_cap_url2),
caps.get_access_url('ivo://ivoa.net/std/VOSI#capabilities'))
self.assertEqual(
'http://{}/availability'.format(resource_cap_url2),
caps.get_access_url('ivo://ivoa.net/std/VOSI#availability'))
self.assertEqual(
'https://{}'.format(resource_cap_url2),
caps.get_access_url('vos://cadc.nrc.ca~service/CADC/mystnd01'))
def test_process(self):
core.BATCH_SIZE = 3 # size of the batch is 3
obs = [['a', 'b', 'c'], ['d'], []]
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
visitor.get_observation = MagicMock(return_value=MagicMock(
spec=SimpleObservation))
visitor.post_observation = MagicMock()
visitor._get_observations = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht')
self.assertEqual(4, len(visited))
self.assertEqual(4, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
obs = [['a', 'b', 'c'], ['d', 'e', 'f'], []]
visitor._get_observations = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht')
self.assertEqual(6, len(visited))
self.assertEqual(6, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
# make it return different status. errorplugin returns according to the
# id of the observation: True for 'UPDATE', False for 'SKIP' and
# raises exception for 'ERROR'
obs_ids = [['UPDATE', 'SKIP', 'ERROR'], []]
obs = [
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP'),
SimpleObservation(collection='TEST', observation_id='ERROR')
]
visitor._get_observations = MagicMock(side_effect=obs_ids)
visitor.get_observation = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
'cfht')
self.assertEqual(3, len(visited))
self.assertEqual(1, len(updated))
self.assertEqual(1, len(skipped))
self.assertEqual(1, len(failed))
# repeat with other obs
obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']]
obs = [
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP'),
SimpleObservation(collection='TEST', observation_id='ERROR'),
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP')
]
visitor._get_observations = MagicMock(side_effect=obs_ids)
visitor.get_observation = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
'cfht')
self.assertEqual(5, len(visited))
self.assertEqual(2, len(updated))
self.assertEqual(2, len(skipped))
self.assertEqual(1, len(failed))
# repeat but halt on first ERROR -> process only 3 observations
obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']]
obs = [
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP'),
SimpleObservation(collection='TEST', observation_id='ERROR'),
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP')
]
visitor._get_observations = MagicMock(side_effect=obs_ids)
visitor.get_observation = MagicMock(side_effect=obs)
with self.assertRaises(SystemError):
visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
'cfht',
halt_on_error=True)
# test with time boundaries
core.BATCH_SIZE = 3 # size of the batch is 3
response = MagicMock()
response.text = """ARCHIVE\ta\t2011-01-01T11:00:00.000
ARCHIVE\tb\t211-01-01T11:00:10.000
ARCHIVE\tc\t2011-01-01T12:00:00.000"""
response2 = MagicMock()
response2.text = """ARCHIVE\td\t2011-02-02T11:00:00.000"""
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
visitor.get_observation = MagicMock(return_value=MagicMock(
spec=SimpleObservation))
visitor.post_observation = MagicMock()
visitor._repo_client.get = MagicMock(side_effect=[response, response2])
start = '2010-10-10T12:00:00.000'
end = '2012-12-12T11:11:11.000'
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht',
start=util.str2ivoa(start),
end=util.str2ivoa(end))
self.assertEqual(4, len(visited))
self.assertEqual(4, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
calls = [
call((core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'),
params={
'START': start,
'END': end,
'MAXREC': 3
}),
call(
(core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'),
params={
'START': '2011-01-01T12:00:00.000',
# datetime of the last record in the batch
'END': end,
'MAXREC': 3
})
]
visitor._repo_client.get.assert_has_calls(calls)
def test_get_caps(self, get_mock, file_mock, file_modtime_mock):
"""
Tests the capabilities part of WsCapabilities
"""
# test when registry information is read from the server
# (cache is outdated)
service = 'myservice'
resource_id = 'ivo://canfar.phys.uvic.ca/{}'.format(service)
resource_cap_url = 'www.canfar.net/myservice'
# set the modified time of the cache file to 0 to make sure the
# info is retrieved from server
file_modtime_mock.return_value = 0
# test anonymous access
fh_mock = Mock()
file_mock.write = fh_mock
response = Mock(
text=capabilities__content.replace('WS_URL', resource_cap_url))
get_mock.return_value = response
caps = ws.WsCapabilities(Mock(resource_id=resource_id,
subject=auth.Subject()))
# mock _get_capability_url to return some url without attempting
# to access the server
def get_url():
return 'http://some.url/capabilities'
caps._get_capability_url = get_url
caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url)
self.assertEqual('http://{}/capabilities'.format(resource_cap_url),
caps.get_access_url(
'ivo://ivoa.net/std/VOSI#capabilities'))
self.assertEqual('http://{}/availability'.format(resource_cap_url),
caps.get_access_url(
'ivo://ivoa.net/std/VOSI#availability'))
self.assertEqual('http://{}/pub'.format(resource_cap_url),
caps.get_access_url(
'vos://cadc.nrc.ca~service/CADC/mystnd01'))
resource_url = urlparse(resource_id)
file_mock.assert_called_once_with(os.path.join(ws.CACHE_LOCATION,
resource_url.netloc,
resource_url.path.strip(
'/')), 'w')
# TODO not sure why need to access write this way
file_mock().__enter__.return_value.write.assert_called_once_with(
capabilities__content.replace('WS_URL', resource_cap_url))
# repeat for basic auth subject. Mock the netrc library to
# prevent a lookup for $HOME/.netrc
with patch('cadcutils.net.auth.netrclib'):
client = Mock(resource_id=resource_id,
subject=auth.Subject(netrc=True))
client.get.return_value = response
caps = ws.WsCapabilities(client)
caps._get_capability_url = get_url
# capabilities works even if it has only one anonymous interface
self.assertEqual('http://{}/capabilities'.format(resource_cap_url),
caps.get_access_url(
'ivo://ivoa.net/std/VOSI#capabilities'))
# same for availability
self.assertEqual('http://{}/availability'.format(resource_cap_url),
caps.get_access_url(
'ivo://ivoa.net/std/VOSI#availability'))
# repeat for https
with patch('os.path.isfile'):
client = Mock(resource_id=resource_id,
subject=auth.Subject(certificate='somecert.pem'))
client.get.return_value = response
caps = ws.WsCapabilities(client)
caps._get_capability_url = get_url
# capabilities works even if it has only one anonymous interface
self.assertEqual('http://{}/capabilities'.format(resource_cap_url),
caps.get_access_url(
'ivo://ivoa.net/std/VOSI#capabilities'))
# same for availability
self.assertEqual('http://{}/availability'.format(resource_cap_url),
caps.get_access_url(
'ivo://ivoa.net/std/VOSI#availability'))
self.assertEqual('https://{}'.format(resource_cap_url),
caps.get_access_url(
'vos://cadc.nrc.ca~service/CADC/mystnd01'))
# test when capabilities information is retrieved from the cache file
get_mock.reset_mock()
get_mock.return_value = None
file_modtime_mock.reset_mock()
file_mock.reset_mock()
resource_cap_url2 = 'http://www.canfar.net/myservice2'
cache_content2 = capabilities__content.replace('WS_URL',
resource_cap_url2)
file_modtime_mock.return_value = time.time()
file_mock().__enter__.return_value.read.return_value = cache_content2
client = Mock(resource_id=resource_id, subject=auth.Subject())
caps = ws.WsCapabilities(client)
caps._get_capability_url = get_url
caps.caps_urls[service] = '{}/capabilities'.format(resource_cap_url2)
self.assertEqual('http://{}/capabilities'.format(resource_cap_url2),
caps.get_access_url(
'ivo://ivoa.net/std/VOSI#capabilities'))
self.assertEqual('http://{}/availability'.format(resource_cap_url2),
caps.get_access_url(
'ivo://ivoa.net/std/VOSI#availability'))
self.assertEqual('http://{}/pub'.format(resource_cap_url2),
caps.get_access_url(
'vos://cadc.nrc.ca~service/CADC/mystnd01'))
resource_url = urlparse(resource_id)
file_mock().__enter__.return_value.read.return_value = \
capabilities__content.replace('WS_URL', resource_cap_url2)
# repeat for basic auth subject. Mock the netrc library to prevent a
# lookup for $HOME/.netrc
with patch('cadcutils.net.auth.netrclib'):
client = Mock(resource_id=resource_id,
subject=auth.Subject(netrc=True))
caps = ws.WsCapabilities(client)
caps._get_capability_url = get_url
# does not work with user-password of the subject set above
self.assertEqual('http://{}/capabilities'.format(resource_cap_url2),
caps.get_access_url(
'ivo://ivoa.net/std/VOSI#capabilities'))
self.assertEqual('http://{}/availability'.format(resource_cap_url2),
caps.get_access_url(
'ivo://ivoa.net/std/VOSI#availability'))
self.assertEqual('http://{}/auth'.format(resource_cap_url2),
caps.get_access_url(
'vos://cadc.nrc.ca~service/CADC/mystnd01'))
# repeat for https
with patch('os.path.isfile'):
client = Mock(resource_id=resource_id,
subject=auth.Subject(certificate='somecert.pem'))
caps = ws.WsCapabilities(client)
caps._get_capability_url = get_url
# does not work with user-password of the subject set above
self.assertEqual('http://{}/capabilities'.format(resource_cap_url2),
caps.get_access_url(
'ivo://ivoa.net/std/VOSI#capabilities'))
self.assertEqual('http://{}/availability'.format(resource_cap_url2),
caps.get_access_url(
'ivo://ivoa.net/std/VOSI#availability'))
self.assertEqual('https://{}'.format(resource_cap_url2),
caps.get_access_url(
'vos://cadc.nrc.ca~service/CADC/mystnd01'))
