def test_plugin_class(self):
# plugin class does not change the observation
collection = 'cfht'
observation_id = '7000000o'
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
obs = SimpleObservation(collection, observation_id)
expect_obs = copy.deepcopy(obs)
visitor._load_plugin_class(os.path.join(THIS_DIR, 'passplugin.py'))
visitor.plugin.update(obs)
self.assertEqual(expect_obs, obs)
# plugin class adds a plane to the observation
visitor = CAOM2RepoClient(auth.Subject(), level)
obs = SimpleObservation('cfht', '7000000o')
expect_obs = copy.deepcopy(obs)
visitor._load_plugin_class(os.path.join(THIS_DIR, 'addplaneplugin.py'))
visitor.plugin.update(obs)
self.assertNotEqual(expect_obs, obs)
self.assertEqual(len(expect_obs.planes) + 1, len(obs.planes))
# non-existent the plugin file
with self.assertRaises(Exception):
visitor._load_plugin_class(os.path.join(THIS_DIR, 'blah.py'))
# non-existent ObservationUpdater class in the plugin file
with self.assertRaises(Exception):
visitor._load_plugin_class(
os.path.join(THIS_DIR, 'test_visitor.py'))
# non-existent update method in ObservationUpdater class
with self.assertRaises(Exception):
visitor._load_plugin_class(
os.path.join(THIS_DIR, 'noupdateplugin.py'))
def test_multiprocess_with_more_different_statuses(self, get_mock):
core.BATCH_SIZE = 3 # size of the batch is 3
# make it return different status. errorplugin returns according to the
# id of the observation: True for 'UPDATE', False for 'SKIP' and
# raises exception for 'ERROR'
obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']]
get_mock.side_effect = self.mock_get_observation
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
visitor.get_observation = PickableMagicMock(
return_value=PickableMagicMock(spec=SimpleObservation))
visitor.post_observation = PickableMagicMock()
visitor._get_observations = PickableMagicMock(side_effect=obs_ids)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
'cfht',
start=None,
end=None,
obs_file=None,
nthreads=3)
try:
self.assertEqual(5, len(visited))
self.assertEqual(2, len(updated))
self.assertEqual(2, len(skipped))
self.assertEqual(1, len(failed))
finally:
# lp.join()
logging.info("DONE")
def test_multiprocess_with_unexpected_type_error(self, get_mock):
core.BATCH_SIZE = 3 # size of the batch is 3
obs_ids = [['a', 'b', 'c'], ['d'], []]
get_mock.side_effect = \
self.mock_get_observation_with_unexpected_type_error
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
visitor.get_observation = PickableMagicMock(
return_value=PickableMagicMock(spec=SimpleObservation))
visitor.post_observation = PickableMagicMock()
visitor._get_observations = PickableMagicMock(side_effect=obs_ids)
try:
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht',
start=None,
end=None,
obs_file=None,
nthreads=3,
halt_on_error=True)
except TypeError as e:
self.assertTrue("unexpected TypeError" in str(e))
finally:
logging.info("DONE")
def test_visit_retry_on_412(self):
# observation changed on server while visited
core.BATCH_SIZE = 3 # size of the batch is 3
obs = [['a'], []]
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
observation = SimpleObservation('cfht', 'a')
observation.acc_meta_checksum = ChecksumURI('md5:abc')
visitor.get_observation = MagicMock(
side_effect=[observation, observation])
exception_412 = exceptions.UnexpectedException()
exception_412.orig_exception = Mock()
exception_412.orig_exception.response = Mock(status_code=412)
visitor.post_observation = MagicMock(side_effect=[exception_412, None])
visitor._get_observations = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht')
self.assertEqual(1, len(visited))
self.assertEqual(1, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
# get and post called twice to recover from error HTTP status 412 -
# precondition
self.assertEqual(2, visitor.get_observation.call_count)
self.assertEqual(2, visitor.post_observation.call_count)
visitor.post_observation.assert_called_with(
observation, observation.acc_meta_checksum.uri)
def test_create_and_visit(self):
#logger.setLevel(logging.DEBUG)
logger.info('-- START: test_create_and_visit --')
start = datetime.now()
name = obs_name = 'caom2pyinttest{}'.format(start.microsecond)
try:
env_a = os.environ['A']
cert_file = env_a + '/test-certificates/x509_CADCAuthtest1.pem'
subject = auth.Subject(certificate=cert_file)
client = CAOM2RepoClient(subject)
# create one observation for today
algorithm = observation.SimpleObservation._DEFAULT_ALGORITHM_NAME
logger.debug("test obs name {}".format(name))
obs = observation.SimpleObservation("TEST", obs_name)
obs.algorithm = algorithm
client.put_observation(obs)
plugin = os.path.join(THIS_DIR, 'visitor-plugin.py')
(visited, updated, skipped, failed) = client.visit(plugin, 'TEST', start=start, halt_on_error=True)
logger.debug("observations visited: {}".format(len(visited)))
self.assertGreater(len(visited), 0, msg="No Observations Visited")
finally:
try:
client.delete_observation("TEST", name)
except:
logger.warning('Failed to delete test observation, continuing')
logger.info('-- END :test_create_and_visit --')
def test_post_observation(self, mock_conn, caps_mock):
caps_mock.get_service_host.return_value = 'some.host.com'
caps_mock.return_value.get_access_url.return_value =\
'http://serviceurl/caom2repo/auth'
collection = 'cfht'
observation_id = '7000000o'
service = 'caom2repo'
service_url = 'www.cadc.nrc.ca'
obs = SimpleObservation(collection, observation_id)
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(netrc='somenetrc'),
level,
host=service_url)
response = MagicMock()
response.status = 200
mock_conn.return_value = response
iobuffer = BytesIO()
ObservationWriter().write(obs, iobuffer)
obsxml = iobuffer.getvalue()
response.content = obsxml
visitor.post_observation(obs)
self.assertEqual('POST', mock_conn.call_args[0][0].method)
self.assertEqual(
'/{}/auth/{}/{}'.format(service, collection, observation_id),
mock_conn.call_args[0][0].path_url)
self.assertEqual('application/xml',
mock_conn.call_args[0][0].headers['Content-Type'])
self.assertEqual(obsxml, mock_conn.call_args[0][0].body)
# signal problems
http_error = requests.HTTPError()
response.status_code = 500
http_error.response = response
response.raise_for_status.side_effect = [http_error]
with self.assertRaises(exceptions.InternalServerException):
visitor.update(obs)
# temporary transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
response.raise_for_status.side_effect = [http_error, None]
visitor.post_observation(obs)
# permanent transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
def raise_error():
raise http_error
response.raise_for_status.side_effect = raise_error
with self.assertRaises(exceptions.HttpException):
visitor.post_observation(obs)
def test_get_observations(self, mock_get, caps_mock):
# This is almost similar to the previous test except that it gets
# observations matching a collection and start/end criteria
# Also, patch the CAOM2RepoClient now.
caps_mock.get_service_host.return_value = 'some.host.com'
caps_mock.return_value.get_access_url.return_value =\
'http://serviceurl/caom2repo/pub'
response = MagicMock()
response.status_code = 200
last_datetime = '2000-10-10T12:30:00.333'
response.text = \
('CFHT\t700000o\t2000-10-10T12:20:11.123\t'
'3e00ca6129dc8358315015204ab9fe15\nCFHT\t700001o\t' +
last_datetime + '\t3e00ca6129dc8358315015204ab9fe15')
mock_get.return_value = response
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
end_date = util.utils.str2ivoa(last_datetime)
expect_observations = ['700000o', '700001o']
self.assertEqual(expect_observations,
visitor._get_observations('cfht'))
self.assertEqual(end_date, visitor._start)
mock_get.assert_called_once_with(
('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2',
'cfht'),
params={'MAXREC': core.BATCH_SIZE})
mock_get.reset_mock()
visitor._get_observations('cfht',
end=datetime.strptime(
'2000-11-11', '%Y-%m-%d'))
mock_get.assert_called_once_with(
('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2',
'cfht'),
params={
'END': '2000-11-11T00:00:00.000',
'MAXREC': core.BATCH_SIZE
})
mock_get.reset_mock()
visitor._get_observations(
'cfht',
start=datetime.strptime('2000-11-11', '%Y-%m-%d'),
end=datetime.strptime('2000-11-12', '%Y-%m-%d'))
mock_get.assert_called_once_with(
('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2',
'cfht'),
params={
'START': '2000-11-11T00:00:00.000',
'END': '2000-11-12T00:00:00.000',
'MAXREC': core.BATCH_SIZE
})
def test_get_observation(self, mock_get, caps_mock):
caps_mock.get_service_host.return_value = 'some.host.com'
caps_mock.return_value.get_access_url.return_value =\
'http://serviceurl/caom2repo/pub'
collection = 'cfht'
observation_id = '7000000o'
service_url = 'www.cadc.nrc.ca/caom2repo'
obs = SimpleObservation(collection, observation_id)
writer = ObservationWriter()
ibuffer = BytesIO()
writer.write(obs, ibuffer)
response = MagicMock()
response.status_code = 200
response.content = ibuffer.getvalue()
mock_get.return_value = response
ibuffer.seek(0) # reposition the buffer for reading
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url)
self.assertEqual(obs,
visitor.get_observation(collection, observation_id))
# signal problems
http_error = requests.HTTPError()
response.status_code = 500
http_error.response = response
response.raise_for_status.side_effect = [http_error]
with self.assertRaises(exceptions.InternalServerException):
visitor.get_observation(collection, observation_id)
# temporary transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
response.raise_for_status.side_effect = [http_error, None]
visitor.read(collection, observation_id)
# permanent transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
def raise_error():
raise http_error
response.raise_for_status.side_effect = raise_error
with self.assertRaises(exceptions.HttpException):
visitor.get_observation(collection, observation_id)
def test_delete_observation(self, mock_conn, caps_mock):
caps_mock.get_service_host.return_value = 'some.host.com'
caps_mock.return_value.get_access_url.return_value =\
'http://serviceurl/caom2repo/pub'
collection = 'cfht'
observation_id = '7000000o'
service_url = 'www.cadc.nrc.ca'
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url)
response = MagicMock()
response.status = 200
mock_conn.return_value = response
visitor.delete_observation(collection, observation_id)
self.assertEqual('DELETE', mock_conn.call_args[0][0].method)
# signal problems
http_error = requests.HTTPError()
response.status_code = 500
http_error.response = response
response.raise_for_status.side_effect = [http_error]
with self.assertRaises(exceptions.InternalServerException):
visitor.delete(collection, observation_id)
# temporary transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
response.raise_for_status.side_effect = [http_error, None]
visitor.delete_observation(collection, observation_id)
# permanent transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
def raise_error():
raise http_error
response.raise_for_status.side_effect = raise_error
with self.assertRaises(exceptions.HttpException):
visitor.delete_observation(collection, observation_id)
def test_get_obs_from_file(self):
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
# no start or end
with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file:
obs_id_list = visitor._get_obs_from_file(obs_file, None, None,
False)
self.assertEqual('obs_id_1', obs_id_list[0])
self.assertEqual('obs_id_2', obs_id_list[1])
self.assertEqual('obs_id_3', obs_id_list[2])
# last_modified_date is earlier than start
with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file:
obs_id_list = visitor._get_obs_from_file(
obs_file, util.str2ivoa('2000-10-11T12:30:00.333'), None,
False)
self.assertEqual('obs_id_1', obs_id_list[0])
# last_modified_date is between start and end
with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file:
obs_id_list = visitor._get_obs_from_file(
obs_file, util.str2ivoa('2000-10-9T12:30:00.333'),
util.str2ivoa('2016-10-11T12:30:00.333'), False)
self.assertEqual('obs_id_1', obs_id_list[0])
self.assertEqual('obs_id_2', obs_id_list[1])
# last_modified_date is after end
with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file:
obs_id_list = visitor._get_obs_from_file(
obs_file, util.str2ivoa('2000-10-9T12:30:00.333'),
util.str2ivoa('2017-10-11T12:30:00.333'), False)
self.assertEqual('obs_id_1', obs_id_list[0])
self.assertEqual('obs_id_2', obs_id_list[1])
self.assertEqual('obs_id_3', obs_id_list[2])
# error in file
with open(os.path.join(THIS_DIR, 'data/obs_id_error.txt')) as obs_file:
with self.assertRaises(Exception):
obs_id_list = visitor._get_obs_from_file(
obs_file, util.str2ivoa('2000-10-9T12:30:00.333'),
util.str2ivoa('2016-10-11T12:30:00.333'), True)
def test_multiprocess_with_more_obs_id(self, client_mock):
core.BATCH_SIZE = 3 # size of the batch is 3
obs_ids = [['a', 'b', 'c'], ['d', 'e', 'f'], []]
client_mock.return_value.get_observation.side_effect = \
self.mock_get_observation
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
visitor.get_observation = PickableMagicMock(
return_value=PickableMagicMock(spec=SimpleObservation))
visitor.post_observation = PickableMagicMock()
visitor._get_observations = PickableMagicMock(side_effect=obs_ids)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht',
start=None,
end=None,
obs_file=None,
nthreads=3)
try:
self.assertEqual(6, len(visited))
self.assertEqual(6, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
self.assertTrue('a' in visited)
self.assertTrue('b' in visited)
self.assertTrue('c' in visited)
self.assertTrue('d' in visited)
self.assertTrue('e' in visited)
self.assertTrue('f' in visited)
self.assertFalse('g' in visited)
self.assertTrue('a' in updated)
self.assertTrue('b' in updated)
self.assertTrue('c' in updated)
self.assertTrue('d' in updated)
self.assertTrue('e' in updated)
self.assertTrue('f' in updated)
self.assertFalse('g' in updated)
finally:
# lp.join()
logging.info("DONE")
def test_shortcuts(self):
level = logging.DEBUG
target = CAOM2RepoClient(auth.Subject(), level)
obs = SimpleObservation('CFHT', 'abc')
target.put_observation = Mock()
target.create(obs)
target.put_observation.assert_called_with(obs)
target.get_observation = Mock()
target.read('CFHT', 'abc')
target.get_observation.assert_called_with('CFHT', 'abc')
target.post_observation = Mock()
target.update(obs)
target.post_observation.assert_called_with(obs)
target.delete_observation = Mock()
target.delete('CFHT', 'abc')
target.delete_observation.assert_called_with('CFHT', 'abc')
def test_process(self):
core.BATCH_SIZE = 3 # size of the batch is 3
obs = [['a', 'b', 'c'], ['d'], []]
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
visitor.get_observation = MagicMock(return_value=MagicMock(
spec=SimpleObservation))
visitor.post_observation = MagicMock()
visitor._get_observations = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht')
self.assertEqual(4, len(visited))
self.assertEqual(4, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
obs = [['a', 'b', 'c'], ['d', 'e', 'f'], []]
visitor._get_observations = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht')
self.assertEqual(6, len(visited))
self.assertEqual(6, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
# make it return different status. errorplugin returns according to the
# id of the observation: True for 'UPDATE', False for 'SKIP' and
# raises exception for 'ERROR'
obs_ids = [['UPDATE', 'SKIP', 'ERROR'], []]
obs = [
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP'),
SimpleObservation(collection='TEST', observation_id='ERROR')
]
visitor._get_observations = MagicMock(side_effect=obs_ids)
visitor.get_observation = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
'cfht')
self.assertEqual(3, len(visited))
self.assertEqual(1, len(updated))
self.assertEqual(1, len(skipped))
self.assertEqual(1, len(failed))
# repeat with other obs
obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']]
obs = [
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP'),
SimpleObservation(collection='TEST', observation_id='ERROR'),
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP')
]
visitor._get_observations = MagicMock(side_effect=obs_ids)
visitor.get_observation = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
'cfht')
self.assertEqual(5, len(visited))
self.assertEqual(2, len(updated))
self.assertEqual(2, len(skipped))
self.assertEqual(1, len(failed))
# repeat but halt on first ERROR -> process only 3 observations
obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']]
obs = [
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP'),
SimpleObservation(collection='TEST', observation_id='ERROR'),
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP')
]
visitor._get_observations = MagicMock(side_effect=obs_ids)
visitor.get_observation = MagicMock(side_effect=obs)
with self.assertRaises(SystemError):
visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
'cfht',
halt_on_error=True)
# test with time boundaries
core.BATCH_SIZE = 3 # size of the batch is 3
response = MagicMock()
response.text = """ARCHIVE\ta\t2011-01-01T11:00:00.000
ARCHIVE\tb\t211-01-01T11:00:10.000
ARCHIVE\tc\t2011-01-01T12:00:00.000"""
response2 = MagicMock()
response2.text = """ARCHIVE\td\t2011-02-02T11:00:00.000"""
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
visitor.get_observation = MagicMock(return_value=MagicMock(
spec=SimpleObservation))
visitor.post_observation = MagicMock()
visitor._repo_client.get = MagicMock(side_effect=[response, response2])
start = '2010-10-10T12:00:00.000'
end = '2012-12-12T11:11:11.000'
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht',
start=util.str2ivoa(start),
end=util.str2ivoa(end))
self.assertEqual(4, len(visited))
self.assertEqual(4, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
calls = [
call((core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'),
params={
'START': start,
'END': end,
'MAXREC': 3
}),
call(
(core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'),
params={
'START': '2011-01-01T12:00:00.000',
# datetime of the last record in the batch
'END': end,
'MAXREC': 3
})
]
visitor._repo_client.get.assert_has_calls(calls)