def test_plugin_class(self): # plugin class does not change the observation collection = 'cfht' observation_id = '7000000o' level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) obs = SimpleObservation(collection, observation_id) expect_obs = copy.deepcopy(obs) visitor._load_plugin_class(os.path.join(THIS_DIR, 'passplugin.py')) visitor.plugin.update(obs) self.assertEqual(expect_obs, obs) # plugin class adds a plane to the observation visitor = CAOM2RepoClient(auth.Subject(), level) obs = SimpleObservation('cfht', '7000000o') expect_obs = copy.deepcopy(obs) visitor._load_plugin_class(os.path.join(THIS_DIR, 'addplaneplugin.py')) visitor.plugin.update(obs) self.assertNotEqual(expect_obs, obs) self.assertEqual(len(expect_obs.planes) + 1, len(obs.planes)) # non-existent the plugin file with self.assertRaises(Exception): visitor._load_plugin_class(os.path.join(THIS_DIR, 'blah.py')) # non-existent ObservationUpdater class in the plugin file with self.assertRaises(Exception): visitor._load_plugin_class( os.path.join(THIS_DIR, 'test_visitor.py')) # non-existent update method in ObservationUpdater class with self.assertRaises(Exception): visitor._load_plugin_class( os.path.join(THIS_DIR, 'noupdateplugin.py'))
def test_multiprocess_with_more_different_statuses(self, get_mock): core.BATCH_SIZE = 3 # size of the batch is 3 # make it return different status. errorplugin returns according to the # id of the observation: True for 'UPDATE', False for 'SKIP' and # raises exception for 'ERROR' obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']] get_mock.side_effect = self.mock_get_observation level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) visitor.get_observation = PickableMagicMock( return_value=PickableMagicMock(spec=SimpleObservation)) visitor.post_observation = PickableMagicMock() visitor._get_observations = PickableMagicMock(side_effect=obs_ids) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht', start=None, end=None, obs_file=None, nthreads=3) try: self.assertEqual(5, len(visited)) self.assertEqual(2, len(updated)) self.assertEqual(2, len(skipped)) self.assertEqual(1, len(failed)) finally: # lp.join() logging.info("DONE")
def test_multiprocess_with_unexpected_type_error(self, get_mock): core.BATCH_SIZE = 3 # size of the batch is 3 obs_ids = [['a', 'b', 'c'], ['d'], []] get_mock.side_effect = \ self.mock_get_observation_with_unexpected_type_error level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) visitor.get_observation = PickableMagicMock( return_value=PickableMagicMock(spec=SimpleObservation)) visitor.post_observation = PickableMagicMock() visitor._get_observations = PickableMagicMock(side_effect=obs_ids) try: (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht', start=None, end=None, obs_file=None, nthreads=3, halt_on_error=True) except TypeError as e: self.assertTrue("unexpected TypeError" in str(e)) finally: logging.info("DONE")
def test_visit_retry_on_412(self): # observation changed on server while visited core.BATCH_SIZE = 3 # size of the batch is 3 obs = [['a'], []] level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) observation = SimpleObservation('cfht', 'a') observation.acc_meta_checksum = ChecksumURI('md5:abc') visitor.get_observation = MagicMock( side_effect=[observation, observation]) exception_412 = exceptions.UnexpectedException() exception_412.orig_exception = Mock() exception_412.orig_exception.response = Mock(status_code=412) visitor.post_observation = MagicMock(side_effect=[exception_412, None]) visitor._get_observations = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') self.assertEqual(1, len(visited)) self.assertEqual(1, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) # get and post called twice to recover from error HTTP status 412 - # precondition self.assertEqual(2, visitor.get_observation.call_count) self.assertEqual(2, visitor.post_observation.call_count) visitor.post_observation.assert_called_with( observation, observation.acc_meta_checksum.uri)
def test_create_and_visit(self): #logger.setLevel(logging.DEBUG) logger.info('-- START: test_create_and_visit --') start = datetime.now() name = obs_name = 'caom2pyinttest{}'.format(start.microsecond) try: env_a = os.environ['A'] cert_file = env_a + '/test-certificates/x509_CADCAuthtest1.pem' subject = auth.Subject(certificate=cert_file) client = CAOM2RepoClient(subject) # create one observation for today algorithm = observation.SimpleObservation._DEFAULT_ALGORITHM_NAME logger.debug("test obs name {}".format(name)) obs = observation.SimpleObservation("TEST", obs_name) obs.algorithm = algorithm client.put_observation(obs) plugin = os.path.join(THIS_DIR, 'visitor-plugin.py') (visited, updated, skipped, failed) = client.visit(plugin, 'TEST', start=start, halt_on_error=True) logger.debug("observations visited: {}".format(len(visited))) self.assertGreater(len(visited), 0, msg="No Observations Visited") finally: try: client.delete_observation("TEST", name) except: logger.warning('Failed to delete test observation, continuing') logger.info('-- END :test_create_and_visit --')
def test_post_observation(self, mock_conn, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/auth' collection = 'cfht' observation_id = '7000000o' service = 'caom2repo' service_url = 'www.cadc.nrc.ca' obs = SimpleObservation(collection, observation_id) level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(netrc='somenetrc'), level, host=service_url) response = MagicMock() response.status = 200 mock_conn.return_value = response iobuffer = BytesIO() ObservationWriter().write(obs, iobuffer) obsxml = iobuffer.getvalue() response.content = obsxml visitor.post_observation(obs) self.assertEqual('POST', mock_conn.call_args[0][0].method) self.assertEqual( '/{}/auth/{}/{}'.format(service, collection, observation_id), mock_conn.call_args[0][0].path_url) self.assertEqual('application/xml', mock_conn.call_args[0][0].headers['Content-Type']) self.assertEqual(obsxml, mock_conn.call_args[0][0].body) # signal problems http_error = requests.HTTPError() response.status_code = 500 http_error.response = response response.raise_for_status.side_effect = [http_error] with self.assertRaises(exceptions.InternalServerException): visitor.update(obs) # temporary transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response response.raise_for_status.side_effect = [http_error, None] visitor.post_observation(obs) # permanent transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response def raise_error(): raise http_error response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.post_observation(obs)
def test_get_observations(self, mock_get, caps_mock): # This is almost similar to the previous test except that it gets # observations matching a collection and start/end criteria # Also, patch the CAOM2RepoClient now. caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/pub' response = MagicMock() response.status_code = 200 last_datetime = '2000-10-10T12:30:00.333' response.text = \ ('CFHT\t700000o\t2000-10-10T12:20:11.123\t' '3e00ca6129dc8358315015204ab9fe15\nCFHT\t700001o\t' + last_datetime + '\t3e00ca6129dc8358315015204ab9fe15') mock_get.return_value = response level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) end_date = util.utils.str2ivoa(last_datetime) expect_observations = ['700000o', '700001o'] self.assertEqual(expect_observations, visitor._get_observations('cfht')) self.assertEqual(end_date, visitor._start) mock_get.assert_called_once_with( ('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2', 'cfht'), params={'MAXREC': core.BATCH_SIZE}) mock_get.reset_mock() visitor._get_observations('cfht', end=datetime.strptime( '2000-11-11', '%Y-%m-%d')) mock_get.assert_called_once_with( ('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2', 'cfht'), params={ 'END': '2000-11-11T00:00:00.000', 'MAXREC': core.BATCH_SIZE }) mock_get.reset_mock() visitor._get_observations( 'cfht', start=datetime.strptime('2000-11-11', '%Y-%m-%d'), end=datetime.strptime('2000-11-12', '%Y-%m-%d')) mock_get.assert_called_once_with( ('vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.2', 'cfht'), params={ 'START': '2000-11-11T00:00:00.000', 'END': '2000-11-12T00:00:00.000', 'MAXREC': core.BATCH_SIZE })
def test_get_observation(self, mock_get, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/pub' collection = 'cfht' observation_id = '7000000o' service_url = 'www.cadc.nrc.ca/caom2repo' obs = SimpleObservation(collection, observation_id) writer = ObservationWriter() ibuffer = BytesIO() writer.write(obs, ibuffer) response = MagicMock() response.status_code = 200 response.content = ibuffer.getvalue() mock_get.return_value = response ibuffer.seek(0) # reposition the buffer for reading level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url) self.assertEqual(obs, visitor.get_observation(collection, observation_id)) # signal problems http_error = requests.HTTPError() response.status_code = 500 http_error.response = response response.raise_for_status.side_effect = [http_error] with self.assertRaises(exceptions.InternalServerException): visitor.get_observation(collection, observation_id) # temporary transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response response.raise_for_status.side_effect = [http_error, None] visitor.read(collection, observation_id) # permanent transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response def raise_error(): raise http_error response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.get_observation(collection, observation_id)
def test_delete_observation(self, mock_conn, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/pub' collection = 'cfht' observation_id = '7000000o' service_url = 'www.cadc.nrc.ca' level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url) response = MagicMock() response.status = 200 mock_conn.return_value = response visitor.delete_observation(collection, observation_id) self.assertEqual('DELETE', mock_conn.call_args[0][0].method) # signal problems http_error = requests.HTTPError() response.status_code = 500 http_error.response = response response.raise_for_status.side_effect = [http_error] with self.assertRaises(exceptions.InternalServerException): visitor.delete(collection, observation_id) # temporary transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response response.raise_for_status.side_effect = [http_error, None] visitor.delete_observation(collection, observation_id) # permanent transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response def raise_error(): raise http_error response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.delete_observation(collection, observation_id)
def test_get_obs_from_file(self): level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) # no start or end with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file: obs_id_list = visitor._get_obs_from_file(obs_file, None, None, False) self.assertEqual('obs_id_1', obs_id_list[0]) self.assertEqual('obs_id_2', obs_id_list[1]) self.assertEqual('obs_id_3', obs_id_list[2]) # last_modified_date is earlier than start with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file: obs_id_list = visitor._get_obs_from_file( obs_file, util.str2ivoa('2000-10-11T12:30:00.333'), None, False) self.assertEqual('obs_id_1', obs_id_list[0]) # last_modified_date is between start and end with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file: obs_id_list = visitor._get_obs_from_file( obs_file, util.str2ivoa('2000-10-9T12:30:00.333'), util.str2ivoa('2016-10-11T12:30:00.333'), False) self.assertEqual('obs_id_1', obs_id_list[0]) self.assertEqual('obs_id_2', obs_id_list[1]) # last_modified_date is after end with open(os.path.join(THIS_DIR, 'data/obs_id.txt')) as obs_file: obs_id_list = visitor._get_obs_from_file( obs_file, util.str2ivoa('2000-10-9T12:30:00.333'), util.str2ivoa('2017-10-11T12:30:00.333'), False) self.assertEqual('obs_id_1', obs_id_list[0]) self.assertEqual('obs_id_2', obs_id_list[1]) self.assertEqual('obs_id_3', obs_id_list[2]) # error in file with open(os.path.join(THIS_DIR, 'data/obs_id_error.txt')) as obs_file: with self.assertRaises(Exception): obs_id_list = visitor._get_obs_from_file( obs_file, util.str2ivoa('2000-10-9T12:30:00.333'), util.str2ivoa('2016-10-11T12:30:00.333'), True)
def test_multiprocess_with_more_obs_id(self, client_mock): core.BATCH_SIZE = 3 # size of the batch is 3 obs_ids = [['a', 'b', 'c'], ['d', 'e', 'f'], []] client_mock.return_value.get_observation.side_effect = \ self.mock_get_observation level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) visitor.get_observation = PickableMagicMock( return_value=PickableMagicMock(spec=SimpleObservation)) visitor.post_observation = PickableMagicMock() visitor._get_observations = PickableMagicMock(side_effect=obs_ids) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht', start=None, end=None, obs_file=None, nthreads=3) try: self.assertEqual(6, len(visited)) self.assertEqual(6, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) self.assertTrue('a' in visited) self.assertTrue('b' in visited) self.assertTrue('c' in visited) self.assertTrue('d' in visited) self.assertTrue('e' in visited) self.assertTrue('f' in visited) self.assertFalse('g' in visited) self.assertTrue('a' in updated) self.assertTrue('b' in updated) self.assertTrue('c' in updated) self.assertTrue('d' in updated) self.assertTrue('e' in updated) self.assertTrue('f' in updated) self.assertFalse('g' in updated) finally: # lp.join() logging.info("DONE")
def test_shortcuts(self): level = logging.DEBUG target = CAOM2RepoClient(auth.Subject(), level) obs = SimpleObservation('CFHT', 'abc') target.put_observation = Mock() target.create(obs) target.put_observation.assert_called_with(obs) target.get_observation = Mock() target.read('CFHT', 'abc') target.get_observation.assert_called_with('CFHT', 'abc') target.post_observation = Mock() target.update(obs) target.post_observation.assert_called_with(obs) target.delete_observation = Mock() target.delete('CFHT', 'abc') target.delete_observation.assert_called_with('CFHT', 'abc')
def test_process(self): core.BATCH_SIZE = 3 # size of the batch is 3 obs = [['a', 'b', 'c'], ['d'], []] level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) visitor.get_observation = MagicMock(return_value=MagicMock( spec=SimpleObservation)) visitor.post_observation = MagicMock() visitor._get_observations = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') self.assertEqual(4, len(visited)) self.assertEqual(4, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) obs = [['a', 'b', 'c'], ['d', 'e', 'f'], []] visitor._get_observations = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') self.assertEqual(6, len(visited)) self.assertEqual(6, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) # make it return different status. errorplugin returns according to the # id of the observation: True for 'UPDATE', False for 'SKIP' and # raises exception for 'ERROR' obs_ids = [['UPDATE', 'SKIP', 'ERROR'], []] obs = [ SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP'), SimpleObservation(collection='TEST', observation_id='ERROR') ] visitor._get_observations = MagicMock(side_effect=obs_ids) visitor.get_observation = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht') self.assertEqual(3, len(visited)) self.assertEqual(1, len(updated)) self.assertEqual(1, len(skipped)) self.assertEqual(1, len(failed)) # repeat with other obs obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']] obs = [ SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP'), SimpleObservation(collection='TEST', observation_id='ERROR'), SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP') ] visitor._get_observations = MagicMock(side_effect=obs_ids) visitor.get_observation = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht') self.assertEqual(5, len(visited)) self.assertEqual(2, len(updated)) self.assertEqual(2, len(skipped)) self.assertEqual(1, len(failed)) # repeat but halt on first ERROR -> process only 3 observations obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']] obs = [ SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP'), SimpleObservation(collection='TEST', observation_id='ERROR'), SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP') ] visitor._get_observations = MagicMock(side_effect=obs_ids) visitor.get_observation = MagicMock(side_effect=obs) with self.assertRaises(SystemError): visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht', halt_on_error=True) # test with time boundaries core.BATCH_SIZE = 3 # size of the batch is 3 response = MagicMock() response.text = """ARCHIVE\ta\t2011-01-01T11:00:00.000 ARCHIVE\tb\t211-01-01T11:00:10.000 ARCHIVE\tc\t2011-01-01T12:00:00.000""" response2 = MagicMock() response2.text = """ARCHIVE\td\t2011-02-02T11:00:00.000""" level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) visitor.get_observation = MagicMock(return_value=MagicMock( spec=SimpleObservation)) visitor.post_observation = MagicMock() visitor._repo_client.get = MagicMock(side_effect=[response, response2]) start = '2010-10-10T12:00:00.000' end = '2012-12-12T11:11:11.000' (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht', start=util.str2ivoa(start), end=util.str2ivoa(end)) self.assertEqual(4, len(visited)) self.assertEqual(4, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) calls = [ call((core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'), params={ 'START': start, 'END': end, 'MAXREC': 3 }), call( (core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'), params={ 'START': '2011-01-01T12:00:00.000', # datetime of the last record in the batch 'END': end, 'MAXREC': 3 }) ] visitor._repo_client.get.assert_has_calls(calls)