def post_observation(self, observation, orig_checksum=None): """ Updates an observation in the CAOM2 repo :param observation: observation to update :param orig_checksum: the checksum of the observation to be updated. Posting this value prevents race conditions when observations are updated concurrently :return: updated observation """ assert observation.collection is not None assert observation.observation_id is not None path = '/{}/{}'.format(observation.collection, observation.observation_id) self.logger.debug('POST {}'.format(path)) ibuffer = BytesIO() ObservationWriter(False, False, 'caom2', self.namespace).write( observation, ibuffer) obs_xml = ibuffer.getvalue() headers = {'Content-Type': 'application/xml'} if orig_checksum: headers['If-Match'] = orig_checksum self._repo_client.post( (self.capability_id, path), headers=headers, data=obs_xml) self.logger.debug('Successfully updated Observation')
def test_get_observation(self, mock_get, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/pub' collection = 'cfht' observation_id = '7000000o' service_url = 'www.cadc.nrc.ca/caom2repo' obs = SimpleObservation(collection, observation_id) writer = ObservationWriter() ibuffer = BytesIO() writer.write(obs, ibuffer) response = MagicMock() response.status_code = 200 response.content = ibuffer.getvalue() mock_get.return_value = response ibuffer.seek(0) # reposition the buffer for reading level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url) self.assertEqual(obs, visitor.get_observation(collection, observation_id)) # signal problems http_error = requests.HTTPError() response.status_code = 500 http_error.response = response response.raise_for_status.side_effect = [http_error] with self.assertRaises(exceptions.InternalServerException): visitor.get_observation(collection, observation_id) # temporary transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response response.raise_for_status.side_effect = [http_error, None] visitor.read(collection, observation_id) # permanent transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response def raise_error(): raise http_error response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.get_observation(collection, observation_id)
def __init__(self): """ Create a repository object. """ self.client = CAOM2RepoClient() self.reader = ObservationReader(True) self.writer = ObservationWriter(True)
def test_get_observation(self, mock_get, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/pub' collection = 'cfht' observation_id = '7000000o' service_url = 'www.cadc.nrc.ca/caom2repo' obs = SimpleObservation(collection, observation_id) writer = ObservationWriter() ibuffer = BytesIO() writer.write(obs, ibuffer) response = MagicMock() response.status_code = 200 response.content = ibuffer.getvalue() mock_get.return_value = response ibuffer.seek(0) # reposition the buffer for reading level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url) self.assertEquals(obs, visitor.get_observation(collection, observation_id)) # signal problems http_error = requests.HTTPError() response.status_code = 500 http_error.response = response response.raise_for_status.side_effect = [http_error] with self.assertRaises(exceptions.InternalServerException): visitor.get_observation(collection, observation_id) # temporary transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response response.raise_for_status.side_effect = [http_error, None] visitor.read(collection, observation_id) # permanent transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response def raise_error(): raise http_error response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.get_observation(collection, observation_id)
def test_post_observation(self, mock_conn, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/auth' collection = 'cfht' observation_id = '7000000o' service = 'caom2repo' service_url = 'www.cadc.nrc.ca' obs = SimpleObservation(collection, observation_id) level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(netrc='somenetrc'), level, host=service_url) response = MagicMock() response.status = 200 mock_conn.return_value = response iobuffer = BytesIO() ObservationWriter().write(obs, iobuffer) obsxml = iobuffer.getvalue() response.content = obsxml visitor.post_observation(obs) self.assertEqual('POST', mock_conn.call_args[0][0].method) self.assertEqual( '/{}/auth/{}/{}'.format(service, collection, observation_id), mock_conn.call_args[0][0].path_url) self.assertEqual('application/xml', mock_conn.call_args[0][0].headers['Content-Type']) self.assertEqual(obsxml, mock_conn.call_args[0][0].body) # signal problems http_error = requests.HTTPError() response.status_code = 500 http_error.response = response response.raise_for_status.side_effect = [http_error] with self.assertRaises(exceptions.InternalServerException): visitor.update(obs) # temporary transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response response.raise_for_status.side_effect = [http_error, None] visitor.post_observation(obs) # permanent transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response def raise_error(): raise http_error response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.post_observation(obs)
def put_observation(self, observation): """ Add an observation to the CAOM2 repo :param observation: observation to add to the CAOM2 repo :return: Added observation """ assert observation.collection is not None assert observation.observation_id is not None path = '/{}/{}'.format(observation.collection, observation.observation_id) self.logger.debug('PUT {}'.format(path)) ibuffer = BytesIO() ObservationWriter().write(observation, ibuffer) obs_xml = ibuffer.getvalue() headers = {'Content-Type': 'application/xml'} self._repo_client.put( (CAOM2REPO_OBS_CAPABILITY_ID, path), headers=headers, data=obs_xml) self.logger.debug('Successfully put Observation')
def post_observation(self, observation): """ Updates an observation in the CAOM2 repo :param observation: observation to update :return: updated observation """ assert observation.collection is not None assert observation.observation_id is not None path = '/{}/{}'.format(observation.collection, observation.observation_id) logging.debug('POST {}'.format(path)) ibuffer = BytesIO() ObservationWriter().write(observation, ibuffer) obs_xml = ibuffer.getvalue() headers = {'Content-Type': 'application/xml'} response = self._repo_client.post((CAOM2REPO_OBS_CAPABILITY_ID, path), headers=headers, data=obs_xml) logging.debug('Successfully updated Observation\n')
def post_observation(self, observation): """ Updates an observation in the CAOM2 repo :param observation: observation to update :return: updated observation """ assert observation.collection is not None assert observation.observation_id is not None path = '/{}/{}'.format(observation.collection, observation.observation_id) self.logger.debug('POST {}'.format(path)) ibuffer = BytesIO() ObservationWriter(False, False, 'caom2', self.namespace).write( observation, ibuffer) obs_xml = ibuffer.getvalue() headers = {'Content-Type': 'application/xml'} self._repo_client.post( (self.capability_id, path), headers=headers, data=obs_xml) self.logger.debug('Successfully updated Observation')
def main_app(): parser = util.get_base_parser(version=version.version, default_resource_id=DEFAULT_RESOURCE_ID) parser.description = ( 'Client for a CAOM2 repo. In addition to CRUD (Create, Read, Update ' 'and Delete) operations it also implements a visitor operation that ' 'allows for updating multiple observations in a collection') parser.add_argument("-s", "--server", help='URL of the CAOM2 repo server') parser.formatter_class = argparse.RawTextHelpFormatter subparsers = parser.add_subparsers(dest='cmd') create_parser = subparsers.add_parser( 'create', description='Create a new observation', help='Create a new observation') create_parser.add_argument('observation', help='XML file containing the observation', type=argparse.FileType('r')) read_parser = subparsers.add_parser( 'read', description='Read an existing observation', help='Read an existing observation') read_parser.add_argument('--output', '-o', help='destination file', required=False) read_parser.add_argument('collection', help='collection name in CAOM2 repo') read_parser.add_argument('observationID', help='observation identifier') update_parser = subparsers.add_parser( 'update', description='Update an existing observation', help='Update an existing observation') update_parser.add_argument('observation', help='XML file containing the observation', type=argparse.FileType('r')) delete_parser = subparsers.add_parser( 'delete', description='Delete an existing observation', help='Delete an existing observation') delete_parser.add_argument('collection', help='collection name in CAOM2 repo') delete_parser.add_argument('observationID', help='observation identifier') # Note: RawTextHelpFormatter allows for the use of newline in epilog visit_parser = subparsers.add_parser( 'visit', formatter_class=argparse.RawTextHelpFormatter, description='Visit observations in a collection', help='Visit observations in a collection') visit_parser.add_argument('--plugin', required=True, type=argparse.FileType('r'), help='plugin class to update each observation') visit_parser.add_argument('--start', type=str2date, help=('earliest observation to visit ' '(UTC IVOA format: YYYY-mm-ddTH:M:S)')) visit_parser.add_argument( '--end', type=str2date, help='latest observation to visit (UTC IVOA format: YYYY-mm-ddTH:M:S)') visit_parser.add_argument( '--obs_file', help='file containing observations to be visited', type=argparse.FileType('r')) visit_parser.add_argument( '--threads', type=int, choices=xrange(2, 10), help='number of working threads used by the visitor when getting ' + 'observations, range is 2 to 10') visit_parser.add_argument( '--halt-on-error', action='store_true', help='stop visitor on first update exception raised by plugin') visit_parser.add_argument('collection', help='data collection in CAOM2 repo') visit_parser.epilog = \ """ Minimum plugin file format: ---- from caom2 import Observation class ObservationUpdater: def update(self, observation, **kwargs): assert isinstance(observation, Observation), ( 'observation {} is not an Observation'.format(observation)) # custom code to update the observation # other arguments passed by the calling code to the update # method: # kwargs['subject'] - user authentication that caom2repo was # invoked with ---- """ args = parser.parse_args() if len(sys.argv) < 2: parser.print_usage(file=sys.stderr) sys.stderr.write("{}: error: too few arguments\n".format(APP_NAME)) sys.exit(-1) if args.verbose: level = logging.INFO elif args.debug: level = logging.DEBUG else: level = logging.WARN subject = net.Subject.from_cmd_line_args(args) server = None if args.server: server = args.server multiprocessing.Manager() logging.basicConfig( format='%(asctime)s %(process)d %(levelname)-8s %(name)-12s ' + '%(funcName)s %(message)s', level=level, stream=sys.stdout) logger = logging.getLogger('main_app') client = CAOM2RepoClient(subject, level, args.resource_id, host=server) if args.cmd == 'visit': print("Visit") logger.debug( "Call visitor with plugin={}, start={}, end={}, collection={}, " + "obs_file={}, threads={}".format(args.plugin.name, args.start, args.end, args.collection, args.obs_file, args.threads)) try: (visited, updated, skipped, failed) = \ client.visit(args.plugin.name, args.collection, start=args.start, end=args.end, obs_file=args.obs_file, nthreads=args.threads, halt_on_error=args.halt_on_error) finally: if args.obs_file is not None: args.obs_file.close() logger.info( 'Visitor stats: visited/updated/skipped/errors: {}/{}/{}/{}'. format(len(visited), len(updated), len(skipped), len(failed))) elif args.cmd == 'create': logger.info("Create") obs_reader = ObservationReader() client.put_observation(obs_reader.read(args.observation)) elif args.cmd == 'read': logger.info("Read") observation = client.get_observation(args.collection, args.observationID) observation_writer = ObservationWriter() if args.output: observation_writer.write(observation, args.output) else: observation_writer.write(observation, sys.stdout) elif args.cmd == 'update': logger.info("Update") obs_reader = ObservationReader() # TODO not sure if need to read in string first client.post_observation(obs_reader.read(args.observation)) else: logger.info("Delete") client.delete_observation(collection=args.collection, observation_id=args.observationID) logger.info("DONE")
def test_main_app(self, client_mock): collection = 'cfht' observation_id = '7000000o' ifile = '/tmp/inputobs' obs = SimpleObservation(collection, observation_id) # test create with open(ifile, 'wb') as infile: ObservationWriter().write(obs, infile) sys.argv = [ "caom2tools", "create", '--resource-id', 'ivo://ca.nrc.ca/resource', ifile ] core.main_app() client_mock.return_value.put_observation.assert_called_with(obs) # test update sys.argv = [ "caom2tools", "update", '--resource-id', 'ivo://ca.nrc.ca/resource', ifile ] core.main_app() client_mock.return_value.post_observation.assert_called_with(obs) # test read sys.argv = [ "caom2tools", "read", '--resource-id', 'ivo://ca.nrc.ca/resource', collection, observation_id ] client_mock.return_value.get_observation.return_value = obs client_mock.return_value.namespace = obs_reader_writer.CAOM24_NAMESPACE core.main_app() client_mock.return_value.get_observation.\ assert_called_with(collection, observation_id) # repeat with output argument sys.argv = [ "caom2tools", "read", '--resource-id', 'ivo://ca.nrc.ca/resource', "--output", ifile, collection, observation_id ] client_mock.return_value.get_observation.return_value = obs core.main_app() client_mock.return_value.get_observation.\ assert_called_with(collection, observation_id) os.remove(ifile) # test delete sys.argv = [ "caom2tools", "delete", '--resource-id', 'ivo://ca.nrc.ca/resource', collection, observation_id ] core.main_app() client_mock.return_value.delete_observation.assert_called_with( collection=collection, observation_id=observation_id) # test visit # get the absolute path to be able to run the tests with the # astropy frameworks plugin_file = THIS_DIR + "/passplugin.py" sys.argv = [ "caom2tools", "visit", '--resource-id', 'ivo://ca.nrc.ca/resource', "--plugin", plugin_file, "--start", "2012-01-01T11:22:33", "--end", "2013-01-01T11:33:22", collection ] client_mock.return_value.visit.return_value = ['1'], ['1'], [], [] with open(plugin_file, 'r') as infile: core.main_app() client_mock.return_value.visit.assert_called_with( ANY, collection, halt_on_error=False, nthreads=None, obs_file=None, start=core.str2date("2012-01-01T11:22:33"), end=core.str2date("2013-01-01T11:33:22")) # repeat visit test with halt-on-error sys.argv = [ "caom2tools", "visit", '--resource-id', 'ivo://ca.nrc.ca/resource', "--plugin", plugin_file, '--halt-on-error', "--start", "2012-01-01T11:22:33", "--end", "2013-01-01T11:33:22", collection ] client_mock.return_value.visit.return_value = ['1'], ['1'], [], [] with open(plugin_file, 'r') as infile: core.main_app() client_mock.return_value.visit.assert_called_with( ANY, collection, halt_on_error=True, nthreads=None, obs_file=None, start=core.str2date("2012-01-01T11:22:33"), end=core.str2date("2013-01-01T11:33:22"))
def main_app(): parser = util.get_base_parser(version=version.version, default_resource_id=DEFAULT_RESOURCE_ID) parser.description = ( 'Client for a CAOM2 repo. In addition to CRUD (Create, Read, Update and Delete) ' 'operations it also implements a visitor operation that allows for updating ' 'multiple observations in a collection') parser.add_argument("-s", "--server", help='URL of the CAOM2 repo server') parser.formatter_class = argparse.RawTextHelpFormatter subparsers = parser.add_subparsers(dest='cmd') create_parser = subparsers.add_parser( 'create', description='Create a new observation', help='Create a new observation') create_parser.add_argument('observation', help='XML file containing the observation', type=argparse.FileType('r')) read_parser = subparsers.add_parser( 'read', description='Read an existing observation', help='Read an existing observation') read_parser.add_argument('--output', '-o', help='destination file', required=False) read_parser.add_argument('collection', help='collection name in CAOM2 repo') read_parser.add_argument('observationID', help='observation identifier') update_parser = subparsers.add_parser( 'update', description='Update an existing observation', help='Update an existing observation') update_parser.add_argument('observation', help='XML file containing the observation', type=argparse.FileType('r')) delete_parser = subparsers.add_parser( 'delete', description='Delete an existing observation', help='Delete an existing observation') delete_parser.add_argument('collection', help='collection name in CAOM2 repo') delete_parser.add_argument('observationID', help='observation identifier') # Note: RawTextHelpFormatter allows for the use of newline in epilog visit_parser = subparsers.add_parser( 'visit', formatter_class=argparse.RawTextHelpFormatter, description='Visit observations in a collection', help='Visit observations in a collection') visit_parser.add_argument('--plugin', required=True, type=argparse.FileType('r'), help='plugin class to update each observation') visit_parser.add_argument( '--start', type=str2date, help='earliest observation to visit (UTC IVOA format: YYYY-mm-ddTH:M:S)' ) visit_parser.add_argument( '--end', type=str2date, help='latest observation to visit (UTC IVOA format: YYYY-mm-ddTH:M:S)') visit_parser.add_argument( '--halt-on-error', action='store_true', help='stop visitor on first update exception raised by plugin') visit_parser.add_argument('collection', help='data collection in CAOM2 repo') visit_parser.epilog =\ """ Minimum plugin file format: ---- from caom2 import Observation class ObservationUpdater: def update(self, observation): assert isinstance(observation, Observation), ( 'observation {} is not an Observation'.format(observation)) # custom code to update the observation ---- """ args = parser.parse_args() if args.verbose: logging.basicConfig(level=logging.INFO, stream=sys.stdout) elif args.debug: logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) else: logging.basicConfig(level=logging.WARN, stream=sys.stdout) subject = net.Subject.from_cmd_line_args(args) server = None if args.server: server = args.server client = CAOM2RepoClient(subject, args.resource_id, host=server) if args.cmd == 'visit': print("Visit") logging.debug( "Call visitor with plugin={}, start={}, end={}, collection={}". format(args.plugin.name, args.start, args.end, args.collection)) (visited, updated, skipped, failed) = \ client.visit(args.plugin.name, args.collection, start=args.start, end=args.end, halt_on_error=args.halt_on_error) logging.info( 'Visitor stats: visited/updated/skipped/errors: {}/{}/{}/{}'. format(len(visited), len(updated), len(skipped), len(failed))) elif args.cmd == 'create': logging.info("Create") obs_reader = ObservationReader() client.put_observation(obs_reader.read(args.observation)) elif args.cmd == 'read': logging.info("Read") observation = client.get_observation(args.collection, args.observationID) observation_writer = ObservationWriter() if args.output: with open(args.output, 'wb') as obsfile: observation_writer.write(observation, obsfile) else: observation_writer.write(observation, sys.stdout) elif args.cmd == 'update': logging.info("Update") obs_reader = ObservationReader() # TODO not sure if need to read in string first client.post_observation(obs_reader.read(args.observation)) else: logging.info("Delete") client.delete_observation(collection=args.collection, observation_id=args.observationID) logging.info("DONE")
class Repository(object): """ Wrapper manager class for the caom2repoClient utility. Public Interface: There are only three public methods 1) The constructor 2) process, a context manager for use in a with statement 3) remove, to remove an observation from the repository The get and put methods are nominally private, and the implementation may change to suit the details of the caom2repoClient class. Notes: The caom2repoClient has four methods to get, put, update and remove an observation. The get, put and update actions require that state be maintained: * If the observation does exist, the final call to push the observation back into the repository must be an update. * If an observation does not exist in the repository, the final call to push the observation into the repository must be a put. """ def __init__(self): """ Create a repository object. """ self.client = CAOM2RepoClient() self.reader = ObservationReader(True) self.writer = ObservationWriter(True) @contextmanager def process(self, uri, allow_remove=False, dry_run=False): """ Context manager to fetch and store a CAOM-2 observation. Arguments: uri: a CAOM-2 URI identifing an observation that may or may not exist allow_remove: if the updated observation is empty (contains no planes) then the observation is removed. Otherwise this is an error. dry_run: disable putting the replacement observation if true Yields: An ObservationWrapper either containing the observation, if it already exists in CAOM-2, or None otherwise. The new/updated/replacement observation should be placed back into this container in the body of the with block. If None is placed into the container then no update will be performed. Exceptions: CAOMError on failure Usage: Pseudocode illustrating the intended usage:: repository = Repository() for observationID in mycollection: uri = <make uri from collection and observationID> with repository.process(uri) as wrapper: if wrapper.observation is None: wrapper.observation = SimpleObservation(...) <perform some operation>(wrapper.observation) """ wrapper = ObservationWrapper(self.get(uri)) exists = wrapper.observation is not None # If the observation already exists, make a note of the planes # present. existing_planes = None if exists: existing_planes = set(wrapper.observation.planes.keys()) yield wrapper if wrapper.observation is not None: if len(wrapper.observation.planes) == 0: # All planes have been removed from the observation: can # we remove it? if allow_remove: # Only need to remove it if it already existed. if exists: logger.info('No planes left: removing record %s', uri) if not dry_run: self.remove(uri) else: # If removal wasn't allowed, raise an error. raise CAOMError( 'processed CAOM-2 record contains no planes') else: # There are planes: put/update the observation. if exists: # First check whether planes have been removed. if existing_planes.issubset( set(wrapper.observation.planes.keys())): logger.debug('No planes have been removed: updating') else: # It seems that the CAOM-2 repository service does not # always notice the removal of planes. CADC therefore # recommend removing and re-putting the observation # in this case. logger.info('Planes removed: deleting and re-putting') if not dry_run: self.remove(uri) exists = False if not dry_run: self.put(uri, wrapper.observation, exists) def get(self, uri): """ Get an observation from the CAOM-2 repository Arguments: uri: a CAOM-2 URI identifing an observation that may or may not exist. Returns: The CAOM-2 observation object, or None if it does not exist yet. Exceptions: CAOMError on failure to fetch the observation (but not if the only error is that it doesn't exist). """ if isinstance(uri, ObservationURI): myuri = uri.uri elif isinstance(uri, str): myuri = uri else: myuri = str(uri) try: logger.debug('Getting CAOM-2 record: %s', myuri) xml = self.client.get_xml(myuri) logger.debug('Parsing CAOM-2 record') with BytesIO(xml) as f: observation = self.reader.read(f) return observation except CAOM2RepoNotFound: logger.debug('CAOM-2 record not found') return None except CAOM2RepoError: logger.exception('error fetching observation from CAOM-2') raise CAOMError('failed to fetch observation from CAOM-2') except Exception as e: logger.exception( 'unexpected exception fetching observation from CAOM-2') raise CAOMError('failed to fetch observation from CAOM-2') def put(self, uri, observation, exists): """ Put or update an observation into the CAOM-2 repository. Arguments: uri: the CAOM-2 URI of the observation observation: the CAOM-2 observation object exists: if True, use update, else use put Exceptions: CAOMError on failure to write the observation to the repository """ if isinstance(uri, ObservationURI): myuri = uri.uri elif isinstance(uri, str): myuri = uri else: myuri = str(uri) logger.debug('Serializing CAOM-2 record') with BytesIO() as f: self.writer.write(observation, f) xml = f.getvalue() try: if exists: logger.debug('Updating CAOM-2 record: %s', myuri) self.client.update_xml(myuri, xml) else: logger.debug('Putting new CAOM-2 record: %s', myuri) self.client.put_xml(myuri, xml) except CAOM2RepoError: logger.exception('error putting/updating observation in CAOM-2') raise CAOMError('failed to put/update observation in CAOM-2') def remove(self, uri): """ Remove an observation from the CAOM-2 repository. Arguments: uri: the CAOM-2 URI of the observation Exceptions: CAOMError on failure """ if isinstance(uri, ObservationURI): myuri = uri.uri elif isinstance(uri, str): myuri = uri else: myuri = str(uri) try: logger.debug('Removing CAOM-2 record: %s', myuri) self.client.remove(myuri) except CAOM2RepoError: logger.exception('error removing observation from CAOM-2') raise CAOMError('failed to remove observation from CAOM-2')