def test_get_observation(self, mock_get, caps_mock):
caps_mock.get_service_host.return_value = 'some.host.com'
caps_mock.return_value.get_access_url.return_value =\
'http://serviceurl/caom2repo/pub'
collection = 'cfht'
observation_id = '7000000o'
service_url = 'www.cadc.nrc.ca/caom2repo'
obs = SimpleObservation(collection, observation_id)
writer = ObservationWriter()
ibuffer = BytesIO()
writer.write(obs, ibuffer)
response = MagicMock()
response.status_code = 200
response.content = ibuffer.getvalue()
mock_get.return_value = response
ibuffer.seek(0) # reposition the buffer for reading
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url)
self.assertEqual(obs,
visitor.get_observation(collection, observation_id))
# signal problems
http_error = requests.HTTPError()
response.status_code = 500
http_error.response = response
response.raise_for_status.side_effect = [http_error]
with self.assertRaises(exceptions.InternalServerException):
visitor.get_observation(collection, observation_id)
# temporary transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
response.raise_for_status.side_effect = [http_error, None]
visitor.read(collection, observation_id)
# permanent transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
def raise_error():
raise http_error
response.raise_for_status.side_effect = raise_error
with self.assertRaises(exceptions.HttpException):
visitor.get_observation(collection, observation_id)
def test_get_observation(self, mock_get, caps_mock):
caps_mock.get_service_host.return_value = 'some.host.com'
caps_mock.return_value.get_access_url.return_value =\
'http://serviceurl/caom2repo/pub'
collection = 'cfht'
observation_id = '7000000o'
service_url = 'www.cadc.nrc.ca/caom2repo'
obs = SimpleObservation(collection, observation_id)
writer = ObservationWriter()
ibuffer = BytesIO()
writer.write(obs, ibuffer)
response = MagicMock()
response.status_code = 200
response.content = ibuffer.getvalue()
mock_get.return_value = response
ibuffer.seek(0) # reposition the buffer for reading
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url)
self.assertEquals(obs,
visitor.get_observation(collection, observation_id))
# signal problems
http_error = requests.HTTPError()
response.status_code = 500
http_error.response = response
response.raise_for_status.side_effect = [http_error]
with self.assertRaises(exceptions.InternalServerException):
visitor.get_observation(collection, observation_id)
# temporary transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
response.raise_for_status.side_effect = [http_error, None]
visitor.read(collection, observation_id)
# permanent transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
def raise_error(): raise http_error
response.raise_for_status.side_effect = raise_error
with self.assertRaises(exceptions.HttpException):
visitor.get_observation(collection, observation_id)
def main_app():
parser = util.get_base_parser(version=version.version,
default_resource_id=DEFAULT_RESOURCE_ID)
parser.description = (
'Client for a CAOM2 repo. In addition to CRUD (Create, Read, Update '
'and Delete) operations it also implements a visitor operation that '
'allows for updating multiple observations in a collection')
parser.add_argument("-s", "--server", help='URL of the CAOM2 repo server')
parser.formatter_class = argparse.RawTextHelpFormatter
subparsers = parser.add_subparsers(dest='cmd')
create_parser = subparsers.add_parser(
'create',
description='Create a new observation',
help='Create a new observation')
create_parser.add_argument('observation',
help='XML file containing the observation',
type=argparse.FileType('r'))
read_parser = subparsers.add_parser(
'read',
description='Read an existing observation',
help='Read an existing observation')
read_parser.add_argument('--output',
'-o',
help='destination file',
required=False)
read_parser.add_argument('collection',
help='collection name in CAOM2 repo')
read_parser.add_argument('observationID', help='observation identifier')
update_parser = subparsers.add_parser(
'update',
description='Update an existing observation',
help='Update an existing observation')
update_parser.add_argument('observation',
help='XML file containing the observation',
type=argparse.FileType('r'))
delete_parser = subparsers.add_parser(
'delete',
description='Delete an existing observation',
help='Delete an existing observation')
delete_parser.add_argument('collection',
help='collection name in CAOM2 repo')
delete_parser.add_argument('observationID', help='observation identifier')
# Note: RawTextHelpFormatter allows for the use of newline in epilog
visit_parser = subparsers.add_parser(
'visit',
formatter_class=argparse.RawTextHelpFormatter,
description='Visit observations in a collection',
help='Visit observations in a collection')
visit_parser.add_argument('--plugin',
required=True,
type=argparse.FileType('r'),
help='plugin class to update each observation')
visit_parser.add_argument('--start',
type=str2date,
help=('earliest observation to visit '
'(UTC IVOA format: YYYY-mm-ddTH:M:S)'))
visit_parser.add_argument(
'--end',
type=str2date,
help='latest observation to visit (UTC IVOA format: YYYY-mm-ddTH:M:S)')
visit_parser.add_argument(
'--obs_file',
help='file containing observations to be visited',
type=argparse.FileType('r'))
visit_parser.add_argument(
'--threads',
type=int,
choices=xrange(2, 10),
help='number of working threads used by the visitor when getting ' +
'observations, range is 2 to 10')
visit_parser.add_argument(
'--halt-on-error',
action='store_true',
help='stop visitor on first update exception raised by plugin')
visit_parser.add_argument('collection',
help='data collection in CAOM2 repo')
visit_parser.epilog = \
"""
Minimum plugin file format:
----
from caom2 import Observation
class ObservationUpdater:
def update(self, observation, **kwargs):
assert isinstance(observation, Observation), (
'observation {} is not an Observation'.format(observation))
# custom code to update the observation
# other arguments passed by the calling code to the update
# method:
# kwargs['subject'] - user authentication that caom2repo was
# invoked with
----
"""
args = parser.parse_args()
if len(sys.argv) < 2:
parser.print_usage(file=sys.stderr)
sys.stderr.write("{}: error: too few arguments\n".format(APP_NAME))
sys.exit(-1)
if args.verbose:
level = logging.INFO
elif args.debug:
level = logging.DEBUG
else:
level = logging.WARN
subject = net.Subject.from_cmd_line_args(args)
server = None
if args.server:
server = args.server
multiprocessing.Manager()
logging.basicConfig(
format='%(asctime)s %(process)d %(levelname)-8s %(name)-12s ' +
'%(funcName)s %(message)s',
level=level,
stream=sys.stdout)
logger = logging.getLogger('main_app')
client = CAOM2RepoClient(subject, level, args.resource_id, host=server)
if args.cmd == 'visit':
print("Visit")
logger.debug(
"Call visitor with plugin={}, start={}, end={}, collection={}, " +
"obs_file={}, threads={}".format(args.plugin.name, args.start,
args.end, args.collection,
args.obs_file, args.threads))
try:
(visited, updated, skipped, failed) = \
client.visit(args.plugin.name, args.collection,
start=args.start,
end=args.end, obs_file=args.obs_file,
nthreads=args.threads,
halt_on_error=args.halt_on_error)
finally:
if args.obs_file is not None:
args.obs_file.close()
logger.info(
'Visitor stats: visited/updated/skipped/errors: {}/{}/{}/{}'.
format(len(visited), len(updated), len(skipped), len(failed)))
elif args.cmd == 'create':
logger.info("Create")
obs_reader = ObservationReader()
client.put_observation(obs_reader.read(args.observation))
elif args.cmd == 'read':
logger.info("Read")
observation = client.get_observation(args.collection,
args.observationID)
observation_writer = ObservationWriter()
if args.output:
observation_writer.write(observation, args.output)
else:
observation_writer.write(observation, sys.stdout)
elif args.cmd == 'update':
logger.info("Update")
obs_reader = ObservationReader()
# TODO not sure if need to read in string first
client.post_observation(obs_reader.read(args.observation))
else:
logger.info("Delete")
client.delete_observation(collection=args.collection,
observation_id=args.observationID)
logger.info("DONE")
def main_app():
parser = util.get_base_parser(version=version.version,
default_resource_id=DEFAULT_RESOURCE_ID)
parser.description = (
'Client for a CAOM2 repo. In addition to CRUD (Create, Read, Update and Delete) '
'operations it also implements a visitor operation that allows for updating '
'multiple observations in a collection')
parser.add_argument("-s", "--server", help='URL of the CAOM2 repo server')
parser.formatter_class = argparse.RawTextHelpFormatter
subparsers = parser.add_subparsers(dest='cmd')
create_parser = subparsers.add_parser(
'create',
description='Create a new observation',
help='Create a new observation')
create_parser.add_argument('observation',
help='XML file containing the observation',
type=argparse.FileType('r'))
read_parser = subparsers.add_parser(
'read',
description='Read an existing observation',
help='Read an existing observation')
read_parser.add_argument('--output',
'-o',
help='destination file',
required=False)
read_parser.add_argument('collection',
help='collection name in CAOM2 repo')
read_parser.add_argument('observationID', help='observation identifier')
update_parser = subparsers.add_parser(
'update',
description='Update an existing observation',
help='Update an existing observation')
update_parser.add_argument('observation',
help='XML file containing the observation',
type=argparse.FileType('r'))
delete_parser = subparsers.add_parser(
'delete',
description='Delete an existing observation',
help='Delete an existing observation')
delete_parser.add_argument('collection',
help='collection name in CAOM2 repo')
delete_parser.add_argument('observationID', help='observation identifier')
# Note: RawTextHelpFormatter allows for the use of newline in epilog
visit_parser = subparsers.add_parser(
'visit',
formatter_class=argparse.RawTextHelpFormatter,
description='Visit observations in a collection',
help='Visit observations in a collection')
visit_parser.add_argument('--plugin',
required=True,
type=argparse.FileType('r'),
help='plugin class to update each observation')
visit_parser.add_argument(
'--start',
type=str2date,
help='earliest observation to visit (UTC IVOA format: YYYY-mm-ddTH:M:S)'
)
visit_parser.add_argument(
'--end',
type=str2date,
help='latest observation to visit (UTC IVOA format: YYYY-mm-ddTH:M:S)')
visit_parser.add_argument(
'--halt-on-error',
action='store_true',
help='stop visitor on first update exception raised by plugin')
visit_parser.add_argument('collection',
help='data collection in CAOM2 repo')
visit_parser.epilog =\
"""
Minimum plugin file format:
----
from caom2 import Observation
class ObservationUpdater:
def update(self, observation):
assert isinstance(observation, Observation), (
'observation {} is not an Observation'.format(observation))
# custom code to update the observation
----
"""
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
elif args.debug:
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
else:
logging.basicConfig(level=logging.WARN, stream=sys.stdout)
subject = net.Subject.from_cmd_line_args(args)
server = None
if args.server:
server = args.server
client = CAOM2RepoClient(subject, args.resource_id, host=server)
if args.cmd == 'visit':
print("Visit")
logging.debug(
"Call visitor with plugin={}, start={}, end={}, collection={}".
format(args.plugin.name, args.start, args.end, args.collection))
(visited, updated, skipped, failed) = \
client.visit(args.plugin.name, args.collection, start=args.start, end=args.end,
halt_on_error=args.halt_on_error)
logging.info(
'Visitor stats: visited/updated/skipped/errors: {}/{}/{}/{}'.
format(len(visited), len(updated), len(skipped), len(failed)))
elif args.cmd == 'create':
logging.info("Create")
obs_reader = ObservationReader()
client.put_observation(obs_reader.read(args.observation))
elif args.cmd == 'read':
logging.info("Read")
observation = client.get_observation(args.collection,
args.observationID)
observation_writer = ObservationWriter()
if args.output:
with open(args.output, 'wb') as obsfile:
observation_writer.write(observation, obsfile)
else:
observation_writer.write(observation, sys.stdout)
elif args.cmd == 'update':
logging.info("Update")
obs_reader = ObservationReader()
# TODO not sure if need to read in string first
client.post_observation(obs_reader.read(args.observation))
else:
logging.info("Delete")
client.delete_observation(collection=args.collection,
observation_id=args.observationID)
logging.info("DONE")
class Repository(object):
"""
Wrapper manager class for the caom2repoClient utility.
Public Interface:
There are only three public methods
1) The constructor
2) process, a context manager for use in a with statement
3) remove, to remove an observation from the repository
The get and put methods are nominally private, and the implementation may
change to suit the details of the caom2repoClient class.
Notes:
The caom2repoClient has four methods to get, put, update and remove
an observation. The get, put and update actions require that state be
maintained:
* If the observation does exist, the final call to push the observation
back into the repository must be an update.
* If an observation does not exist in the repository, the final call to
push the observation into the repository must be a put.
"""
def __init__(self):
"""
Create a repository object.
"""
self.client = CAOM2RepoClient()
self.reader = ObservationReader(True)
self.writer = ObservationWriter(True)
@contextmanager
def process(self, uri, allow_remove=False, dry_run=False):
"""
Context manager to fetch and store a CAOM-2 observation.
Arguments:
uri: a CAOM-2 URI identifing an observation that may or may not exist
allow_remove: if the updated observation is empty (contains no planes)
then the observation is removed. Otherwise this is an error.
dry_run: disable putting the replacement observation if true
Yields:
An ObservationWrapper either containing the observation, if it already
exists in CAOM-2, or None otherwise. The new/updated/replacement
observation should be placed back into this container in the
body of the with block. If None is placed into the container then
no update will be performed.
Exceptions:
CAOMError
on failure
Usage:
Pseudocode illustrating the intended usage::
repository = Repository()
for observationID in mycollection:
uri = <make uri from collection and observationID>
with repository.process(uri) as wrapper:
if wrapper.observation is None:
wrapper.observation = SimpleObservation(...)
<perform some operation>(wrapper.observation)
"""
wrapper = ObservationWrapper(self.get(uri))
exists = wrapper.observation is not None
# If the observation already exists, make a note of the planes
# present.
existing_planes = None
if exists:
existing_planes = set(wrapper.observation.planes.keys())
yield wrapper
if wrapper.observation is not None:
if len(wrapper.observation.planes) == 0:
# All planes have been removed from the observation: can
# we remove it?
if allow_remove:
# Only need to remove it if it already existed.
if exists:
logger.info('No planes left: removing record %s', uri)
if not dry_run:
self.remove(uri)
else:
# If removal wasn't allowed, raise an error.
raise CAOMError(
'processed CAOM-2 record contains no planes')
else:
# There are planes: put/update the observation.
if exists:
# First check whether planes have been removed.
if existing_planes.issubset(
set(wrapper.observation.planes.keys())):
logger.debug('No planes have been removed: updating')
else:
# It seems that the CAOM-2 repository service does not
# always notice the removal of planes. CADC therefore
# recommend removing and re-putting the observation
# in this case.
logger.info('Planes removed: deleting and re-putting')
if not dry_run:
self.remove(uri)
exists = False
if not dry_run:
self.put(uri, wrapper.observation, exists)
def get(self, uri):
"""
Get an observation from the CAOM-2 repository
Arguments:
uri: a CAOM-2 URI identifing an observation that may or may not exist.
Returns:
The CAOM-2 observation object, or None if it does not exist yet.
Exceptions:
CAOMError
on failure to fetch the observation (but not if the only error
is that it doesn't exist).
"""
if isinstance(uri, ObservationURI):
myuri = uri.uri
elif isinstance(uri, str):
myuri = uri
else:
myuri = str(uri)
try:
logger.debug('Getting CAOM-2 record: %s', myuri)
xml = self.client.get_xml(myuri)
logger.debug('Parsing CAOM-2 record')
with BytesIO(xml) as f:
observation = self.reader.read(f)
return observation
except CAOM2RepoNotFound:
logger.debug('CAOM-2 record not found')
return None
except CAOM2RepoError:
logger.exception('error fetching observation from CAOM-2')
raise CAOMError('failed to fetch observation from CAOM-2')
except Exception as e:
logger.exception(
'unexpected exception fetching observation from CAOM-2')
raise CAOMError('failed to fetch observation from CAOM-2')
def put(self, uri, observation, exists):
"""
Put or update an observation into the CAOM-2 repository.
Arguments:
uri: the CAOM-2 URI of the observation
observation: the CAOM-2 observation object
exists: if True, use update, else use put
Exceptions:
CAOMError on failure to write the observation to the repository
"""
if isinstance(uri, ObservationURI):
myuri = uri.uri
elif isinstance(uri, str):
myuri = uri
else:
myuri = str(uri)
logger.debug('Serializing CAOM-2 record')
with BytesIO() as f:
self.writer.write(observation, f)
xml = f.getvalue()
try:
if exists:
logger.debug('Updating CAOM-2 record: %s', myuri)
self.client.update_xml(myuri, xml)
else:
logger.debug('Putting new CAOM-2 record: %s', myuri)
self.client.put_xml(myuri, xml)
except CAOM2RepoError:
logger.exception('error putting/updating observation in CAOM-2')
raise CAOMError('failed to put/update observation in CAOM-2')
def remove(self, uri):
"""
Remove an observation from the CAOM-2 repository.
Arguments:
uri: the CAOM-2 URI of the observation
Exceptions:
CAOMError on failure
"""
if isinstance(uri, ObservationURI):
myuri = uri.uri
elif isinstance(uri, str):
myuri = uri
else:
myuri = str(uri)
try:
logger.debug('Removing CAOM-2 record: %s', myuri)
self.client.remove(myuri)
except CAOM2RepoError:
logger.exception('error removing observation from CAOM-2')
raise CAOMError('failed to remove observation from CAOM-2')