def test_visit_retry_on_412(self):
# observation changed on server while visited
core.BATCH_SIZE = 3 # size of the batch is 3
obs = [['a'], []]
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
observation = SimpleObservation('cfht', 'a')
observation.acc_meta_checksum = ChecksumURI('md5:abc')
visitor.get_observation = MagicMock(
side_effect=[observation, observation])
exception_412 = exceptions.UnexpectedException()
exception_412.orig_exception = Mock()
exception_412.orig_exception.response = Mock(status_code=412)
visitor.post_observation = MagicMock(side_effect=[exception_412, None])
visitor._get_observations = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht')
self.assertEqual(1, len(visited))
self.assertEqual(1, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
# get and post called twice to recover from error HTTP status 412 -
# precondition
self.assertEqual(2, visitor.get_observation.call_count)
self.assertEqual(2, visitor.post_observation.call_count)
visitor.post_observation.assert_called_with(
observation, observation.acc_meta_checksum.uri)
def test_plugin_class(self):
# plugin class does not change the observation
collection = 'cfht'
observation_id = '7000000o'
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
obs = SimpleObservation(collection, observation_id)
expect_obs = copy.deepcopy(obs)
visitor._load_plugin_class(os.path.join(THIS_DIR, 'passplugin.py'))
visitor.plugin.update(obs)
self.assertEqual(expect_obs, obs)
# plugin class adds a plane to the observation
visitor = CAOM2RepoClient(auth.Subject(), level)
obs = SimpleObservation('cfht', '7000000o')
expect_obs = copy.deepcopy(obs)
visitor._load_plugin_class(os.path.join(THIS_DIR, 'addplaneplugin.py'))
visitor.plugin.update(obs)
self.assertNotEqual(expect_obs, obs)
self.assertEqual(len(expect_obs.planes) + 1, len(obs.planes))
# non-existent the plugin file
with self.assertRaises(Exception):
visitor._load_plugin_class(os.path.join(THIS_DIR, 'blah.py'))
# non-existent ObservationUpdater class in the plugin file
with self.assertRaises(Exception):
visitor._load_plugin_class(
os.path.join(THIS_DIR, 'test_visitor.py'))
# non-existent update method in ObservationUpdater class
with self.assertRaises(Exception):
visitor._load_plugin_class(
os.path.join(THIS_DIR, 'noupdateplugin.py'))
def test_post_observation(self, mock_conn, caps_mock):
caps_mock.get_service_host.return_value = 'some.host.com'
caps_mock.return_value.get_access_url.return_value =\
'http://serviceurl/caom2repo/auth'
collection = 'cfht'
observation_id = '7000000o'
service = 'caom2repo'
service_url = 'www.cadc.nrc.ca'
obs = SimpleObservation(collection, observation_id)
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(netrc='somenetrc'),
level,
host=service_url)
response = MagicMock()
response.status = 200
mock_conn.return_value = response
iobuffer = BytesIO()
ObservationWriter().write(obs, iobuffer)
obsxml = iobuffer.getvalue()
response.content = obsxml
visitor.post_observation(obs)
self.assertEqual('POST', mock_conn.call_args[0][0].method)
self.assertEqual(
'/{}/auth/{}/{}'.format(service, collection, observation_id),
mock_conn.call_args[0][0].path_url)
self.assertEqual('application/xml',
mock_conn.call_args[0][0].headers['Content-Type'])
self.assertEqual(obsxml, mock_conn.call_args[0][0].body)
# signal problems
http_error = requests.HTTPError()
response.status_code = 500
http_error.response = response
response.raise_for_status.side_effect = [http_error]
with self.assertRaises(exceptions.InternalServerException):
visitor.update(obs)
# temporary transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
response.raise_for_status.side_effect = [http_error, None]
visitor.post_observation(obs)
# permanent transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
def raise_error():
raise http_error
response.raise_for_status.side_effect = raise_error
with self.assertRaises(exceptions.HttpException):
visitor.post_observation(obs)
def test_get_observation(self, mock_get, caps_mock):
caps_mock.get_service_host.return_value = 'some.host.com'
caps_mock.return_value.get_access_url.return_value =\
'http://serviceurl/caom2repo/pub'
collection = 'cfht'
observation_id = '7000000o'
service_url = 'www.cadc.nrc.ca/caom2repo'
obs = SimpleObservation(collection, observation_id)
writer = ObservationWriter()
ibuffer = BytesIO()
writer.write(obs, ibuffer)
response = MagicMock()
response.status_code = 200
response.content = ibuffer.getvalue()
mock_get.return_value = response
ibuffer.seek(0) # reposition the buffer for reading
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url)
self.assertEqual(obs,
visitor.get_observation(collection, observation_id))
# signal problems
http_error = requests.HTTPError()
response.status_code = 500
http_error.response = response
response.raise_for_status.side_effect = [http_error]
with self.assertRaises(exceptions.InternalServerException):
visitor.get_observation(collection, observation_id)
# temporary transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
response.raise_for_status.side_effect = [http_error, None]
visitor.read(collection, observation_id)
# permanent transient errors
http_error = requests.HTTPError()
response.status_code = 503
http_error.response = response
def raise_error():
raise http_error
response.raise_for_status.side_effect = raise_error
with self.assertRaises(exceptions.HttpException):
visitor.get_observation(collection, observation_id)
def test_shortcuts(self):
level = logging.DEBUG
target = CAOM2RepoClient(auth.Subject(), level)
obs = SimpleObservation('CFHT', 'abc')
target.put_observation = Mock()
target.create(obs)
target.put_observation.assert_called_with(obs)
target.get_observation = Mock()
target.read('CFHT', 'abc')
target.get_observation.assert_called_with('CFHT', 'abc')
target.post_observation = Mock()
target.update(obs)
target.post_observation.assert_called_with(obs)
target.delete_observation = Mock()
target.delete('CFHT', 'abc')
target.delete_observation.assert_called_with('CFHT', 'abc')
def build_observation(self,
observation,
common,
subsystem,
files):
"""
Construct a simple observation from the available metadata
Since we are dealing with raw data, the algorithm = "exposure"
by default, a change in notation for the JCMT.
Arguments:
obsid obsid from COMMON to be used as the observationID
common dictionary containing fields common to the observation
subsystem dictionary containing fields from ACSIS or SCUBA2
files dictionary containing artifact info dictionaries
"""
collection = self.collection
observationID = self.obsid
logger.debug('PROGRESS: build observationID = %s', self.obsid)
if observation is None:
observation = SimpleObservation(collection,
observationID)
# Determine data quality metrics for this observation.
data_quality = DataQuality(Quality.JUNK) \
if OMPState.is_caom_junk(common['quality']) else None
requirement_status = Requirements(Status.FAIL) \
if OMPState.is_caom_fail(common['quality']) else None
# "Requirements" is an observation-level attribute, so fill it in now.
observation.requirements = requirement_status
# Every ACSSIS and SCUBA2 observation has an obsnum in COMMON.
observation.sequence_number = common['obsnum']
observation.meta_release = common['release_date']
# The observation type is derived from COMMON.obs_type and
# COMMON.sam_mode
if common['obs_type'] == "science":
# raster is a synonym for scan
if common['sam_mode'] == "raster":
observation.obs_type = "scan"
else:
observation.obs_type = common['sam_mode']
else:
observation.obs_type = common['obs_type']
# set the observation intent
observation.intent = intent(common['obs_type'],
common['backend'])
proposal = Proposal(common['project'])
if common['pi'] is not None:
proposal.pi_name = common['pi']
if common['survey'] is not None:
proposal.project = common['survey']
if common['title'] is not None:
proposal.title = truncate_string(common['title'], 80)
observation.proposal = proposal
environment = Environment()
if common['atstart'] is not None:
make_env = True
environment.ambient_temp = common['atstart']
if common['elstart'] is not None:
environment.elevation = common['elstart']
if common['humstart'] is not None:
if common['humstart'] < 0.0:
environment.humidity = 0.0
elif common['humstart'] > 100.0:
environment.humidity = 100.0
else:
environment.humidity = common['humstart']
if common['seeingst'] is not None and common['seeingst'] > 0.0:
environment.seeing = common['seeingst']
if common['tau225st'] is not None:
environment.tau = common['tau225st']
environment.wavelength_tau = raw.SpeedOfLight/225.0e9
observation.environment = environment
frontend = common['instrume'].upper()
backend = common['backend'].upper()
if common['inbeam']:
inbeam = common['inbeam'].upper()
else:
inbeam = ''
instrument = Instrument(instrument_name(frontend,
backend,
inbeam))
instrument.keywords.update(self.instrument_keywords)
observation.instrument = instrument
hybrid = {}
if backend in ['ACSIS', 'DAS', 'AOSC']:
keys = sorted(subsystem.keys())
beamsize = 0.0
for key in keys:
productID = self.productID_dict[str(key)]
# Convert restfreq from GHz to Hz
restfreq = 1.0e9 * subsystem[key]['restfreq']
iffreq = subsystem[key]['iffreq']
ifchansp = subsystem[key]['ifchansp']
if productID not in hybrid:
hybrid[productID] = {}
hybrid[productID]['restfreq'] = restfreq
hybrid[productID]['iffreq'] = iffreq
hybrid[productID]['ifchansp'] = ifchansp
this_hybrid = hybrid[productID]
if 'freq_sig_lower' in this_hybrid:
this_hybrid['freq_sig_lower'] = \
min(subsystem[key]['freq_sig_lower'],
this_hybrid['freq_sig_lower'])
else:
this_hybrid['freq_sig_lower'] = \
subsystem[key]['freq_sig_lower']
if 'freq_sig_upper' in this_hybrid:
this_hybrid['freq_sig_upper'] = \
max(subsystem[key]['freq_sig_upper'],
this_hybrid['freq_sig_upper'])
else:
this_hybrid['freq_sig_upper'] = \
subsystem[key]['freq_sig_upper']
if 'freq_img_lower' in this_hybrid:
this_hybrid['freq_img_lower'] = \
min(subsystem[key]['freq_img_lower'],
this_hybrid['freq_img_lower'])
else:
this_hybrid['freq_img_lower'] = \
subsystem[key]['freq_img_lower']
if 'freq_img_upper' in this_hybrid:
this_hybrid['freq_img_upper'] = \
max(subsystem[key]['freq_img_upper'],
this_hybrid['freq_img_upper'])
else:
this_hybrid['freq_img_upper'] = \
subsystem[key]['freq_img_upper']
this_hybrid['meanfreq'] = (this_hybrid['freq_sig_lower'] +
this_hybrid['freq_sig_upper'])/2.0
# Compute maximum beam size for this observation in degrees
# frequencies are in GHz
# The scale factor is:
# 206264.8 ["/r] * sqrt(pi/2) * c [m GHz]/ 15 [m]
beamsize = max(beamsize, 1.435 / this_hybrid['meanfreq'])
else:
# Compute beam size in degrees for 850 micron array
# filter is in microns
# The scale factor is:
# pi/180 * sqrt(pi/2) * 1e-6 * lambda [um]/ 15 [m]
beamsize = 4.787e-6 * 850.0
if (observation.obs_type not in (
'flatfield', 'noise', 'setup', 'skydip')
and common['object']):
# The target is not significant for the excluded kinds of
# observation, even if supplied in COMMON
if common['object']:
targetname = target_name(common['object'])
target = Target(targetname)
if common['obsra'] is None or common['obsdec'] is None:
target.moving = True
target_position = None
else:
target.moving = False
target_position = TargetPosition(Point(common['obsra'],
common['obsdec']),
'ICRS',
2000.0)
observation.target_position = target_position
if common['standard'] is not None:
target.standard = True if common['standard'] else False
if backend != 'SCUBA-2':
subsysnr = min(subsystem.keys())
if subsystem[subsysnr]['zsource'] is not None:
target.redshift = subsystem[subsysnr]['zsource']
observation.target = target
telescope = Telescope(u'JCMT')
telescope.geo_location_x = common['obsgeo_x']
telescope.geo_location_y = common['obsgeo_y']
telescope.geo_location_z = common['obsgeo_z']
observation.telescope = telescope
# Delete any existing raw planes, since we will construct
# new ones from scratch. For all other planes, update the
# "data quality" since this is a plane-level attribute.
preview = defaultdict(list)
for productID in list(observation.planes):
if productID[0:3] == 'raw':
old_plane = observation.planes.pop(productID)
# Keep a list of the preview artifacts.
for old_artifact in old_plane.artifacts.values():
old_product_type = old_artifact.product_type
if ((old_product_type is ProductType.PREVIEW)
or (old_product_type is ProductType.THUMBNAIL)):
preview[productID].append(old_artifact)
else:
observation.planes[productID].quality = data_quality
# Use key for the numeric value of subsysnr here for brevity and
# to distinguish it from the string representation that will be
# named subsysnr in this section
for key in sorted(subsystem.keys()):
productID = self.productID_dict[str(key)]
obsid_subsysnr = subsystem[key]['obsid_subsysnr']
logger.debug('Processing subsystem %s: %s, %s',
key, obsid_subsysnr, productID)
# This plane might already have been created in a hybrid-mode
# observation, use it if it exists
if productID not in observation.planes:
observation.planes.add(Plane(productID))
plane = observation.planes[productID]
else:
plane = observation.planes[productID]
# set the release dates
plane.meta_release = common['release_date']
plane.data_release = common['release_date']
# all JCMT raw data is in a non-FITS format
plane.calibration_level = CalibrationLevel.RAW_INSTRUMENTAL
# set the plane data quality
plane.quality = data_quality
# For JCMT raw data, all artifacts have the same WCS
for file_info in files[obsid_subsysnr]:
file_name = file_info['name']
file_id = make_file_id_jcmt(file_name)
uri = 'ad:JCMT/' + file_id
if observation.intent == ObservationIntentType.SCIENCE:
artifact_product_type = ProductType.SCIENCE
else:
artifact_product_type = ProductType.CALIBRATION
artifact = Artifact(
uri, product_type=artifact_product_type,
release_type=ReleaseType.DATA,
content_type=determine_mime_type(file_name))
if file_info['size'] is not None:
artifact.content_length = file_info['size']
if file_info['md5sum'] is not None:
artifact.content_checksum = ChecksumURI(
'md5:{}'.format(file_info['md5sum']))
artifact.meta_release = common['release_date']
# There is only one part and one chunk for raw data
artifact.parts.add(Part('0'))
chunk = Chunk()
artifact.meta_release = common['release_date']
artifact.parts['0'].meta_release = common['release_date']
chunk.meta_release = common['release_date']
spatial_wcs = self.build_spatial_wcs(common, beamsize)
if spatial_wcs is not None:
chunk.position = spatial_wcs
chunk.energy = self.build_spectral_wcs(
common, subsystem[key], hybrid.get(productID))
chunk.time = self.build_temporal_wcs(common)
# Chunk is done, so append it to the part
artifact.parts['0'].chunks.append(chunk)
# and append the atrifact to the plane
plane.artifacts.add(artifact)
# Restore saved previews
for artifact in preview[productID]:
logger.debug(
'Retaining old preview/thumbnail: %s', artifact.uri)
plane.artifacts.add(artifact)
return observation
def test_main_app(self, client_mock):
collection = 'cfht'
observation_id = '7000000o'
ifile = '/tmp/inputobs'
obs = SimpleObservation(collection, observation_id)
# test create
with open(ifile, 'wb') as infile:
ObservationWriter().write(obs, infile)
sys.argv = [
"caom2tools", "create", '--resource-id',
'ivo://ca.nrc.ca/resource', ifile
]
core.main_app()
client_mock.return_value.put_observation.assert_called_with(obs)
# test update
sys.argv = [
"caom2tools", "update", '--resource-id',
'ivo://ca.nrc.ca/resource', ifile
]
core.main_app()
client_mock.return_value.post_observation.assert_called_with(obs)
# test read
sys.argv = [
"caom2tools", "read", '--resource-id', 'ivo://ca.nrc.ca/resource',
collection, observation_id
]
client_mock.return_value.get_observation.return_value = obs
client_mock.return_value.namespace = obs_reader_writer.CAOM24_NAMESPACE
core.main_app()
client_mock.return_value.get_observation.\
assert_called_with(collection, observation_id)
# repeat with output argument
sys.argv = [
"caom2tools", "read", '--resource-id', 'ivo://ca.nrc.ca/resource',
"--output", ifile, collection, observation_id
]
client_mock.return_value.get_observation.return_value = obs
core.main_app()
client_mock.return_value.get_observation.\
assert_called_with(collection, observation_id)
os.remove(ifile)
# test delete
sys.argv = [
"caom2tools", "delete", '--resource-id',
'ivo://ca.nrc.ca/resource', collection, observation_id
]
core.main_app()
client_mock.return_value.delete_observation.assert_called_with(
collection=collection, observation_id=observation_id)
# test visit
# get the absolute path to be able to run the tests with the
# astropy frameworks
plugin_file = THIS_DIR + "/passplugin.py"
sys.argv = [
"caom2tools", "visit", '--resource-id', 'ivo://ca.nrc.ca/resource',
"--plugin", plugin_file, "--start", "2012-01-01T11:22:33", "--end",
"2013-01-01T11:33:22", collection
]
client_mock.return_value.visit.return_value = ['1'], ['1'], [], []
with open(plugin_file, 'r') as infile:
core.main_app()
client_mock.return_value.visit.assert_called_with(
ANY,
collection,
halt_on_error=False,
nthreads=None,
obs_file=None,
start=core.str2date("2012-01-01T11:22:33"),
end=core.str2date("2013-01-01T11:33:22"))
# repeat visit test with halt-on-error
sys.argv = [
"caom2tools", "visit", '--resource-id', 'ivo://ca.nrc.ca/resource',
"--plugin", plugin_file, '--halt-on-error', "--start",
"2012-01-01T11:22:33", "--end", "2013-01-01T11:33:22", collection
]
client_mock.return_value.visit.return_value = ['1'], ['1'], [], []
with open(plugin_file, 'r') as infile:
core.main_app()
client_mock.return_value.visit.assert_called_with(
ANY,
collection,
halt_on_error=True,
nthreads=None,
obs_file=None,
start=core.str2date("2012-01-01T11:22:33"),
end=core.str2date("2013-01-01T11:33:22"))
def mock_get_observation(self, collection, observationID):
return SimpleObservation(collection, observationID)
def test_process(self):
core.BATCH_SIZE = 3 # size of the batch is 3
obs = [['a', 'b', 'c'], ['d'], []]
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
visitor.get_observation = MagicMock(return_value=MagicMock(
spec=SimpleObservation))
visitor.post_observation = MagicMock()
visitor._get_observations = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht')
self.assertEqual(4, len(visited))
self.assertEqual(4, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
obs = [['a', 'b', 'c'], ['d', 'e', 'f'], []]
visitor._get_observations = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht')
self.assertEqual(6, len(visited))
self.assertEqual(6, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
# make it return different status. errorplugin returns according to the
# id of the observation: True for 'UPDATE', False for 'SKIP' and
# raises exception for 'ERROR'
obs_ids = [['UPDATE', 'SKIP', 'ERROR'], []]
obs = [
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP'),
SimpleObservation(collection='TEST', observation_id='ERROR')
]
visitor._get_observations = MagicMock(side_effect=obs_ids)
visitor.get_observation = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
'cfht')
self.assertEqual(3, len(visited))
self.assertEqual(1, len(updated))
self.assertEqual(1, len(skipped))
self.assertEqual(1, len(failed))
# repeat with other obs
obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']]
obs = [
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP'),
SimpleObservation(collection='TEST', observation_id='ERROR'),
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP')
]
visitor._get_observations = MagicMock(side_effect=obs_ids)
visitor.get_observation = MagicMock(side_effect=obs)
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
'cfht')
self.assertEqual(5, len(visited))
self.assertEqual(2, len(updated))
self.assertEqual(2, len(skipped))
self.assertEqual(1, len(failed))
# repeat but halt on first ERROR -> process only 3 observations
obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']]
obs = [
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP'),
SimpleObservation(collection='TEST', observation_id='ERROR'),
SimpleObservation(collection='TEST', observation_id='UPDATE'),
SimpleObservation(collection='TEST', observation_id='SKIP')
]
visitor._get_observations = MagicMock(side_effect=obs_ids)
visitor.get_observation = MagicMock(side_effect=obs)
with self.assertRaises(SystemError):
visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'),
'cfht',
halt_on_error=True)
# test with time boundaries
core.BATCH_SIZE = 3 # size of the batch is 3
response = MagicMock()
response.text = """ARCHIVE\ta\t2011-01-01T11:00:00.000
ARCHIVE\tb\t211-01-01T11:00:10.000
ARCHIVE\tc\t2011-01-01T12:00:00.000"""
response2 = MagicMock()
response2.text = """ARCHIVE\td\t2011-02-02T11:00:00.000"""
level = logging.DEBUG
visitor = CAOM2RepoClient(auth.Subject(), level)
visitor.get_observation = MagicMock(return_value=MagicMock(
spec=SimpleObservation))
visitor.post_observation = MagicMock()
visitor._repo_client.get = MagicMock(side_effect=[response, response2])
start = '2010-10-10T12:00:00.000'
end = '2012-12-12T11:11:11.000'
(visited, updated, skipped,
failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'),
'cfht',
start=util.str2ivoa(start),
end=util.str2ivoa(end))
self.assertEqual(4, len(visited))
self.assertEqual(4, len(updated))
self.assertEqual(0, len(skipped))
self.assertEqual(0, len(failed))
calls = [
call((core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'),
params={
'START': start,
'END': end,
'MAXREC': 3
}),
call(
(core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'),
params={
'START': '2011-01-01T12:00:00.000',
# datetime of the last record in the batch
'END': end,
'MAXREC': 3
})
]
visitor._repo_client.get.assert_has_calls(calls)
