def test_visit_retry_on_412(self): # observation changed on server while visited core.BATCH_SIZE = 3 # size of the batch is 3 obs = [['a'], []] level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) observation = SimpleObservation('cfht', 'a') observation.acc_meta_checksum = ChecksumURI('md5:abc') visitor.get_observation = MagicMock( side_effect=[observation, observation]) exception_412 = exceptions.UnexpectedException() exception_412.orig_exception = Mock() exception_412.orig_exception.response = Mock(status_code=412) visitor.post_observation = MagicMock(side_effect=[exception_412, None]) visitor._get_observations = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') self.assertEqual(1, len(visited)) self.assertEqual(1, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) # get and post called twice to recover from error HTTP status 412 - # precondition self.assertEqual(2, visitor.get_observation.call_count) self.assertEqual(2, visitor.post_observation.call_count) visitor.post_observation.assert_called_with( observation, observation.acc_meta_checksum.uri)
def test_plugin_class(self): # plugin class does not change the observation collection = 'cfht' observation_id = '7000000o' level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) obs = SimpleObservation(collection, observation_id) expect_obs = copy.deepcopy(obs) visitor._load_plugin_class(os.path.join(THIS_DIR, 'passplugin.py')) visitor.plugin.update(obs) self.assertEqual(expect_obs, obs) # plugin class adds a plane to the observation visitor = CAOM2RepoClient(auth.Subject(), level) obs = SimpleObservation('cfht', '7000000o') expect_obs = copy.deepcopy(obs) visitor._load_plugin_class(os.path.join(THIS_DIR, 'addplaneplugin.py')) visitor.plugin.update(obs) self.assertNotEqual(expect_obs, obs) self.assertEqual(len(expect_obs.planes) + 1, len(obs.planes)) # non-existent the plugin file with self.assertRaises(Exception): visitor._load_plugin_class(os.path.join(THIS_DIR, 'blah.py')) # non-existent ObservationUpdater class in the plugin file with self.assertRaises(Exception): visitor._load_plugin_class( os.path.join(THIS_DIR, 'test_visitor.py')) # non-existent update method in ObservationUpdater class with self.assertRaises(Exception): visitor._load_plugin_class( os.path.join(THIS_DIR, 'noupdateplugin.py'))
def test_post_observation(self, mock_conn, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/auth' collection = 'cfht' observation_id = '7000000o' service = 'caom2repo' service_url = 'www.cadc.nrc.ca' obs = SimpleObservation(collection, observation_id) level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(netrc='somenetrc'), level, host=service_url) response = MagicMock() response.status = 200 mock_conn.return_value = response iobuffer = BytesIO() ObservationWriter().write(obs, iobuffer) obsxml = iobuffer.getvalue() response.content = obsxml visitor.post_observation(obs) self.assertEqual('POST', mock_conn.call_args[0][0].method) self.assertEqual( '/{}/auth/{}/{}'.format(service, collection, observation_id), mock_conn.call_args[0][0].path_url) self.assertEqual('application/xml', mock_conn.call_args[0][0].headers['Content-Type']) self.assertEqual(obsxml, mock_conn.call_args[0][0].body) # signal problems http_error = requests.HTTPError() response.status_code = 500 http_error.response = response response.raise_for_status.side_effect = [http_error] with self.assertRaises(exceptions.InternalServerException): visitor.update(obs) # temporary transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response response.raise_for_status.side_effect = [http_error, None] visitor.post_observation(obs) # permanent transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response def raise_error(): raise http_error response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.post_observation(obs)
def test_get_observation(self, mock_get, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' caps_mock.return_value.get_access_url.return_value =\ 'http://serviceurl/caom2repo/pub' collection = 'cfht' observation_id = '7000000o' service_url = 'www.cadc.nrc.ca/caom2repo' obs = SimpleObservation(collection, observation_id) writer = ObservationWriter() ibuffer = BytesIO() writer.write(obs, ibuffer) response = MagicMock() response.status_code = 200 response.content = ibuffer.getvalue() mock_get.return_value = response ibuffer.seek(0) # reposition the buffer for reading level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level, host=service_url) self.assertEqual(obs, visitor.get_observation(collection, observation_id)) # signal problems http_error = requests.HTTPError() response.status_code = 500 http_error.response = response response.raise_for_status.side_effect = [http_error] with self.assertRaises(exceptions.InternalServerException): visitor.get_observation(collection, observation_id) # temporary transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response response.raise_for_status.side_effect = [http_error, None] visitor.read(collection, observation_id) # permanent transient errors http_error = requests.HTTPError() response.status_code = 503 http_error.response = response def raise_error(): raise http_error response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.get_observation(collection, observation_id)
def test_shortcuts(self): level = logging.DEBUG target = CAOM2RepoClient(auth.Subject(), level) obs = SimpleObservation('CFHT', 'abc') target.put_observation = Mock() target.create(obs) target.put_observation.assert_called_with(obs) target.get_observation = Mock() target.read('CFHT', 'abc') target.get_observation.assert_called_with('CFHT', 'abc') target.post_observation = Mock() target.update(obs) target.post_observation.assert_called_with(obs) target.delete_observation = Mock() target.delete('CFHT', 'abc') target.delete_observation.assert_called_with('CFHT', 'abc')
def build_observation(self, observation, common, subsystem, files): """ Construct a simple observation from the available metadata Since we are dealing with raw data, the algorithm = "exposure" by default, a change in notation for the JCMT. Arguments: obsid obsid from COMMON to be used as the observationID common dictionary containing fields common to the observation subsystem dictionary containing fields from ACSIS or SCUBA2 files dictionary containing artifact info dictionaries """ collection = self.collection observationID = self.obsid logger.debug('PROGRESS: build observationID = %s', self.obsid) if observation is None: observation = SimpleObservation(collection, observationID) # Determine data quality metrics for this observation. data_quality = DataQuality(Quality.JUNK) \ if OMPState.is_caom_junk(common['quality']) else None requirement_status = Requirements(Status.FAIL) \ if OMPState.is_caom_fail(common['quality']) else None # "Requirements" is an observation-level attribute, so fill it in now. observation.requirements = requirement_status # Every ACSSIS and SCUBA2 observation has an obsnum in COMMON. observation.sequence_number = common['obsnum'] observation.meta_release = common['release_date'] # The observation type is derived from COMMON.obs_type and # COMMON.sam_mode if common['obs_type'] == "science": # raster is a synonym for scan if common['sam_mode'] == "raster": observation.obs_type = "scan" else: observation.obs_type = common['sam_mode'] else: observation.obs_type = common['obs_type'] # set the observation intent observation.intent = intent(common['obs_type'], common['backend']) proposal = Proposal(common['project']) if common['pi'] is not None: proposal.pi_name = common['pi'] if common['survey'] is not None: proposal.project = common['survey'] if common['title'] is not None: proposal.title = truncate_string(common['title'], 80) observation.proposal = proposal environment = Environment() if common['atstart'] is not None: make_env = True environment.ambient_temp = common['atstart'] if common['elstart'] is not None: environment.elevation = common['elstart'] if common['humstart'] is not None: if common['humstart'] < 0.0: environment.humidity = 0.0 elif common['humstart'] > 100.0: environment.humidity = 100.0 else: environment.humidity = common['humstart'] if common['seeingst'] is not None and common['seeingst'] > 0.0: environment.seeing = common['seeingst'] if common['tau225st'] is not None: environment.tau = common['tau225st'] environment.wavelength_tau = raw.SpeedOfLight/225.0e9 observation.environment = environment frontend = common['instrume'].upper() backend = common['backend'].upper() if common['inbeam']: inbeam = common['inbeam'].upper() else: inbeam = '' instrument = Instrument(instrument_name(frontend, backend, inbeam)) instrument.keywords.update(self.instrument_keywords) observation.instrument = instrument hybrid = {} if backend in ['ACSIS', 'DAS', 'AOSC']: keys = sorted(subsystem.keys()) beamsize = 0.0 for key in keys: productID = self.productID_dict[str(key)] # Convert restfreq from GHz to Hz restfreq = 1.0e9 * subsystem[key]['restfreq'] iffreq = subsystem[key]['iffreq'] ifchansp = subsystem[key]['ifchansp'] if productID not in hybrid: hybrid[productID] = {} hybrid[productID]['restfreq'] = restfreq hybrid[productID]['iffreq'] = iffreq hybrid[productID]['ifchansp'] = ifchansp this_hybrid = hybrid[productID] if 'freq_sig_lower' in this_hybrid: this_hybrid['freq_sig_lower'] = \ min(subsystem[key]['freq_sig_lower'], this_hybrid['freq_sig_lower']) else: this_hybrid['freq_sig_lower'] = \ subsystem[key]['freq_sig_lower'] if 'freq_sig_upper' in this_hybrid: this_hybrid['freq_sig_upper'] = \ max(subsystem[key]['freq_sig_upper'], this_hybrid['freq_sig_upper']) else: this_hybrid['freq_sig_upper'] = \ subsystem[key]['freq_sig_upper'] if 'freq_img_lower' in this_hybrid: this_hybrid['freq_img_lower'] = \ min(subsystem[key]['freq_img_lower'], this_hybrid['freq_img_lower']) else: this_hybrid['freq_img_lower'] = \ subsystem[key]['freq_img_lower'] if 'freq_img_upper' in this_hybrid: this_hybrid['freq_img_upper'] = \ max(subsystem[key]['freq_img_upper'], this_hybrid['freq_img_upper']) else: this_hybrid['freq_img_upper'] = \ subsystem[key]['freq_img_upper'] this_hybrid['meanfreq'] = (this_hybrid['freq_sig_lower'] + this_hybrid['freq_sig_upper'])/2.0 # Compute maximum beam size for this observation in degrees # frequencies are in GHz # The scale factor is: # 206264.8 ["/r] * sqrt(pi/2) * c [m GHz]/ 15 [m] beamsize = max(beamsize, 1.435 / this_hybrid['meanfreq']) else: # Compute beam size in degrees for 850 micron array # filter is in microns # The scale factor is: # pi/180 * sqrt(pi/2) * 1e-6 * lambda [um]/ 15 [m] beamsize = 4.787e-6 * 850.0 if (observation.obs_type not in ( 'flatfield', 'noise', 'setup', 'skydip') and common['object']): # The target is not significant for the excluded kinds of # observation, even if supplied in COMMON if common['object']: targetname = target_name(common['object']) target = Target(targetname) if common['obsra'] is None or common['obsdec'] is None: target.moving = True target_position = None else: target.moving = False target_position = TargetPosition(Point(common['obsra'], common['obsdec']), 'ICRS', 2000.0) observation.target_position = target_position if common['standard'] is not None: target.standard = True if common['standard'] else False if backend != 'SCUBA-2': subsysnr = min(subsystem.keys()) if subsystem[subsysnr]['zsource'] is not None: target.redshift = subsystem[subsysnr]['zsource'] observation.target = target telescope = Telescope(u'JCMT') telescope.geo_location_x = common['obsgeo_x'] telescope.geo_location_y = common['obsgeo_y'] telescope.geo_location_z = common['obsgeo_z'] observation.telescope = telescope # Delete any existing raw planes, since we will construct # new ones from scratch. For all other planes, update the # "data quality" since this is a plane-level attribute. preview = defaultdict(list) for productID in list(observation.planes): if productID[0:3] == 'raw': old_plane = observation.planes.pop(productID) # Keep a list of the preview artifacts. for old_artifact in old_plane.artifacts.values(): old_product_type = old_artifact.product_type if ((old_product_type is ProductType.PREVIEW) or (old_product_type is ProductType.THUMBNAIL)): preview[productID].append(old_artifact) else: observation.planes[productID].quality = data_quality # Use key for the numeric value of subsysnr here for brevity and # to distinguish it from the string representation that will be # named subsysnr in this section for key in sorted(subsystem.keys()): productID = self.productID_dict[str(key)] obsid_subsysnr = subsystem[key]['obsid_subsysnr'] logger.debug('Processing subsystem %s: %s, %s', key, obsid_subsysnr, productID) # This plane might already have been created in a hybrid-mode # observation, use it if it exists if productID not in observation.planes: observation.planes.add(Plane(productID)) plane = observation.planes[productID] else: plane = observation.planes[productID] # set the release dates plane.meta_release = common['release_date'] plane.data_release = common['release_date'] # all JCMT raw data is in a non-FITS format plane.calibration_level = CalibrationLevel.RAW_INSTRUMENTAL # set the plane data quality plane.quality = data_quality # For JCMT raw data, all artifacts have the same WCS for file_info in files[obsid_subsysnr]: file_name = file_info['name'] file_id = make_file_id_jcmt(file_name) uri = 'ad:JCMT/' + file_id if observation.intent == ObservationIntentType.SCIENCE: artifact_product_type = ProductType.SCIENCE else: artifact_product_type = ProductType.CALIBRATION artifact = Artifact( uri, product_type=artifact_product_type, release_type=ReleaseType.DATA, content_type=determine_mime_type(file_name)) if file_info['size'] is not None: artifact.content_length = file_info['size'] if file_info['md5sum'] is not None: artifact.content_checksum = ChecksumURI( 'md5:{}'.format(file_info['md5sum'])) artifact.meta_release = common['release_date'] # There is only one part and one chunk for raw data artifact.parts.add(Part('0')) chunk = Chunk() artifact.meta_release = common['release_date'] artifact.parts['0'].meta_release = common['release_date'] chunk.meta_release = common['release_date'] spatial_wcs = self.build_spatial_wcs(common, beamsize) if spatial_wcs is not None: chunk.position = spatial_wcs chunk.energy = self.build_spectral_wcs( common, subsystem[key], hybrid.get(productID)) chunk.time = self.build_temporal_wcs(common) # Chunk is done, so append it to the part artifact.parts['0'].chunks.append(chunk) # and append the atrifact to the plane plane.artifacts.add(artifact) # Restore saved previews for artifact in preview[productID]: logger.debug( 'Retaining old preview/thumbnail: %s', artifact.uri) plane.artifacts.add(artifact) return observation
def test_main_app(self, client_mock): collection = 'cfht' observation_id = '7000000o' ifile = '/tmp/inputobs' obs = SimpleObservation(collection, observation_id) # test create with open(ifile, 'wb') as infile: ObservationWriter().write(obs, infile) sys.argv = [ "caom2tools", "create", '--resource-id', 'ivo://ca.nrc.ca/resource', ifile ] core.main_app() client_mock.return_value.put_observation.assert_called_with(obs) # test update sys.argv = [ "caom2tools", "update", '--resource-id', 'ivo://ca.nrc.ca/resource', ifile ] core.main_app() client_mock.return_value.post_observation.assert_called_with(obs) # test read sys.argv = [ "caom2tools", "read", '--resource-id', 'ivo://ca.nrc.ca/resource', collection, observation_id ] client_mock.return_value.get_observation.return_value = obs client_mock.return_value.namespace = obs_reader_writer.CAOM24_NAMESPACE core.main_app() client_mock.return_value.get_observation.\ assert_called_with(collection, observation_id) # repeat with output argument sys.argv = [ "caom2tools", "read", '--resource-id', 'ivo://ca.nrc.ca/resource', "--output", ifile, collection, observation_id ] client_mock.return_value.get_observation.return_value = obs core.main_app() client_mock.return_value.get_observation.\ assert_called_with(collection, observation_id) os.remove(ifile) # test delete sys.argv = [ "caom2tools", "delete", '--resource-id', 'ivo://ca.nrc.ca/resource', collection, observation_id ] core.main_app() client_mock.return_value.delete_observation.assert_called_with( collection=collection, observation_id=observation_id) # test visit # get the absolute path to be able to run the tests with the # astropy frameworks plugin_file = THIS_DIR + "/passplugin.py" sys.argv = [ "caom2tools", "visit", '--resource-id', 'ivo://ca.nrc.ca/resource', "--plugin", plugin_file, "--start", "2012-01-01T11:22:33", "--end", "2013-01-01T11:33:22", collection ] client_mock.return_value.visit.return_value = ['1'], ['1'], [], [] with open(plugin_file, 'r') as infile: core.main_app() client_mock.return_value.visit.assert_called_with( ANY, collection, halt_on_error=False, nthreads=None, obs_file=None, start=core.str2date("2012-01-01T11:22:33"), end=core.str2date("2013-01-01T11:33:22")) # repeat visit test with halt-on-error sys.argv = [ "caom2tools", "visit", '--resource-id', 'ivo://ca.nrc.ca/resource', "--plugin", plugin_file, '--halt-on-error', "--start", "2012-01-01T11:22:33", "--end", "2013-01-01T11:33:22", collection ] client_mock.return_value.visit.return_value = ['1'], ['1'], [], [] with open(plugin_file, 'r') as infile: core.main_app() client_mock.return_value.visit.assert_called_with( ANY, collection, halt_on_error=True, nthreads=None, obs_file=None, start=core.str2date("2012-01-01T11:22:33"), end=core.str2date("2013-01-01T11:33:22"))
def mock_get_observation(self, collection, observationID): return SimpleObservation(collection, observationID)
def test_process(self): core.BATCH_SIZE = 3 # size of the batch is 3 obs = [['a', 'b', 'c'], ['d'], []] level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) visitor.get_observation = MagicMock(return_value=MagicMock( spec=SimpleObservation)) visitor.post_observation = MagicMock() visitor._get_observations = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') self.assertEqual(4, len(visited)) self.assertEqual(4, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) obs = [['a', 'b', 'c'], ['d', 'e', 'f'], []] visitor._get_observations = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') self.assertEqual(6, len(visited)) self.assertEqual(6, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) # make it return different status. errorplugin returns according to the # id of the observation: True for 'UPDATE', False for 'SKIP' and # raises exception for 'ERROR' obs_ids = [['UPDATE', 'SKIP', 'ERROR'], []] obs = [ SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP'), SimpleObservation(collection='TEST', observation_id='ERROR') ] visitor._get_observations = MagicMock(side_effect=obs_ids) visitor.get_observation = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht') self.assertEqual(3, len(visited)) self.assertEqual(1, len(updated)) self.assertEqual(1, len(skipped)) self.assertEqual(1, len(failed)) # repeat with other obs obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']] obs = [ SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP'), SimpleObservation(collection='TEST', observation_id='ERROR'), SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP') ] visitor._get_observations = MagicMock(side_effect=obs_ids) visitor.get_observation = MagicMock(side_effect=obs) (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht') self.assertEqual(5, len(visited)) self.assertEqual(2, len(updated)) self.assertEqual(2, len(skipped)) self.assertEqual(1, len(failed)) # repeat but halt on first ERROR -> process only 3 observations obs_ids = [['UPDATE', 'SKIP', 'ERROR'], ['UPDATE', 'SKIP']] obs = [ SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP'), SimpleObservation(collection='TEST', observation_id='ERROR'), SimpleObservation(collection='TEST', observation_id='UPDATE'), SimpleObservation(collection='TEST', observation_id='SKIP') ] visitor._get_observations = MagicMock(side_effect=obs_ids) visitor.get_observation = MagicMock(side_effect=obs) with self.assertRaises(SystemError): visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht', halt_on_error=True) # test with time boundaries core.BATCH_SIZE = 3 # size of the batch is 3 response = MagicMock() response.text = """ARCHIVE\ta\t2011-01-01T11:00:00.000 ARCHIVE\tb\t211-01-01T11:00:10.000 ARCHIVE\tc\t2011-01-01T12:00:00.000""" response2 = MagicMock() response2.text = """ARCHIVE\td\t2011-02-02T11:00:00.000""" level = logging.DEBUG visitor = CAOM2RepoClient(auth.Subject(), level) visitor.get_observation = MagicMock(return_value=MagicMock( spec=SimpleObservation)) visitor.post_observation = MagicMock() visitor._repo_client.get = MagicMock(side_effect=[response, response2]) start = '2010-10-10T12:00:00.000' end = '2012-12-12T11:11:11.000' (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht', start=util.str2ivoa(start), end=util.str2ivoa(end)) self.assertEqual(4, len(visited)) self.assertEqual(4, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) calls = [ call((core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'), params={ 'START': start, 'END': end, 'MAXREC': 3 }), call( (core.CURRENT_CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'), params={ 'START': '2011-01-01T12:00:00.000', # datetime of the last record in the batch 'END': end, 'MAXREC': 3 }) ] visitor._repo_client.get.assert_has_calls(calls)