def test_augment_artifact_position_from_blueprint(): test_blueprint = ObsBlueprint(position_axes=(1, 2)) test_blueprint.set('Chunk.positionAxis1', '1') test_blueprint.set('Chunk.positionAxis2', '2') test_blueprint.set('Chunk.position.axis.axis1.ctype', 'GLON-CAR') test_blueprint.set('Chunk.position.axis.axis1.cunit', 'deg') test_blueprint.set('Chunk.position.axis.axis2.ctype', 'GLAT-CAR') test_blueprint.set('Chunk.position.axis.axis2.cunit', 'deg') test_blueprint.set('Chunk.position.axis.function.cd11', '-0.004999999') test_blueprint.set('Chunk.position.axis.function.cd12', '0.0') test_blueprint.set('Chunk.position.axis.function.cd21', '0.0') test_blueprint.set('Chunk.position.axis.function.cd22', '0.004999999') test_blueprint.set('Chunk.position.axis.function.dimension.naxis1', '1') test_blueprint.set('Chunk.position.axis.function.dimension.naxis2', '1') test_blueprint.set('Chunk.position.axis.range.start.coord1.pix', '513.0') test_blueprint.set('Chunk.position.axis.range.start.coord1.val', '128.7499990027') test_blueprint.set('Chunk.position.axis.range.start.coord2.pix', '513.0') test_blueprint.set('Chunk.position.axis.range.start.coord2.val', '-0.9999999922536') test_fitsparser = FitsParser(sample_file_4axes, test_blueprint, uri='test_parser') test_chunk = Chunk() test_fitsparser._try_position_with_blueprint(test_chunk, 0) ex = _get_from_str_xml(EXPECTED_POSITION_XML, ObservationReader()._get_spatial_wcs, 'position') result = get_differences(ex, test_chunk.position) assert result is None
def test_get_observation(): """ NOTE: This hits the ALMA service and it will only work with Internet connection and the ALMA service being up and running. :return: """ # access ALMA to get a set of observations and verifies them dir = tempfile.mkdtemp() obs_file = os.path.join(dir, 'obs.xml') writer = obs_reader_writer.ObservationWriter(validate=True) reader = obs_reader_writer.ObservationReader() for id in ALMA_OBS_IDS: obs = collection.get_observation(id) caomvalidator.validate(obs) # write it to a temporary file to make sure it passes the xml # validation too writer.write(obs, obs_file) benchmark_obs_file = os.path.join(DATA_DIR, '{}.xml'.format(id)) # An easy way to update the benchmark files when the mapping algorithm # is updated. Briefly uncomment the following line to update the # files and comment it back after. # writer.write(obs, benchmark_obs_file) # compare with what we are expecting expected_obs = reader.read(benchmark_obs_file) assert not get_differences(expected_obs, obs) # check that a RuntimeError with status_code 403 is thrown when trying to # get a proprietary observation. Hopefully the used observation will stay # proprietary until the release date in year 3000 ... for id in ALMA_PROPRIETARY_OBS_IDS: with pytest.raises(RuntimeError) as e_info: collection.get_observation(id) assert 403 == e_info.value.status_code
def compare_obs(): logging.getLogger().setLevel(logging.INFO) parser = ArgumentParser(add_help=False, formatter_class=RawDescriptionHelpFormatter) parser.description = ('Call get_differences for an observation, ' 'one from Operations, and one from the sandbox. ' 'Assumes there is a $HOME/.netrc file for ' 'access control. Cleans up after itself. ' 'Expected is the operational version, actual ' 'is the sandbox version.') parser.add_argument('--collection', help='For example, OMM') parser.add_argument( '--observationID', help=('The one that might contain ' 'words, not the UUID.'), ) args = parser.parse_args() logging.info(args) cwd = os.getcwd() sb_model_fqn = os.path.join(cwd, f'{args.observationID}.sb.xml') ops_model_fqn = os.path.join(cwd, f'{args.observationID}.ops.xml') sb_obs = _get_obs(args.collection, args.observationID, sb_model_fqn, RESOURCE_ID) ops_obs = _get_obs(args.collection, args.observationID, ops_model_fqn) result = get_differences(ops_obs, sb_obs, parent=None) if result is not None: logging.error('::') logging.error('::') logging.error('Expected is from the operational version, actual is ' 'the sandbox version.') logging.error('\n'.join(x for x in result)) else: logging.info('The observations appear to be the same')
def test_augment_energy(): bp = ObsBlueprint(energy_axis=1) test_fitsparser = FitsParser(sample_file_4axes, bp) artifact = Artifact('ad:{}/{}'.format('TEST', sample_file_4axes), ProductType.SCIENCE, ReleaseType.DATA) test_fitsparser.augment_artifact(artifact) energy = artifact.parts['0'].chunks[0].energy ex = _get_from_str_xml(EXPECTED_ENERGY_XML, ObservationReader()._get_spectral_wcs, 'energy') result = get_differences(ex, energy) assert result is None, repr(energy)
def test_augment_polarization(): test_fitsparser = FitsParser(sample_file_4axes, ObsBlueprint(polarization_axis=1)) artifact = Artifact('ad:{}/{}'.format('TEST', sample_file_4axes), ProductType.SCIENCE, ReleaseType.DATA) test_fitsparser.augment_artifact(artifact) polarization = artifact.parts['0'].chunks[0].polarization ex = _get_from_str_xml(EXPECTED_POLARIZATION_XML, ObservationReader()._get_polarization_wcs, 'polarization') result = get_differences(ex, polarization) assert result is None, result
def test_augment_artifact_time(): test_fitsparser = FitsParser(sample_file_time_axes, ObsBlueprint(time_axis=1)) artifact = Artifact('ad:{}/{}'.format('TEST', sample_file_time_axes), ProductType.SCIENCE, ReleaseType.DATA) test_fitsparser.augment_artifact(artifact) assert artifact.parts is not None assert len(artifact.parts) == 6 test_part = artifact.parts['1'] test_chunk = test_part.chunks.pop() assert test_chunk is not None assert test_chunk.time is not None ex = _get_from_str_xml(EXPECTED_CFHT_WIRCAM_RAW_GUIDE_CUBE_TIME, ObservationReader()._get_temporal_wcs, 'time') result = get_differences(ex, test_chunk.time) assert result is None
def test_file_scheme_uris(): """ Tests that local files as URIs will be accepted and processed.""" fname = 'file://{}'.format(sample_file_4axes) with patch('sys.stdout', new_callable=BytesIO) as stdout_mock: sys.argv = [ 'fits2caom2', '--observation', 'test_collection_id', 'test_observation_id', '--productID', 'test_product_id', '--config', java_config_file, '--override', test_override, fname ] main_app() if stdout_mock.getvalue(): expected = _get_obs(EXPECTED_FILE_SCHEME_XML) actual = _get_obs(stdout_mock.getvalue().decode('ascii')) result = get_differences(expected, actual, 'Observation') assert result is None
def test_augment_artifact(): test_blueprint = ObsBlueprint(position_axes=(1, 2)) test_fitsparser = FitsParser(sample_file_4axes, test_blueprint) artifact = Artifact('ad:{}/{}'.format('TEST', sample_file_4axes), ProductType.SCIENCE, ReleaseType.DATA) test_fitsparser.augment_artifact(artifact) assert artifact.parts is not None assert len(artifact.parts) == 1 test_part = artifact.parts['0'] test_chunk = test_part.chunks.pop() assert test_chunk is not None assert test_chunk.position is not None ex = _get_from_str_xml(EXPECTED_POSITION_XML, ObservationReader()._get_spatial_wcs, 'position') result = get_differences(ex, test_chunk.position) assert result is None
def test_main_app(): # test_name = '' # basename = os.path.basename(test_name) # product_id = basename.split('.fits')[0] # lineage = _get_lineage(product_id, basename) # output_file = '{}.actual.xml'.format(test_name) # local = _get_local(test_name) # plugin = PLUGIN # # with patch('caom2utils.fits2caom2.CadcDataClient') as data_client_mock: # def get_file_info(archive, file_id): # if '_prev' in file_id: # return {'size': 10290, # 'md5sum': md5('-37'.encode()).hexdigest(), # 'type': 'image/jpeg'} # else: # return {'size': 37, # 'md5sum': md5('-37'.encode()).hexdigest(), # 'type': 'application/octet-stream'} # data_client_mock.return_value.get_file_info.side_effect = \ # get_file_info obs_id = 'f060_s00001' file_name = '20190805T024026_f060_s00001.h5' local = os.path.join(TEST_DATA_DIR, file_name) plugin = None product_id = 'pixel' output_file = '{}/{}.xml'.format(TEST_DATA_DIR, obs_id) lineage = '{}/ad:TAOSII/{}'.format(product_id, file_name) sys.argv = \ ('{} --no_validate --local {} ' '--plugin {} --module {} --observation TAOSII {} -o {} ' '--lineage {}'. format(main_app.APPLICATION, local, plugin, plugin, obs_id, output_file, lineage)).split() print(sys.argv) main_app.main_app2() obs_path = os.path.join(THIS_DIR, 'stuff.xml') expected = mc.read_obs_from_file(obs_path) actual = mc.read_obs_from_file(output_file) result = get_differences(actual, expected, 'Observation') if result: msg = 'Differences found in observation {}\n{}'. \ format(expected.observation_id, '\n'.join( [r for r in result])) raise AssertionError(msg)
def test_generic_parser(): """ Tests that GenericParser will be created.""" fname = 'file://{}'.format(text_file) with patch('sys.stdout', new_callable=BytesIO) as stdout_mock: sys.argv = [ 'fits2caom2', '--local', fname, '--observation', 'test_collection_id', 'test_observation_id', '--productID', 'test_product_id', '--config', java_config_file, '--override', text_override, fname ] main_app() if stdout_mock.getvalue(): expected = _get_obs(EXPECTED_GENERIC_PARSER_FILE_SCHEME_XML) actual = _get_obs(stdout_mock.getvalue().decode('ascii')) result = get_differences(expected, actual, 'Observation') assert result is None
def test_augment_observation(): test_obs_blueprint = ObsBlueprint(position_axes=(1, 2)) test_obs_blueprint.set('Observation.target.name', 'CGPS Mosaic MA1') test_obs_blueprint.set('Observation.target.standard', False) test_obs_blueprint.set('Observation.telescope.name', 'DRAO-ST') test_obs_blueprint.set('Observation.instrument.name', 'DRAO-ST') test_obs_blueprint.set('Observation.telescope.geoLocationX', '-2100330.87517') test_obs_blueprint.set('Observation.telescope.geoLocationY', '-3694247.82445') test_obs_blueprint.set('Observation.telescope.geoLocationZ', '4741018.33097') test_obs_blueprint.set('Plane.dataProductType', 'cube') test_obs_blueprint.set('Artifact.productType', 'info') test_obs_blueprint.set('Artifact.releaseType', 'data') test_obs_blueprint.set('Plane.calibrationLevel', '2') test_fitsparser = FitsParser(sample_file_4axes_obs, test_obs_blueprint) test_fitsparser.blueprint = test_obs_blueprint test_obs = SimpleObservation('collection', 'MA1_DRAO-ST', Algorithm('exposure')) test_fitsparser.augment_observation(test_obs, sample_file_4axes_uri, product_id='HI-line') assert test_obs is not None assert test_obs.planes is not None assert len(test_obs.planes) == 1 test_plane = test_obs.planes['HI-line'] assert test_plane.artifacts is not None assert len(test_plane.artifacts) == 1 test_artifact = test_plane.artifacts[sample_file_4axes_uri] assert test_artifact is not None test_part = test_artifact.parts['0'] # remove the chunk bit, as it's part of other tests - # results in <caom2:chunks/> xml output test_part.chunks.pop() output = BytesIO() ow = ObservationWriter(False, False, "caom2", obs_reader_writer.CAOM20_NAMESPACE) ow.write(test_obs, output) result = output.getvalue().decode('UTF-8') output.close() expected = _get_obs(EXPECTED_OBS_XML) actual = _get_obs(result) diff_result = get_differences(expected, actual, 'Observation') assert diff_result is None
def test_augment_artifact_polarization_from_blueprint(): test_blueprint = ObsBlueprint(polarization_axis=1) test_blueprint.set('Chunk.polarizationAxis', '1') test_blueprint.set('Chunk.polarization.axis.axis.ctype', 'STOKES') test_blueprint.set('Chunk.polarization.axis.function.refCoord.pix', '1.0') test_blueprint.set('Chunk.polarization.axis.function.refCoord.val', '1.0') test_blueprint.set('Chunk.polarization.axis.function.delta', '1.0') test_blueprint.set('Chunk.polarization.axis.function.naxis', '1') test_fitsparser = FitsParser(sample_file_4axes, test_blueprint, uri='test_parser') test_chunk = Chunk() test_fitsparser._try_polarization_with_blueprint(test_chunk, 0) ex = _get_from_str_xml(EXPECTED_POLARIZATION_XML, ObservationReader()._get_polarization_wcs, 'polarization') result = get_differences(ex, test_chunk.polarization) assert result is None
def test_augment_artifact_energy_from_blueprint(): test_blueprint = ObsBlueprint(energy_axis=1) test_blueprint.set('Chunk.energyAxis', 1) test_blueprint.set('Chunk.energy.specsys', 'LSRK') test_blueprint.set('Chunk.energy.axis.axis.ctype', 'VRAD') test_blueprint.set('Chunk.energy.axis.axis.cunit', 'm / s') test_blueprint.set('Chunk.energy.axis.function.refCoord.pix', '145.0') test_blueprint.set('Chunk.energy.axis.function.refCoord.val', '-60000.0') test_blueprint.set('Chunk.energy.axis.function.delta', '-824.46002') test_blueprint.set('Chunk.energy.axis.function.naxis', '1') test_fitsparser = FitsParser(sample_file_4axes, test_blueprint, uri='ad:TEST/test_blueprint') test_chunk = Chunk() test_fitsparser._try_energy_with_blueprint(test_chunk, 0) ex = _get_from_str_xml(EXPECTED_ENERGY_XML, ObservationReader()._get_spectral_wcs, 'energy') result = get_differences(ex, test_chunk.energy) assert result is None
def _do_botched_compare(actual_fqn, expected_fqn): """Compare artifacts for .fits files only. Rationalize the keys so that the schema change from GEM to GEMINI is irrelevant.""" actual = mc.read_obs_from_file(actual_fqn) expected = mc.read_obs_from_file(expected_fqn) result = _compare_keys(expected.planes.keys(), actual.planes.keys(), 'plane') if result is None: for plane in actual.planes.values(): e_plane = expected.planes[plane.product_id] expected_a_keys = _rationalize_keys(e_plane.artifacts.keys()) actual_a_keys = _rationalize_keys(plane.artifacts.keys()) result = _compare_keys(expected_a_keys, actual_a_keys, f'artifact.{plane.product_id}') if result is None: for artifact in plane.artifacts.values(): if 'gemini:GEM/' in artifact.uri: # ignore the artifacts that are named with the # 'old' URIs continue expected_id = artifact.uri.replace('GEMINI/', 'GEM/') expected_artifact = e_plane.artifacts[expected_id] expected_artifact.uri = artifact.uri if expected_id in e_plane.artifacts.keys(): temp = get_differences(artifact, expected_artifact, 'Artifact') if temp is not None: if result is None: result = temp else: result = f'{result}\n{temp}' else: temp = f'Artifact {expected_id} in actual ' \ f'but not expected.' if result is None: result = temp else: result = f'{result}\n{temp}' else: result = f'Got {len(actual.planes)} planes. Expected ' \ f'{len(expected.planes)} planes.' return result
def test_augment_artifact_time_from_blueprint(): test_blueprint = ObsBlueprint(time_axis=1) test_blueprint.set('Chunk.timeAxis', '1') test_blueprint.set('Chunk.time.exposure', '0.02') test_blueprint.set('Chunk.time.resolution', '0.02') test_blueprint.set('Chunk.time.timesys', 'UTC') test_blueprint.set('Chunk.time.axis.axis.ctype', 'TIME') test_blueprint.set('Chunk.time.axis.axis.cunit', 'd') test_blueprint.set('Chunk.time.axis.error.syser', '1e-07') test_blueprint.set('Chunk.time.axis.error.rnder', '1e-07') test_blueprint.set('Chunk.time.axis.function.refCoord.pix', '0.5') test_blueprint.set('Chunk.time.axis.function.refCoord.val', '56789.4298069') test_blueprint.set('Chunk.time.axis.function.delta', '2.31481e-07') test_blueprint.set('Chunk.time.axis.function.naxis', '1') test_fitsparser = FitsParser(sample_file_4axes, test_blueprint, uri='ad:TEST/test_blueprint') test_chunk = Chunk() test_fitsparser._try_time_with_blueprint(test_chunk, 0) ex = _get_from_str_xml(EXPECTED_CFHT_WIRCAM_RAW_GUIDE_CUBE_TIME, ObservationReader()._get_temporal_wcs, 'time') result = get_differences(ex, test_chunk.time) assert result is None
def test_value_repair_cache(): test_subject = mc.ValueRepairCache() assert test_subject is not None, 'expect a result' test_observation = mc.read_obs_from_file( os.path.join(tc.TEST_DATA_DIR, 'value_repair_start.xml')) test_product_id = 'GN2001BQ013-04' test_artifact_uri = 'gemini:GEM/GN2001BQ013-04.fits' test_part = '0' test_chunk_index = 0 test_plane = test_observation.planes[test_product_id] test_artifact = test_plane.artifacts[test_artifact_uri] test_part = test_artifact.parts[test_part] test_chunk = test_part.chunks[test_chunk_index] assert test_observation.type == 'Dark', 'repair initial condition' assert test_observation.proposal.pi_name == 'jjk', 'pi name ic' assert test_plane.meta_release == datetime(1990, 1, 1, 0, 0), 'plane meta release ic' assert math.isclose( test_chunk.position.axis.function.ref_coord.coord1.pix, 512.579594886106, ), 'position pix ic' assert test_artifact.uri == test_artifact_uri, 'artifact uri ic' assert test_part.product_type is ProductType.CALIBRATION, 'part ic' assert test_chunk.position.coordsys == 'ICRS', 'un-changed ic' assert test_observation.intent is None, 'None ic' assert test_observation.environment.seeing is None, 'None ic' assert test_chunk.position.axis.axis1.ctype == 'RA---TAN', 'unchanged ic' test_subject.repair(test_observation) assert test_observation.type == 'DARK', 'repair failed' assert (test_observation.proposal.pi_name == 'JJ Kavelaars' ), 'proposal pi name repair failed' assert test_plane.meta_release == datetime( 2000, 1, 1, 0, 0), 'plane meta release repair failed' assert math.isclose( test_chunk.position.axis.function.ref_coord.coord1.pix, 512.57959987654321, ), 'position pix repair failed' assert (test_artifact.uri == 'cadc:GEMINI/GN2001BQ013-04.fits' ), f'uri repair failed {test_artifact.uri}' assert test_part.product_type is None, 'product type repair failed' # check that values that do not match are un-changed assert test_chunk.position.coordsys == 'ICRS', 'should be un-changed' # check that None values are not set - it's a _repair_ function, after # all, and the code cannot repair something that does not exist assert (test_observation.intent == ObservationIntentType.SCIENCE ), 'None value should be set, since "none" was the original type' assert (test_observation.environment.seeing is None), 'None remains None because the original is a specific value' assert test_chunk.position.axis.axis1.ctype == 'RA---TAN', 'unchanged post' with pytest.raises(mc.CadcException): # pre-condition of 'Unexpected repair key' error test_subject._value_repair = {'unknown': 'unknown'} test_subject.repair(test_observation) with pytest.raises(mc.CadcException): # try to set an attribute that cannot be assigned test_subject._value_repair = { 'observation.instrument.name': { 'gmos': 'GMOS-N' }, } test_subject.repair(test_observation) # pre-condition of 'Could not figure out attribute name' the attribute is # not set in the test observation, so the observation should remain # unchanged test_observation = mc.read_obs_from_file( os.path.join(tc.TEST_DATA_DIR, 'value_repair_start.xml')) test_subject._value_repair = {'chunk.observable.dependent': 'not_found'} test_subject.repair(test_observation) test_compare_observation = mc.read_obs_from_file( os.path.join(tc.TEST_DATA_DIR, 'value_repair_start.xml')) test_diff = get_differences(test_compare_observation, test_observation) assert test_diff is None, 'expect no comparison error'
def test_augment_artifact_bounds_range_from_blueprint(): test_blueprint = ObsBlueprint(energy_axis=1, time_axis=2, polarization_axis=3, position_axes=(4, 5)) test_blueprint.set('Chunk.energy.axis.range.start.pix', '145.0') test_blueprint.set('Chunk.energy.axis.range.start.val', '-60000.0') test_blueprint.set('Chunk.energy.axis.range.end.pix', '-824.46002') test_blueprint.set('Chunk.energy.axis.range.end.val', '1') test_blueprint.set('Chunk.time.axis.range.start.pix', '145.0') test_blueprint.set('Chunk.time.axis.range.start.val', '-60000.0') test_blueprint.set('Chunk.time.axis.range.end.pix', '-824.46002') test_blueprint.set('Chunk.time.axis.range.end.val', '1') test_blueprint.set('Chunk.polarization.axis.range.start.pix', '145.0') test_blueprint.set('Chunk.polarization.axis.range.start.val', '-60000.0') test_blueprint.set('Chunk.polarization.axis.range.end.pix', '-824.46002') test_blueprint.set('Chunk.polarization.axis.range.end.val', '1') test_blueprint.set('Chunk.position.axis.range.start.coord1.pix', '145.0') test_blueprint.set('Chunk.position.axis.range.start.coord1.val', '-60000.0') test_blueprint.set('Chunk.position.axis.range.end.coord1.pix', '-824.46002') test_blueprint.set('Chunk.position.axis.range.end.coord1.val', '1') test_blueprint.set('Chunk.position.axis.range.start.coord2.pix', '145.0') test_blueprint.set('Chunk.position.axis.range.start.coord2.val', '-60000.0') test_blueprint.set('Chunk.position.axis.range.end.coord2.pix', '-824.46002') test_blueprint.set('Chunk.position.axis.range.end.coord2.val', '1') test_fitsparser = FitsParser(sample_file_4axes, test_blueprint, uri='ad:TEST/test_blueprint') test_chunk = Chunk() test_chunk.energy = SpectralWCS(CoordAxis1D(Axis('WAVE', 'm')), 'TOPOCENT') test_chunk.time = TemporalWCS(CoordAxis1D(Axis('TIME', 'd'))) test_chunk.polarization = PolarizationWCS(CoordAxis1D(Axis('STOKES'))) test_chunk.position = SpatialWCS( CoordAxis2D(Axis('RA', 'deg'), Axis('DEC', 'deg'))) test_fitsparser._try_range_with_blueprint(test_chunk, 0) assert test_chunk.energy.axis.range is not None, \ 'chunk.energy.axis.range should be declared' assert test_chunk.time.axis.range is not None, \ 'chunk.time.axis.range should be declared' assert test_chunk.polarization.axis.range is not None, \ 'chunk.polarization.axis.range should be declared' assert test_chunk.position.axis.range is not None, \ 'chunk.position.axis.range should be declared' ex = _get_from_str_xml(EXPECTED_ENERGY_RANGE_BOUNDS_XML, ObservationReader()._get_spectral_wcs, 'energy') assert ex is not None, \ 'energy string from expected output should be declared' result = get_differences(ex, test_chunk.energy) assert result is None ex = _get_from_str_xml(EXPECTED_TIME_RANGE_BOUNDS_XML, ObservationReader()._get_temporal_wcs, 'time') assert ex is not None, \ 'time string from expected output should be declared' result = get_differences(ex, test_chunk.time) assert result is None ex = _get_from_str_xml(EXPECTED_POL_RANGE_BOUNDS_XML, ObservationReader()._get_polarization_wcs, 'polarization') assert ex is not None, \ 'polarization string from expected output should be declared' result = get_differences(ex, test_chunk.polarization) assert result is None ex = _get_from_str_xml(EXPECTED_POS_RANGE_BOUNDS_XML, ObservationReader()._get_spatial_wcs, 'position') assert ex is not None, \ 'position string from expected output should be declared' result = get_differences(ex, test_chunk.position) assert result is None