def test_rountrip(): source_file_path = os.path.join(THIS_DIR, TEST_DATA, 'SampleComposite-CAOM-2.3.xml') reader = obs_reader_writer.ObservationReader(True) with open(source_file_path, 'r'): obs = reader.read(source_file_path) filename = tempfile.TemporaryFile() writer = obs_reader_writer.ObservationWriter(True) writer.write(obs, filename) # go back to the beginning of the file filename.seek(0) obs = reader.read(filename) for plane in obs.planes.values(): for artifact in plane.artifacts.values(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) assert chunk.acc_meta_checksum == get_acc_meta_checksum( chunk) assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum(part) assert artifact.meta_checksum == get_meta_checksum(artifact) assert artifact.acc_meta_checksum == get_acc_meta_checksum( artifact) assert plane.meta_checksum == get_meta_checksum(plane) assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) # check observation assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum == get_acc_meta_checksum(obs)
def test_get_observation(): """ NOTE: This hits the ALMA service and it will only work with Internet connection and the ALMA service being up and running. :return: """ # access ALMA to get a set of observations and verifies them dir = tempfile.mkdtemp() obs_file = os.path.join(dir, 'obs.xml') writer = obs_reader_writer.ObservationWriter(validate=True) reader = obs_reader_writer.ObservationReader() for id in ALMA_OBS_IDS: obs = collection.get_observation(id) caomvalidator.validate(obs) # write it to a temporary file to make sure it passes the xml # validation too writer.write(obs, obs_file) benchmark_obs_file = os.path.join(DATA_DIR, '{}.xml'.format(id)) # An easy way to update the benchmark files when the mapping algorithm # is updated. Briefly uncomment the following line to update the # files and comment it back after. # writer.write(obs, benchmark_obs_file) # compare with what we are expecting expected_obs = reader.read(benchmark_obs_file) assert not get_differences(expected_obs, obs) # check that a RuntimeError with status_code 403 is thrown when trying to # get a proprietary observation. Hopefully the used observation will stay # proprietary until the release date in year 3000 ... for id in ALMA_PROPRIETARY_OBS_IDS: with pytest.raises(RuntimeError) as e_info: collection.get_observation(id) assert 403 == e_info.value.status_code
def test_cadc_uri(si_mock, ws_mock): def _get_mock(id_ignore, dest, fhead): dest.write( b"""SIMPLE = T / Written by IDL: Fri Oct 6 01:48:35 2017 BITPIX = -32 / Bits per pixel NAXIS = 2 / Number of dimensions NAXIS1 = 2048 / NAXIS2 = 2048 / DATATYPE= 'REDUC ' /Data type, SCIENCE/CALIB/REJECT/FOCUS/TEST END""") def _info_mock(uri): return FileInfo(id=uri, size=12, file_type='application/fits', md5sum='abc') si_mock.return_value.cadcget.side_effect = _get_mock si_mock.return_value.cadcinfo.side_effect = _info_mock working_dir = os.path.join(tc.TESTDATA_DIR, 'si') out_fqn = os.path.join(working_dir, 'test_out.xml') bp_fqn = os.path.join(working_dir, 'si.blueprint') if os.path.exists(out_fqn): os.unlink(out_fqn) sys.argv = ('caom2gen --debug -o {} --no_validate ' '--resource-id ivo://cadc.nrc.ca/test ' '--observation TEST_COLLECTION TEST_OBS_ID ' '--lineage test_product_id/cadc:TEST/test_file.fits ' '--blueprint {}'.format(out_fqn, bp_fqn)).split() caom2blueprint.caom2gen() assert os.path.exists(out_fqn), 'expect output file' obs_reader = obs_reader_writer.ObservationReader() obs = obs_reader.read(out_fqn) assert obs is not None, 'expect an Observation' assert obs.algorithm.name == 'exposure', 'wrong algorithm construction' if os.path.exists(out_fqn): os.unlink(out_fqn)
def test_round_trip(): source_file_path = os.path.join(THIS_DIR, TEST_DATA, 'SampleComposite-CAOM-2.3.xml') reader = obs_reader_writer.ObservationReader(True) with open(source_file_path, 'r'): obs = reader.read(source_file_path) filename = tempfile.TemporaryFile() writer = obs_reader_writer.ObservationWriter( True, namespace=obs_reader_writer.CAOM23_NAMESPACE) writer.write(obs, filename) # go back to the beginning of the file filename.seek(0) obs = reader.read(filename) _common_check(obs) # check observation assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum == get_acc_meta_checksum(obs)
def test_compatibility_simple_obs(): # tests loads a previously generated observation and checks the checksums # against the previously calculated (in Java) checksums logger = logging.getLogger('checksum') level = logger.getEffectiveLevel() logger.setLevel(logging.DEBUG) source_file_path = os.path.join(THIS_DIR, TEST_DATA, 'SampleSimple-CAOM-2.3.xml') reader = obs_reader_writer.ObservationReader(True) with open(source_file_path, 'r'): obs = reader.read(source_file_path) writer = obs_reader_writer.ObservationWriter( True, namespace=obs_reader_writer.CAOM23_NAMESPACE) writer.write(obs, '/tmp/test.xml') _common_check(obs) # check observation assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum == get_acc_meta_checksum(obs) logger.setLevel(level)
def test_checksum_diff(): source_file_path = os.path.join(THIS_DIR, TEST_DATA, 'SampleComposite-CAOM-2.3.xml') output_file = tempfile.NamedTemporaryFile() sys.argv = 'caom2_checksum -d -o {} {}'.format( output_file.name, source_file_path).split() with patch('sys.stdout', new_callable=StringIO) as stdout_mock: checksum_diff() output = stdout_mock.getvalue() assert 'mismatch' not in output, '{} should have correct checksum'.format( source_file_path) assert 'chunk' in output assert 'part' in output assert 'artifact' in output assert 'plane' in output assert 'observation' in output # original observation and the one outputed should be identical reader = obs_reader_writer.ObservationReader() expected = reader.read(source_file_path) actual = reader.read(output_file.name) assert get_acc_meta_checksum(expected) == get_acc_meta_checksum(actual)
def test_compatibility(): # tests loads a previously generated observation and checks the checksums # against the previously calculated (in Java) checksums source_file_path = os.path.join(THIS_DIR, TEST_DATA, 'SampleComposite-CAOM-2.3.xml') reader = obs_reader_writer.ObservationReader(True) with open(source_file_path, 'r'): obs = reader.read(source_file_path) writer = obs_reader_writer.ObservationWriter( True, namespace=obs_reader_writer.CAOM23_NAMESPACE) writer.write(obs, '/tmp/test.xml') _common_check(obs) # check observation assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum == get_acc_meta_checksum(obs) # white spaces around strings should not affect the checksum obs.algorithm = ' {}\t\n'.format(obs.algorithm.name) assert obs.meta_checksum == get_meta_checksum(obs) # now change some attributes and see how the checksums start to diverge old_val = obs.collection obs.collection = 'OTHER' _common_check(obs) assert obs.meta_checksum != get_meta_checksum(obs) assert obs.acc_meta_checksum != get_acc_meta_checksum(obs) obs.collection = old_val # now change a plane aplane = list(obs.planes.values())[0] old_val = aplane.product_id aplane.product_id = 'TESTPRODID' for plane in obs.planes.values(): for artifact in plane.artifacts.values(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) assert chunk.acc_meta_checksum == get_acc_meta_checksum( chunk) assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum(part) assert artifact.meta_checksum == get_meta_checksum(artifact) assert artifact.acc_meta_checksum == get_acc_meta_checksum( artifact) if plane._id == aplane._id: assert plane.meta_checksum != get_meta_checksum(plane) assert plane.acc_meta_checksum != get_acc_meta_checksum(plane) else: assert plane.meta_checksum == get_meta_checksum(plane) assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum != get_acc_meta_checksum(obs) aplane.product_id = old_val # change an artifact anartifact = list(aplane.artifacts.values())[0] old_val = anartifact.content_length anartifact.content_length = 3344 for plane in obs.planes.values(): for artifact in plane.artifacts.values(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) assert chunk.acc_meta_checksum == get_acc_meta_checksum( chunk) assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum(part) if artifact._id == anartifact._id: assert artifact.meta_checksum != get_meta_checksum(artifact) assert artifact.acc_meta_checksum != get_acc_meta_checksum( artifact) else: assert artifact.meta_checksum == get_meta_checksum(artifact) assert artifact.acc_meta_checksum == get_acc_meta_checksum( artifact) assert plane.meta_checksum == get_meta_checksum(plane) if plane._id == aplane._id: assert plane.acc_meta_checksum != get_acc_meta_checksum(plane) else: assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum != get_acc_meta_checksum(obs) anartifact.content_length = old_val apart = list(anartifact.parts.values())[0] old_val = apart.name apart.name = 'therealpart' for plane in obs.planes.values(): for artifact in plane.artifacts.values(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) assert chunk.acc_meta_checksum == get_acc_meta_checksum( chunk) if part._id == apart._id: assert part.meta_checksum != get_meta_checksum(part) assert part.acc_meta_checksum != get_acc_meta_checksum( part) else: assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum( part) assert artifact.meta_checksum == get_meta_checksum(artifact) if artifact._id == anartifact._id: assert artifact.acc_meta_checksum != get_acc_meta_checksum( artifact) else: assert artifact.acc_meta_checksum == get_acc_meta_checksum( artifact) assert plane.meta_checksum == get_meta_checksum(plane) if plane._id == aplane._id: assert plane.acc_meta_checksum != get_acc_meta_checksum(plane) else: assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum != get_acc_meta_checksum(obs) apart.name = old_val achunk = list(apart.chunks)[0] old_val = chunk.naxis if old_val == 5: achunk.naxis = 4 else: achunk.naxis = old_val + 1 for plane in obs.planes.values(): for artifact in plane.artifacts.values(): for part in artifact.parts.values(): for chunk in part.chunks: if chunk._id == achunk._id: assert chunk.meta_checksum != get_meta_checksum(chunk) assert chunk.acc_meta_checksum !=\ get_acc_meta_checksum(chunk) else: assert chunk.meta_checksum == get_meta_checksum(chunk) assert chunk.acc_meta_checksum ==\ get_acc_meta_checksum(chunk) assert part.meta_checksum == get_meta_checksum(part) if part._id == apart._id: assert part.acc_meta_checksum != get_acc_meta_checksum( part) else: assert part.acc_meta_checksum == get_acc_meta_checksum( part) assert artifact.meta_checksum == get_meta_checksum(artifact) if artifact._id == anartifact._id: assert artifact.acc_meta_checksum != get_acc_meta_checksum( artifact) else: assert artifact.acc_meta_checksum == get_acc_meta_checksum( artifact) assert plane.meta_checksum == get_meta_checksum(plane) if plane._id == aplane._id: assert plane.acc_meta_checksum != get_acc_meta_checksum(plane) else: assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum != get_acc_meta_checksum(obs) achunk.naxis = old_val # update the checksums and everything should match again update_meta_checksum(obs) _common_check(obs)
def _get_obs(from_xml_string): etree.parse = Mock(return_value=etree.ElementTree( etree.fromstring(from_xml_string.encode('ascii')))) obsr = obs_reader_writer.ObservationReader() obs = obsr.read(None) return obs
def test_compatibility(): # tests loads a previously generated observation and checks the checksums against the previously # calculated (in Java) checksums source_file_path = os.path.join(THIS_DIR, TEST_DATA, 'SampleComposite-CAOM-2.3.xml') reader = obs_reader_writer.ObservationReader(True) with open(source_file_path, 'r') as f: obs = reader.read(source_file_path) for plane in obs.planes.values(): for artifact in plane.artifacts.values(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) assert chunk.acc_meta_checksum == get_acc_meta_checksum( chunk) assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum(part) assert artifact.meta_checksum == get_meta_checksum(artifact) assert artifact.acc_meta_checksum == get_acc_meta_checksum( artifact) assert plane.meta_checksum == get_meta_checksum(plane) assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) # check observation assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum == get_acc_meta_checksum(obs) # now change some attributes and see how the checksums start to diverge old_val = obs.collection obs.collection = 'OTHER' for plane in obs.planes.values(): for artifact in plane.artifacts.values(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) assert chunk.acc_meta_checksum == get_acc_meta_checksum( chunk) assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum(part) assert artifact.meta_checksum == get_meta_checksum(artifact) assert artifact.acc_meta_checksum == get_acc_meta_checksum( artifact) assert plane.meta_checksum == get_meta_checksum(plane) assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) assert obs.meta_checksum != get_meta_checksum(obs) assert obs.acc_meta_checksum != get_acc_meta_checksum(obs) obs.collection = old_val # now change a plane aplane = list(obs.planes.values())[0] old_val = aplane.product_id aplane.product_id = 'TESTPRODID' for plane in obs.planes.values(): for artifact in plane.artifacts.values(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) assert chunk.acc_meta_checksum == get_acc_meta_checksum( chunk) assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum(part) assert artifact.meta_checksum == get_meta_checksum(artifact) assert artifact.acc_meta_checksum == get_acc_meta_checksum( artifact) if plane._id == aplane._id: assert plane.meta_checksum != get_meta_checksum(plane) assert plane.acc_meta_checksum != get_acc_meta_checksum(plane) else: assert plane.meta_checksum == get_meta_checksum(plane) assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum != get_acc_meta_checksum(obs) aplane.product_id = old_val # change an artifact anartifact = list(aplane.artifacts.values())[0] old_val = anartifact.content_length anartifact.content_length = 3344 for plane in obs.planes.values(): for artifact in plane.artifacts.values(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) assert chunk.acc_meta_checksum == get_acc_meta_checksum( chunk) assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum(part) if artifact._id == anartifact._id: assert artifact.meta_checksum != get_meta_checksum(artifact) assert artifact.acc_meta_checksum != get_acc_meta_checksum( artifact) else: assert artifact.meta_checksum == get_meta_checksum(artifact) assert artifact.acc_meta_checksum == get_acc_meta_checksum( artifact) assert plane.meta_checksum == get_meta_checksum(plane) if plane._id == aplane._id: assert plane.acc_meta_checksum != get_acc_meta_checksum(plane) else: assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum != get_acc_meta_checksum(obs) anartifact.content_length = old_val apart = list(anartifact.parts.values())[0] old_val = apart.name apart.name = 'therealpart' for plane in obs.planes.values(): for artifact in plane.artifacts.values(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) assert chunk.acc_meta_checksum == get_acc_meta_checksum( chunk) if part._id == apart._id: assert part.meta_checksum != get_meta_checksum(part) assert part.acc_meta_checksum != get_acc_meta_checksum( part) else: assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum( part) assert artifact.meta_checksum == get_meta_checksum(artifact) if artifact._id == anartifact._id: assert artifact.acc_meta_checksum != get_acc_meta_checksum( artifact) else: assert artifact.acc_meta_checksum == get_acc_meta_checksum( artifact) assert plane.meta_checksum == get_meta_checksum(plane) if plane._id == aplane._id: assert plane.acc_meta_checksum != get_acc_meta_checksum(plane) else: assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum != get_acc_meta_checksum(obs) apart.name = old_val achunk = list(apart.chunks)[0] old_val = chunk.naxis if old_val == 5: achunk.naxis = 4 else: achunk.naxis = old_val + 1 for plane in obs.planes.values(): for artifact in plane.artifacts.values(): for part in artifact.parts.values(): for chunk in part.chunks: if chunk._id == achunk._id: assert chunk.meta_checksum != get_meta_checksum(chunk) assert chunk.acc_meta_checksum != get_acc_meta_checksum( chunk) else: assert chunk.meta_checksum == get_meta_checksum(chunk) assert chunk.acc_meta_checksum == get_acc_meta_checksum( chunk) assert part.meta_checksum == get_meta_checksum(part) if part._id == apart._id: assert part.acc_meta_checksum != get_acc_meta_checksum( part) else: assert part.acc_meta_checksum == get_acc_meta_checksum( part) assert artifact.meta_checksum == get_meta_checksum(artifact) if artifact._id == anartifact._id: assert artifact.acc_meta_checksum != get_acc_meta_checksum( artifact) else: assert artifact.acc_meta_checksum == get_acc_meta_checksum( artifact) assert plane.meta_checksum == get_meta_checksum(plane) if plane._id == aplane._id: assert plane.acc_meta_checksum != get_acc_meta_checksum(plane) else: assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum != get_acc_meta_checksum(obs) achunk.naxis = old_val
def _read_obs(model_fqn): reader = obs_reader_writer.ObservationReader(False) result = reader.read(model_fqn) return result