def test_rountrip():
source_file_path = os.path.join(THIS_DIR, TEST_DATA,
'SampleComposite-CAOM-2.3.xml')
reader = obs_reader_writer.ObservationReader(True)
with open(source_file_path, 'r'):
obs = reader.read(source_file_path)
filename = tempfile.TemporaryFile()
writer = obs_reader_writer.ObservationWriter(True)
writer.write(obs, filename)
# go back to the beginning of the file
filename.seek(0)
obs = reader.read(filename)
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
for part in artifact.parts.values():
for chunk in part.chunks:
assert chunk.meta_checksum == get_meta_checksum(chunk)
assert chunk.acc_meta_checksum == get_acc_meta_checksum(
chunk)
assert part.meta_checksum == get_meta_checksum(part)
assert part.acc_meta_checksum == get_acc_meta_checksum(part)
assert artifact.meta_checksum == get_meta_checksum(artifact)
assert artifact.acc_meta_checksum == get_acc_meta_checksum(
artifact)
assert plane.meta_checksum == get_meta_checksum(plane)
assert plane.acc_meta_checksum == get_acc_meta_checksum(plane)
# check observation
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum == get_acc_meta_checksum(obs)
def test_get_observation():
"""
NOTE: This hits the ALMA service and it will only work with Internet
connection and the ALMA service being up and running.
:return:
"""
# access ALMA to get a set of observations and verifies them
dir = tempfile.mkdtemp()
obs_file = os.path.join(dir, 'obs.xml')
writer = obs_reader_writer.ObservationWriter(validate=True)
reader = obs_reader_writer.ObservationReader()
for id in ALMA_OBS_IDS:
obs = collection.get_observation(id)
caomvalidator.validate(obs)
# write it to a temporary file to make sure it passes the xml
# validation too
writer.write(obs, obs_file)
benchmark_obs_file = os.path.join(DATA_DIR, '{}.xml'.format(id))
# An easy way to update the benchmark files when the mapping algorithm
# is updated. Briefly uncomment the following line to update the
# files and comment it back after.
# writer.write(obs, benchmark_obs_file)
# compare with what we are expecting
expected_obs = reader.read(benchmark_obs_file)
assert not get_differences(expected_obs, obs)
# check that a RuntimeError with status_code 403 is thrown when trying to
# get a proprietary observation. Hopefully the used observation will stay
# proprietary until the release date in year 3000 ...
for id in ALMA_PROPRIETARY_OBS_IDS:
with pytest.raises(RuntimeError) as e_info:
collection.get_observation(id)
assert 403 == e_info.value.status_code
def test_cadc_uri(si_mock, ws_mock):
def _get_mock(id_ignore, dest, fhead):
dest.write(
b"""SIMPLE = T / Written by IDL: Fri Oct 6 01:48:35 2017
BITPIX = -32 / Bits per pixel
NAXIS = 2 / Number of dimensions
NAXIS1 = 2048 /
NAXIS2 = 2048 /
DATATYPE= 'REDUC ' /Data type, SCIENCE/CALIB/REJECT/FOCUS/TEST
END""")
def _info_mock(uri):
return FileInfo(id=uri,
size=12,
file_type='application/fits',
md5sum='abc')
si_mock.return_value.cadcget.side_effect = _get_mock
si_mock.return_value.cadcinfo.side_effect = _info_mock
working_dir = os.path.join(tc.TESTDATA_DIR, 'si')
out_fqn = os.path.join(working_dir, 'test_out.xml')
bp_fqn = os.path.join(working_dir, 'si.blueprint')
if os.path.exists(out_fqn):
os.unlink(out_fqn)
sys.argv = ('caom2gen --debug -o {} --no_validate '
'--resource-id ivo://cadc.nrc.ca/test '
'--observation TEST_COLLECTION TEST_OBS_ID '
'--lineage test_product_id/cadc:TEST/test_file.fits '
'--blueprint {}'.format(out_fqn, bp_fqn)).split()
caom2blueprint.caom2gen()
assert os.path.exists(out_fqn), 'expect output file'
obs_reader = obs_reader_writer.ObservationReader()
obs = obs_reader.read(out_fqn)
assert obs is not None, 'expect an Observation'
assert obs.algorithm.name == 'exposure', 'wrong algorithm construction'
if os.path.exists(out_fqn):
os.unlink(out_fqn)
def test_round_trip():
source_file_path = os.path.join(THIS_DIR, TEST_DATA,
'SampleComposite-CAOM-2.3.xml')
reader = obs_reader_writer.ObservationReader(True)
with open(source_file_path, 'r'):
obs = reader.read(source_file_path)
filename = tempfile.TemporaryFile()
writer = obs_reader_writer.ObservationWriter(
True, namespace=obs_reader_writer.CAOM23_NAMESPACE)
writer.write(obs, filename)
# go back to the beginning of the file
filename.seek(0)
obs = reader.read(filename)
_common_check(obs)
# check observation
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum == get_acc_meta_checksum(obs)
def test_compatibility_simple_obs():
# tests loads a previously generated observation and checks the checksums
# against the previously calculated (in Java) checksums
logger = logging.getLogger('checksum')
level = logger.getEffectiveLevel()
logger.setLevel(logging.DEBUG)
source_file_path = os.path.join(THIS_DIR, TEST_DATA,
'SampleSimple-CAOM-2.3.xml')
reader = obs_reader_writer.ObservationReader(True)
with open(source_file_path, 'r'):
obs = reader.read(source_file_path)
writer = obs_reader_writer.ObservationWriter(
True, namespace=obs_reader_writer.CAOM23_NAMESPACE)
writer.write(obs, '/tmp/test.xml')
_common_check(obs)
# check observation
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum == get_acc_meta_checksum(obs)
logger.setLevel(level)
def test_checksum_diff():
source_file_path = os.path.join(THIS_DIR, TEST_DATA,
'SampleComposite-CAOM-2.3.xml')
output_file = tempfile.NamedTemporaryFile()
sys.argv = 'caom2_checksum -d -o {} {}'.format(
output_file.name, source_file_path).split()
with patch('sys.stdout', new_callable=StringIO) as stdout_mock:
checksum_diff()
output = stdout_mock.getvalue()
assert 'mismatch' not in output, '{} should have correct checksum'.format(
source_file_path)
assert 'chunk' in output
assert 'part' in output
assert 'artifact' in output
assert 'plane' in output
assert 'observation' in output
# original observation and the one outputed should be identical
reader = obs_reader_writer.ObservationReader()
expected = reader.read(source_file_path)
actual = reader.read(output_file.name)
assert get_acc_meta_checksum(expected) == get_acc_meta_checksum(actual)
def test_compatibility():
# tests loads a previously generated observation and checks the checksums
# against the previously calculated (in Java) checksums
source_file_path = os.path.join(THIS_DIR, TEST_DATA,
'SampleComposite-CAOM-2.3.xml')
reader = obs_reader_writer.ObservationReader(True)
with open(source_file_path, 'r'):
obs = reader.read(source_file_path)
writer = obs_reader_writer.ObservationWriter(
True, namespace=obs_reader_writer.CAOM23_NAMESPACE)
writer.write(obs, '/tmp/test.xml')
_common_check(obs)
# check observation
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum == get_acc_meta_checksum(obs)
# white spaces around strings should not affect the checksum
obs.algorithm = ' {}\t\n'.format(obs.algorithm.name)
assert obs.meta_checksum == get_meta_checksum(obs)
# now change some attributes and see how the checksums start to diverge
old_val = obs.collection
obs.collection = 'OTHER'
_common_check(obs)
assert obs.meta_checksum != get_meta_checksum(obs)
assert obs.acc_meta_checksum != get_acc_meta_checksum(obs)
obs.collection = old_val
# now change a plane
aplane = list(obs.planes.values())[0]
old_val = aplane.product_id
aplane.product_id = 'TESTPRODID'
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
for part in artifact.parts.values():
for chunk in part.chunks:
assert chunk.meta_checksum == get_meta_checksum(chunk)
assert chunk.acc_meta_checksum == get_acc_meta_checksum(
chunk)
assert part.meta_checksum == get_meta_checksum(part)
assert part.acc_meta_checksum == get_acc_meta_checksum(part)
assert artifact.meta_checksum == get_meta_checksum(artifact)
assert artifact.acc_meta_checksum == get_acc_meta_checksum(
artifact)
if plane._id == aplane._id:
assert plane.meta_checksum != get_meta_checksum(plane)
assert plane.acc_meta_checksum != get_acc_meta_checksum(plane)
else:
assert plane.meta_checksum == get_meta_checksum(plane)
assert plane.acc_meta_checksum == get_acc_meta_checksum(plane)
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum != get_acc_meta_checksum(obs)
aplane.product_id = old_val
# change an artifact
anartifact = list(aplane.artifacts.values())[0]
old_val = anartifact.content_length
anartifact.content_length = 3344
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
for part in artifact.parts.values():
for chunk in part.chunks:
assert chunk.meta_checksum == get_meta_checksum(chunk)
assert chunk.acc_meta_checksum == get_acc_meta_checksum(
chunk)
assert part.meta_checksum == get_meta_checksum(part)
assert part.acc_meta_checksum == get_acc_meta_checksum(part)
if artifact._id == anartifact._id:
assert artifact.meta_checksum != get_meta_checksum(artifact)
assert artifact.acc_meta_checksum != get_acc_meta_checksum(
artifact)
else:
assert artifact.meta_checksum == get_meta_checksum(artifact)
assert artifact.acc_meta_checksum == get_acc_meta_checksum(
artifact)
assert plane.meta_checksum == get_meta_checksum(plane)
if plane._id == aplane._id:
assert plane.acc_meta_checksum != get_acc_meta_checksum(plane)
else:
assert plane.acc_meta_checksum == get_acc_meta_checksum(plane)
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum != get_acc_meta_checksum(obs)
anartifact.content_length = old_val
apart = list(anartifact.parts.values())[0]
old_val = apart.name
apart.name = 'therealpart'
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
for part in artifact.parts.values():
for chunk in part.chunks:
assert chunk.meta_checksum == get_meta_checksum(chunk)
assert chunk.acc_meta_checksum == get_acc_meta_checksum(
chunk)
if part._id == apart._id:
assert part.meta_checksum != get_meta_checksum(part)
assert part.acc_meta_checksum != get_acc_meta_checksum(
part)
else:
assert part.meta_checksum == get_meta_checksum(part)
assert part.acc_meta_checksum == get_acc_meta_checksum(
part)
assert artifact.meta_checksum == get_meta_checksum(artifact)
if artifact._id == anartifact._id:
assert artifact.acc_meta_checksum != get_acc_meta_checksum(
artifact)
else:
assert artifact.acc_meta_checksum == get_acc_meta_checksum(
artifact)
assert plane.meta_checksum == get_meta_checksum(plane)
if plane._id == aplane._id:
assert plane.acc_meta_checksum != get_acc_meta_checksum(plane)
else:
assert plane.acc_meta_checksum == get_acc_meta_checksum(plane)
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum != get_acc_meta_checksum(obs)
apart.name = old_val
achunk = list(apart.chunks)[0]
old_val = chunk.naxis
if old_val == 5:
achunk.naxis = 4
else:
achunk.naxis = old_val + 1
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
for part in artifact.parts.values():
for chunk in part.chunks:
if chunk._id == achunk._id:
assert chunk.meta_checksum != get_meta_checksum(chunk)
assert chunk.acc_meta_checksum !=\
get_acc_meta_checksum(chunk)
else:
assert chunk.meta_checksum == get_meta_checksum(chunk)
assert chunk.acc_meta_checksum ==\
get_acc_meta_checksum(chunk)
assert part.meta_checksum == get_meta_checksum(part)
if part._id == apart._id:
assert part.acc_meta_checksum != get_acc_meta_checksum(
part)
else:
assert part.acc_meta_checksum == get_acc_meta_checksum(
part)
assert artifact.meta_checksum == get_meta_checksum(artifact)
if artifact._id == anartifact._id:
assert artifact.acc_meta_checksum != get_acc_meta_checksum(
artifact)
else:
assert artifact.acc_meta_checksum == get_acc_meta_checksum(
artifact)
assert plane.meta_checksum == get_meta_checksum(plane)
if plane._id == aplane._id:
assert plane.acc_meta_checksum != get_acc_meta_checksum(plane)
else:
assert plane.acc_meta_checksum == get_acc_meta_checksum(plane)
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum != get_acc_meta_checksum(obs)
achunk.naxis = old_val
# update the checksums and everything should match again
update_meta_checksum(obs)
_common_check(obs)
def _get_obs(from_xml_string):
etree.parse = Mock(return_value=etree.ElementTree(
etree.fromstring(from_xml_string.encode('ascii'))))
obsr = obs_reader_writer.ObservationReader()
obs = obsr.read(None)
return obs
def test_compatibility():
# tests loads a previously generated observation and checks the checksums against the previously
# calculated (in Java) checksums
source_file_path = os.path.join(THIS_DIR, TEST_DATA,
'SampleComposite-CAOM-2.3.xml')
reader = obs_reader_writer.ObservationReader(True)
with open(source_file_path, 'r') as f:
obs = reader.read(source_file_path)
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
for part in artifact.parts.values():
for chunk in part.chunks:
assert chunk.meta_checksum == get_meta_checksum(chunk)
assert chunk.acc_meta_checksum == get_acc_meta_checksum(
chunk)
assert part.meta_checksum == get_meta_checksum(part)
assert part.acc_meta_checksum == get_acc_meta_checksum(part)
assert artifact.meta_checksum == get_meta_checksum(artifact)
assert artifact.acc_meta_checksum == get_acc_meta_checksum(
artifact)
assert plane.meta_checksum == get_meta_checksum(plane)
assert plane.acc_meta_checksum == get_acc_meta_checksum(plane)
# check observation
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum == get_acc_meta_checksum(obs)
# now change some attributes and see how the checksums start to diverge
old_val = obs.collection
obs.collection = 'OTHER'
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
for part in artifact.parts.values():
for chunk in part.chunks:
assert chunk.meta_checksum == get_meta_checksum(chunk)
assert chunk.acc_meta_checksum == get_acc_meta_checksum(
chunk)
assert part.meta_checksum == get_meta_checksum(part)
assert part.acc_meta_checksum == get_acc_meta_checksum(part)
assert artifact.meta_checksum == get_meta_checksum(artifact)
assert artifact.acc_meta_checksum == get_acc_meta_checksum(
artifact)
assert plane.meta_checksum == get_meta_checksum(plane)
assert plane.acc_meta_checksum == get_acc_meta_checksum(plane)
assert obs.meta_checksum != get_meta_checksum(obs)
assert obs.acc_meta_checksum != get_acc_meta_checksum(obs)
obs.collection = old_val
# now change a plane
aplane = list(obs.planes.values())[0]
old_val = aplane.product_id
aplane.product_id = 'TESTPRODID'
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
for part in artifact.parts.values():
for chunk in part.chunks:
assert chunk.meta_checksum == get_meta_checksum(chunk)
assert chunk.acc_meta_checksum == get_acc_meta_checksum(
chunk)
assert part.meta_checksum == get_meta_checksum(part)
assert part.acc_meta_checksum == get_acc_meta_checksum(part)
assert artifact.meta_checksum == get_meta_checksum(artifact)
assert artifact.acc_meta_checksum == get_acc_meta_checksum(
artifact)
if plane._id == aplane._id:
assert plane.meta_checksum != get_meta_checksum(plane)
assert plane.acc_meta_checksum != get_acc_meta_checksum(plane)
else:
assert plane.meta_checksum == get_meta_checksum(plane)
assert plane.acc_meta_checksum == get_acc_meta_checksum(plane)
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum != get_acc_meta_checksum(obs)
aplane.product_id = old_val
# change an artifact
anartifact = list(aplane.artifacts.values())[0]
old_val = anartifact.content_length
anartifact.content_length = 3344
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
for part in artifact.parts.values():
for chunk in part.chunks:
assert chunk.meta_checksum == get_meta_checksum(chunk)
assert chunk.acc_meta_checksum == get_acc_meta_checksum(
chunk)
assert part.meta_checksum == get_meta_checksum(part)
assert part.acc_meta_checksum == get_acc_meta_checksum(part)
if artifact._id == anartifact._id:
assert artifact.meta_checksum != get_meta_checksum(artifact)
assert artifact.acc_meta_checksum != get_acc_meta_checksum(
artifact)
else:
assert artifact.meta_checksum == get_meta_checksum(artifact)
assert artifact.acc_meta_checksum == get_acc_meta_checksum(
artifact)
assert plane.meta_checksum == get_meta_checksum(plane)
if plane._id == aplane._id:
assert plane.acc_meta_checksum != get_acc_meta_checksum(plane)
else:
assert plane.acc_meta_checksum == get_acc_meta_checksum(plane)
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum != get_acc_meta_checksum(obs)
anartifact.content_length = old_val
apart = list(anartifact.parts.values())[0]
old_val = apart.name
apart.name = 'therealpart'
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
for part in artifact.parts.values():
for chunk in part.chunks:
assert chunk.meta_checksum == get_meta_checksum(chunk)
assert chunk.acc_meta_checksum == get_acc_meta_checksum(
chunk)
if part._id == apart._id:
assert part.meta_checksum != get_meta_checksum(part)
assert part.acc_meta_checksum != get_acc_meta_checksum(
part)
else:
assert part.meta_checksum == get_meta_checksum(part)
assert part.acc_meta_checksum == get_acc_meta_checksum(
part)
assert artifact.meta_checksum == get_meta_checksum(artifact)
if artifact._id == anartifact._id:
assert artifact.acc_meta_checksum != get_acc_meta_checksum(
artifact)
else:
assert artifact.acc_meta_checksum == get_acc_meta_checksum(
artifact)
assert plane.meta_checksum == get_meta_checksum(plane)
if plane._id == aplane._id:
assert plane.acc_meta_checksum != get_acc_meta_checksum(plane)
else:
assert plane.acc_meta_checksum == get_acc_meta_checksum(plane)
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum != get_acc_meta_checksum(obs)
apart.name = old_val
achunk = list(apart.chunks)[0]
old_val = chunk.naxis
if old_val == 5:
achunk.naxis = 4
else:
achunk.naxis = old_val + 1
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
for part in artifact.parts.values():
for chunk in part.chunks:
if chunk._id == achunk._id:
assert chunk.meta_checksum != get_meta_checksum(chunk)
assert chunk.acc_meta_checksum != get_acc_meta_checksum(
chunk)
else:
assert chunk.meta_checksum == get_meta_checksum(chunk)
assert chunk.acc_meta_checksum == get_acc_meta_checksum(
chunk)
assert part.meta_checksum == get_meta_checksum(part)
if part._id == apart._id:
assert part.acc_meta_checksum != get_acc_meta_checksum(
part)
else:
assert part.acc_meta_checksum == get_acc_meta_checksum(
part)
assert artifact.meta_checksum == get_meta_checksum(artifact)
if artifact._id == anartifact._id:
assert artifact.acc_meta_checksum != get_acc_meta_checksum(
artifact)
else:
assert artifact.acc_meta_checksum == get_acc_meta_checksum(
artifact)
assert plane.meta_checksum == get_meta_checksum(plane)
if plane._id == aplane._id:
assert plane.acc_meta_checksum != get_acc_meta_checksum(plane)
else:
assert plane.acc_meta_checksum == get_acc_meta_checksum(plane)
assert obs.meta_checksum == get_meta_checksum(obs)
assert obs.acc_meta_checksum != get_acc_meta_checksum(obs)
achunk.naxis = old_val
def _read_obs(model_fqn):
reader = obs_reader_writer.ObservationReader(False)
result = reader.read(model_fqn)
return result
