def _art_mock(
fqn,
product_type,
release_type,
uri,
art_ignore,
):
if '_th' in uri:
return Artifact(
uri,
ProductType.PREVIEW,
ReleaseType.DATA,
'image/jpeg',
13,
ChecksumURI('md5:13'),
)
else:
return Artifact(
uri,
ProductType.PREVIEW,
ReleaseType.DATA,
'image/jpeg',
12,
ChecksumURI('md5:12'),
)
def copy_artifact(from_artifact, features=None):
"""Make a copy of an artifact. This works around the CAOM2 constraint
'org.postgresql.util.PSQLException: ERROR: duplicate key value violates
unique constraint "artifact_pkey"', when trying to use the same artifact
information in different planes (e.g. when referring to the same
thumbnail and preview files).
:param from_artifact Artifact of which to make a shallow copy
:param features which version of CAOM to use
:return a copy of the from_artifact, with parts set to None
"""
if features is not None and features.supports_latest_caom:
copy = Artifact(uri=from_artifact.uri,
product_type=from_artifact.product_type,
release_type=from_artifact.release_type,
content_type=from_artifact.content_type,
content_length=from_artifact.content_length,
content_checksum=from_artifact.content_checksum,
content_release=from_artifact.content_release,
content_read_groups=from_artifact.content_read_groups,
parts=None)
else:
copy = Artifact(uri=from_artifact.uri,
product_type=from_artifact.product_type,
release_type=from_artifact.release_type,
content_type=from_artifact.content_type,
content_length=from_artifact.content_length,
content_checksum=from_artifact.content_checksum,
parts=None)
return copy
def test_augment_polarization(test_file):
test_fitsparser = FitsParser(test_file, ObsBlueprint(polarization_axis=1))
artifact = Artifact('ad:{}/{}'.format('TEST', test_file),
ProductType.SCIENCE, ReleaseType.DATA)
test_fitsparser.augment_artifact(artifact)
polarization = artifact.parts['0'].chunks[0].polarization
check_xml(ObservationWriter()._add_polarization_wcs_element, polarization,
EXPECTED_POLARIZATION_XML)
def test_augment_energy(test_file):
test_fitsparser = FitsParser(test_file)
artifact = Artifact('ad:{}/{}'.format('TEST', test_file),
ProductType.SCIENCE, ReleaseType.DATA)
test_fitsparser.augment_artifact(artifact)
energy = artifact.parts['0'].chunks[0].energy
energy.bandpassName = '21 cm' # user set attribute
check_xml(ObservationWriter()._add_spectral_wcs_element, energy,
EXPECTED_ENERGY_XML)
def test_augment_energy():
bp = ObsBlueprint(energy_axis=1)
test_fitsparser = FitsParser(sample_file_4axes, bp)
artifact = Artifact('ad:{}/{}'.format('TEST', sample_file_4axes),
ProductType.SCIENCE, ReleaseType.DATA)
test_fitsparser.augment_artifact(artifact)
energy = artifact.parts['0'].chunks[0].energy
ex = _get_from_str_xml(EXPECTED_ENERGY_XML,
ObservationReader()._get_spectral_wcs, 'energy')
result = get_differences(ex, energy)
assert result is None, repr(energy)
def test_augment_polarization():
test_fitsparser = FitsParser(sample_file_4axes,
ObsBlueprint(polarization_axis=1))
artifact = Artifact('ad:{}/{}'.format('TEST', sample_file_4axes),
ProductType.SCIENCE, ReleaseType.DATA)
test_fitsparser.augment_artifact(artifact)
polarization = artifact.parts['0'].chunks[0].polarization
ex = _get_from_str_xml(EXPECTED_POLARIZATION_XML,
ObservationReader()._get_polarization_wcs,
'polarization')
result = get_differences(ex, polarization)
assert result is None, result
def test_augment_artifact_time(test_file, expected):
test_fitsparser = FitsParser(test_file, ObsBlueprint(time_axis=1))
artifact = Artifact('ad:{}/{}'.format('TEST', test_file),
ProductType.SCIENCE, ReleaseType.DATA)
test_fitsparser.augment_artifact(artifact)
assert artifact.parts is not None
assert len(artifact.parts) == 6
test_part = artifact.parts['1']
test_chunk = test_part.chunks.pop()
assert test_chunk is not None
assert test_chunk.time is not None
check_xml(ObservationWriter()._add_temporal_wcs_element, test_chunk.time,
expected)
def test_augment_artifact(test_file):
test_fitsparser = FitsParser(test_file, ObsBlueprint(position_axis=(1, 2)))
artifact = Artifact('ad:{}/{}'.format('TEST', test_file),
ProductType.SCIENCE, ReleaseType.DATA)
test_fitsparser.augment_artifact(artifact)
assert artifact.parts is not None
assert len(artifact.parts) == 1
test_part = artifact.parts['0']
test_chunk = test_part.chunks.pop()
assert test_chunk is not None
assert test_chunk.position is not None
check_xml(ObservationWriter()._add_spatial_wcs_element,
test_chunk.position, EXPECTED_POSITION_XML)
def test_augment_artifact_time():
test_fitsparser = FitsParser(sample_file_time_axes,
ObsBlueprint(time_axis=1))
artifact = Artifact('ad:{}/{}'.format('TEST', sample_file_time_axes),
ProductType.SCIENCE, ReleaseType.DATA)
test_fitsparser.augment_artifact(artifact)
assert artifact.parts is not None
assert len(artifact.parts) == 6
test_part = artifact.parts['1']
test_chunk = test_part.chunks.pop()
assert test_chunk is not None
assert test_chunk.time is not None
ex = _get_from_str_xml(EXPECTED_CFHT_WIRCAM_RAW_GUIDE_CUBE_TIME,
ObservationReader()._get_temporal_wcs, 'time')
result = get_differences(ex, test_chunk.time)
assert result is None
def test_augment_artifact():
test_blueprint = ObsBlueprint(position_axes=(1, 2))
test_fitsparser = FitsParser(sample_file_4axes, test_blueprint)
artifact = Artifact('ad:{}/{}'.format('TEST', sample_file_4axes),
ProductType.SCIENCE, ReleaseType.DATA)
test_fitsparser.augment_artifact(artifact)
assert artifact.parts is not None
assert len(artifact.parts) == 1
test_part = artifact.parts['0']
test_chunk = test_part.chunks.pop()
assert test_chunk is not None
assert test_chunk.position is not None
ex = _get_from_str_xml(EXPECTED_POSITION_XML,
ObservationReader()._get_spatial_wcs, 'position')
result = get_differences(ex, test_chunk.position)
assert result is None
def test_compute(self):
# _choose_product returns Artifact.product (SCIENCE),
# user_chunk = False
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.SCIENCE
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
actual_axis = wcs_util.CustomAxisUtil.compute(artifacts)
expected_axis = None
self.assertEqual(expected_axis, actual_axis)
# _choose_product returns Artifact.product (CALIBRATION),
# user_chunk = False
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.SCIENCE
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.SCIENCE
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.CALIBRATION
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
actual_axis = wcs_util.CustomAxisUtil.compute(artifacts)
expected_axis = None
self.assertEqual(expected_axis, actual_axis)
# _choose_product returns Part.product (SCIENCE),
# user_chunk = False
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.SCIENCE
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.SCIENCE
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.PREVIEW
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
actual_axis = wcs_util.CustomAxisUtil.compute(artifacts)
expected_axis = None
self.assertEqual(expected_axis, actual_axis)
# _choose_product returns Part.product (CALIBRATION),
# user_chunk = False
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.SCIENCE
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.PREVIEW
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
actual_axis = wcs_util.CustomAxisUtil.compute(artifacts)
expected_axis = None
self.assertEqual(expected_axis, actual_axis)
# _choose_product returns Chunk.product (SCIENCE), user_chunk = False
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.SCIENCE
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.PREVIEW
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.PREVIEW
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
actual_axis = wcs_util.CustomAxisUtil.compute(artifacts)
expected_axis = None
self.assertEqual(expected_axis, actual_axis)
# _choose_product returns Chunk.product (CALIBRATION),
# user_chunk = False
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.PREVIEW
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.PREVIEW
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
actual_axis = wcs_util.CustomAxisUtil.compute(artifacts)
expected_axis = None
self.assertEqual(expected_axis, actual_axis)
# _choose_product returns None, user_chunk = False
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.PREVIEW
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.PREVIEW
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.PREVIEW
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
actual_axis = wcs_util.CustomAxisUtil.compute(artifacts)
expected_axis = None
self.assertEqual(expected_axis, actual_axis)
# _choose_product returns Artifact.product (SCIENCE),
# user_chunk = True, Chunk.custom is None
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.SCIENCE
test_chunk.custom = None
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.SCIENCE
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.SCIENCE
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
expected_axis = None
actual_axis = wcs_util.CustomAxisUtil.compute(artifacts)
self.assertEqual(expected_axis, actual_axis)
# _choose_product returns Artifact.product (SCIENCE),
# user_chunk = True, Chunk.custom is not None
# bad Chunk.custom.axis.axis.ctype
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.SCIENCE
test_chunk.custom = CustomTestUtil.bad_ctype_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.SCIENCE
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.SCIENCE
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
with pytest.raises(ValueError) as ex:
actual_axis = wcs_util.CustomAxisUtil.compute(artifacts)
assert ('Unsupported CTYPE:' in str(ex.value))
# _choose_product returns Artifact.product (SCIENCE),
# user_chunk = True, Chunk.custom is not None
# first_ctype == Chunk.custom.axis.axis.ctype
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.SCIENCE
test_chunk.custom = CustomTestUtil.good_wcs_with_function()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.SCIENCE
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.SCIENCE
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
expected_ctype = "RM"
expected_sample = Interval(-49.5, 19950.5)
expected_samples = [expected_sample]
expected_bounds = Interval(-49.5, 19950.5, expected_samples)
expected_dimension = 200
expected_axis = plane.CustomAxis(expected_ctype, expected_bounds,
expected_dimension)
actual_axis = wcs_util.CustomAxisUtil.compute(artifacts)
self.assertEqual(expected_axis.ctype, actual_axis.ctype)
self.assertEqual(expected_axis.bounds.lower, actual_axis.bounds.lower)
self.assertEqual(expected_axis.bounds.upper, actual_axis.bounds.upper)
self.assertEqual(expected_axis.bounds.samples[0].lower,
actual_axis.bounds.samples[0].lower)
self.assertEqual(expected_axis.bounds.samples[0].upper,
actual_axis.bounds.samples[0].upper)
self.assertEqual(expected_axis.dimension, actual_axis.dimension)
# _choose_product returns Artifact.product (SCIENCE),
# user_chunk = True, Chunk.custom is not None
# first_ctype == Chunk.custom.axis.axis.ctype
test_chunk_1 = Chunk()
test_chunk_1.product_type = chunk.ProductType.SCIENCE
test_chunk_1.custom = CustomTestUtil.good_wcs_with_function()
test_chunk_2 = Chunk()
test_chunk_2.product_type = chunk.ProductType.SCIENCE
test_chunk_2.custom = CustomTestUtil.good_wcs_with_function()
test_chunk_2.custom.axis.axis.ctype = "FARADAY"
test_chunks = TypedList(Chunk, test_chunk_1, test_chunk_2)
part_name = "test_part"
part_product_type = chunk.ProductType.SCIENCE
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.SCIENCE
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
expected_ctype = "RM"
expected_sample = Interval(-49.5, 19950.5)
expected_samples = [expected_sample]
expected_bounds = Interval(-49.5, 19950.5, expected_samples)
expected_dimension = 200
expected_axis = plane.CustomAxis(expected_ctype, expected_bounds,
expected_dimension)
with pytest.raises(ValueError) as ex:
actual_axis = wcs_util.CustomAxisUtil.compute(artifacts)
assert ('CTYPE must be the same across all Artifacts' in str(ex.value))
def test_compute_bounds(self):
# user_chunk = False
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.SCIENCE
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.SCIENCE
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.SCIENCE
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = None
expected_ctype = "RM"
actual_bounds = wcs_util.CustomAxisUtil.compute_bounds(
artifacts, product_type, expected_ctype)
expected_bounds = None
self.assertEqual(expected_bounds, actual_bounds)
# user_chunk = True, current_type != expected_ctype
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs_with_function()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.CALIBRATION
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "FARADAY"
with pytest.raises(ValueError) as ex:
wcs_util.CustomAxisUtil.compute_bounds(artifacts, product_type,
expected_ctype)
assert ('CTYPE must be the same across all Artifacts' in str(ex.value))
# user_chunk = True, range is not None
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs_with_range()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.CALIBRATION
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "RM"
actual_interval = wcs_util.CustomAxisUtil.compute_bounds(
artifacts, product_type, expected_ctype)
expected_interval = Interval(1.1, 11.1)
self.assertEqual(expected_interval.lower, actual_interval.lower)
self.assertEqual(expected_interval.upper, actual_interval.upper)
# user_chunk = True, get_num_pixels: bounds with 3 samples that
# traverses _merge_into_list completely
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs_with_bounds_3_samples()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.CALIBRATION
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "RM"
actual_interval = wcs_util.CustomAxisUtil.compute_bounds(
artifacts, product_type, expected_ctype)
expected_interval = Interval(-1.2, 11.2)
self.assertEqual(expected_interval.lower, actual_interval.lower)
self.assertEqual(expected_interval.upper, actual_interval.upper)
# user_chunk = True, function is not None
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs_with_function()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.CALIBRATION
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "RM"
actual_interval = wcs_util.CustomAxisUtil.compute_bounds(
artifacts, product_type, expected_ctype)
expected_interval = Interval(-49.5, 19950.5)
self.assertEqual(expected_interval.lower, actual_interval.lower)
self.assertEqual(expected_interval.upper, actual_interval.upper)
def test_compute_dimension_from_wcs(self):
# bounds is None
bounds = None
artifacts = None
product_type = None
expected_ctype = None
actual_dimension = wcs_util.CustomAxisUtil.compute_dimension_from_wcs(
bounds, artifacts, product_type, expected_ctype)
expected_dimension = None
self.assertEqual(expected_dimension, actual_dimension)
# bounds is not None, user_chunk = True, current_type != expected_ctype
bounds = Interval(1.1, 11.1)
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs_with_function()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.CALIBRATION
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "FARADAY"
with pytest.raises(ValueError) as ex:
wcs_util.CustomAxisUtil.compute_dimension_from_wcs(
bounds, artifacts, product_type, expected_ctype)
assert ('CTYPE must be the same across all Artifacts' in str(ex.value))
# bounds is not None, user_chunk = True, current_type is not None and
# current_type == expected_ctype, ss >= scale, num = 1
bounds = Interval(1.1, 11.1)
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs_with_negative_delta()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.CALIBRATION
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "RM"
actual_dimension = wcs_util.CustomAxisUtil.compute_dimension_from_wcs(
bounds, artifacts, product_type, expected_ctype)
expected_dimension = 200
self.assertEqual(expected_dimension, actual_dimension)
# bounds is not None, user_chunk = False, sw = None
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.SCIENCE
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.SCIENCE
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.SCIENCE
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = None
expected_ctype = "RM"
actual_dimension = wcs_util.CustomAxisUtil.compute_dimension_from_wcs(
bounds, artifacts, product_type, expected_ctype)
expected_dimension = None
self.assertEqual(expected_dimension, actual_dimension)
# bounds is not None, user_chunk = True, current_type is not None and
# current_type == expected_ctype, ss >= scale, num = 2
bounds = Interval(1.1, 11.1)
test_chunk1 = Chunk()
test_chunk1.product_type = chunk.ProductType.CALIBRATION
test_chunk1.custom = CustomTestUtil.good_wcs_with_negative_delta()
test_chunk2 = Chunk()
test_chunk2.product_type = chunk.ProductType.CALIBRATION
test_chunk2.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk1, test_chunk2)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.CALIBRATION
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "RM"
actual_dimension = wcs_util.CustomAxisUtil.compute_dimension_from_wcs(
bounds, artifacts, product_type, expected_ctype)
expected_dimension = 500
self.assertEqual(expected_dimension, actual_dimension)
def test_compute_dimension_from_range_bounds(self):
# user_chunk = False, matches is None
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.SCIENCE
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.SCIENCE
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.SCIENCE
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = None
expected_ctype = "RM"
actual_num_pixels = \
wcs_util.CustomAxisUtil.compute_dimension_from_range_bounds(
artifacts, product_type, expected_ctype)
expected_num_pixels = None
self.assertEqual(expected_num_pixels, actual_num_pixels)
# user_chunk = False, ctype not match
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.SCIENCE
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.SCIENCE
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.SCIENCE
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "RM"
actual_num_pixels = \
wcs_util.CustomAxisUtil.compute_dimension_from_range_bounds(
artifacts, product_type, expected_ctype)
expected_num_pixels = None
self.assertEqual(expected_num_pixels, actual_num_pixels)
# user_chunk = False, ptype not match
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.SCIENCE
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.SCIENCE
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "RM"
actual_num_pixels = \
wcs_util.CustomAxisUtil.compute_dimension_from_range_bounds(
artifacts, product_type, expected_ctype)
expected_num_pixels = None
self.assertEqual(expected_num_pixels, actual_num_pixels)
# user_chunk = False, atype not match
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.SCIENCE
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "RM"
actual_num_pixels = \
wcs_util.CustomAxisUtil.compute_dimension_from_range_bounds(
artifacts, product_type, expected_ctype)
expected_num_pixels = None
self.assertEqual(expected_num_pixels, actual_num_pixels)
# user_chunk = True, current_type != expected_ctype
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs_with_function()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.CALIBRATION
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "FARADAY"
with pytest.raises(ValueError) as ex:
wcs_util.CustomAxisUtil.compute_dimension_from_range_bounds(
artifacts, product_type, expected_ctype)
assert ('CTYPE must be the same across all Artifacts' in str(ex.value))
# user_chunk = True, get_num_pixels: range is not None
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs_with_range()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.CALIBRATION
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "RM"
actual_num_pixels = \
wcs_util.CustomAxisUtil.compute_dimension_from_range_bounds(
artifacts, product_type, expected_ctype)
expected_num_pixels = 10
self.assertEqual(expected_num_pixels, actual_num_pixels)
# user_chunk = True, get_num_pixels: bounds with 3 samples that
# traverses _merge_into_list completely
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs_with_bounds_3_samples()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.CALIBRATION
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "RM"
actual_num_pixels = \
wcs_util.CustomAxisUtil.compute_dimension_from_range_bounds(
artifacts, product_type, expected_ctype)
expected_num_pixels = 11
self.assertEqual(expected_num_pixels, actual_num_pixels)
# user_chunk = True, range = None, bounds = None, use_func and
# function = None
test_chunk = Chunk()
test_chunk.product_type = chunk.ProductType.CALIBRATION
test_chunk.custom = CustomTestUtil.good_wcs()
test_chunks = TypedList(Chunk, test_chunk)
part_name = "test_part"
part_product_type = chunk.ProductType.CALIBRATION
part = Part(part_name, part_product_type, test_chunks)
uri = 'mast:HST/product/test_file.jpg'
artifact_product_type = chunk.ProductType.CALIBRATION
release_type = ReleaseType.DATA
artifact = Artifact(uri, artifact_product_type, release_type)
artifact.parts = TypedOrderedDict((Part), (part_name, part))
artifacts = TypedList(Artifact, artifact)
product_type = chunk.ProductType.CALIBRATION
expected_ctype = "RM"
actual_num_pixels = \
wcs_util.CustomAxisUtil.compute_dimension_from_range_bounds(
artifacts, product_type, expected_ctype)
expected_num_pixels = None
self.assertEqual(expected_num_pixels, actual_num_pixels)
def build_observation(db_content, observation, md_name):
override = read_md_pk(md_name)
fqn = override.get('fqn')
almaca_name = AlmacaName(fname_on_disk=fqn)
# logging.error(db_content.colnames)
# logging.error('fqn is {}'.format(fqn))
field_index = _get_index(almaca_name, db_content)
# field_index = 0
if observation is None:
observation = _build_obs(override, db_content, fqn, field_index,
almaca_name, md_name)
provenance = get_provenance(almaca_name)
provenance.inputs.add(
PlaneURI('caom:ALMA/A001_X88b_X23/A001_X88b_X23-raw'))
# HK 07-02-20
# I'm looking at the very first entry, A002_Xb999fd_X602.SCI.J1851+0035.
# The time bounds listed under all of the second-level planes correspond
# to a date of Oct 20, 2016, which agrees with the observing date I pull
# up on listobs. But in the top level plane, the metaRelease date is
# listed as Oct 12, 2016. As we discussed earlier this week, it doesn't
# make sense to have the meta data released before the observation was
# even taken. Using the 'end time' of the observation that's already
# pulled for a lower plane, and putting that as the metaRelease date in
# the top level would be a good solution. NB: since all spws are observed
# simultaneously, you'll get the same answer for whichever of the
# [high/low]res_spw[X] entries that you pull the information from.
input_meta_data = read_md_pk(almaca_name.input_ms_metadata)
meta_release = mc.to_float(input_meta_data.get('end_date'))
meta_release = time.Time(meta_release, format='mjd')
meta_release.format = 'isot'
meta_release_dt = mc.make_time(meta_release.value)
release_date = db_content['Release date'][field_index]
if release_date is None:
raise mc.CadcException('No release date for {}'.format(fqn))
else:
release_date = time.Time(release_date).to_datetime()
logging.error('Add plane {} to {}'.format(almaca_name.product_id,
almaca_name.obs_id))
plane = Plane(product_id=almaca_name.product_id,
data_release=release_date,
meta_release=meta_release_dt,
provenance=provenance)
plane.position = build_position(db_content, field_index, md_name)
plane.energy = build_energy(override)
plane.polarization = None
plane.time = build_time(override, almaca_name)
# HK 14-08-2019
# dataProductType should be 'visibility'
plane.data_product_type = DataProductType.VISIBILITY
plane.calibration_level = CalibrationLevel.CALIBRATED
observation.planes.add(plane)
observation.meta_release = plane.meta_release
# TODO hard-coded
observation.members.add(ObservationURI('caom:ALMA/A001_X88b_X23'))
# HK 29-07-19
# qa/ contains images, plots, and web page status views generated
# during the original (non-CANFAR) calibration of the raw data.
# We may want to consider retaining these files as well, as they give
# a more advanced user an easier way to check on data quality,
# potential issues with calibration, etc. I believe they come
# packaged with the rest of the 'products' tarball on the archive,
# so they would be obtainable even if we do not keep a copy. These
# files are fairly small.
# TODO override.get('artifact_uri')
artifact = Artifact(uri=almaca_name.uri,
product_type=almaca_name.intent,
release_type=ReleaseType.DATA,
content_type='application/x-tar',
content_length=None)
plane.artifacts.add(artifact)
return observation