def test_decompose_uri():
test_uri = 'ad:STARS/galaxies.fits.gz'
scheme, path, f_name = mc.decompose_uri(test_uri)
assert scheme == 'ad', 'expected ad'
assert path == 'STARS', 'expected STARS'
assert f_name == 'galaxies.fits.gz', f'wrong f_name {f_name}'
test_uri = 'vos:ngvs/merge/NGVS-2-4.m.z.Mg002.weight.fits.fz'
scheme, path, f_name = mc.decompose_uri(test_uri)
assert scheme == 'vos', 'expected vos'
assert path == 'ngvs/merge', 'expected ngvs/merge'
assert (f_name == 'NGVS-2-4.m.z.Mg002.weight.fits.fz'
), f'wrong f_name {f_name}'
with pytest.raises(mc.CadcException):
mc.decompose_uri('')
def get_time_axis_val(params):
header = params.get('header')
uri = params.get('uri')
ignore_scheme, ignore_path, f_name = mc.decompose_uri(uri)
keyword = 'DATE-OBS'
if f_name.startswith('a'):
keyword = 'CLOCKVAL'
return ac.get_datetime(header.get(keyword))
def _get_obs_intent(uri):
# DB 07-08-20
# The co-added products (the ones with ‘g’ in the file name?) the Intent
# should be set to calibration.
ignore_scheme, ignore_path, file_name = mc.decompose_uri(uri)
prefix = obs_file_relationship.get_prefix(file_name)
result = ObservationIntentType.SCIENCE
if 'g' in prefix:
result = ObservationIntentType.CALIBRATION
return result
def visit(observation, **kwargs):
"""
If there are artifacts with the same name, but different case, prefer
the lower case artifact, and remove the upper-case one.
:param observation: Observation instance - check all it's artifacts
:param kwargs:
"""
mc.check_param(observation, Observation)
artifact_count = 0
plane_count = 0
if len(observation.planes.values()) > 1:
all_artifact_keys = cc.get_all_artifact_keys(observation)
all_artifact_keys_lower = [ii.lower() for ii in all_artifact_keys]
set_artifact_keys_lower = set(all_artifact_keys_lower)
delete_these_artifacts = []
if len(all_artifact_keys) != len(set_artifact_keys_lower):
for entry in set_artifact_keys_lower:
ignore_scheme, ignore_path, file_name = mc.decompose_uri(entry)
file_id = obs_file_relationship.remove_extensions(file_name)
# it's the suffix that has the different case, so use it
# to figure out which artifacts shouldn't exist
suffixes = obs_file_relationship.get_suffix(
file_id, observation.observation_id)
for key in all_artifact_keys:
for suffix in suffixes:
if suffix.upper() in key:
# get the fits, previews, thumbnails as well
delete_these_artifacts.append(key)
delete_these_planes = []
for entry in delete_these_artifacts:
for plane in observation.planes.values():
if entry in plane.artifacts.keys():
plane.artifacts.pop(entry)
logging.info(f'Removing {entry} from {plane.product_id}.')
artifact_count += 1
if len(plane.artifacts.keys()) == 0:
delete_these_planes.append(plane.product_id)
for entry in set(delete_these_planes):
observation.planes.pop(entry)
logging.info(
f'Removing {entry} from {observation.observation_id}.')
plane_count += 1
logging.info(
f'Completed cleanup for {observation.observation_id}. Removed '
f'{artifact_count} artifacts and {plane_count} planes.')
result = {
'artifacts': artifact_count,
'planes': plane_count,
}
return observation
def mock_get_data_label(uri):
ignore_scheme, ignore_collection, f_name = mc.decompose_uri(uri)
file_id = GemName.remove_extensions(f_name)
temp = mock_get_obs_metadata(file_id)
result = None
for ii in temp:
y = obs_file_relationship.remove_extensions(ii.get('filename'))
if y == file_id:
result = ii.get('data_label')
break
return result
def get(self, source, dest_fqn):
"""
:param source: str - artifact uri
:param dest_fqn: str - fully-qualified file system name
"""
working_dir = os.path.dirname(dest_fqn)
f_name = os.path.basename(dest_fqn)
scheme_ignore, archive, f_name_ignore = mc.decompose_uri(source)
mc.data_get(
self._cadc_client,
working_dir,
f_name,
archive,
self._observable.metrics,
)
def accumulate_bp(bp, uri):
"""Configure the telescope-specific ObsBlueprint at the CAOM model
Observation level."""
logging.debug('Begin accumulate_bp.')
scheme, collection, f_name = mc.decompose_uri(uri)
if f_name.startswith('d'):
_accumulate_dao_bp(bp)
else:
_accumulate_skycam_bp(bp)
meta_producer = mc.get_version(APPLICATION)
bp.set('Observation.metaProducer', meta_producer)
bp.set('Plane.metaProducer', meta_producer)
bp.set('Artifact.metaProducer', meta_producer)
bp.set('Chunk.metaProducer', meta_producer)
bp.clear('Observation.algorithm.name')
bp.set('Observation.telescope.name', 'get_telescope_name(header)')
bp.set('Observation.telescope.geoLocationX', 'get_geo_x(header)')
bp.set('Observation.telescope.geoLocationY', 'get_geo_y(header)')
bp.set('Observation.telescope.geoLocationZ', 'get_geo_z(header)')
bp.set('Chunk.time.axis.axis.ctype', 'TIME')
bp.set('Chunk.time.axis.axis.cunit', 'd')
bp.set('Chunk.time.axis.function.naxis', '1')
bp.set('Chunk.time.axis.function.delta', 'get_time_axis_delta(header)')
bp.set('Chunk.time.axis.function.refCoord.pix', '0.5')
bp.set('Chunk.time.axis.function.refCoord.val',
'get_time_axis_val(params)')
bp.set('Chunk.observable.axis.axis.ctype', 'FLUX')
bp.set('Chunk.observable.axis.axis.cunit', 'COUNTS')
bp.set('Chunk.observable.axis.function.refCoord.pix', 1)
bp.set('Chunk.energy.axis.axis.ctype', 'WAVE')
bp.set('Chunk.energy.axis.axis.cunit', 'Angstrom')
bp.set('Chunk.energy.specsys', 'TOPOCENT')
bp.set('Chunk.energy.ssysobs', 'TOPOCENT')
bp.set('Chunk.energy.ssyssrc', 'TOPOCENT')
# derived observations
if dn.DAOName.is_derived(uri):
bp.set('CompositeObservation.members', 'get_members(header)')
bp.add_fits_attribute('Observation.algorithm.name', 'PROCNAME')
logging.debug('Done accumulate_bp.')
def accumulate_bp(bp, uri):
"""Configure the telescope-specific ObsBlueprint at the CAOM model
Observation level."""
logging.debug(f'Begin accumulate_bp for {uri}.')
bp.configure_position_axes((1, 2))
scheme, archive, file_name = mc.decompose_uri(uri)
storage_name = sn.get_storage_name(file_name, file_name)
if sn.is_ngvs(uri):
_accumulate_ngvs_bp(bp, storage_name)
else:
_accumulate_mp_bp(bp, storage_name)
# they're all DerivedObservations
bp.set('DerivedObservation.members', {})
bp.set('Observation.type', 'OBJECT')
bp.set('Observation.proposal.id', 'get_proposal_id(header)')
bp.clear('Plane.metaRelease')
bp.add_fits_attribute('Plane.metaRelease', 'REL_DATE')
bp.clear('Chunk.position.resolution')
bp.add_fits_attribute('Chunk.position.resolution', 'FINALIQ')
bp.set('Observation.instrument.name', INSTRUMENT)
bp.set('Observation.telescope.name', 'CFHT 3.6m')
x, y, z = ac.get_geocentric_location('cfht')
bp.set('Observation.telescope.geoLocationX', x)
bp.set('Observation.telescope.geoLocationY', y)
bp.set('Observation.telescope.geoLocationZ', z)
bp.set('Plane.calibrationLevel', 'get_calibration_level(uri)')
bp.set('Plane.dataProductType', 'get_data_product_type(uri)')
bp.set('Plane.provenance.producer', 'CADC')
bp.set('Artifact.productType', 'get_artifact_product_type(uri)')
if storage_name.collection == sn.MP_COLLECTION:
_accumulate_mp_bp(bp, storage_name)
else:
_accumulate_ngvs_bp(bp, storage_name)
logging.debug('Done accumulate_bp.')
def get(self, uri):
"""
:param uri: Artifact URI at CADC
"""
ignore_scheme, ignore_collection, f_name = mc.decompose_uri(uri)
if self._connected:
result = None
if self._use_local_files:
result = self._check_local(f_name)
if result is None:
result = self._check_caom2(uri)
if result is None:
result = self._check_remote(uri)
else:
result = self._check_local(f_name)
repaired_data_label = None
if result is not None:
repaired_data_label = obs_file_relationship.repair_data_label(
f_name, result
)
return repaired_data_label
def _later_version_at_cadc(entry, caom_client, metrics):
later_version_found = False
storage_name = VlassName(file_name=entry, entry=entry)
caom_at_cadc = mc.repo_get(caom_client, COLLECTION,
storage_name.obs_id, metrics)
if caom_at_cadc is not None:
for plane in caom_at_cadc.planes.values():
for artifact in plane.artifacts.values():
if 'jpg' in artifact.uri:
continue
ignore_scheme, ignore_collection, f_name = mc.decompose_uri(
artifact.uri)
vlass_name = VlassName(file_name=f_name)
if vlass_name.version > storage_name.version:
# there's a later version at CADC, everything is good
# ignore the failure report
later_version_found = True
break
if later_version_found:
break
return later_version_found
def get_storage_name(file_name, entry):
"""
:param file_name may be a file name or a URI. The parameter has to be
called 'file_name', because there's a constructor call that
expects that.
:param entry str what came from the todo.txt file
:return: The StorageName extension class for the entry
"""
uri = None
temp = file_name
scheme = 'ad'
if '/' in file_name:
# parameter is a URI, adjust accordingly
scheme, archive, f_name = mc.decompose_uri(file_name)
temp = f_name
uri = file_name
if is_ngvs(temp):
result = NGVSName(file_name=temp, scheme=scheme, entry=entry)
else:
result = MEGAPIPEName(file_name=temp, entry=entry)
result.file_uri = uri
return result
def __init__(self, file_name=None, artifact_uri=None, entry=None):
if file_name:
self.fname_in_ad = file_name
self._file_name = file_name
self._file_id = mc.StorageName.remove_extensions(file_name)
self._assign_bits()
super(PHANGSName, self).__init__(self._obs_id,
COLLECTION,
PHANGSName.PHANGS_NAME_PATTERN,
fname_on_disk=file_name,
compression='',
entry=entry)
elif artifact_uri:
scheme, path, file_name = mc.decompose_uri(artifact_uri)
self._file_name = file_name
self._file_id = mc.StorageName.remove_extensions(file_name)
self._assign_bits()
super(PHANGSName, self).__init__(self._obs_id,
COLLECTION,
PHANGSName.PHANGS_NAME_PATTERN,
fname_on_disk=file_name,
compression='',
entry=entry)
print(':::1 - Find the name of a file to test with.')
ops_query = f"SELECT O.observationID, A.uri " \
f"FROM caom2.Observation as O " \
f"JOIN caom2.Plane as P on O.obsID = P.obsID " \
f"JOIN caom2.Artifact as A on P.planeID = A.planeID " \
f"WHERE O.collection = '{archive}' " \
f"AND A.uri like '%.fits%' " \
f"LIMIT 1"
ops_buffer = io.StringIO()
ops_client.query(ops_query, output_file=ops_buffer, data_only=True, response_format='csv')
ops_table = Table.read(ops_buffer.getvalue().split('\n'), format='csv')
if len(ops_table) == 1:
obs_id = ops_table[0]['observationID']
uri = ops_table[0]['uri']
ignore_scheme, ignore_path, f_name = mc.decompose_uri(uri)
print(f':::Looking for {obs_id} and {f_name}')
else:
print(f':::No observation records found for collection {archive}')
sys.exit(-1)
obs = caom_client.read(archive, obs_id)
obs_fqn = f'/usr/src/app/expected.{obs_id}.xml'
mc.write_obs_to_file(obs, obs_fqn)
print(f':::2 - Get {f_name}')
config = mc.Config()
config.get_executors()
data_client = CadcDataClient(subject)
metrics = mc.Metrics(config)
mc.data_get(data_client, '/usr/src/app', f_name, collection, metrics)