def _update_plane_provenance(observation, plane, headers):
logging.debug(f'Begin _update_plane_provenance for {plane.product_id} with'
f'observation type: {observation.type}.')
if observation.type in ['object', 'flat', 'comparison']:
f_name = headers[0].get('BIAS')
if f_name is not None:
bias_name = dn.DAOName(file_name=f_name)
plane_uri = _make_uris(bias_name.obs_id, bias_name.product_id)
plane.provenance.inputs.add(plane_uri)
if observation.type in ['object', 'comparison']:
f_name = headers[0].get('FLAT')
if f_name is not None:
flat_name = dn.DAOName(file_name=f_name)
plane_uri = _make_uris(flat_name.obs_id, flat_name.product_id)
plane.provenance.inputs.add(plane_uri)
# referral to raw plane
plane_uri = _make_uris(observation.observation_id,
observation.observation_id)
plane.provenance.inputs.add(plane_uri)
if observation.type == 'object':
f_name = headers[0].get('DCLOG1')
if f_name is not None:
ref_spec1_name = dn.DAOName(file_name=f_name.split()[2])
plane_uri = _make_uris(ref_spec1_name.obs_id,
ref_spec1_name.product_id)
plane.provenance.inputs.add(plane_uri)
if headers[0].get('DCLOG2') is not None:
ref_spec1_name = dn.DAOName(
file_name=headers[0].get('DCLOG2').split()[2])
plane_uri = _make_uris(ref_spec1_name.obs_id,
ref_spec1_name.product_id)
plane.provenance.inputs.add(plane_uri)
logging.debug(f'End _update_plane_provenance.')
def generate_plots(self, obs_id):
count = 0
storage_name = dn.DAOName(file_name=self._science_file)
science_fqn = os.path.join(self._working_dir, storage_name.file_name)
preview = storage_name.prev
preview_fqn = os.path.join(self._working_dir, preview)
thumb = storage_name.thumb
thumb_fqn = os.path.join(self._working_dir, thumb)
hdu_list = fits.open(science_fqn)
header = hdu_list[0].header
if ('e' in storage_name.file_name or 'p' in storage_name.file_name
or 'v' in storage_name.file_name):
count += self._do_cal_processed(hdu_list, header, science_fqn,
storage_name, preview_fqn,
thumb_fqn, obs_id)
elif storage_name.file_name.startswith('a'):
count += DAOPreview._do_skycam(science_fqn, preview_fqn, thumb_fqn)
else:
count += self._do_sci(hdu_list, header, storage_name, science_fqn,
preview_fqn, thumb_fqn)
hdu_list.close()
if count == 2:
self.add_preview(storage_name.thumb_uri, storage_name.thumb,
ProductType.THUMBNAIL)
self.add_preview(storage_name.prev_uri, storage_name.prev,
ProductType.PREVIEW)
self.add_to_delete(thumb_fqn)
self.add_to_delete(preview_fqn)
return count
def _get_uris(args):
result = []
if args.lineage:
for ii in args.lineage:
ignore, uri = mc.decompose_lineage(ii)
result.append(uri)
elif args.local:
for ii in args.local:
obs_id = mc.StorageName.remove_extensions(os.path.basename(ii))
uri = dn.DAOName(obs_id=obs_id).file_uri
result.append(uri)
elif args.observation:
uri = dn.DAOName(obs_id=args.observation[1]).file_uri
result.append(uri)
else:
raise mc.CadcException(f'Could not define uri from these args {args}')
return result
def _repair_provenance_value(value, obs_id):
logging.debug(f'Begin _repair_provenance_value for {obs_id}')
# values look like:
# FLAT_1 = 'dao_c122_2007_000916.fits'
# FLAT_2 = 'dao_c122_2007_000917.fits'
# FLAT_3 = 'dao_c122_2007_000918.fits'
# FLAT_4 = 'dao_c122_2007_000919.fits'
#
# OR
#
# ZERO_18 = 'dao_c122_2016_012728.fits'
# ZERO_19 = 'dao_c122_2016_012729.fits'
# ZERO_20 = 'dao_c122_2016_012730.fits'
dao_name = dn.DAOName(file_name=value)
prov_prod_id = dao_name.product_id
prov_obs_id = dao_name.obs_id
logging.debug(f'End _repair_provenance_value')
return prov_obs_id, prov_prod_id
def test_visit():
# this should result in three new artifacts being added to every plane:
# one for a thumbnail and two for previews (one zoom)
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
test_files = {
# processed spectrum
'visit_obs_start_v.xml': [
'dao_c122_2007_000882_v.fits',
'dao_c122_2007_000882.fits.gz',
],
# processed image
'visit_obs_start_a.xml': [
'dao_c182_2016_004034_a.fits',
'dao_c182_2016_004034.fits.gz',
],
'dao_r182_1989_000369.xml': ['dao_r182_1989_000369.fits.gz'],
'dao_r122_1989_003111.xml': ['dao_r122_1989_003111.fits.gz'],
'dao_c122_2017_011124.xml': ['dao_c122_2017_011124.fits.gz'],
'dao_c182_2017_010870.xml': ['dao_c182_2017_010870.fits.gz'],
'dao_c182_2017_019322.xml': ['dao_c182_2017_019322.fits.gz'],
'dao_c182_2017_016292.xml': ['dao_c182_2017_016292.fits.gz'],
'visit_obs_start_e.xml': [
'dao_c122_2007_000881.fits.gz',
'dao_c122_2007_000881_e.fits',
],
'sky_cam_start.xml': ['a2020_06_17_07_00_01.fits']
}
test_checksums = {
'cadc:DAO/dao_c122_2007_000881_e_256.png':
'md5:e68a3c389bbdf53cde0cca073c7a37c1',
'cadc:DAO/dao_c122_2007_000881_e_1024.png':
'md5:48065b2feb0fda3d7561c02b2890efaa',
'cadc:DAO/dao_c122_2007_000882_256.png':
'md5:1268e5b79e463ca75580d7e538d88a2a',
'cadc:DAO/dao_c122_2007_000882_1024.png':
'md5:5694ee03486899d3b14b404caab22505',
'cadc:DAO/dao_c122_2017_011124_256.png':
'md5:9793efcdeec043be8f0c77c7bf875cca',
'cadc:DAO/dao_c122_2017_011124_1024.png':
'md5:4d77005756fbed7fc3ae525591cc07db',
'cadc:DAO/dao_c182_2016_004034_256.png':
'md5:5c6b6816f8c0a54a0ff041b29247bdd5',
'cadc:DAO/dao_c182_2016_004034_1024.png':
'md5:455bddac53488acca46758fbc1d2b096',
'cadc:DAO/dao_c182_2017_010870_256.png':
'md5:106a6e0837fd00e971a37462266c2f66',
'cadc:DAO/dao_c182_2017_010870_1024.png':
'md5:bc92d495aa0e56eb4ea35f409932391e',
'cadc:DAO/dao_c182_2017_016292_256.png':
'md5:ef6bcc6501db0f64dfc7741f5ab37870',
'cadc:DAO/dao_c182_2017_016292_1024.png':
'md5:6c425b3606dd598be33b61dfef51d132',
'cadc:DAO/dao_c182_2017_019322_256.png':
'md5:dfca85e675f29a33ecdbabc91ed274bf',
'cadc:DAO/dao_c182_2017_019322_1024.png':
'md5:2818ddf6acb0f36655fbdfc9e19098e7',
'cadc:DAO/dao_r122_1989_003111_256.png':
'md5:7cb2344a84d195228d7fb866a83d031a',
'cadc:DAO/dao_r122_1989_003111_1024.png':
'md5:a864bc14955bc1f8524e8829c3ee2ee9',
'cadc:DAO/dao_r182_1989_000369_256.png':
'md5:5ef8f53784376b31a12dccb8671aca5e',
'cadc:DAO/dao_r182_1989_000369_1024.png':
'md5:2e14aefda2a74b5575f6a4d8d1b1b3cb',
'cadc:DAO/dao_c122_2007_000882_v_1024.png':
'md5:37cfb96bce27da93ede75eadabad7b8f',
'cadc:DAO/a2020_06_17_07_00_01_1024.png':
'md5:37cfb96bce27da93ede75eadabad7b8f',
}
kwargs = {
'working_directory': TEST_FILES_DIR,
'cadc_client': None,
'stream': 'stream',
'observable': test_observable,
}
for entry in glob.glob(f'{TEST_FILES_DIR}/*.png'):
os.unlink(entry)
for key, value in test_files.items():
obs = mc.read_obs_from_file(
f'{test_fits2caom2_augmentation.TEST_DATA_DIR}/previews/{key}')
for f_name in value:
test_name = dao_name.DAOName(f_name)
kwargs['storage_name'] = test_name
try:
ignore = preview_augmentation.visit(obs, **kwargs)
f_name_list = [test_name.prev_uri, test_name.thumb_uri]
for p in f_name_list:
artifact = obs.planes[test_name.product_id].artifacts[p]
# assert artifact.content_checksum.uri ==
# test_checksums[p], \
# f'wrong checksum {p} {artifact.content_checksum} ' \
# f'{test_checksums[p]}'
except Exception as e:
assert False, f'key {key} value {value} f_name {f_name} {str(e)}'
def update(observation, **kwargs):
"""Called to fill multiple CAOM model elements and/or attributes (an n:n
relationship between TDM attributes and CAOM attributes). Must have this
signature for import_module loading and execution.
:param observation A CAOM Observation model instance.
:param **kwargs Everything else."""
logging.debug('Begin update.')
mc.check_param(observation, Observation)
headers = kwargs.get('headers')
fqn = kwargs.get('fqn')
uri = kwargs.get('uri')
dao_name = None
if uri is not None:
dao_name = dn.DAOName(artifact_uri=uri)
if fqn is not None:
dao_name = dn.DAOName(file_name=os.path.basename(fqn))
if dao_name is None:
raise mc.CadcException(f'Need one of fqn or uri defined for '
f'{observation.observation_id}')
# correct the *_axis values
for plane in observation.planes.values():
for artifact in plane.artifacts.values():
if (artifact.uri.replace('.gz', '') != dao_name.file_uri.replace(
'.gz', '')):
continue
for part in artifact.parts.values():
for chunk in part.chunks:
time_delta = get_time_axis_delta(headers[0])
cc.undo_astropy_cdfix_call(chunk, time_delta)
if dao_name.file_name.startswith('d'):
if plane.data_product_type == DataProductType.SPECTRUM:
if (dn.DAOName.is_unprocessed_reticon(artifact.uri)
or dn.DAOName.is_derived(artifact.uri)
and observation.type == 'flat'):
cc.reset_energy(chunk)
if not artifact.product_type == ProductType.SCIENCE:
if observation.type == 'dark':
chunk.position_axis_1 = 3
chunk.position_axis_2 = 4
else:
cc.reset_position(chunk)
# no energy for calibration?
if (observation.type
not in ['flat', 'comparison', 'dark']):
cc.reset_energy(chunk)
# DB 04-03-21
# If WAVELENG isn’t present (for unprocessed
# spectra) then all energy metadata should be
# ignored
if (not dn.DAOName.is_processed(artifact.uri)
and headers[0].get('WAVELENG') is None):
cc.reset_energy(chunk)
else: # DataProductType.IMAGE
if dn.DAOName.override_provenance(artifact.uri):
plane.provenance.producer = 'Spaceguard_C'
# no observable axis when image
cc.reset_observable(chunk)
if artifact.product_type == ProductType.CALIBRATION:
if observation.type != 'dark':
cc.reset_position(chunk)
if observation.type not in ['flat', 'dark']:
cc.reset_energy(chunk)
# WCS axis wisdom from Pat:
#
# In general, assigning axis indices above the value of
# naxis is allowed but more or less pointless. The
# only use case that would justify it is that in a FITS
# file there could be a header with NAXIS=2 and
# WCSAXES=4 which would tell the fits reader to look
# for CTYPE1 through 4 and axes 3 and 4 are metadata.
# Assign those values to Chunk only if you care about
# capturing that the extra wcs metadata was really in
# the fits header and so the order could be preserved;
# in general do not assign the 3 and 4.
chunk.energy_axis = None
chunk.observable_axis = None
chunk.time_axis = None
naxis = headers[0].get('NAXIS')
naxis1 = headers[0].get('NAXIS1')
naxis2 = headers[0].get('NAXIS2')
chunk.naxis = None
chunk.position_axis_1 = None
chunk.position_axis_2 = None
if naxis is not None:
if (naxis1 is not None and naxis2 is not None
and naxis == 2
and chunk.position is not None
and plane.data_product_type is
DataProductType.IMAGE):
chunk.naxis = 2
chunk.position_axis_1 = 1
chunk.position_axis_2 = 2
if (naxis1 is not None and naxis == 1
and chunk.energy is not None):
chunk.naxis = 1
chunk.energy_axis = 1
else:
chunk.energy_axis = None
if plane.product_id != dao_name.product_id:
continue
# provenance: inputs vs members
#
# DB - 29-04-20
# The inconsistencies are consistent for both telescope for the
# derived observations: the processed, co-added flats (files with F
# suffix) and co-added biases (files with B suffix). These should
# have the ‘members’ list added to the inputs. From the definition of
# provenance:inputs I’m assuming for the science observations,
# processed comparison arcs, and processed flats having a composite
# flat and/or bias observation as an ‘input’ is okay rather than
# breaking these down into their individual members (since those
# derived observations will all be available in the archive with
# proper provenance provided).
if observation.type == 'flat' and cc.is_composite(headers, 'FLAT_'):
cc.update_plane_provenance(plane, headers, 'FLAT_', dn.COLLECTION,
_repair_provenance_value,
observation.observation_id)
elif observation.type == 'bias' and cc.is_composite(headers, 'ZERO_'):
cc.update_plane_provenance(plane, headers, 'ZERO_', dn.COLLECTION,
_repair_provenance_value,
observation.observation_id)
if dn.DAOName.is_processed(dao_name.file_uri):
_update_plane_provenance(observation, plane, headers)
if (cc.is_composite(headers, 'FLAT_')
or cc.is_composite(headers, 'ZERO_')):
_update_observation_members(observation)
logging.debug('Done update.')
return observation
def visit(observation, **kwargs):
previewer = DAOPreview(mime_type='image/png', **kwargs)
dao_name = dn.DAOName(file_name=previewer.science_file)
return previewer.visit(observation, dao_name)