def _update_cal_provenance(self, observation):
plane_inputs = TypedSet(
PlaneURI,
)
members_inputs = TypedSet(
ObservationURI,
)
for keyword in self._headers[0]:
if keyword.startswith('F_ON') or keyword.startswith('F_OFF'):
value = self._headers[0].get(keyword)
base_name = (
f'C{OmmName.remove_extensions(os.path.basename(value))}'
)
file_id = f'{base_name}_CAL'
obs_member_uri_str = mc.CaomName.make_obs_uri_from_obs_id(
COLLECTION, base_name
)
obs_member_uri = ObservationURI(obs_member_uri_str)
plane_uri = PlaneURI.get_plane_uri(obs_member_uri, file_id)
plane_inputs.add(plane_uri)
members_inputs.add(obs_member_uri)
for plane in observation.planes.values():
if plane.provenance is None:
plane.provenance = Provenance('CPAPIR')
mc.update_typed_set(plane.provenance.inputs, plane_inputs)
mc.update_typed_set(observation.members, members_inputs)
def _update_science_provenance(observation, headers):
members_inputs = TypedSet(ObservationURI, )
plane_inputs = TypedSet(PlaneURI, )
for keyword in headers[0]:
if keyword.startswith('IN_'):
value = headers[0].get(keyword)
base_name = OmmName.remove_extensions(os.path.basename(value))
if base_name.startswith('S'):
# starting 'S' means a science input, 'C' will mean cal
file_id = '{}_SCI'.format(base_name.replace('S', 'C', 1))
elif base_name.startswith('C'):
file_id = '{}_CAL'.format(base_name)
else:
raise mc.CadcException(
'Unknown file naming pattern {}'.format(base_name))
obs_member_uri_str = ec.CaomName.make_obs_uri_from_obs_id(
COLLECTION, file_id)
obs_member_uri = ObservationURI(obs_member_uri_str)
# the product id is the same as the observation id for OMM
plane_uri = PlaneURI.get_plane_uri(obs_member_uri, file_id)
plane_inputs.add(plane_uri)
members_inputs.add(obs_member_uri)
mc.update_typed_set(observation.members, members_inputs)
mc.update_typed_set(
observation.planes[observation.observation_id].provenance.inputs,
plane_inputs)
def update_plane_provenance_from_values(plane, repair, values, collection,
obs_id):
plane_inputs = TypedSet(PlaneURI, )
for value in values:
prov_obs_id, prov_prod_id = repair(value, obs_id)
if prov_obs_id is not None and prov_prod_id is not None:
obs_member_uri_ignore, plane_uri = make_plane_uri(
prov_obs_id, prov_prod_id, collection)
plane_inputs.add(plane_uri)
logging.debug(f'Adding PlaneURI {plane_uri}')
mc.update_typed_set(plane.provenance.inputs, plane_inputs)
def _update_science_provenance(self, observation):
members_inputs = TypedSet(
ObservationURI,
)
plane_inputs = TypedSet(
PlaneURI,
)
# values look like:
# IN_00010= 'S/data/cpapir/data/101116/101116_0088.fits.fits.gz'
# or
# IN_00001= 'S050213_0278.fits.gz' /raw input file (1/5)
# or
# DD - slack - 11-02-20
# Add this new prefix. This will be a much easier fix than changing the
# pipeline and all the headers once more.
#
# ID_00001= 'S/data/cpapir/data/101116/101116_0041.fits.fits.gz'
for keyword in self._headers[0]:
if keyword.startswith('IN_') or keyword.startswith('ID_'):
temp = keyword.split('_')[1]
try:
int(temp)
except ValueError as e:
# skip the keyword ID_PROG
continue
value = self._headers[0].get(keyword)
base_name = OmmName.remove_extensions(os.path.basename(value))
if base_name.startswith('S'):
# starting 'S' means a science input, 'C' will mean cal
base_name = base_name.replace('S', 'C', 1)
file_id = f'{base_name}_SCI'
elif value.startswith('S'):
base_name = f'C{base_name}'
file_id = f'{base_name}_SCI'
elif base_name.startswith('C') or value.startswith('C'):
file_id = f'{base_name}_CAL'
else:
raise mc.CadcException(
f'Unknown file naming pattern {base_name}'
)
obs_member_uri_str = mc.CaomName.make_obs_uri_from_obs_id(
COLLECTION, base_name
)
obs_member_uri = ObservationURI(obs_member_uri_str)
plane_uri = PlaneURI.get_plane_uri(obs_member_uri, file_id)
plane_inputs.add(plane_uri)
members_inputs.add(obs_member_uri)
mc.update_typed_set(observation.members, members_inputs)
for plane in observation.planes.values():
mc.update_typed_set(plane.provenance.inputs, plane_inputs)
def update_observation_members(observation):
"""Add members to Observation from all its Planes.
:param observation Observation instance to add members to
"""
members_inputs = TypedSet(ObservationURI, )
for plane in observation.planes.values():
if (plane.provenance is not None
and plane.provenance.inputs is not None):
for inpt in plane.provenance.inputs:
members_inputs.add(inpt.get_observation_uri())
logging.debug(
f'Adding Observation URI {inpt.get_observation_uri()}')
mc.update_typed_set(observation.members, members_inputs)
def update_observation_members_filtered(observation, filter_fn):
"""Add members to Observation from all its Planes, depending
on the return of the filter_fn.
:param observation Observation instance to add members to
:param filter_fn returns True if a plane input should also be an
Observation member
"""
inputs = []
members_inputs = TypedSet(ObservationURI, )
for plane in observation.planes.values():
if (plane.provenance is not None
and plane.provenance.inputs is not None):
inputs = filter(filter_fn, plane.provenance.inputs)
for entry in inputs:
members_inputs.add(entry.get_observation_uri())
logging.debug(f'Adding Observation URI {entry.get_observation_uri()}')
mc.update_typed_set(observation.members, members_inputs)
def _update_observation_members(self, observation):
"""
Must filter results because:
DB - 11-06-20
For the spectra there is a minor issue with members for master flat,
*_F, observations. The master bias used in the processing, the *_B.fits
file, shouldn’t be a member for the master flats.
The master bias is in the list of inputs though: Inputs for master flat
are the unprocessed flats and the master bias. The master bias is
subtracted pixel-by-pixel from each unprocessed flat as part of the
processing before the flats are then co-added.
The composite/derived master flats (F) and master biases (B) should
never be members. At least for any processing that is currently being
done. For now the only members should those given by the
NCOMBINE x ZERO_# or FLAT_# keyword values.
"""
def filter_fun(x):
result = True
if DAOName.is_master_flat(observation.observation_id):
if DAOName.is_master_bias(x.get_observation_uri().uri):
result = False
return result
inputs = []
members_inputs = TypedSet(ObservationURI,)
for plane in observation.planes.values():
if (
plane.provenance is not None
and plane.provenance.inputs is not None
):
inputs = filter(filter_fun, plane.provenance.inputs)
for entry in inputs:
members_inputs.add(entry.get_observation_uri())
logging.debug(f'Adding Observation URI {entry.get_observation_uri()}')
mc.update_typed_set(observation.members, members_inputs)
def _update_cal_provenance(observation, headers):
plane_inputs = TypedSet(PlaneURI, )
members_inputs = TypedSet(ObservationURI, )
for keyword in headers[0]:
if keyword.startswith('F_ON') or keyword.startswith('F_OFF'):
value = headers[0].get(keyword)
base_name = OmmName.remove_extensions(os.path.basename(value))
file_id = 'C{}_CAL'.format(base_name)
obs_member_uri_str = ec.CaomName.make_obs_uri_from_obs_id(
COLLECTION, file_id)
obs_member_uri = ObservationURI(obs_member_uri_str)
# the product id is the same as the observation id for OMM
plane_uri = PlaneURI.get_plane_uri(obs_member_uri, file_id)
plane_inputs.add(plane_uri)
members_inputs.add(obs_member_uri)
for key in observation.planes:
plane = observation.planes[key]
if plane.provenance is None:
plane.provenance = Provenance('CPAPIR')
mc.update_typed_set(plane.provenance.inputs, plane_inputs)
mc.update_typed_set(observation.members, members_inputs)
def update_plane_provenance_from_values(plane, repair, values, collection,
obs_id):
"""
Add inputs to Planes, based on a list of input values.
:param plane: Plane instance to add inputs to
:param repair: The function to fix the input values, to ensure they
match input observationID values
:param values: list of values to add as inputs, after repair
:param collection: str The collection name for URI construction
:param obs_id: str value for logging only
:return:
"""
logging.debug(f'Begin update_plane_provenance_from_values')
plane_inputs = TypedSet(PlaneURI, )
for value in values:
prov_obs_id, prov_prod_id = repair(value, obs_id)
if prov_obs_id is not None and prov_prod_id is not None:
obs_member_uri_ignore, plane_uri = make_plane_uri(
prov_obs_id, prov_prod_id, collection)
plane_inputs.add(plane_uri)
logging.debug(f'Adding PlaneURI {plane_uri}')
mc.update_typed_set(plane.provenance.inputs, plane_inputs)
logging.debug(f'End update_plane_provenance_from_values')