def get_minimum_header(context, dataset, ignore_cache=False):
"""Given a `dataset` and a `context`, extract relevant header
information from the `dataset`.
"""
dump_mappings(context, ignore_cache=ignore_cache)
ctx = crds.get_pickled_mapping(context) # reviewed
return ctx.get_minimum_header(dataset)
def get_update_map(old_pipeline, updated_rmaps):
"""Given the name of a pipeline context, `old_pipeline`, and a list
of new rmap names, `updated_rmaps`, return the mapping:
{ imap_name : [ updates_for_that_imap, ... ], ... }
Updated rmaps can be rmap names or strings of the form:
<instrument>_<filekind>_"n/a"
e.g. miri_dflat_n/a
"""
pctx = crds.get_pickled_mapping(old_pipeline) # reviewed
updates = {}
for update in sorted(updated_rmaps):
if update.endswith(("_n/a","_N/A")):
instrument, _filekind, na = update.split("_")
else: # should be an rmap name
instrument, _filekind = utils.get_file_properties(pctx.observatory, update)
imap_name = pctx.get_imap(instrument).filename
if imap_name not in updates:
updates[imap_name] = []
assert update not in updates[imap_name], "Duplicate update for " + repr(update)
updates[imap_name].append(update)
return updates
def get_update_map(old_pipeline, updated_rmaps):
"""Given the name of a pipeline context, `old_pipeline`, and a list
of new rmap names, `updated_rmaps`, return the mapping:
{ imap_name : [ updates_for_that_imap, ... ], ... }
Updated rmaps can be rmap names or strings of the form:
<instrument>_<filekind>_"n/a"
e.g. miri_dflat_n/a
"""
pctx = crds.get_pickled_mapping(old_pipeline) # reviewed
updates = {}
for update in sorted(updated_rmaps):
if update.endswith(("_n/a", "_N/A")):
instrument, _filekind, na = update.split("_")
else: # should be an rmap name
instrument, _filekind = utils.get_file_properties(
pctx.observatory, update)
imap_name = pctx.get_imap(instrument).filename
if imap_name not in updates:
updates[imap_name] = []
assert update not in updates[
imap_name], "Duplicate update for " + repr(update)
updates[imap_name].append(update)
return updates
def get_affected(old_pmap, new_pmap, *args, **keys):
"""Examine the diffs between `old_pmap` and `new_pmap` and return sorted lists of affected instruments and types.
Returns { affected_instrument : { affected_type, ... } }
"""
observatory = keys.pop("observatory", crds.get_pickled_mapping(old_pmap).observatory) # reviewed
differ = MappingDifferencer(observatory, old_pmap, new_pmap, *args, **keys)
return differ.get_affected()
def get_affected(old_pmap, new_pmap, *args, **keys):
"""Examine the diffs between `old_pmap` and `new_pmap` and return sorted lists of affected instruments and types.
Returns { affected_instrument : { affected_type, ... } }
"""
observatory = keys.pop("observatory", crds.get_pickled_mapping(old_pmap).observatory) # reviewed
differ = MappingDifferencer(observatory, old_pmap, new_pmap, *args, **keys)
return differ.get_affected()
def observatory_from_context(self):
"""Determine the observatory from `pipeline_context`, based on name if possible."""
if "jwst" in self.pipeline_context:
observatory = "jwst"
elif "hst" in self.pipeline_context:
observatory = "hst"
else:
observatory = crds.get_pickled_mapping(self.pipeline_context).observatory # reviewed
return observatory
def _get_synphot_filename(context, reftype, compname):
pmap = crds.get_pickled_mapping(context, ignore_checksum=True)
rmap = pmap.get_imap(utils.SYNPHOT_INSTRUMENT).get_rmap(reftype)
for key, filename in rmap.selector.todict()["selections"]:
if key[0].lower() == compname:
return filename
return None
def observatory_from_context(self):
"""Determine the observatory from `pipeline_context`, based on name if possible."""
import crds
for observatory in constants.ALL_OBSERVATORIES:
if observatory in self.pipeline_context:
return observatory
else:
observatory = crds.get_pickled_mapping(
self.pipeline_context).observatory # reviewed
return observatory
def refpath_to_parkeys(refpath):
"""Given a key for a TpnInfo's list, return the associated required parkeys."""
from . import locate
keys = []
with log.verbose_warning_on_exception("Can't determine parkeys for",
repr(refpath)):
context = heavy_client.get_context_name("roman")
p = crds.get_pickled_mapping(context) # reviewed
instrument, filekind = locate.get_file_properties(refpath)
keys = p.get_imap(instrument).get_rmap(filekind).get_required_parkeys()
return sorted(keys)
def print_new_files(self):
"""Print the references or mappings which are in the second (new) context and not
the firtst (old) context.
"""
if not config.is_mapping(self.old_file) or not config.is_mapping(self.new_file):
log.error("--print-new-files really only works for mapping differences.")
return -1
old = crds.get_pickled_mapping(self.old_file) # reviewed
new = crds.get_pickled_mapping(self.new_file) # reviewed
old_mappings = set(old.mapping_names())
new_mappings = set(new.mapping_names())
old_references = set(old.reference_names())
new_references = set(new.reference_names())
status = 0
for name in sorted(new_mappings - old_mappings):
print(name)
status = 1
for name in sorted(new_references - old_references):
print(name)
status = 1
return status
def print_new_files(self):
"""Print the references or mappings which are in the second (new) context and not
the firtst (old) context.
"""
if not rmap.is_mapping(self.old_file) or not rmap.is_mapping(self.new_file):
log.error("--print-new-files really only works for mapping differences.")
return -1
old = crds.get_pickled_mapping(self.old_file) # reviewed
new = crds.get_pickled_mapping(self.new_file) # reviewed
old_mappings = set(old.mapping_names())
new_mappings = set(new.mapping_names())
old_references = set(old.reference_names())
new_references = set(new.reference_names())
status = 0
for name in sorted(new_mappings - old_mappings):
print(name)
status = 1
for name in sorted(new_references - old_references):
print(name)
status = 1
return status
def tpninfos_key_to_parkeys(tpn):
"""Given a key for a TpnInfo's list, return the associated required parkeys."""
if "all_" or "_all." in tpn:
return []
with log.verbose_warning_on_exception("Can't determine parkeys for", repr(tpn)):
_mode, context = heavy_client.get_processing_mode("jwst")
p = crds.get_pickled_mapping(context) # reviewed
instrument, suffix = tpn.split(".")[0].split("_")[:2]
filekind = p.locate.suffix_to_filekind(instrument, suffix)
keys = p.get_imap(instrument).get_rmap(filekind).get_required_parkeys()
keys.append("META.INSTRUMENT.NAME")
return sorted(keys)
def tpninfos_key_to_parkeys(tpn):
"""Given a key for a TpnInfo's list, return the associated required parkeys."""
if "all_" or "_all." in tpn:
return []
with log.verbose_warning_on_exception("Can't determine parkeys for",
repr(tpn)):
_mode, context = heavy_client.get_processing_mode("jwst")
p = crds.get_pickled_mapping(context) # reviewed
instrument, suffix = tpn.split(".")[0].split("_")[:2]
filekind = p.locate.suffix_to_filekind(instrument, suffix)
keys = p.get_imap(instrument).get_rmap(filekind).get_required_parkeys()
keys.append("META.INSTRUMENT.NAME")
return sorted(keys)
def get_update_map(old_pipeline, updated_rmaps):
"""Given the name of a pipeline context, `old_pipeline`, and a list
of new rmap names, `updated_rmaps`, return the mapping:
{ imap_name : [ updates_for_that_imap, ... ], ... }
"""
pctx = crds.get_pickled_mapping(old_pipeline) # reviewed
updates = {}
for update in sorted(updated_rmaps):
instrument, _filekind = utils.get_file_properties(pctx.observatory, update)
imap_name = pctx.get_imap(instrument).filename
if imap_name not in updates:
updates[imap_name] = []
assert update not in updates[imap_name], "Duplicate update for " + repr(update)
updates[imap_name].append(update)
return updates
def refpath_to_parkeys(refpath):
"""Given a key for a TpnInfo's list, return the associated required parkeys."""
from . import locate
keys = []
with log.verbose_warning_on_exception("Can't determine parkeys for", repr(refpath)):
context = heavy_client.get_context_name("jwst")
p = crds.get_pickled_mapping(context) # reviewed
instrument, filekind = locate.get_file_properties(refpath)
keys = p.get_imap(instrument).get_rmap(filekind).get_required_parkeys()
# The below turned out to be bad because CRDS-only values aren't in datamodels
# so in this case the datamodels schema are too narrow for these and will reject
# e.g. SYSTEM or it's types.
# keys.append("META.INSTRUMENT.NAME")
# keys.append("META.REFTYPE")
return sorted(keys)
def refpath_to_parkeys(refpath):
"""Given a key for a TpnInfo's list, return the associated required parkeys."""
from . import locate
keys = []
with log.verbose_warning_on_exception("Can't determine parkeys for", repr(refpath)):
context = heavy_client.get_context_name("jwst")
p = crds.get_pickled_mapping(context) # reviewed
instrument, filekind = locate.get_file_properties(refpath)
keys = p.get_imap(instrument).get_rmap(filekind).get_required_parkeys()
# The below turned out to be bad because CRDS-only values aren't in datamodels
# so in this case the datamodels schema are too narrow for these and will reject
# e.g. SYSTEM or it's types.
# keys.append("META.INSTRUMENT.NAME")
# keys.append("META.REFTYPE")
return sorted(keys)
def get_old_bestrefs(self, source):
"""Return the historical best references corresponding to `source`. Always define old bestrefs
in terms of filekind/typename rather than in terms of FITS keyword.
"""
header = add_instrument(self.header(source))
instrument = utils.header_to_instrument(header)
pmap = crds.get_pickled_mapping(self.context)
result = {}
for filekind in pmap.get_imap(instrument).selections:
keyword = pmap.locate.filekind_to_keyword(filekind)
filekind = filekind.upper()
try:
result[filekind] = header[keyword]
except KeyError:
result[filekind] = header.get(filekind, "UNDEFINED")
return result
def insert_references(self):
"""Insert files specified by --references into the appropriate rmaps identified by --source-context."""
self._setup_source_context()
categorized = self.categorize_files(self.args.references)
pmap = crds.get_pickled_mapping(self.source_context) # reviewed
self.args.rmaps = []
for (instrument, filekind) in categorized:
try:
self.args.rmaps.append(pmap.get_imap(instrument).get_rmap(filekind).filename)
except crexc.CrdsError:
log.info("Existing rmap for", (instrument, filekind), "not found. Trying empty spec.")
spec_file = os.path.join(
os.path.dirname(self.obs_pkg.__file__), "specs", instrument + "_" + filekind + ".rmap")
rmapping = rmap.asmapping(spec_file)
log.info("Loaded spec file from", repr(spec_file))
self.args.rmaps.append(spec_file)
self.rmap_apply(insert_rmap_references, categorized=categorized)
def insert_references(self):
"""Insert files specified by --references into the appropriate rmaps identified by --source-context."""
self._setup_source_context()
categorized = self.categorize_files(self.args.references)
pmap = crds.get_pickled_mapping(self.source_context) # reviewed
self.args.rmaps = []
for (instrument, filekind) in categorized:
try:
self.args.rmaps.append(pmap.get_imap(instrument).get_rmap(filekind).filename)
except crexc.CrdsError:
log.info("Existing rmap for", (instrument, filekind), "not found. Trying empty spec.")
spec_file = os.path.join(
os.path.dirname(self.obs_pkg.__file__), "specs", instrument + "_" + filekind + ".rmap")
rmapping = rmap.asmapping(spec_file)
log.info("Loaded spec file from", repr(spec_file))
self.args.rmaps.append(spec_file)
self.rmap_apply(insert_rmap_references, categorized=categorized)
def find_full_match_paths(context, reffile):
"""Return the list of full match paths for `reference` in `context` as a
list of tuples of tuples. Each inner tuple is a (var, value) pair.
Returns [((context_tuples,),(match_tuple,),(useafter_tuple,)), ...]
>>> pp(find_full_match_paths("hst.pmap", "q9e1206kj_bia.fits"))
[((('observatory', 'hst'), ('instrument', 'acs'), ('filekind', 'biasfile')),
(('DETECTOR', 'HRC'),
('CCDAMP', 'A'),
('CCDGAIN', '4.0'),
('APERTURE', '*'),
('NUMCOLS', '1062'),
('NUMROWS', '1044'),
('LTV1', '19.0'),
('LTV2', '20.0'),
('XCORNER', 'N/A'),
('YCORNER', 'N/A'),
('CCDCHIP', 'N/A')),
(('DATE-OBS', '2006-07-04'), ('TIME-OBS', '11:32:35'))),
((('observatory', 'hst'), ('instrument', 'acs'), ('filekind', 'biasfile')),
(('DETECTOR', 'HRC'),
('CCDAMP', 'A'),
('CCDGAIN', '4.0'),
('APERTURE', '*'),
('NUMCOLS', 'N/A'),
('NUMROWS', 'N/A'),
('LTV1', 'N/A'),
('LTV2', 'N/A'),
('XCORNER', 'N/A'),
('YCORNER', 'N/A'),
('CCDCHIP', 'N/A')),
(('DATE-OBS', '2006-07-04'), ('TIME-OBS', '11:32:35')))]
"""
ctx = crds.get_pickled_mapping(context, cached=True) # reviewed
return ctx.file_matches(reffile)
def find_full_match_paths(context, reffile):
"""Return the list of full match paths for `reference` in `context` as a
list of tuples of tuples. Each inner tuple is a (var, value) pair.
Returns [((context_tuples,),(match_tuple,),(useafter_tuple,)), ...]
>>> pp(find_full_match_paths("hst.pmap", "q9e1206kj_bia.fits"))
[((('observatory', 'hst'), ('instrument', 'acs'), ('filekind', 'biasfile')),
(('DETECTOR', 'HRC'),
('CCDAMP', 'A'),
('CCDGAIN', '4.0'),
('APERTURE', '*'),
('NUMCOLS', '1062'),
('NUMROWS', '1044'),
('LTV1', '19.0'),
('LTV2', '20.0'),
('XCORNER', 'N/A'),
('YCORNER', 'N/A'),
('CCDCHIP', 'N/A')),
(('DATE-OBS', '2006-07-04'), ('TIME-OBS', '11:32:35'))),
((('observatory', 'hst'), ('instrument', 'acs'), ('filekind', 'biasfile')),
(('DETECTOR', 'HRC'),
('CCDAMP', 'A'),
('CCDGAIN', '4.0'),
('APERTURE', '*'),
('NUMCOLS', 'N/A'),
('NUMROWS', 'N/A'),
('LTV1', 'N/A'),
('LTV2', 'N/A'),
('XCORNER', 'N/A'),
('YCORNER', 'N/A'),
('CCDCHIP', 'N/A')),
(('DATE-OBS', '2006-07-04'), ('TIME-OBS', '11:32:35')))]
"""
ctx = crds.get_pickled_mapping(context, cached=True) # reviewed
return ctx.file_matches(reffile)
