def _get_reftypes(exp_type, cal_ver=None, context=None):
"""Given `exp_type` and `cal_ver` and `context`, locate the appropriate SYSTEM CRDSCFG
reference file and determine the reference types required to process every pipeline Step
nominally associated with that exp_type.
"""
with log.warn_on_exception("Failed determining required reftypes from",
"EXP_TYPE", srepr(exp_type)):
config_manager = _get_config_manager(context, cal_ver)
return config_manager.exptype_to_reftypes(exp_type)
return []
def get_schema_tpninfos(refpath):
"""Load the list of TPN info tuples corresponding to `instrument` and
`filekind` from it's .tpn file.
"""
with log.warn_on_exception("Failed loading schema constraints for", repr(refpath)):
schema_name = reference_to_schema_name(refpath)
tpns = get_schema_tpns(schema_name)
parkeys = refpath_to_parkeys(refpath)
return [ info for info in tpns if info.name in parkeys ]
return []
def scan_exp_type_coverage():
"""Verify that there is some get_reftypes() response for all available exp_types."""
from . import schema as crds_schema
exp_types = crds_schema.get_exptypes()
for exp_type in exp_types:
if exp_type in ["ANY","N/A"]:
continue
with log.warn_on_exception("failed determining reftypes for", repr(exp_type)):
reftypes = get_reftypes(exp_type)
log.verbose("Reftypes for", repr(exp_type), "=", repr(reftypes))
def scan_exp_type_coverage():
"""Verify that there is some get_reftypes() response for all available exp_types."""
from . import schema as crds_schema
exp_types = crds_schema.get_exptypes()
for exp_type in exp_types:
if exp_type in ["ANY","N/A"]:
continue
with log.warn_on_exception("failed determining reftypes for", repr(exp_type)):
reftypes = _get_reftypes(exp_type)
log.verbose("Reftypes for", repr(exp_type), "=", repr(reftypes))
def header_to_reftypes(header, context=None):
"""Given a dataset `header`, extract the EXP_TYPE or META.EXPOSURE.TYPE keyword
from and use it to look up the reftypes required to process it.
Return a list of reftype names.
"""
with log.warn_on_exception("Failed determining exp_type, cal_ver from header", log.PP(header)):
exp_type, cal_ver = _header_to_exptype_calver(header)
return get_reftypes(exp_type, cal_ver, context)
return []
def get_schema_tpninfos(refpath):
"""Load the list of TPN info tuples corresponding to `instrument` and
`filekind` from it's .tpn file.
"""
with log.warn_on_exception("Failed loading schema constraints for", repr(refpath)):
schema_name = reference_to_schema_name(refpath)
tpns = get_schema_tpns(schema_name)
parkeys = refpath_to_parkeys(refpath)
return [ info for info in tpns if info.name in parkeys ]
return []
def _cat_array_properties(self, path):
"""Print out the CRDS interpretation of every array in `path`, currently FITS only."""
with data_file.fits_open(path) as hdulist:
for i, hdu in enumerate(hdulist):
with log.warn_on_exception("Can't load array properties for HDU[" + str(i) +"]"):
if i > 0:
extname = hdu.header.get("EXTNAME",str(i))
self._cat_banner("CRDS Array Info [" + repr(extname) + "]:",
delim="-", bottom_delim=None)
props = data_file.get_array_properties(path, extname)
props = { prop:value for (prop,value) in props.items() if value is not None }
self._print_lines(path, _pp_lines(props))
def get_reftypes(exp_type, cal_ver=None, context=None):
"""Given `exp_type` and `cal_ver` and `context`, locate the appropriate SYSTEM CRDSCFG
reference file and determine the reference types required to process every pipeline Step
nominally associated with that exp_type.
"""
context = _get_missing_context(context)
cal_ver = _get_missing_calver(cal_ver)
with log.warn_on_exception("Failed determining required reftypes from",
"EXP_TYPE", srepr(exp_type), "CAL_VER", srepr(cal_ver)):
config_manager = _get_config_manager(context, cal_ver)
return config_manager.exptype_to_reftypes(exp_type)
return []
def header_to_reftypes(header, context=None):
"""Given a dataset `header`, extract the EXP_TYPE or META.EXPOSURE.TYPE keyword
from and use it to look up the reftypes required to process it.
Return a list of reftype names.
"""
with log.warn_on_exception("Failed determining reftypes for", log.PP(header)):
exp_type, cal_ver = _header_to_exptype_calver(header)
config_manager = _get_config_manager(context, cal_ver)
pipelines = header_to_pipelines(header, context)
reftypes = set()
for cfg in pipelines:
steps = config_manager.pipeline_cfgs_to_steps[cfg]
for step in steps:
reftypes |= set(config_manager.steps_to_reftypes[step])
return sorted(list(reftypes))
return []
def load_specs(spec_path):
"""Load either the combined .json formatted type specs (preferred) or specs from
individual type spec files (if necessary).
"""
combined_specs_path = os.path.join(spec_path, "combined_specs.json")
if os.path.exists(combined_specs_path):
headers = load_json_specs(combined_specs_path)
else:
headers = load_raw_specs(spec_path)
with log.warn_on_exception("Failed to save type specs .json at", repr(combined_specs_path)):
save_json_specs(headers, combined_specs_path)
type_specs = { instr :
{ filekind : TypeSpec(headers[instr][filekind])
for filekind in headers[instr]
}
for instr in headers
}
return type_specs
def load_specs(spec_path):
"""Load either the combined .json formatted type specs (preferred) or specs from
individual type spec files (if necessary).
"""
combined_specs_path = os.path.join(spec_path, "combined_specs.json")
if os.path.exists(combined_specs_path):
headers = load_json_specs(combined_specs_path)
else:
headers = load_raw_specs(spec_path)
with log.warn_on_exception("Failed to save type specs .json at", repr(combined_specs_path)):
save_json_specs(headers, combined_specs_path)
type_specs = { instr :
{ filekind : TypeSpec(headers[instr][filekind])
for filekind in headers[instr]
}
for instr in headers
}
return type_specs
def _cat_array_properties(self, path):
"""Print out the CRDS interpretation of every array in `path`, currently FITS only."""
with data_file.fits_open(path) as hdulist:
for i, hdu in enumerate(hdulist):
with log.warn_on_exception(
"Can't load array properties for HDU[" + str(i) + "]"):
if i > 0:
extname = hdu.header.get("EXTNAME", str(i))
self._cat_banner("CRDS Array Info [" + repr(extname) +
"]:",
delim="-",
bottom_delim=None)
props = data_file.get_array_properties(path, extname)
props = {
prop: value
for (prop, value) in props.items()
if value is not None
}
self._print_lines(path, _pp_lines(props))
def fetch_sqlite_db(self):
"""Download a SQLite version of the CRDS catalog from the server."""
log.info("Downloading CRDS catalog database file.")
with log.warn_on_exception("Failed updating local CRDS database"):
db_path = api.get_sqlite_db(self.observatory)
log.info("Sqlite3 database file downloaded to:", db_path)
def rmap_insert_references(old_rmap, new_rmap, inserted_references):
"""Given the full path of starting rmap `old_rmap`, modify it by inserting
or replacing all files in `inserted_references` and write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
old_rmap str Filepath of source rmap into which `inserted_references` will be inserted
new_rmap str Filepath of updated rmap written out
inserted_references [ str, ...] List of reference filepaths to be insterted
Note that "inserting" a reference file can result in:
1. adding a new match case,
2. adding a new USEAFTER case
3. exactly replacing an existing reference file.
Other outcomes are also possible for non-standard rmap selector class configurations.
Additional checking:
1. Generates an ERROR if any of the inserted reference files have identical
matching criteria since only one file with those criteria would be added to
the rmap and the other(s) would be "replaced" by their own insertion set.
Note: it is valid/common for an inserted reference to replace a reference
which is already in `old_rmap`. This ERROR only applies to equalities
within the inserted_references list.
2. Generates a WARNING if the matching criteria of any inserted reference
file is a proper subset of inserted or existing references. Thes subsets
will generally lead to the addition of new matching cases. Since CRDS
inherited instances of these "subset overlaps" from HST CDBS, this warning
is only visible with --verbose for HST, they exist. Since this condition
is bad both for understanding rmaps and for runtime complexity and
performance, for JWST the warning is visible without --verbose and will
also generate a runtime ERROR. For JWST there is the expectation that an
offending file submission will either be (a) cancelled and corrected or (b)
provisionally accepted followed by an immediate manual rmap correction.
Provisional acceptance gives the option of f keeping the work
associated with large deliveries where the corrective measure might be to
manually merge overlapping categories with rmap edits.
Return None, `new_rmap` is already the implicit result
"""
new = old = rmap.fetch_mapping(old_rmap, ignore_checksum=True)
inserted_cases = {}
for reference in inserted_references:
log.info("Inserting", os.path.basename(reference), "into",
repr(new.name))
new = new.insert_reference(reference)
baseref = os.path.basename(reference)
with log.warn_on_exception("Failed checking rmap update for",
repr(baseref)):
cases = new.file_matches(baseref)
for fullcase in cases:
case = fullcase[1:]
if case not in inserted_cases:
inserted_cases[case] = baseref
else:
log.error(
"-" * 40 + "\nBoth", srepr(baseref), "and",
srepr(inserted_cases[case]),
"identically match case:\n", log.PP(case), """
Each reference would replace the other in the rmap.
Either reference file matching parameters need correction
or additional matching parameters should be added to the rmap
to enable CRDS to differentiate between the two files.
See the file submission section of the CRDS server user's guide here:
https://jwst-crds.stsci.edu/static/users_guide/index.html
for more explanation.""")
