def get_minimum_header(context, dataset, ignore_cache=False): """Given a `dataset` and a `context`, extract relevant header information from the `dataset`. """ dump_mappings(context, ignore_cache=ignore_cache) ctx = crds.get_pickled_mapping(context) # reviewed return ctx.get_minimum_header(dataset)
def get_update_map(old_pipeline, updated_rmaps): """Given the name of a pipeline context, `old_pipeline`, and a list of new rmap names, `updated_rmaps`, return the mapping: { imap_name : [ updates_for_that_imap, ... ], ... } Updated rmaps can be rmap names or strings of the form: <instrument>_<filekind>_"n/a" e.g. miri_dflat_n/a """ pctx = crds.get_pickled_mapping(old_pipeline) # reviewed updates = {} for update in sorted(updated_rmaps): if update.endswith(("_n/a","_N/A")): instrument, _filekind, na = update.split("_") else: # should be an rmap name instrument, _filekind = utils.get_file_properties(pctx.observatory, update) imap_name = pctx.get_imap(instrument).filename if imap_name not in updates: updates[imap_name] = [] assert update not in updates[imap_name], "Duplicate update for " + repr(update) updates[imap_name].append(update) return updates
def get_update_map(old_pipeline, updated_rmaps): """Given the name of a pipeline context, `old_pipeline`, and a list of new rmap names, `updated_rmaps`, return the mapping: { imap_name : [ updates_for_that_imap, ... ], ... } Updated rmaps can be rmap names or strings of the form: <instrument>_<filekind>_"n/a" e.g. miri_dflat_n/a """ pctx = crds.get_pickled_mapping(old_pipeline) # reviewed updates = {} for update in sorted(updated_rmaps): if update.endswith(("_n/a", "_N/A")): instrument, _filekind, na = update.split("_") else: # should be an rmap name instrument, _filekind = utils.get_file_properties( pctx.observatory, update) imap_name = pctx.get_imap(instrument).filename if imap_name not in updates: updates[imap_name] = [] assert update not in updates[ imap_name], "Duplicate update for " + repr(update) updates[imap_name].append(update) return updates
def get_affected(old_pmap, new_pmap, *args, **keys): """Examine the diffs between `old_pmap` and `new_pmap` and return sorted lists of affected instruments and types. Returns { affected_instrument : { affected_type, ... } } """ observatory = keys.pop("observatory", crds.get_pickled_mapping(old_pmap).observatory) # reviewed differ = MappingDifferencer(observatory, old_pmap, new_pmap, *args, **keys) return differ.get_affected()
def observatory_from_context(self): """Determine the observatory from `pipeline_context`, based on name if possible.""" if "jwst" in self.pipeline_context: observatory = "jwst" elif "hst" in self.pipeline_context: observatory = "hst" else: observatory = crds.get_pickled_mapping(self.pipeline_context).observatory # reviewed return observatory
def _get_synphot_filename(context, reftype, compname): pmap = crds.get_pickled_mapping(context, ignore_checksum=True) rmap = pmap.get_imap(utils.SYNPHOT_INSTRUMENT).get_rmap(reftype) for key, filename in rmap.selector.todict()["selections"]: if key[0].lower() == compname: return filename return None
def observatory_from_context(self): """Determine the observatory from `pipeline_context`, based on name if possible.""" import crds for observatory in constants.ALL_OBSERVATORIES: if observatory in self.pipeline_context: return observatory else: observatory = crds.get_pickled_mapping( self.pipeline_context).observatory # reviewed return observatory
def refpath_to_parkeys(refpath): """Given a key for a TpnInfo's list, return the associated required parkeys.""" from . import locate keys = [] with log.verbose_warning_on_exception("Can't determine parkeys for", repr(refpath)): context = heavy_client.get_context_name("roman") p = crds.get_pickled_mapping(context) # reviewed instrument, filekind = locate.get_file_properties(refpath) keys = p.get_imap(instrument).get_rmap(filekind).get_required_parkeys() return sorted(keys)
def print_new_files(self): """Print the references or mappings which are in the second (new) context and not the firtst (old) context. """ if not config.is_mapping(self.old_file) or not config.is_mapping(self.new_file): log.error("--print-new-files really only works for mapping differences.") return -1 old = crds.get_pickled_mapping(self.old_file) # reviewed new = crds.get_pickled_mapping(self.new_file) # reviewed old_mappings = set(old.mapping_names()) new_mappings = set(new.mapping_names()) old_references = set(old.reference_names()) new_references = set(new.reference_names()) status = 0 for name in sorted(new_mappings - old_mappings): print(name) status = 1 for name in sorted(new_references - old_references): print(name) status = 1 return status
def print_new_files(self): """Print the references or mappings which are in the second (new) context and not the firtst (old) context. """ if not rmap.is_mapping(self.old_file) or not rmap.is_mapping(self.new_file): log.error("--print-new-files really only works for mapping differences.") return -1 old = crds.get_pickled_mapping(self.old_file) # reviewed new = crds.get_pickled_mapping(self.new_file) # reviewed old_mappings = set(old.mapping_names()) new_mappings = set(new.mapping_names()) old_references = set(old.reference_names()) new_references = set(new.reference_names()) status = 0 for name in sorted(new_mappings - old_mappings): print(name) status = 1 for name in sorted(new_references - old_references): print(name) status = 1 return status
def tpninfos_key_to_parkeys(tpn): """Given a key for a TpnInfo's list, return the associated required parkeys.""" if "all_" or "_all." in tpn: return [] with log.verbose_warning_on_exception("Can't determine parkeys for", repr(tpn)): _mode, context = heavy_client.get_processing_mode("jwst") p = crds.get_pickled_mapping(context) # reviewed instrument, suffix = tpn.split(".")[0].split("_")[:2] filekind = p.locate.suffix_to_filekind(instrument, suffix) keys = p.get_imap(instrument).get_rmap(filekind).get_required_parkeys() keys.append("META.INSTRUMENT.NAME") return sorted(keys)
def get_update_map(old_pipeline, updated_rmaps): """Given the name of a pipeline context, `old_pipeline`, and a list of new rmap names, `updated_rmaps`, return the mapping: { imap_name : [ updates_for_that_imap, ... ], ... } """ pctx = crds.get_pickled_mapping(old_pipeline) # reviewed updates = {} for update in sorted(updated_rmaps): instrument, _filekind = utils.get_file_properties(pctx.observatory, update) imap_name = pctx.get_imap(instrument).filename if imap_name not in updates: updates[imap_name] = [] assert update not in updates[imap_name], "Duplicate update for " + repr(update) updates[imap_name].append(update) return updates
def refpath_to_parkeys(refpath): """Given a key for a TpnInfo's list, return the associated required parkeys.""" from . import locate keys = [] with log.verbose_warning_on_exception("Can't determine parkeys for", repr(refpath)): context = heavy_client.get_context_name("jwst") p = crds.get_pickled_mapping(context) # reviewed instrument, filekind = locate.get_file_properties(refpath) keys = p.get_imap(instrument).get_rmap(filekind).get_required_parkeys() # The below turned out to be bad because CRDS-only values aren't in datamodels # so in this case the datamodels schema are too narrow for these and will reject # e.g. SYSTEM or it's types. # keys.append("META.INSTRUMENT.NAME") # keys.append("META.REFTYPE") return sorted(keys)
def get_old_bestrefs(self, source): """Return the historical best references corresponding to `source`. Always define old bestrefs in terms of filekind/typename rather than in terms of FITS keyword. """ header = add_instrument(self.header(source)) instrument = utils.header_to_instrument(header) pmap = crds.get_pickled_mapping(self.context) result = {} for filekind in pmap.get_imap(instrument).selections: keyword = pmap.locate.filekind_to_keyword(filekind) filekind = filekind.upper() try: result[filekind] = header[keyword] except KeyError: result[filekind] = header.get(filekind, "UNDEFINED") return result
def insert_references(self): """Insert files specified by --references into the appropriate rmaps identified by --source-context.""" self._setup_source_context() categorized = self.categorize_files(self.args.references) pmap = crds.get_pickled_mapping(self.source_context) # reviewed self.args.rmaps = [] for (instrument, filekind) in categorized: try: self.args.rmaps.append(pmap.get_imap(instrument).get_rmap(filekind).filename) except crexc.CrdsError: log.info("Existing rmap for", (instrument, filekind), "not found. Trying empty spec.") spec_file = os.path.join( os.path.dirname(self.obs_pkg.__file__), "specs", instrument + "_" + filekind + ".rmap") rmapping = rmap.asmapping(spec_file) log.info("Loaded spec file from", repr(spec_file)) self.args.rmaps.append(spec_file) self.rmap_apply(insert_rmap_references, categorized=categorized)
def find_full_match_paths(context, reffile): """Return the list of full match paths for `reference` in `context` as a list of tuples of tuples. Each inner tuple is a (var, value) pair. Returns [((context_tuples,),(match_tuple,),(useafter_tuple,)), ...] >>> pp(find_full_match_paths("hst.pmap", "q9e1206kj_bia.fits")) [((('observatory', 'hst'), ('instrument', 'acs'), ('filekind', 'biasfile')), (('DETECTOR', 'HRC'), ('CCDAMP', 'A'), ('CCDGAIN', '4.0'), ('APERTURE', '*'), ('NUMCOLS', '1062'), ('NUMROWS', '1044'), ('LTV1', '19.0'), ('LTV2', '20.0'), ('XCORNER', 'N/A'), ('YCORNER', 'N/A'), ('CCDCHIP', 'N/A')), (('DATE-OBS', '2006-07-04'), ('TIME-OBS', '11:32:35'))), ((('observatory', 'hst'), ('instrument', 'acs'), ('filekind', 'biasfile')), (('DETECTOR', 'HRC'), ('CCDAMP', 'A'), ('CCDGAIN', '4.0'), ('APERTURE', '*'), ('NUMCOLS', 'N/A'), ('NUMROWS', 'N/A'), ('LTV1', 'N/A'), ('LTV2', 'N/A'), ('XCORNER', 'N/A'), ('YCORNER', 'N/A'), ('CCDCHIP', 'N/A')), (('DATE-OBS', '2006-07-04'), ('TIME-OBS', '11:32:35')))] """ ctx = crds.get_pickled_mapping(context, cached=True) # reviewed return ctx.file_matches(reffile)