def rmap_apply(self, func, *args, **keys): """Apply `func()` to *args and **keys, adding the pmap, imap, and rmap values associated with the elaboration of args.source_context, args.instruments, args.types. """ keywords = dict(keys) self._setup_source_context() if self.args.rmaps: for rmap_name in self.args.rmaps: with log.error_on_exception("Failed processing rmap", srepr(rmap_name)): log.info("="*20, "Refactoring rmap", srepr(rmap_name), "="*20) rmapping = rmap.load_mapping(rmap_name) new_filename = self._process_rmap(func, rmapping=rmapping, **keywords) self._diff_and_certify(rmapping=rmapping, new_filename=new_filename, source_context=self.source_context, **keywords) else: pmapping = rmap.load_mapping(self.source_context) instruments = pmapping.selections.keys() if "all" in self.args.instruments else self.args.instruments for instr in instruments: with log.augment_exception("Failed loading imap for", repr(instr), "from", repr(self.source_context)): imapping = pmapping.get_imap(instr) types = imapping.selections.keys() if "all" in self.args.types else self.args.types for filekind in types: with log.error_on_exception("Failed processing rmap for", repr(filekind)): #, "from", # repr(imapping.basename), "of", repr(self.source_context)): try: rmapping = imapping.get_rmap(filekind).copy() except crds.exceptions.IrrelevantReferenceTypeError as exc: log.info("Skipping type", srepr(filekind), "as N/A") continue log.info("="*20, "Refactoring rmap", srepr(rmapping.basename), "="*20) new_filename = self._process_rmap(func, rmapping=rmapping, **keywords) self._diff_and_certify(rmapping=rmapping, source_context=self.source_context, new_filename=new_filename, **keywords)
def _process_rmap(self, func, rmapping, *args, **keys): """Execute `func` on a single `rmapping` passing along *args and **keys""" keywords = dict(keys) rmapping_org = rmapping new_filename = rmapping.filename if self.args.inplace else os.path.join(".", rmapping.basename) if os.path.exists(new_filename): log.info("Continuing refactoring from local copy", srepr(new_filename)) rmapping = rmap.load_mapping(new_filename) keywords.update(locals()) fixers = self.args.fixers if fixers: rmapping = rmap.load_mapping(rmapping.filename) keywords.update(locals()) apply_rmap_fixers(*args, **keywords) func(*args, **keywords) return new_filename
def get_file_properties(filename): """Figure out (instrument, filekind, serial) based on `filename` which should be a mapping or FITS reference file. >> get_file_properties("./hst_acs_biasfile_0001.rmap") ('acs', 'biasfile') >> get_file_properties("./hst_acs_biasfile_0001.pmap") Traceback (most recent call last): ... AssertionError: Invalid .pmap filename './hst_acs_biasfile_0001.pmap' >> get_file_properties("test_data/s7g1700gl_dead.fits") """ if rmap.is_mapping(filename): try: return decompose_newstyle_name(filename)[2:4] except Exception: # NOTE: load_mapping more conservative than fetch_mapping used in properties_from_mapping mapping = rmap.load_mapping(filename) return mapping.instrument, mapping.filekind elif config.is_reference(filename): result = get_reference_properties(filename)[2:4] else: try: result = properties_inside_mapping(filename) except Exception as exc: result = get_reference_properties(filename)[2:4] assert result[0] in INSTRUMENTS+[""], "Bad instrument " + \ repr(result[0]) + " in filename " + repr(filename) assert result[1] in FILEKINDS+[""], "Bad filekind " + \ repr(result[1]) + " in filename " + repr(filename) return result
def del_header_value(old_rmap, new_rmap, key): """Set the value of `key` in `filename` to `new_value` and rewrite the rmap. This is potentially lossy since rewriting the rmap may/will lose comments and formatting quirks. """ mapping = rmap.load_mapping(old_rmap) del mapping.header[key] mapping.write(new_rmap)
def apply_rmap_fixers(rmapping, new_filename, fixers, *args, **keys): """Apply the text replacements defined in list of colon separated old:new `fixers` list to `rmapping` writing results to `new_filename`. """ for fixer in fixers: old_text, new_text = fixer.split(":") replace_rmap_text(rmapping, new_filename, old_text, new_text, *args, **keys) rmapping = rmap.load_mapping(new_filename)
def apply_rmap_fixers(rmapping, new_filename, fixers, *args, **keys): """Apply the text replacements defined in list of colon separated old:new `fixers` list to `rmapping` writing results to `new_filename`. """ keys = dict(keys) keys.pop("old_text", None) keys.pop("new_text", None) for fixer in fixers: old_text, new_text = fixer.split(":") rmap.replace_rmap_text(rmapping, new_filename, old_text, new_text, *args, **keys) rmapping = rmap.load_mapping(new_filename)
def from_file(cls, filename): """For historical HST types, build type info from a spec file derived from CDBS specs like reference_file_defs.xml or cdbscatalog.dat. For new CRDS-only types, use a prototype rmap with an enhanced header to define the type. Prototypes should be submissible but should not contain references. """ log.verbose("Loading type spec", repr(filename), verbosity=75) if filename.endswith(".spec"): return cls(utils.evalfile(filename)) else: return cls(rmap.load_mapping(filename).header)
def set_rmap_parkey(rmapping, new_filename, parkey, *args, **keys): """Set the parkey of `rmapping` to `parkey` and write out to `new_filename`. """ log.info("Setting parkey, removing all references from", srepr(rmapping.basename)) pktuple = eval(parkey) required_keywords = tuple(utils.flatten(pktuple)) refnames = rmapping.reference_names() references_headers = { refname : get_refactoring_header(rmapping.filename, refname, required_keywords) for refname in refnames } rmapping = rmap_delete_references(rmapping.filename, new_filename, refnames) log.info("Setting parkey", srepr(parkey), "in", srepr(rmapping.basename)) rmapping.header["parkey"] = pktuple rmapping.write(new_filename) rmapping = rmap.load_mapping(new_filename) rmapping = rmap_insert_references_by_matches(new_filename, new_filename, references_headers) return rmapping
def rmap_delete_references(old_rmap, new_rmap, deleted_references): """Given the full path of starting rmap `old_rmap`, modify it by deleting all files in `deleted_references` and write out the result to `new_rmap`. If no actions are performed, don't write out `new_rmap`. Return new ReferenceMapping named `new_rmap` """ new = old = rmap.load_mapping(old_rmap, ignore_checksum=True) for reference in deleted_references: baseref = os.path.basename(reference) log.info("Deleting", srepr(baseref), "from", srepr(new.name)) with log.augment_exception("In reference", srepr(baseref)): new = new.delete(reference) new.header["derived_from"] = old.basename log.verbose("Writing", srepr(new_rmap)) new.write(new_rmap) formatted = new.format() for reference in deleted_references: reference = os.path.basename(reference) assert reference not in formatted, \ "Rules update failure. Deleted " + srepr(reference) + " still appears in new rmap." return new
def rmap_insert_references_by_matches(old_rmap, new_rmap, references_headers): """Given the full path of starting rmap `old_rmap`, modify it by inserting or replacing all files in dict `references_headers` which maps a reference file basename onto a list of headers under which it should be matched. Write out the result to `new_rmap`. If no actions are performed, don't write out `new_rmap`. Return new ReferenceMapping named `new_rmap` """ new = old = rmap.load_mapping(old_rmap, ignore_checksum=True) for baseref, header in references_headers.items(): with log.augment_exception("In reference", srepr(baseref)): log.info("Inserting", srepr(baseref), "match case", srepr(header), "into", srepr(baseref)) new = new.insert_header_reference(header, baseref) new.header["derived_from"] = old.basename log.verbose("Writing", srepr(new_rmap)) new.write(new_rmap) formatted = new.format() for baseref in references_headers: assert baseref in formatted, \ "Rules update failure. " + srepr(baseref) + " does not appear in new rmap." \ " May be identical match with other submitted references." return new
def rmap_insert_references_by_matches(old_rmap, new_rmap, references_headers): """Given the full path of starting rmap `old_rmap`, modify it by inserting or replacing all files in dict `references_headers` which maps a reference file basename onto a list of headers under which it should be matched. Write out the result to `new_rmap`. If no actions are performed, don't write out `new_rmap`. Return new ReferenceMapping named `new_rmap` """ new = old = rmap.load_mapping(old_rmap, ignore_checksum=True) for baseref, header in references_headers.items(): with log.augment_exception("In reference", srepr(baseref)): log.info("Inserting", srepr(baseref), "into", srepr(old_rmap)) log.verbose("Inserting", srepr(baseref), "match case", srepr(header), "into", srepr(old_rmap)) new = new.insert_header_reference(header, baseref) new.header["derived_from"] = old.basename log.verbose("Writing", srepr(new_rmap)) new.write(new_rmap) formatted = new.format() for baseref in references_headers: assert baseref in formatted, \ "Rules update failure. " + srepr(baseref) + " does not appear in new rmap." \ " May be identical match with other submitted references." return new
def test_missing_required_header_key(self): with self.assertRaises(MissingHeaderKeyError): r = rmap.load_mapping("data/hst_acs_darkfile_missing_key.rmap")
def setUp(self): # Note: load_mapping must deliver a unique copy of the specified rmap super(Test_01_Insert, self).setUp() self.rmap = rmap.load_mapping("tobs_tinstr_tfilekind.rmap") self.original = rmap.load_mapping("tobs_tinstr_tfilekind.rmap")
def test_1_recursive_use_rmap(self): r = rmap.load_mapping(self.result_filename) result = r.get_best_ref(self.lookup_header) log.verbose("recursive lookup result:", result) assert result == self.expected_lookup_result, "Recursively generated rmap produced wrong result."
def test_1_recursive_use_rmap(self): r = rmap.load_mapping(self.result_filename) ref = r.get_best_ref(self.lookup_header) log.verbose("ref:", ref) assert ref.startswith("NOT FOUND list index out of range")