def rmap_apply(self, func, *args, **keys):
"""Apply `func()` to *args and **keys, adding the pmap, imap, and rmap values
associated with the elaboration of args.source_context, args.instruments, args.types.
"""
keywords = dict(keys)
self._setup_source_context()
if self.args.rmaps:
for rmap_name in self.args.rmaps:
with log.error_on_exception("Failed processing rmap", srepr(rmap_name)):
log.info("="*20, "Refactoring rmap", srepr(rmap_name), "="*20)
rmapping = rmap.load_mapping(rmap_name)
new_filename = self._process_rmap(func, rmapping=rmapping, **keywords)
self._diff_and_certify(rmapping=rmapping, new_filename=new_filename,
source_context=self.source_context, **keywords)
else:
pmapping = rmap.load_mapping(self.source_context)
instruments = pmapping.selections.keys() if "all" in self.args.instruments else self.args.instruments
for instr in instruments:
with log.augment_exception("Failed loading imap for", repr(instr), "from",
repr(self.source_context)):
imapping = pmapping.get_imap(instr)
types = imapping.selections.keys() if "all" in self.args.types else self.args.types
for filekind in types:
with log.error_on_exception("Failed processing rmap for", repr(filekind)):
#, "from",
# repr(imapping.basename), "of", repr(self.source_context)):
try:
rmapping = imapping.get_rmap(filekind).copy()
except crds.exceptions.IrrelevantReferenceTypeError as exc:
log.info("Skipping type", srepr(filekind), "as N/A")
continue
log.info("="*20, "Refactoring rmap", srepr(rmapping.basename), "="*20)
new_filename = self._process_rmap(func, rmapping=rmapping, **keywords)
self._diff_and_certify(rmapping=rmapping, source_context=self.source_context,
new_filename=new_filename, **keywords)
def rmap_apply(self, func, *args, **keys):
"""Apply `func()` to *args and **keys, adding the pmap, imap, and rmap values
associated with the elaboration of args.source_context, args.instruments, args.types.
"""
keywords = dict(keys)
self._setup_source_context()
if self.args.rmaps:
for rmap_name in self.args.rmaps:
with log.error_on_exception("Failed processing rmap",
srepr(rmap_name)):
log.info("=" * 20, "Refactoring rmap", srepr(rmap_name),
"=" * 20)
rmapping = rmap.load_mapping(rmap_name)
new_filename = self._process_rmap(func,
rmapping=rmapping,
**keywords)
self._diff_and_certify(rmapping=rmapping,
new_filename=new_filename,
source_context=self.source_context,
**keywords)
else:
pmapping = rmap.load_mapping(self.source_context)
instruments = pmapping.selections.keys(
) if "all" in self.args.instruments else self.args.instruments
for instr in instruments:
with log.augment_exception("Failed loading imap for",
repr(instr), "from",
repr(self.source_context)):
imapping = pmapping.get_imap(instr)
types = imapping.selections.keys(
) if "all" in self.args.types else self.args.types
for filekind in types:
with log.error_on_exception("Failed processing rmap for",
repr(filekind)):
#, "from",
# repr(imapping.basename), "of", repr(self.source_context)):
try:
rmapping = imapping.get_rmap(filekind).copy()
except crds.exceptions.IrrelevantReferenceTypeError as exc:
log.info("Skipping type", srepr(filekind),
"as N/A")
continue
log.info("=" * 20, "Refactoring rmap",
srepr(rmapping.basename), "=" * 20)
new_filename = self._process_rmap(func,
rmapping=rmapping,
**keywords)
self._diff_and_certify(
rmapping=rmapping,
source_context=self.source_context,
new_filename=new_filename,
**keywords)
def verify_checksum(file_):
"""Verify checksums in `file_`."""
log.info("Verifying checksum for", repr(file_))
if config.is_reference(file_):
data_file.verify_checksum(file_)
elif rmap.is_mapping(file_):
if config.CRDS_IGNORE_MAPPING_CHECKSUM.get():
log.warning(
"Mapping checksums are disabled by config.CRDS_IGNORE_MAPPING_CHECKSM."
)
rmap.load_mapping(file_)
else:
raise exceptions.CrdsError(
"File", repr(file_),
"does not appear to be a CRDS reference or mapping file.")
def get_file_properties(filename):
"""Figure out (instrument, filekind, serial) based on `filename` which
should be a mapping or FITS reference file.
>> get_file_properties("./hst_acs_biasfile_0001.rmap")
('acs', 'biasfile')
>> get_file_properties("./hst_acs_biasfile_0001.pmap")
Traceback (most recent call last):
...
AssertionError: Invalid .pmap filename './hst_acs_biasfile_0001.pmap'
>> get_file_properties("test_data/s7g1700gl_dead.fits")
"""
if config.is_mapping(filename):
try:
return decompose_newstyle_name(filename)[2:4]
except Exception:
# NOTE: load_mapping more conservative than fetch_mapping used in properties_from_mapping
mapping = rmap.load_mapping(filename)
return mapping.instrument, mapping.filekind
elif config.is_reference(filename):
result = get_reference_properties(filename)[2:4]
else:
try:
result = properties_inside_mapping(filename)
except Exception as exc:
result = get_reference_properties(filename)[2:4]
assert result[0] in INSTRUMENTS+[""], "Bad instrument " + \
repr(result[0]) + " in filename " + repr(filename)
assert result[1] in FILEKINDS+[""], "Bad filekind " + \
repr(result[1]) + " in filename " + repr(filename)
return result
def _process_rmap(self, func, rmapping, *args, **keys):
"""Execute `func` on a single `rmapping` passing along *args and **keys"""
keywords = dict(keys)
rmapping_org = rmapping
new_filename = rmapping.filename if self.args.inplace else os.path.join(".", rmapping.basename)
if os.path.exists(new_filename):
log.info("Continuing refactoring from local copy", srepr(new_filename))
rmapping = rmap.load_mapping(new_filename)
keywords.update(locals())
fixers = self.args.fixers
if fixers:
rmapping = rmap.load_mapping(rmapping.filename)
keywords.update(locals())
apply_rmap_fixers(*args, **keywords)
func(*args, **keywords)
return new_filename
def del_header_value(old_rmap, new_rmap, key):
"""Set the value of `key` in `filename` to `new_value` and rewrite the rmap.
This is potentially lossy since rewriting the rmap may/will lose comments and
formatting quirks.
"""
mapping = rmap.load_mapping(old_rmap)
del mapping.header[key]
mapping.write(new_rmap)
def _process_rmap(self, func, rmapping, *args, **keys):
"""Execute `func` on a single `rmapping` passing along *args and **keys"""
keywords = dict(keys)
rmapping_org = rmapping
new_filename = rmapping.filename if self.args.inplace else os.path.join(
".", rmapping.basename)
if os.path.exists(new_filename):
log.info("Continuing refactoring from local copy",
srepr(new_filename))
rmapping = rmap.load_mapping(new_filename)
keywords.update(locals())
fixers = self.args.fixers
if fixers:
rmapping = rmap.load_mapping(rmapping.filename)
keywords.update(locals())
apply_rmap_fixers(*args, **keywords)
func(*args, **keywords)
return new_filename
def apply_rmap_fixers(rmapping, new_filename, fixers, *args, **keys):
"""Apply the text replacements defined in list of colon separated
old:new `fixers` list to `rmapping` writing results to `new_filename`.
"""
keys = dict(keys)
keys.pop("old_text", None)
keys.pop("new_text", None)
for fixer in fixers:
old_text, new_text = fixer.split(":")
rmap.replace_rmap_text(rmapping, new_filename, old_text, new_text, *args, **keys)
rmapping = rmap.load_mapping(new_filename)
def from_file(cls, filename):
"""For historical HST types, build type info from a spec file derived from CDBS specs like
reference_file_defs.xml or cdbscatalog.dat. For new CRDS-only types, use a prototype rmap
with an enhanced header to define the type. Prototypes should be submissible but should not
contain references.
"""
log.verbose("Loading type spec", repr(filename), verbosity=75)
if filename.endswith(".spec"):
return cls(utils.evalfile(filename))
else:
return cls(rmap.load_mapping(filename).header)
def from_file(cls, filename):
"""For historical HST types, build type info from a spec file derived from CDBS specs like
reference_file_defs.xml or cdbscatalog.dat. For new CRDS-only types, use a prototype rmap
with an enhanced header to define the type. Prototypes should be submissible but should not
contain references.
"""
log.verbose("Loading type spec", repr(filename), verbosity=75)
if filename.endswith(".spec"):
return cls(utils.evalfile(filename))
else:
return cls(rmap.load_mapping(filename).header)
def apply_rmap_fixers(rmapping, new_filename, fixers, *args, **keys):
"""Apply the text replacements defined in list of colon separated
old:new `fixers` list to `rmapping` writing results to `new_filename`.
"""
keys = dict(keys)
keys.pop("old_text", None)
keys.pop("new_text", None)
for fixer in fixers:
old_text, new_text = fixer.split(":")
rmap.replace_rmap_text(rmapping, new_filename, old_text, new_text,
*args, **keys)
rmapping = rmap.load_mapping(new_filename)
def set_rmap_parkey(rmapping, new_filename, parkey, *args, **keys):
"""Set the parkey of `rmapping` to `parkey` and write out to `new_filename`.
"""
log.info("Setting parkey, removing all references from", srepr(rmapping.basename))
pktuple = eval(parkey)
required_keywords = tuple(utils.flatten(pktuple))
refnames = rmapping.reference_names()
references_headers = { refname : get_refactoring_header(rmapping.filename, refname, required_keywords)
for refname in refnames }
rmapping = rmap_delete_references(rmapping.filename, new_filename, refnames)
log.info("Setting parkey", srepr(parkey), "in", srepr(rmapping.basename))
rmapping.header["parkey"] = pktuple
rmapping.write(new_filename)
rmapping = rmap.load_mapping(new_filename)
rmapping = rmap_insert_references_by_matches(new_filename, new_filename, references_headers)
return rmapping
def rmap_delete_references(old_rmap, new_rmap, deleted_references):
"""Given the full path of starting rmap `old_rmap`, modify it by deleting
all files in `deleted_references` and write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.load_mapping(old_rmap, ignore_checksum=True)
for reference in deleted_references:
baseref = os.path.basename(reference)
log.info("Deleting", srepr(baseref), "from", srepr(new.name))
with log.augment_exception("In reference", srepr(baseref)):
new = new.delete(reference)
new.header["derived_from"] = old.basename
log.verbose("Writing", srepr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for reference in deleted_references:
reference = os.path.basename(reference)
assert reference not in formatted, \
"Rules update failure. Deleted " + srepr(reference) + " still appears in new rmap."
return new
def set_rmap_parkey(rmapping, new_filename, parkey, *args, **keys):
"""Set the parkey of `rmapping` to `parkey` and write out to `new_filename`.
"""
log.info("Setting parkey, removing all references from",
srepr(rmapping.basename))
pktuple = eval(parkey)
required_keywords = tuple(utils.flatten(pktuple))
refnames = rmapping.reference_names()
references_headers = {
refname: get_refactoring_header(rmapping.filename, refname,
required_keywords)
for refname in refnames
}
rmapping = rmap_delete_references(rmapping.filename, new_filename,
refnames)
log.info("Setting parkey", srepr(parkey), "in", srepr(rmapping.basename))
rmapping.header["parkey"] = pktuple
rmapping.write(new_filename)
rmapping = rmap.load_mapping(new_filename)
rmapping = rmap_insert_references_by_matches(new_filename, new_filename,
references_headers)
return rmapping
def rmap_delete_references(old_rmap, new_rmap, deleted_references):
"""Given the full path of starting rmap `old_rmap`, modify it by deleting
all files in `deleted_references` and write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.load_mapping(old_rmap, ignore_checksum=True)
for reference in deleted_references:
baseref = os.path.basename(reference)
log.info("Deleting", srepr(baseref), "from", srepr(new.name))
with log.augment_exception("In reference", srepr(baseref)):
new = new.delete(reference)
new.header["derived_from"] = old.basename
log.verbose("Writing", srepr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for reference in deleted_references:
reference = os.path.basename(reference)
assert reference not in formatted, \
"Rules update failure. Deleted " + srepr(reference) + " still appears in new rmap."
return new
def get_file_properties(filename):
"""Figure out (instrument, filekind, serial) based on `filename` which
should be a mapping or ASDF reference file.
>>> get_file_properties('tests/data/roman_wfi16_f158_flat_small.asdf')
('wfi', 'flat')
>>> get_file_properties('tests/data/roman_wfi_flat_0004.rmap')
('wfi', 'flat')
>>> get_file_properties('tests/data/roman_0001.pmap')
('', '')
>>> get_file_properties('tests/data/ascii_tab.csv') # doctest: +ELLIPSIS
Traceback (most recent call last):
...
TypeError: string indices must be integers
"""
if config.is_mapping(filename):
try:
return decompose_newstyle_name(filename)[2:4]
except Exception:
# NOTE: load_mapping more conservative than fetch_mapping used in properties_from_mapping
mapping = rmap.load_mapping(filename)
return mapping.instrument, mapping.filekind
elif config.is_reference(filename):
result = get_reference_properties(filename)[2:4]
else:
try:
result = properties_inside_mapping(filename)
except Exception as exc:
result = get_reference_properties(filename)[2:4]
assert result[0] in INSTRUMENTS+[""], "Bad instrument " + \
repr(result[0]) + " in filename " + repr(filename)
assert result[1] in FILEKINDS+[""], "Bad filekind " + \
repr(result[1]) + " in filename " + repr(filename)
return result
def rmap_insert_references_by_matches(old_rmap, new_rmap, references_headers):
"""Given the full path of starting rmap `old_rmap`, modify it by inserting
or replacing all files in dict `references_headers` which maps a reference file basename
onto a list of headers under which it should be matched. Write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.load_mapping(old_rmap, ignore_checksum=True)
for baseref, header in references_headers.items():
with log.augment_exception("In reference", srepr(baseref)):
log.info("Inserting", srepr(baseref), "into", srepr(old_rmap))
log.verbose("Inserting", srepr(baseref), "match case", srepr(header), "into", srepr(old_rmap))
new = new.insert_header_reference(header, baseref)
new.header["derived_from"] = old.basename
log.verbose("Writing", srepr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for baseref in references_headers:
assert baseref in formatted, \
"Rules update failure. " + srepr(baseref) + " does not appear in new rmap." \
" May be identical match with other submitted references."
return new
def rmap_insert_references_by_matches(old_rmap, new_rmap, references_headers):
"""Given the full path of starting rmap `old_rmap`, modify it by inserting
or replacing all files in dict `references_headers` which maps a reference file basename
onto a list of headers under which it should be matched. Write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.load_mapping(old_rmap, ignore_checksum=True)
for baseref, header in references_headers.items():
with log.augment_exception("In reference", srepr(baseref)):
log.info("Inserting", srepr(baseref), "into", srepr(old_rmap))
log.verbose("Inserting", srepr(baseref), "match case",
srepr(header), "into", srepr(old_rmap))
new = new.insert_header_reference(header, baseref)
new.header["derived_from"] = old.basename
log.verbose("Writing", srepr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for baseref in references_headers:
assert baseref in formatted, \
"Rules update failure. " + srepr(baseref) + " does not appear in new rmap." \
" May be identical match with other submitted references."
return new
def setUp(self):
# Note: load_mapping must deliver a unique copy of the specified rmap
super(Test_01_Insert, self).setUp()
self.rmap = rmap.load_mapping("tobs_tinstr_tfilekind.rmap")
self.original = rmap.load_mapping("tobs_tinstr_tfilekind.rmap")
def test_1_recursive_use_rmap(self):
r = rmap.load_mapping(self.result_filename)
result = r.get_best_ref(self.lookup_header)
log.verbose("recursive lookup result:", result)
assert result == self.expected_lookup_result, "Recursively generated rmap produced wrong result."
def test_1_recursive_use_rmap(self):
r = rmap.load_mapping(self.result_filename)
ref = r.get_best_ref(self.lookup_header)
log.verbose("ref:", ref)
assert ref.startswith("NOT FOUND list index out of range")
def test_1_recursive_use_rmap(self):
r = rmap.load_mapping(self.result_filename)
result = r.get_best_ref(self.lookup_header)
log.verbose("recursive lookup result:", result)
assert result == self.expected_lookup_result, "Recursively generated rmap produced wrong result."
def setUp(self):
# Note: load_mapping must deliver a unique copy of the specified rmap
super(Test_01_Insert, self).setUp()
self.rmap = rmap.load_mapping("tobs_tinstr_tfilekind.rmap")
self.original = rmap.load_mapping("tobs_tinstr_tfilekind.rmap")
def test_1_recursive_use_rmap(self):
r = rmap.load_mapping(self.result_filename)
ref = r.get_best_ref(self.lookup_header)
log.verbose("ref:", ref)
assert ref.startswith("NOT FOUND list index out of range")
