def purge_mappings(self):
"""Remove all mappings not under pmaps `self.contexts`."""
# rmap.list_mappings lists all mappings in the *local* cache.
# in contrast, client.list_mappings globs available mappings in the server cache.
purge_maps = set(rmap.list_mappings('*.[pir]map', self.observatory))
keep = set(self.get_context_mappings())
self.remove_files(sorted(purge_maps-keep), "mapping")
def purge_mappings(self):
"""Remove all mappings not under pmaps `self.contexts`."""
# rmap.list_mappings lists all mappings in the *local* cache.
# in contrast, client.list_mappings globs available mappings in the server cache.
purge_maps = set(rmap.list_mappings('*.[pir]map', self.observatory))
keep = set(self.get_context_mappings())
self.remove_files(sorted(purge_maps-keep), "mapping")
def test_rmap_list_mappings(self):
os.environ["CRDS_MAPPATH_SINGLE"] = self.data_dir
self.assertEqual(rmap.list_mappings("*.imap", "hst"), [
'hst_acs.imap', 'hst_acs_0001.imap', 'hst_acs_0002.imap',
'hst_acs_9999.imap', 'hst_cos.imap', 'hst_nicmos.imap',
'hst_stis.imap', 'hst_wfc3.imap', 'hst_wfpc2.imap',
'jwst_fgs_na.imap', 'jwst_miri_omit.imap',
'jwst_niriss_na_omit.imap'
])
def all_filekinds():
filekinds = set()
for context in rmap.list_mappings("*.pmap", "jwst"):
p = crds.get_cached_mapping(context)
for i in p.selections.values():
for r in i.selections.values():
if r.filekind not in filekinds:
log.info("Adding", repr(r.filekind), "from", repr(context))
filekinds.add(r.filekind)
return sorted(filekinds)
def test_purge_mappings(self):
self.run_script("crds.sync --contexts hst_cos_deadtab.rmap --fetch-references")
self.run_script("crds.sync --organize=flat")
r = crds.get_cached_mapping("hst_cos_deadtab.rmap")
self.assertEqual(r.reference_names(), ["s7g1700gl_dead.fits", "s7g1700ql_dead.fits"])
self.assertEqual(rmap.list_references("*", "hst"), ["s7g1700gl_dead.fits", "s7g1700ql_dead.fits"])
self.assert_crds_exists("s7g1700gl_dead.fits")
self.assert_crds_exists("s7g1700ql_dead.fits")
self.run_script("crds.sync --contexts hst_acs_imphttab.rmap --fetch-references --purge-mappings --purge-references")
self.assertEqual(rmap.list_references("*", "hst"), ['w3m1716tj_imp.fits', 'w3m17170j_imp.fits', 'w3m17171j_imp.fits'])
self.assertEqual(rmap.list_mappings("*", "hst"), ['hst_acs_imphttab.rmap'])
def get_updated_files(context1, context2):
"""Return the sorted list of files names which are in `context2` (or any intermediate context)
but not in `context1`. context2 > context1.
"""
extension1 = os.path.splitext(context1)[1]
extension2 = os.path.splitext(context2)[1]
assert extension1 == extension2, "Only compare mappings of same type/extension."
old_map = rmap.get_cached_mapping(context1)
old_files = set(old_map.mapping_names() + old_map.reference_names())
all_mappings = rmap.list_mappings("*"+extension1, old_map.observatory)
updated = set()
context1, context2 = os.path.basename(context1), os.path.basename(context2)
for new in all_mappings:
new = os.path.basename(new)
if context1 < new <= context2:
new_map = rmap.get_cached_mapping(new)
updated |= set(new_map.mapping_names() + new_map.reference_names())
return sorted(list(updated - old_files))
def get_updated_files(context1, context2):
"""Return the sorted list of files names which are in `context2` (or any intermediate context)
but not in `context1`. context2 > context1.
"""
extension1 = os.path.splitext(context1)[1]
extension2 = os.path.splitext(context2)[1]
assert extension1 == extension2, "Only compare mappings of same type/extension."
old_map = crds.get_cached_mapping(context1)
old_files = set(old_map.mapping_names() + old_map.reference_names())
all_mappings = rmap.list_mappings("*" + extension1, old_map.observatory)
updated = set()
context1, context2 = os.path.basename(context1), os.path.basename(context2)
for new in all_mappings:
new = os.path.basename(new)
if context1 < new <= context2:
new_map = crds.get_cached_mapping(new)
updated |= set(new_map.mapping_names() + new_map.reference_names())
return sorted(list(updated - old_files))
def test_rmap_list_mappings(self):
os.environ["CRDS_MAPPATH_SINGLE"] = self.data_dir
self.assertEqual(rmap.list_mappings("*.imap", "hst"), [
'hst_acs.imap', 'hst_acs_0001.imap', 'hst_acs_0002.imap', 'hst_acs_9999.imap',
'hst_cos.imap', 'hst_nicmos.imap', 'hst_stis.imap', 'hst_wfc3.imap', 'hst_wfpc2.imap',
'jwst_fgs_na.imap', 'jwst_miri_omit.imap', 'jwst_niriss_na_omit.imap'])
def gen_all_specs():
for context in reversed(rmap.list_mappings("*.pmap", "jwst")):
gen_specs(context)
check_specs()
def list_cached_mappings(self):
"""List the mapping paths in the local cache."""
_print_list(rmap.list_mappings("*.*map", self.observatory, full_path=self.args.full_path))
def check_naming_consistency(checked_instrument=None, exhaustive_mapping_check=False):
"""Dev function to compare the properties returned by name decomposition
to the properties determined by file contents and make sure they're the same.
Also checks rmap membership.
>> from crds.tests import test_config
>> old_config = test_config.setup()
>> check_naming_consistency("acs")
>> check_naming_consistency("cos")
>> check_naming_consistency("nicmos")
>> check_naming_consistency("stis")
>> check_naming_consistency("wfc3")
>> check_naming_consistency("wfpc2")
>> test_config.cleanup(old_config)
"""
from crds import certify
for ref in rmap.list_references("*", observatory="hst", full_path=True):
with log.error_on_exception("Failed processing:", repr(ref)):
_path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path(ref)
if checked_instrument is not None and instrument != checked_instrument:
continue
if data_file.is_geis_data(ref):
if os.path.exists(data_file.get_conjugate(ref)):
continue
else:
log.warning("No GEIS header for", repr(ref))
log.verbose("Processing:", instrument, filekind, ref)
_path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header(ref)
if instrument != instrument2:
log.error("Inconsistent instruments", repr(instrument), "vs.", repr(instrument2),
"for", repr(ref))
if filekind != filekind2:
log.error("Inconsistent filekinds", repr(filekind), "vs.", repr(filekind2),
"for", repr(ref))
for pmap_name in reversed(sorted(rmap.list_mappings("*.pmap", observatory="hst"))):
pmap = crds.get_cached_mapping(pmap_name)
r = certify.find_governing_rmap(pmap_name, ref)
if not r:
continue
if r.instrument != instrument:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with name derived instrument", repr(instrument), "for", repr(ref), "in", repr(pmap_name))
if r.filekind != filekind:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with name derived filekind", repr(filekind), "for", repr(ref), "in", repr(pmap_name))
if r.instrument != instrument2:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with content derived instrument", repr(instrument2), "for", repr(ref), "in", repr(pmap_name))
if r.filekind != filekind2:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with content derived filekind", repr(filekind2), "for", repr(ref), "in", repr(pmap_name))
if not exhaustive_mapping_check:
break
else:
log.error("Orphan reference", repr(ref), "not found under any context.")
def check_naming_consistency(checked_instrument=None,
exhaustive_mapping_check=False):
"""Dev function to compare the properties returned by name decomposition
to the properties determined by file contents and make sure they're the same.
Also checks rmap membership.
>> from crds.tests import test_config
>> old_config = test_config.setup()
>> check_naming_consistency("acs")
>> check_naming_consistency("cos")
>> check_naming_consistency("nicmos")
>> check_naming_consistency("stis")
>> check_naming_consistency("wfc3")
>> check_naming_consistency("wfpc2")
>> test_config.cleanup(old_config)
"""
from crds import certify
for ref in rmap.list_references("*", observatory="hst", full_path=True):
with log.error_on_exception("Failed processing:", repr(ref)):
_path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path(
ref)
if checked_instrument is not None and instrument != checked_instrument:
continue
if data_file.is_geis_data(ref):
if os.path.exists(data_file.get_conjugate(ref)):
continue
else:
log.warning("No GEIS header for", repr(ref))
log.verbose("Processing:", instrument, filekind, ref)
_path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header(
ref)
if instrument != instrument2:
log.error("Inconsistent instruments", repr(instrument), "vs.",
repr(instrument2), "for", repr(ref))
if filekind != filekind2:
log.error("Inconsistent filekinds", repr(filekind), "vs.",
repr(filekind2), "for", repr(ref))
for pmap_name in reversed(
sorted(rmap.list_mappings("*.pmap", observatory="hst"))):
pmap = crds.get_cached_mapping(pmap_name)
r = certify.find_governing_rmap(pmap_name, ref)
if not r:
continue
if r.instrument != instrument:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with name derived instrument",
repr(instrument), "for", repr(ref), "in",
repr(pmap_name))
if r.filekind != filekind:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with name derived filekind",
repr(filekind), "for", repr(ref), "in",
repr(pmap_name))
if r.instrument != instrument2:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with content derived instrument",
repr(instrument2), "for", repr(ref), "in",
repr(pmap_name))
if r.filekind != filekind2:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with content derived filekind",
repr(filekind2), "for", repr(ref), "in",
repr(pmap_name))
if not exhaustive_mapping_check:
break
else:
log.error("Orphan reference", repr(ref),
"not found under any context.")
