def test_purge_mappings(self):
self.run_script("crds.sync --contexts hst_cos_deadtab.rmap --fetch-references")
self.run_script("crds.sync --organize=flat")
r = crds.get_cached_mapping("hst_cos_deadtab.rmap")
self.assertEqual(r.reference_names(), ["s7g1700gl_dead.fits", "s7g1700ql_dead.fits"])
self.assertEqual(rmap.list_references("*", "hst"), ["s7g1700gl_dead.fits", "s7g1700ql_dead.fits"])
self.assert_crds_exists("s7g1700gl_dead.fits")
self.assert_crds_exists("s7g1700ql_dead.fits")
self.run_script("crds.sync --contexts hst_acs_imphttab.rmap --fetch-references --purge-mappings --purge-references")
self.assertEqual(rmap.list_references("*", "hst"), ['w3m1716tj_imp.fits', 'w3m17170j_imp.fits', 'w3m17171j_imp.fits'])
self.assertEqual(rmap.list_mappings("*", "hst"), ['hst_acs_imphttab.rmap'])
def test_rmap_list_references(self):
os.environ["CRDS_REFPATH_SINGLE"] = self.data_dir
config.CRDS_REF_SUBDIR_MODE = "flat"
self.assertEqual(rmap.list_references("*.r1h", "hst"), [
'dbu1405fu.r1h', 'dbu1405iu.r1h', 'e1b09593u.r1h', 'e1b09594u.r1h',
'valid.r1h'
])
def organize_references(self, new_mode):
"""Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`.
This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument.
"""
old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True)
old_mode = config.get_crds_ref_subdir_mode(self.observatory)
log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode))
config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory)
new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change.
for refpath in old_refpaths:
with log.error_on_exception("Failed relocating:", repr(refpath)):
desired_loc = rmap.locate_file(os.path.basename(refpath), observatory=self.observatory)
if desired_loc != refpath:
if os.path.exists(desired_loc):
if not self.args.organize_delete_junk:
log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath))
continue
utils.remove(desired_loc, observatory=self.observatory)
if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)):
log.info("Relocating", repr(refpath), "to", repr(desired_loc))
shutil.move(refpath, desired_loc)
else:
if old_mode != new_mode:
log.warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode))
else:
log.warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath))
if new_mode == "flat" and old_mode == "instrument":
log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.")
for instrument in self.locator.INSTRUMENTS:
self.remove_dir(instrument)
def organize_references(self, new_mode):
"""Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`.
This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument.
"""
old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True)
old_mode = config.get_crds_ref_subdir_mode(self.observatory)
log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode))
config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory)
new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change.
for refpath in old_refpaths:
with log.error_on_exception("Failed relocating:", repr(refpath)):
desired_loc = rmap.locate_file(os.path.basename(refpath), observatory=self.observatory)
if desired_loc != refpath:
if os.path.exists(desired_loc):
if not self.args.organize_delete_junk:
log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath))
continue
utils.remove(desired_loc, observatory=self.observatory)
if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)):
log.info("Relocating", repr(refpath), "to", repr(desired_loc))
shutil.move(refpath, desired_loc)
else:
if old_mode != new_mode:
log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode))
else:
log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath))
if new_mode == "flat" and old_mode == "instrument":
log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.")
for instrument in self.locator.INSTRUMENTS:
self.remove_dir(instrument)
def purge_references(self, keep=None):
"""Remove all references not references under pmaps `self.contexts`."""
purge_refs = set(rmap.list_references("*", self.observatory))
if keep is None:
keep = set(self.get_context_references())
else:
keep = set(keep)
self.remove_files(sorted(purge_refs - keep), "reference")
def purge_references(self, keep=None):
"""Remove all references not references under pmaps `self.contexts`."""
purge_refs = set(rmap.list_references("*", self.observatory))
if keep is None:
keep = set(self.get_context_references())
else:
keep = set(keep)
self.remove_files(sorted(purge_refs - keep), "reference")
def fetch_references(self, references):
"""Gets all references required to support `only_contexts`. Removes
all references from the CRDS reference cache which are not required for
`only_contexts`.
"""
if not self.contexts:
return
if self.args.readonly_cache:
already_have = set(rmap.list_references("*", self.observatory))
fetched = [ x for x in sorted(set(references)-set(already_have)) if not x.startswith("NOT FOUND") ]
if fetched:
log.info("READONLY CACHE would fetch references:", repr(fetched))
with log.info_on_exception("Reference size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "references totaling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
self.dump_files(self.contexts[0], references)
def fetch_references(self, references):
"""Gets all references required to support `only_contexts`. Removes
all references from the CRDS reference cache which are not required for
`only_contexts`.
"""
if not self.contexts:
return
if self.args.readonly_cache:
already_have = set(rmap.list_references("*", self.observatory))
fetched = [ x for x in sorted(set(references)-set(already_have)) if not x.startswith("NOT FOUND") ]
if fetched:
log.info("READONLY CACHE would fetch references:", repr(fetched))
with log.info_on_exception("Reference size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "references totaling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
self.dump_files(self.contexts[0], references)
def test_rmap_list_references(self):
os.environ["CRDS_REFPATH_SINGLE"] = self.data_dir
config.CRDS_REF_SUBDIR_MODE = "flat"
self.assertEqual(rmap.list_references("*.r1h", "hst"), ['dbu1405fu.r1h', 'dbu1405iu.r1h', 'e1b09593u.r1h', 'e1b09594u.r1h', 'valid.r1h'])
def list_cached_references(self):
"""List the reference paths in the local cache."""
_print_list(rmap.list_references("*", self.observatory, full_path=self.args.full_path))
def check_naming_consistency(checked_instrument=None, exhaustive_mapping_check=False):
"""Dev function to compare the properties returned by name decomposition
to the properties determined by file contents and make sure they're the same.
Also checks rmap membership.
>> from crds.tests import test_config
>> old_config = test_config.setup()
>> check_naming_consistency("acs")
>> check_naming_consistency("cos")
>> check_naming_consistency("nicmos")
>> check_naming_consistency("stis")
>> check_naming_consistency("wfc3")
>> check_naming_consistency("wfpc2")
>> test_config.cleanup(old_config)
"""
from crds import certify
for ref in rmap.list_references("*", observatory="hst", full_path=True):
with log.error_on_exception("Failed processing:", repr(ref)):
_path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path(ref)
if checked_instrument is not None and instrument != checked_instrument:
continue
if data_file.is_geis_data(ref):
if os.path.exists(data_file.get_conjugate(ref)):
continue
else:
log.warning("No GEIS header for", repr(ref))
log.verbose("Processing:", instrument, filekind, ref)
_path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header(ref)
if instrument != instrument2:
log.error("Inconsistent instruments", repr(instrument), "vs.", repr(instrument2),
"for", repr(ref))
if filekind != filekind2:
log.error("Inconsistent filekinds", repr(filekind), "vs.", repr(filekind2),
"for", repr(ref))
for pmap_name in reversed(sorted(rmap.list_mappings("*.pmap", observatory="hst"))):
pmap = crds.get_cached_mapping(pmap_name)
r = certify.find_governing_rmap(pmap_name, ref)
if not r:
continue
if r.instrument != instrument:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with name derived instrument", repr(instrument), "for", repr(ref), "in", repr(pmap_name))
if r.filekind != filekind:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with name derived filekind", repr(filekind), "for", repr(ref), "in", repr(pmap_name))
if r.instrument != instrument2:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with content derived instrument", repr(instrument2), "for", repr(ref), "in", repr(pmap_name))
if r.filekind != filekind2:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with content derived filekind", repr(filekind2), "for", repr(ref), "in", repr(pmap_name))
if not exhaustive_mapping_check:
break
else:
log.error("Orphan reference", repr(ref), "not found under any context.")
def check_naming_consistency(checked_instrument=None,
exhaustive_mapping_check=False):
"""Dev function to compare the properties returned by name decomposition
to the properties determined by file contents and make sure they're the same.
Also checks rmap membership.
>> from crds.tests import test_config
>> old_config = test_config.setup()
>> check_naming_consistency("acs")
>> check_naming_consistency("cos")
>> check_naming_consistency("nicmos")
>> check_naming_consistency("stis")
>> check_naming_consistency("wfc3")
>> check_naming_consistency("wfpc2")
>> test_config.cleanup(old_config)
"""
from crds import certify
for ref in rmap.list_references("*", observatory="hst", full_path=True):
with log.error_on_exception("Failed processing:", repr(ref)):
_path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path(
ref)
if checked_instrument is not None and instrument != checked_instrument:
continue
if data_file.is_geis_data(ref):
if os.path.exists(data_file.get_conjugate(ref)):
continue
else:
log.warning("No GEIS header for", repr(ref))
log.verbose("Processing:", instrument, filekind, ref)
_path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header(
ref)
if instrument != instrument2:
log.error("Inconsistent instruments", repr(instrument), "vs.",
repr(instrument2), "for", repr(ref))
if filekind != filekind2:
log.error("Inconsistent filekinds", repr(filekind), "vs.",
repr(filekind2), "for", repr(ref))
for pmap_name in reversed(
sorted(rmap.list_mappings("*.pmap", observatory="hst"))):
pmap = crds.get_cached_mapping(pmap_name)
r = certify.find_governing_rmap(pmap_name, ref)
if not r:
continue
if r.instrument != instrument:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with name derived instrument",
repr(instrument), "for", repr(ref), "in",
repr(pmap_name))
if r.filekind != filekind:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with name derived filekind",
repr(filekind), "for", repr(ref), "in",
repr(pmap_name))
if r.instrument != instrument2:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with content derived instrument",
repr(instrument2), "for", repr(ref), "in",
repr(pmap_name))
if r.filekind != filekind2:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with content derived filekind",
repr(filekind2), "for", repr(ref), "in",
repr(pmap_name))
if not exhaustive_mapping_check:
break
else:
log.error("Orphan reference", repr(ref),
"not found under any context.")
def list_cached_references(self):
"""List the reference paths in the local cache."""
_print_list(rmap.list_references("*", self.observatory, full_path=self.args.full_path))