def test_purge_mappings(self):
self.run_script("crds.sync --contexts hst_cos_deadtab.rmap --fetch-references")
self.run_script("crds.sync --organize=flat")
r = crds.get_cached_mapping("hst_cos_deadtab.rmap")
self.assertEqual(r.reference_names(), ["s7g1700gl_dead.fits", "s7g1700ql_dead.fits"])
self.assertEqual(rmap.list_references("*", "hst"), ["s7g1700gl_dead.fits", "s7g1700ql_dead.fits"])
self.assert_crds_exists("s7g1700gl_dead.fits")
self.assert_crds_exists("s7g1700ql_dead.fits")
self.run_script("crds.sync --contexts hst_acs_imphttab.rmap --fetch-references --purge-mappings --purge-references")
self.assertEqual(rmap.list_references("*", "hst"), ['w3m1716tj_imp.fits', 'w3m17170j_imp.fits', 'w3m17171j_imp.fits'])
self.assertEqual(rmap.list_mappings("*", "hst"), ['hst_acs_imphttab.rmap'])
def fetch_files(self, context, files):
"""Downloads `files` as needed relative to `context` nominally used to identify
the observatory to the server. If the CRDS cache is currently configured as READONLY,
prints an estimate about which files will download and the total size. The estimate
does not (currently) include the nested mappings associated with the closure of `files`,
but the dominant costs of downloads are reference files.
"""
files = set([os.path.basename(_file) for _file in files])
if config.get_cache_readonly():
log.info("READONLY CACHE estimating required downloads.")
if not self.args.ignore_cache:
already_have = (set(rmap.list_references("*", self.observatory)) |
set(rmap.list_mappings("*", self.observatory)))
else:
already_have = set()
fetched = [ x for x in sorted(files - already_have) if not x.startswith("NOT FOUND") ]
for _file in files:
if _file in already_have:
log.verbose("File", repr(_file), "is already in the CRDS cache.", verbosity=55)
else:
log.verbose("File", repr(_file), "would be downloaded.", verbosity=55)
if fetched:
with log.info_on_exception("File size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "files totalling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
log.info("READONLY CACHE no reference downloads expected.")
else:
self.dump_files(context, files, self.args.ignore_cache)
def organize_references(self, new_mode):
"""Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`.
This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument.
"""
old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True)
old_mode = config.get_crds_ref_subdir_mode(self.observatory)
log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode))
config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory)
new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change.
for refpath in old_refpaths:
with log.error_on_exception("Failed relocating:", repr(refpath)):
desired_loc = rmap.locate_file(os.path.basename(refpath), observatory=self.observatory)
if desired_loc != refpath:
if os.path.exists(desired_loc):
if not self.args.organize_delete_junk:
log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath))
continue
utils.remove(desired_loc, observatory=self.observatory)
if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)):
log.info("Relocating", repr(refpath), "to", repr(desired_loc))
shutil.move(refpath, desired_loc)
else:
if old_mode != new_mode:
log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode))
else:
log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath))
if new_mode == "flat" and old_mode == "instrument":
log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.")
for instrument in self.locator.INSTRUMENTS:
self.remove_dir(instrument)
def organize_references(self, new_mode):
"""Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`.
This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument.
"""
old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True)
old_mode = config.get_crds_ref_subdir_mode(self.observatory)
log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode))
config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory)
new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change.
for refpath in old_refpaths:
with log.error_on_exception("Failed relocating:", repr(refpath)):
desired_loc = config.locate_file(os.path.basename(refpath), observatory=self.observatory)
if desired_loc != refpath:
if os.path.exists(desired_loc):
if not self.args.organize_delete_junk:
log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath))
continue
utils.remove(desired_loc, observatory=self.observatory)
if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)):
log.info("Relocating", repr(refpath), "to", repr(desired_loc))
shutil.move(refpath, desired_loc)
else:
if old_mode != new_mode:
log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode))
else:
log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath))
if new_mode == "flat" and old_mode == "instrument":
log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.")
for instrument in self.locator.INSTRUMENTS:
self.remove_dir(instrument)
def fetch_files(self, context, files):
"""Downloads `files` as needed relative to `context` nominally used to identify
the observatory to the server. If the CRDS cache is currently configured as READONLY,
prints an estimate about which files will download and the total size. The estimate
does not (currently) include the nested mappings associated with the closure of `files`,
but the dominant costs of downloads are reference files.
"""
files = set([os.path.basename(_file) for _file in files])
if config.get_cache_readonly():
log.info("READONLY CACHE estimating required downloads.")
if not self.args.ignore_cache:
already_have = (set(rmap.list_references("*", self.observatory)) |
set(rmap.list_mappings("*", self.observatory)))
else:
already_have = set()
fetched = [ x for x in sorted(files - already_have) if not x.startswith("NOT FOUND") ]
for _file in files:
if _file in already_have:
log.verbose("File", repr(_file), "is already in the CRDS cache.", verbosity=55)
else:
log.verbose("File", repr(_file), "would be downloaded.", verbosity=55)
if fetched:
with log.info_on_exception("File size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "files totalling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
log.info("READONLY CACHE no reference downloads expected.")
else:
self.dump_files(context, files, self.args.ignore_cache)
def purge_references(self, keep=None):
"""Remove all references not references under pmaps `self.contexts`."""
purge_refs = set(rmap.list_references("*", self.observatory))
if keep is None:
keep = set(self.get_context_references())
else:
keep = set(keep)
self.remove_files(sorted(purge_refs - keep), "reference")
def purge_references(self, keep=None):
"""Remove all references not references under pmaps `self.contexts`."""
purge_refs = set(rmap.list_references("*", self.observatory))
if keep is None:
keep = set(self.get_context_references())
else:
keep = set(keep)
self.remove_files(sorted(purge_refs - keep), "reference")
def check_naming_consistency(checked_instrument=None,
exhaustive_mapping_check=False):
"""Dev function to compare the properties returned by name decomposition
to the properties determined by file contents and make sure they're the same.
Also checks rmap membership.
>> from crds.tests import test_config
>> old_config = test_config.setup()
>> check_naming_consistency("acs")
>> check_naming_consistency("cos")
>> check_naming_consistency("nicmos")
>> check_naming_consistency("stis")
>> check_naming_consistency("wfc3")
>> check_naming_consistency("wfpc2")
>> test_config.cleanup(old_config)
"""
from crds import certify
for ref in rmap.list_references("*", observatory="hst", full_path=True):
with log.error_on_exception("Failed processing:", repr(ref)):
_path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path(
ref)
if checked_instrument is not None and instrument != checked_instrument:
continue
if data_file.is_geis_data(ref):
if os.path.exists(data_file.get_conjugate(ref)):
continue
else:
log.warning("No GEIS header for", repr(ref))
log.verbose("Processing:", instrument, filekind, ref)
_path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header(
ref)
if instrument != instrument2:
log.error("Inconsistent instruments", repr(instrument), "vs.",
repr(instrument2), "for", repr(ref))
if filekind != filekind2:
log.error("Inconsistent filekinds", repr(filekind), "vs.",
repr(filekind2), "for", repr(ref))
for pmap_name in reversed(
sorted(rmap.list_mappings("*.pmap", observatory="hst"))):
r = certify.certify.find_governing_rmap(pmap_name, ref)
if not r:
continue
if r.instrument != instrument:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with name derived instrument",
repr(instrument), "for", repr(ref), "in",
repr(pmap_name))
if r.filekind != filekind:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with name derived filekind",
repr(filekind), "for", repr(ref), "in",
repr(pmap_name))
if r.instrument != instrument2:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with content derived instrument",
repr(instrument2), "for", repr(ref), "in",
repr(pmap_name))
if r.filekind != filekind2:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with content derived filekind",
repr(filekind2), "for", repr(ref), "in",
repr(pmap_name))
if not exhaustive_mapping_check:
break
else:
log.error("Orphan reference", repr(ref),
"not found under any context.")
def list_cached_references(self):
"""List the reference paths in the local cache."""
_print_list(rmap.list_references("*", self.observatory, full_path=self.args.full_path))
def list_cached_references(self):
"""List the reference paths in the local cache."""
_print_list(
rmap.list_references("*",
self.observatory,
full_path=self.args.full_path))
def check_naming_consistency(checked_instrument=None, exhaustive_mapping_check=False):
"""Dev function to compare the properties returned by name decomposition
to the properties determined by file contents and make sure they're the same.
Also checks rmap membership.
>> from crds.tests import test_config
>> old_config = test_config.setup()
>> check_naming_consistency("acs")
>> check_naming_consistency("cos")
>> check_naming_consistency("nicmos")
>> check_naming_consistency("stis")
>> check_naming_consistency("wfc3")
>> check_naming_consistency("wfpc2")
>> test_config.cleanup(old_config)
"""
from crds import certify
for ref in rmap.list_references("*", observatory="hst", full_path=True):
with log.error_on_exception("Failed processing:", repr(ref)):
_path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path(ref)
if checked_instrument is not None and instrument != checked_instrument:
continue
if data_file.is_geis_data(ref):
if os.path.exists(data_file.get_conjugate(ref)):
continue
else:
log.warning("No GEIS header for", repr(ref))
log.verbose("Processing:", instrument, filekind, ref)
_path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header(ref)
if instrument != instrument2:
log.error("Inconsistent instruments", repr(instrument), "vs.", repr(instrument2),
"for", repr(ref))
if filekind != filekind2:
log.error("Inconsistent filekinds", repr(filekind), "vs.", repr(filekind2),
"for", repr(ref))
for pmap_name in reversed(sorted(rmap.list_mappings("*.pmap", observatory="hst"))):
r = certify.find_governing_rmap(pmap_name, ref)
if not r:
continue
if r.instrument != instrument:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with name derived instrument", repr(instrument), "for", repr(ref), "in", repr(pmap_name))
if r.filekind != filekind:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with name derived filekind", repr(filekind), "for", repr(ref), "in", repr(pmap_name))
if r.instrument != instrument2:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with content derived instrument", repr(instrument2), "for", repr(ref), "in", repr(pmap_name))
if r.filekind != filekind2:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with content derived filekind", repr(filekind2), "for", repr(ref), "in", repr(pmap_name))
if not exhaustive_mapping_check:
break
else:
log.error("Orphan reference", repr(ref), "not found under any context.")