def fetch_files(self, context, files):
"""Downloads `files` as needed relative to `context` nominally used to identify
the observatory to the server. If the CRDS cache is currently configured as READONLY,
prints an estimate about which files will download and the total size. The estimate
does not (currently) include the nested mappings associated with the closure of `files`,
but the dominant costs of downloads are reference files.
"""
files = set([os.path.basename(_file) for _file in files])
if config.get_cache_readonly():
log.info("READONLY CACHE estimating required downloads.")
if not self.args.ignore_cache:
already_have = (set(rmap.list_references("*", self.observatory)) |
set(rmap.list_mappings("*", self.observatory)))
else:
already_have = set()
fetched = [ x for x in sorted(files - already_have) if not x.startswith("NOT FOUND") ]
for _file in files:
if _file in already_have:
log.verbose("File", repr(_file), "is already in the CRDS cache.", verbosity=55)
else:
log.verbose("File", repr(_file), "would be downloaded.", verbosity=55)
if fetched:
with log.info_on_exception("File size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "files totalling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
log.info("READONLY CACHE no reference downloads expected.")
else:
self.dump_files(context, files, self.args.ignore_cache)
def fetch_files(self, context, files):
"""Downloads `files` as needed relative to `context` nominally used to identify
the observatory to the server. If the CRDS cache is currently configured as READONLY,
prints an estimate about which files will download and the total size. The estimate
does not (currently) include the nested mappings associated with the closure of `files`,
but the dominant costs of downloads are reference files.
"""
files = set([os.path.basename(_file) for _file in files])
if config.get_cache_readonly():
log.info("READONLY CACHE estimating required downloads.")
if not self.args.ignore_cache:
already_have = (set(rmap.list_references("*", self.observatory)) |
set(rmap.list_mappings("*", self.observatory)))
else:
already_have = set()
fetched = [ x for x in sorted(files - already_have) if not x.startswith("NOT FOUND") ]
for _file in files:
if _file in already_have:
log.verbose("File", repr(_file), "is already in the CRDS cache.", verbosity=55)
else:
log.verbose("File", repr(_file), "would be downloaded.", verbosity=55)
if fetched:
with log.info_on_exception("File size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "files totalling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
log.info("READONLY CACHE no reference downloads expected.")
else:
self.dump_files(context, files, self.args.ignore_cache)
def purge_mappings(self):
"""Remove all mappings not under pmaps `self.contexts`."""
# rmap.list_mappings lists all mappings in the *local* cache.
# in contrast, client.list_mappings globs available mappings in the server cache.
purge_maps = set(rmap.list_mappings('*.[pir]map', self.observatory))
keep = set(self.get_context_mappings())
self.remove_files(sorted(purge_maps-keep), "mapping")
def purge_mappings(self):
"""Remove all mappings not under pmaps `self.contexts`."""
# rmap.list_mappings lists all mappings in the *local* cache.
# in contrast, client.list_mappings globs available mappings in the server cache.
purge_maps = set(rmap.list_mappings('*.[pir]map', self.observatory))
keep = set(self.get_context_mappings())
self.remove_files(sorted(purge_maps-keep), "mapping")
def load_all_mappings(observatory, pattern="*map"):
"""Return a dictionary mapping the names of all CRDS Mappings matching `pattern`
onto the loaded Mapping object.
"""
all_mappings = rmap.list_mappings(pattern, observatory)
loaded = {}
for name in all_mappings:
with log.error_on_exception("Failed loading", repr(name)):
loaded[name] = rmap.get_cached_mapping(name)
return loaded
def test_purge_mappings(self):
self.run_script("crds.sync --contexts hst_cos_deadtab.rmap --fetch-references")
self.run_script("crds.sync --organize=flat")
r = crds.get_cached_mapping("hst_cos_deadtab.rmap")
self.assertEqual(r.reference_names(), ["s7g1700gl_dead.fits", "s7g1700ql_dead.fits"])
self.assertEqual(rmap.list_references("*", "hst"), ["s7g1700gl_dead.fits", "s7g1700ql_dead.fits"])
self.assert_crds_exists("s7g1700gl_dead.fits")
self.assert_crds_exists("s7g1700ql_dead.fits")
self.run_script("crds.sync --contexts hst_acs_imphttab.rmap --fetch-references --purge-mappings --purge-references")
self.assertEqual(rmap.list_references("*", "hst"), ['w3m1716tj_imp.fits', 'w3m17170j_imp.fits', 'w3m17171j_imp.fits'])
self.assertEqual(rmap.list_mappings("*", "hst"), ['hst_acs_imphttab.rmap'])
def get_updated_files(context1, context2):
"""Return the sorted list of files names which are in `context2` (or any intermediate context)
but not in `context1`. context2 > context1.
"""
extension1 = os.path.splitext(context1)[1]
extension2 = os.path.splitext(context2)[1]
assert extension1 == extension2, "Only compare mappings of same type/extension."
old_map = crds.get_cached_mapping(context1)
old_files = set(old_map.mapping_names() + old_map.reference_names())
all_mappings = rmap.list_mappings("*"+extension1, old_map.observatory)
updated = set()
context1, context2 = os.path.basename(context1), os.path.basename(context2)
for new in all_mappings:
new = os.path.basename(new)
if context1 < new <= context2:
new_map = crds.get_cached_mapping(new)
updated |= set(new_map.mapping_names() + new_map.reference_names())
return sorted(list(updated - old_files))
def get_updated_files(context1, context2):
"""Return the sorted list of files names which are in `context2` (or any intermediate context)
but not in `context1`. context2 > context1.
"""
extension1 = os.path.splitext(context1)[1]
extension2 = os.path.splitext(context2)[1]
assert extension1 == extension2, "Only compare mappings of same type/extension."
old_map = crds.get_cached_mapping(context1)
old_files = set(old_map.mapping_names() + old_map.reference_names())
all_mappings = rmap.list_mappings("*"+extension1, old_map.observatory)
updated = set()
context1, context2 = os.path.basename(context1), os.path.basename(context2)
for new in all_mappings:
new = os.path.basename(new)
if context1 < new <= context2:
new_map = crds.get_cached_mapping(new)
updated |= set(new_map.mapping_names() + new_map.reference_names())
return sorted(list(updated - old_files))
def check_naming_consistency(checked_instrument=None,
exhaustive_mapping_check=False):
"""Dev function to compare the properties returned by name decomposition
to the properties determined by file contents and make sure they're the same.
Also checks rmap membership.
>> from crds.tests import test_config
>> old_config = test_config.setup()
>> check_naming_consistency("acs")
>> check_naming_consistency("cos")
>> check_naming_consistency("nicmos")
>> check_naming_consistency("stis")
>> check_naming_consistency("wfc3")
>> check_naming_consistency("wfpc2")
>> test_config.cleanup(old_config)
"""
from crds import certify
for ref in rmap.list_references("*", observatory="hst", full_path=True):
with log.error_on_exception("Failed processing:", repr(ref)):
_path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path(
ref)
if checked_instrument is not None and instrument != checked_instrument:
continue
if data_file.is_geis_data(ref):
if os.path.exists(data_file.get_conjugate(ref)):
continue
else:
log.warning("No GEIS header for", repr(ref))
log.verbose("Processing:", instrument, filekind, ref)
_path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header(
ref)
if instrument != instrument2:
log.error("Inconsistent instruments", repr(instrument), "vs.",
repr(instrument2), "for", repr(ref))
if filekind != filekind2:
log.error("Inconsistent filekinds", repr(filekind), "vs.",
repr(filekind2), "for", repr(ref))
for pmap_name in reversed(
sorted(rmap.list_mappings("*.pmap", observatory="hst"))):
r = certify.certify.find_governing_rmap(pmap_name, ref)
if not r:
continue
if r.instrument != instrument:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with name derived instrument",
repr(instrument), "for", repr(ref), "in",
repr(pmap_name))
if r.filekind != filekind:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with name derived filekind",
repr(filekind), "for", repr(ref), "in",
repr(pmap_name))
if r.instrument != instrument2:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with content derived instrument",
repr(instrument2), "for", repr(ref), "in",
repr(pmap_name))
if r.filekind != filekind2:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with content derived filekind",
repr(filekind2), "for", repr(ref), "in",
repr(pmap_name))
if not exhaustive_mapping_check:
break
else:
log.error("Orphan reference", repr(ref),
"not found under any context.")
def list_cached_mappings(self):
"""List the mapping paths in the local cache."""
_print_list(rmap.list_mappings("*.*map", self.observatory, full_path=self.args.full_path))
def list_cached_mappings(self):
"""List the mapping paths in the local cache."""
_print_list(
rmap.list_mappings("*.*map",
self.observatory,
full_path=self.args.full_path))
def check_naming_consistency(checked_instrument=None, exhaustive_mapping_check=False):
"""Dev function to compare the properties returned by name decomposition
to the properties determined by file contents and make sure they're the same.
Also checks rmap membership.
>> from crds.tests import test_config
>> old_config = test_config.setup()
>> check_naming_consistency("acs")
>> check_naming_consistency("cos")
>> check_naming_consistency("nicmos")
>> check_naming_consistency("stis")
>> check_naming_consistency("wfc3")
>> check_naming_consistency("wfpc2")
>> test_config.cleanup(old_config)
"""
from crds import certify
for ref in rmap.list_references("*", observatory="hst", full_path=True):
with log.error_on_exception("Failed processing:", repr(ref)):
_path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path(ref)
if checked_instrument is not None and instrument != checked_instrument:
continue
if data_file.is_geis_data(ref):
if os.path.exists(data_file.get_conjugate(ref)):
continue
else:
log.warning("No GEIS header for", repr(ref))
log.verbose("Processing:", instrument, filekind, ref)
_path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header(ref)
if instrument != instrument2:
log.error("Inconsistent instruments", repr(instrument), "vs.", repr(instrument2),
"for", repr(ref))
if filekind != filekind2:
log.error("Inconsistent filekinds", repr(filekind), "vs.", repr(filekind2),
"for", repr(ref))
for pmap_name in reversed(sorted(rmap.list_mappings("*.pmap", observatory="hst"))):
r = certify.find_governing_rmap(pmap_name, ref)
if not r:
continue
if r.instrument != instrument:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with name derived instrument", repr(instrument), "for", repr(ref), "in", repr(pmap_name))
if r.filekind != filekind:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with name derived filekind", repr(filekind), "for", repr(ref), "in", repr(pmap_name))
if r.instrument != instrument2:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with content derived instrument", repr(instrument2), "for", repr(ref), "in", repr(pmap_name))
if r.filekind != filekind2:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with content derived filekind", repr(filekind2), "for", repr(ref), "in", repr(pmap_name))
if not exhaustive_mapping_check:
break
else:
log.error("Orphan reference", repr(ref), "not found under any context.")