def download_files(self, downloads, localpaths):
"""Serial file-by-file download."""
download_metadata = get_download_metadata()
self.info_map = {}
for filename in downloads:
self.info_map[filename] = download_metadata.get(
filename, "NOT FOUND unknown to server")
if config.writable_cache_or_verbose(
"Readonly cache, skipping download of (first 5):",
repr(downloads[:5]),
verbosity=70):
bytes_so_far = 0
total_files = len(downloads)
total_bytes = get_total_bytes(self.info_map)
for nth_file, name in enumerate(downloads):
try:
if "NOT FOUND" in self.info_map[name]:
raise CrdsDownloadError(
"file is not known to CRDS server.")
bytes, path = self.catalog_file_size(
name), localpaths[name]
log.info(
file_progress("Fetching", name, path, bytes,
bytes_so_far, total_bytes, nth_file,
total_files))
self.download(name, path)
bytes_so_far += os.stat(path).st_size
except Exception as exc:
if self.raise_exceptions:
raise
else:
log.error("Failure downloading file", repr(name), ":",
str(exc))
return bytes_so_far
return 0
def locate_dir(instrument, mode=None):
"""Locate the instrument specific directory for a reference file."""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="tobs")
else:
config.check_crds_ref_subdir_mode(mode)
crds_refpath = config.get_crds_refpath("tobs")
prefix = get_env_prefix(instrument)
if mode == "legacy": # Locate cached files at the appropriate CDBS-style iref$ locations
try:
rootdir = os.environ[prefix]
except KeyError:
try:
rootdir = os.environ[prefix[:-1]]
except KeyError as exc:
raise KeyError("Reference location not defined for",
repr(instrument), ". Did you configure",
repr(prefix) + "?") from exc
elif mode == "instrument": # use simple names inside CRDS cache.
rootdir = os.path.join(crds_refpath, instrument)
refdir = os.path.join(crds_refpath, prefix[:-1])
if not os.path.exists(refdir):
if config.writable_cache_or_verbose(
"Skipping making instrument directory link for",
repr(instrument)):
log.verbose("Creating legacy cache link", repr(refdir), "-->",
repr(rootdir))
utils.ensure_dir_exists(rootdir + "/locate_dir.fits")
os.symlink(rootdir, refdir)
elif mode == "flat": # use original flat cache structure, all instruments in same directory.
rootdir = crds_refpath
else:
raise ValueError("Unhandled reference file location mode " +
repr(mode))
return rootdir
def locate_dir(instrument, mode=None):
"""Locate the instrument specific directory for a reference file."""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="hst")
else:
config.check_crds_ref_subdir_mode(mode)
crds_refpath = config.get_crds_refpath("hst")
prefix = get_env_prefix(instrument)
if mode == "legacy": # Locate cached files at the appropriate CDBS-style iref$ locations
try:
rootdir = os.environ[prefix]
except KeyError:
try:
rootdir = os.environ[prefix[:-1]]
except KeyError:
raise KeyError("Reference location not defined for " + repr(instrument) +
". Did you configure " + repr(prefix) + "?")
elif mode == "instrument" and instrument != "synphot": # use simple names inside CRDS cache.
rootdir = os.path.join(crds_refpath, instrument)
refdir = os.path.join(crds_refpath, prefix[:-1])
if not os.path.exists(refdir):
if config.writable_cache_or_verbose("Skipping making instrument directory link for", repr(instrument)):
log.verbose("Creating legacy cache link", repr(refdir), "-->", repr(rootdir))
with log.verbose_warning_on_exception("Failed creating legacy symlink:", refdir, "-->", rootdir):
utils.ensure_dir_exists(rootdir + "/locate_dir.fits")
os.symlink(rootdir, refdir)
elif mode == "instrument" and instrument == "synphot":
rootdir = os.path.join(crds_refpath, instrument)
elif mode == "flat": # use original flat cache structure, all instruments in same directory.
rootdir = crds_refpath
else:
raise ValueError("Unhandled reference file location mode " + repr(mode))
return rootdir
def main(self):
"""Synchronize files."""
# clear any mutliprocessing locks associated with the CRDS cache.
if self.args.clear_locks:
crds_cache_locking.clear_cache_locks()
return log.errors()
self.handle_misc_switches() # simple side effects only
# if explicitly requested, or the cache is suspect or being ignored, clear
# cached context pickles.
if self.args.clear_pickles or self.args.ignore_cache or self.args.repair_files:
self.clear_pickles()
# utility to change cache structure, e.g. add instrument subdirs.
# do this before syncing anything under the current mode.
if self.args.organize:
self.organize_references(self.args.organize)
# fetching and verifying files both require a server connection.
self.require_server_connection()
# primary sync'ing occurs here, both mappings and references as well as odd-ball
# server sqlite3 database download.
verify_file_list = self.file_transfers()
# verification is relative to sync'ed files, and can include file replacement if
# defects are found.
if (self.args.check_files or self.args.check_sha1sum or self.args.repair_files or
self.args.purge_blacklisted or self.args.purge_rejected):
self.verify_files(verify_file_list)
# context pickles should only be (re)generated after mappings are fully sync'ed and verified
if self.args.save_pickles:
self.pickle_contexts(self.contexts)
# update CRDS cache config area, including stored version of operational context.
# implement pipeline support functions of context update verify and echo
# If --output-dir was specified, do not update cache config.
if self.args.output_dir:
log.verbose_warning("Used --output-dir, skipping cache server_config update including default context and bad files.")
if config.writable_cache_or_verbose("skipping removal of ref_cache_subdir_mode file."):
os.remove(config.get_crds_ref_subdir_file(self.observatory))
else:
self.update_context()
self.report_stats()
log.standard_status()
return log.errors()
def locate_dir(instrument, mode=None):
"""Locate the instrument specific directory for a reference file.
The mode=None test case is disabled because it mysteriously causes these tests to
fail when running the runtests script:
ERROR: test_throughput_lookup_generation (crds.tests.test_synphot_lookup_generator.TestSynphotLookupGenerator)
FAIL: Doctest: crds.tests.test_bad_files.dt_bad_references_fast_mode
FAIL: Doctest: crds.tests.test_bad_files.dt_bad_rules_jwst_getreferences_warning
FAIL: Doctest: crds.tests.test_certify.certify_recursive
FAIL: Doctest: crds.tests.test_certify.certify_table_comparison_context
FAIL: Doctest: crds.tests.test_heavy_client.dt_getreferences_ignore_cache
FAIL: Doctest: crds.tests.test_list.dt_list_cached_references
FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test
FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test
XXXX TODO: Enable the mode=None test case and resolve the ensuing test failures in other modules.
>> locate_dir('wfi', None) # doctest: +ELLIPSIS
'.../references/roman/wfi'
>>> locate_dir('wfi', 'instrument') # doctest: +ELLIPSIS
'.../references/roman/wfi'
>>> locate_dir('wfi', 'flat') # doctest: +ELLIPSIS
'.../references/roman'
>>> locate_dir('wfi', 'other') # doctest: +ELLIPSIS
Traceback (most recent call last):
...
AssertionError: Invalid CRDS cache subdirectory mode = 'other'
"""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="roman")
else:
config.check_crds_ref_subdir_mode(mode)
crds_refpath = config.get_crds_refpath("roman")
if mode == "instrument": # use simple names inside CRDS cache.
rootdir = os.path.join(crds_refpath, instrument.lower())
if not os.path.exists(rootdir):
if config.writable_cache_or_verbose(
"Skipping making instrument directory link for",
repr(instrument)):
utils.ensure_dir_exists(rootdir + "/locate_dir.fits")
elif mode == "flat": # use original flat cache structure, all instruments in same directory.
rootdir = crds_refpath
else:
raise ValueError("Unhandled reference file location mode " +
repr(mode))
return rootdir
def locate_dir(instrument, mode=None):
"""Locate the instrument specific directory for a reference file."""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="jwst")
else:
config.check_crds_ref_subdir_mode(mode)
crds_refpath = config.get_crds_refpath("jwst")
if mode == "instrument": # use simple names inside CRDS cache.
rootdir = os.path.join(crds_refpath, instrument.lower())
if not os.path.exists(rootdir):
if config.writable_cache_or_verbose("Skipping making instrument directory link for", repr(instrument)):
utils.ensure_dir_exists(rootdir + "/locate_dir.fits")
elif mode == "flat": # use original flat cache structure, all instruments in same directory.
rootdir = crds_refpath
else:
raise ValueError("Unhandled reference file location mode " + repr(mode))
return rootdir
def download_files(self, downloads, localpaths):
"""Serial file-by-file download."""
self.info_map = get_file_info_map(
self.observatory, downloads, ["size", "rejected", "blacklisted", "state", "sha1sum", "instrument"])
if config.writable_cache_or_verbose("Readonly cache, skipping download of (first 5):", repr(downloads[:5]), verbosity=70):
bytes_so_far = 0
total_files = len(downloads)
total_bytes = get_total_bytes(self.info_map)
for nth_file, name in enumerate(downloads):
try:
if "NOT FOUND" in self.info_map[name]:
raise CrdsDownloadError("file is not known to CRDS server.")
bytes, path = self.catalog_file_size(name), localpaths[name]
log.info(file_progress("Fetching", name, path, bytes, bytes_so_far, total_bytes, nth_file, total_files))
self.download(name, path)
bytes_so_far += os.stat(path).st_size
except Exception as exc:
if self.raise_exceptions:
raise
else:
log.error("Failure downloading file", repr(name), ":", str(exc))
return bytes_so_far
return 0
