def locate_dir(instrument, mode=None):
"""Locate the instrument specific directory for a reference file."""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="tobs")
else:
config.check_crds_ref_subdir_mode(mode)
crds_refpath = config.get_crds_refpath("tobs")
prefix = get_env_prefix(instrument)
if mode == "legacy": # Locate cached files at the appropriate CDBS-style iref$ locations
try:
rootdir = os.environ[prefix]
except KeyError:
try:
rootdir = os.environ[prefix[:-1]]
except KeyError as exc:
raise KeyError("Reference location not defined for",
repr(instrument), ". Did you configure",
repr(prefix) + "?") from exc
elif mode == "instrument": # use simple names inside CRDS cache.
rootdir = os.path.join(crds_refpath, instrument)
refdir = os.path.join(crds_refpath, prefix[:-1])
if not os.path.exists(refdir):
if config.writable_cache_or_verbose(
"Skipping making instrument directory link for",
repr(instrument)):
log.verbose("Creating legacy cache link", repr(refdir), "-->",
repr(rootdir))
utils.ensure_dir_exists(rootdir + "/locate_dir.fits")
os.symlink(rootdir, refdir)
elif mode == "flat": # use original flat cache structure, all instruments in same directory.
rootdir = crds_refpath
else:
raise ValueError("Unhandled reference file location mode " +
repr(mode))
return rootdir
def locate_dir(instrument, mode=None):
"""Locate the instrument specific directory for a reference file."""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="hst")
else:
config.check_crds_ref_subdir_mode(mode)
crds_refpath = config.get_crds_refpath("hst")
prefix = get_env_prefix(instrument)
if mode == "legacy": # Locate cached files at the appropriate CDBS-style iref$ locations
try:
rootdir = os.environ[prefix]
except KeyError:
try:
rootdir = os.environ[prefix[:-1]]
except KeyError:
raise KeyError("Reference location not defined for " + repr(instrument) +
". Did you configure " + repr(prefix) + "?")
elif mode == "instrument" and instrument != "synphot": # use simple names inside CRDS cache.
rootdir = os.path.join(crds_refpath, instrument)
refdir = os.path.join(crds_refpath, prefix[:-1])
if not os.path.exists(refdir):
if config.writable_cache_or_verbose("Skipping making instrument directory link for", repr(instrument)):
log.verbose("Creating legacy cache link", repr(refdir), "-->", repr(rootdir))
with log.verbose_warning_on_exception("Failed creating legacy symlink:", refdir, "-->", rootdir):
utils.ensure_dir_exists(rootdir + "/locate_dir.fits")
os.symlink(rootdir, refdir)
elif mode == "instrument" and instrument == "synphot":
rootdir = os.path.join(crds_refpath, instrument)
elif mode == "flat": # use original flat cache structure, all instruments in same directory.
rootdir = crds_refpath
else:
raise ValueError("Unhandled reference file location mode " + repr(mode))
return rootdir
def _write_last_processed(self, hist):
"""Write down the history tuple of the last context processed."""
log.verbose("Saving last processed:", repr(hist))
log.verbose("Storing last processed state at", repr(self.last_processed_path))
utils.ensure_dir_exists(self.last_processed_path)
with open(self.last_processed_path, "w+") as last:
last.write(str(hist))
def get_sqlite_db(observatory):
"""Download the CRDS database as a SQLite database."""
assert not config.get_cache_readonly(), "Readonly cache, updating the SQLite database cannot be done."""
encoded_compressed_data = S.get_sqlite_db(observatory)
data = zlib.decompress(base64.b64decode(encoded_compressed_data))
path = config.get_sqlite3_db_path(observatory)
utils.ensure_dir_exists(path)
with open(path, "wb+") as db_out:
db_out.write(data)
return path
def get_sqlite_db(observatory):
"""Download the CRDS database as a SQLite database."""
assert not config.get_cache_readonly(), "Readonly cache, updating the SQLite database cannot be done."""
encoded_compressed_data = S.get_sqlite_db(observatory)
data = zlib.decompress(base64.b64decode(encoded_compressed_data))
path = config.get_sqlite3_db_path(observatory)
utils.ensure_dir_exists(path)
with open(path, "wb+") as db_out:
db_out.write(data)
return path
def locate_dir(instrument, mode=None):
"""Locate the instrument specific directory for a reference file.
The mode=None test case is disabled because it mysteriously causes these tests to
fail when running the runtests script:
ERROR: test_throughput_lookup_generation (crds.tests.test_synphot_lookup_generator.TestSynphotLookupGenerator)
FAIL: Doctest: crds.tests.test_bad_files.dt_bad_references_fast_mode
FAIL: Doctest: crds.tests.test_bad_files.dt_bad_rules_jwst_getreferences_warning
FAIL: Doctest: crds.tests.test_certify.certify_recursive
FAIL: Doctest: crds.tests.test_certify.certify_table_comparison_context
FAIL: Doctest: crds.tests.test_heavy_client.dt_getreferences_ignore_cache
FAIL: Doctest: crds.tests.test_list.dt_list_cached_references
FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test
FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test
XXXX TODO: Enable the mode=None test case and resolve the ensuing test failures in other modules.
>> locate_dir('wfi', None) # doctest: +ELLIPSIS
'.../references/roman/wfi'
>>> locate_dir('wfi', 'instrument') # doctest: +ELLIPSIS
'.../references/roman/wfi'
>>> locate_dir('wfi', 'flat') # doctest: +ELLIPSIS
'.../references/roman'
>>> locate_dir('wfi', 'other') # doctest: +ELLIPSIS
Traceback (most recent call last):
...
AssertionError: Invalid CRDS cache subdirectory mode = 'other'
"""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="roman")
else:
config.check_crds_ref_subdir_mode(mode)
crds_refpath = config.get_crds_refpath("roman")
if mode == "instrument": # use simple names inside CRDS cache.
rootdir = os.path.join(crds_refpath, instrument.lower())
if not os.path.exists(rootdir):
if config.writable_cache_or_verbose(
"Skipping making instrument directory link for",
repr(instrument)):
utils.ensure_dir_exists(rootdir + "/locate_dir.fits")
elif mode == "flat": # use original flat cache structure, all instruments in same directory.
rootdir = crds_refpath
else:
raise ValueError("Unhandled reference file location mode " +
repr(mode))
return rootdir
def locate_dir(instrument, mode=None):
"""Locate the instrument specific directory for a reference file."""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="jwst")
else:
config.check_crds_ref_subdir_mode(mode)
crds_refpath = config.get_crds_refpath("jwst")
if mode == "instrument": # use simple names inside CRDS cache.
rootdir = os.path.join(crds_refpath, instrument.lower())
if not os.path.exists(rootdir):
if config.writable_cache_or_verbose("Skipping making instrument directory link for", repr(instrument)):
utils.ensure_dir_exists(rootdir + "/locate_dir.fits")
elif mode == "flat": # use original flat cache structure, all instruments in same directory.
rootdir = crds_refpath
else:
raise ValueError("Unhandled reference file location mode " + repr(mode))
return rootdir
def download(self, name, localpath):
"""Download a single file."""
# This code is complicated by the desire to blow away failed downloads. For the specific
# case of KeyboardInterrupt, the file needs to be blown away, but the interrupt should not
# be re-characterized so it is still un-trapped elsewhere under normal idioms which try *not*
# to trap KeyboardInterrupt.
assert not config.get_cache_readonly(), "Readonly cache, cannot download files " + repr(name)
try:
utils.ensure_dir_exists(localpath)
return proxy.apply_with_retries(self.download_core, name, localpath)
except Exception as exc:
self.remove_file(localpath)
raise CrdsDownloadError(
"Error fetching data for", srepr(name),
"at CRDS server", srepr(get_crds_server()),
"with mode", srepr(config.get_download_mode()),
":", str(exc)) from exc
except: # mainly for control-c, catch it and throw it.
self.remove_file(localpath)
raise
def download(self, name, localpath):
"""Download a single file."""
# This code is complicated by the desire to blow away failed downloads. For the specific
# case of KeyboardInterrupt, the file needs to be blown away, but the interrupt should not
# be re-characterized so it is still un-trapped elsewhere under normal idioms which try *not*
# to trap KeyboardInterrupt.
assert not config.get_cache_readonly(), "Readonly cache, cannot download files " + repr(name)
try:
utils.ensure_dir_exists(localpath)
return proxy.apply_with_retries(self.download_core, name, localpath)
except Exception as exc:
self.remove_file(localpath)
raise CrdsDownloadError(
"Error fetching data for", srepr(name),
"at CRDS server", srepr(get_crds_server()),
"with mode", srepr(config.get_download_mode()),
":", str(exc)) from exc
except: # mainly for control-c, catch it and throw it.
self.remove_file(localpath)
raise
def cross_link_cdbs_paths(self, syn_name_map):
"""Hard link files from the downloaded CRDS cache structure to the
classic CDBS-style directory locations specified by `syn_name_map`.
On OS-X and Linux this creates files which are referenced from two
different paths on the file system. Deleting one path or the other
leaves behind a normal file referenced from only one location.
syn_name_map dict { syn_basename : pysyn_filepath, ... }
returns None
"""
for reference in syn_name_map:
with log.error_on_exception(
"Failed linking", repr(reference), "to CDBS directory."):
crds_filepath = os.path.abspath(self.imap.locate_file(reference))
cdbs_filepath = syn_name_map[reference]
utils.ensure_dir_exists(cdbs_filepath)
if not os.path.exists(cdbs_filepath):
os.link(crds_filepath, cdbs_filepath)
log.verbose("Linked", cdbs_filepath, "-->", crds_filepath)
else:
log.verbose("Skipped existing", cdbs_filepath)
def cross_link_cdbs_paths(self, syn_name_map):
"""Hard link files from the downloaded CRDS cache structure to the
classic CDBS-style directory locations specified by `syn_name_map`.
On OS-X and Linux this creates files which are referenced from two
different paths on the file system. Deleting one path or the other
leaves behind a normal file referenced from only one location.
syn_name_map dict { syn_basename : pysyn_filepath, ... }
returns None
"""
for reference in syn_name_map:
with log.error_on_exception(
"Failed linking", repr(reference), "to CDBS directory."):
crds_filepath = os.path.abspath(self.imap.locate_file(reference))
cdbs_filepath = syn_name_map[reference]
utils.ensure_dir_exists(cdbs_filepath)
if not os.path.exists(cdbs_filepath):
os.link(crds_filepath, cdbs_filepath)
log.verbose("Linked", cdbs_filepath, "-->", crds_filepath)
else:
log.verbose("Skipped existing", cdbs_filepath)
