def organize_references(self, new_mode):
"""Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`.
This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument.
"""
old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True)
old_mode = config.get_crds_ref_subdir_mode(self.observatory)
log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode))
config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory)
new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change.
for refpath in old_refpaths:
with log.error_on_exception("Failed relocating:", repr(refpath)):
desired_loc = rmap.locate_file(os.path.basename(refpath), observatory=self.observatory)
if desired_loc != refpath:
if os.path.exists(desired_loc):
if not self.args.organize_delete_junk:
log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath))
continue
utils.remove(desired_loc, observatory=self.observatory)
if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)):
log.info("Relocating", repr(refpath), "to", repr(desired_loc))
shutil.move(refpath, desired_loc)
else:
if old_mode != new_mode:
log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode))
else:
log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath))
if new_mode == "flat" and old_mode == "instrument":
log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.")
for instrument in self.locator.INSTRUMENTS:
self.remove_dir(instrument)
def organize_references(self, new_mode):
"""Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`.
This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument.
"""
old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True)
old_mode = config.get_crds_ref_subdir_mode(self.observatory)
log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode))
config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory)
new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change.
for refpath in old_refpaths:
with log.error_on_exception("Failed relocating:", repr(refpath)):
desired_loc = config.locate_file(os.path.basename(refpath), observatory=self.observatory)
if desired_loc != refpath:
if os.path.exists(desired_loc):
if not self.args.organize_delete_junk:
log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath))
continue
utils.remove(desired_loc, observatory=self.observatory)
if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)):
log.info("Relocating", repr(refpath), "to", repr(desired_loc))
shutil.move(refpath, desired_loc)
else:
if old_mode != new_mode:
log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode))
else:
log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath))
if new_mode == "flat" and old_mode == "instrument":
log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.")
for instrument in self.locator.INSTRUMENTS:
self.remove_dir(instrument)
def list_config(self):
"""Print out configuration info about the current environment and server."""
info = config.get_crds_env_vars()
real_paths = config.get_crds_actual_paths(self.observatory)
server = self.server_info
current_server_url = api.get_crds_server()
cache_subdir_mode = config.get_crds_ref_subdir_mode(self.observatory)
pyinfo = _get_python_info()
_print_dict("CRDS Environment", info)
_print_dict("CRDS Client Config", {
"server_url" : current_server_url,
"cache_subdir_mode": cache_subdir_mode,
"readonly_cache": self.readonly_cache,
"effective_context": heavy_client.get_context_name(self.observatory),
"crds" : repr(crds),
"version": heavy_client.version_info()
})
_print_dict("CRDS Actual Paths", real_paths)
_print_dict("CRDS Server Info", server,
["observatory", "status", "connected", "operational_context", "last_synced",
"reference_url", "mapping_url", "effective_mode"])
if self.observatory == "hst":
cal_vars = { var : os.environ[var] for var in os.environ
if len(var) == 4 and var.lower().endswith("ref") }
_print_dict("Calibration Environment", cal_vars)
_print_dict("Python Environment", pyinfo)
def locate_dir(instrument, mode=None):
"""Locate the instrument specific directory for a reference file."""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="tobs")
else:
config.check_crds_ref_subdir_mode(mode)
crds_refpath = config.get_crds_refpath("tobs")
prefix = get_env_prefix(instrument)
if mode == "legacy": # Locate cached files at the appropriate CDBS-style iref$ locations
try:
rootdir = os.environ[prefix]
except KeyError:
try:
rootdir = os.environ[prefix[:-1]]
except KeyError as exc:
raise KeyError("Reference location not defined for",
repr(instrument), ". Did you configure",
repr(prefix) + "?") from exc
elif mode == "instrument": # use simple names inside CRDS cache.
rootdir = os.path.join(crds_refpath, instrument)
refdir = os.path.join(crds_refpath, prefix[:-1])
if not os.path.exists(refdir):
if config.writable_cache_or_verbose(
"Skipping making instrument directory link for",
repr(instrument)):
log.verbose("Creating legacy cache link", repr(refdir), "-->",
repr(rootdir))
utils.ensure_dir_exists(rootdir + "/locate_dir.fits")
os.symlink(rootdir, refdir)
elif mode == "flat": # use original flat cache structure, all instruments in same directory.
rootdir = crds_refpath
else:
raise ValueError("Unhandled reference file location mode " +
repr(mode))
return rootdir
def locate_dir(instrument, mode=None):
"""Locate the instrument specific directory for a reference file."""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="hst")
else:
config.check_crds_ref_subdir_mode(mode)
crds_refpath = config.get_crds_refpath("hst")
prefix = get_env_prefix(instrument)
if mode == "legacy": # Locate cached files at the appropriate CDBS-style iref$ locations
try:
rootdir = os.environ[prefix]
except KeyError:
try:
rootdir = os.environ[prefix[:-1]]
except KeyError:
raise KeyError("Reference location not defined for " + repr(instrument) +
". Did you configure " + repr(prefix) + "?")
elif mode == "instrument" and instrument != "synphot": # use simple names inside CRDS cache.
rootdir = os.path.join(crds_refpath, instrument)
refdir = os.path.join(crds_refpath, prefix[:-1])
if not os.path.exists(refdir):
if config.writable_cache_or_verbose("Skipping making instrument directory link for", repr(instrument)):
log.verbose("Creating legacy cache link", repr(refdir), "-->", repr(rootdir))
with log.verbose_warning_on_exception("Failed creating legacy symlink:", refdir, "-->", rootdir):
utils.ensure_dir_exists(rootdir + "/locate_dir.fits")
os.symlink(rootdir, refdir)
elif mode == "instrument" and instrument == "synphot":
rootdir = os.path.join(crds_refpath, instrument)
elif mode == "flat": # use original flat cache structure, all instruments in same directory.
rootdir = crds_refpath
else:
raise ValueError("Unhandled reference file location mode " + repr(mode))
return rootdir
def locate_dir(instrument, mode=None):
"""Locate the instrument specific directory for a reference file.
The mode=None test case is disabled because it mysteriously causes these tests to
fail when running the runtests script:
ERROR: test_throughput_lookup_generation (crds.tests.test_synphot_lookup_generator.TestSynphotLookupGenerator)
FAIL: Doctest: crds.tests.test_bad_files.dt_bad_references_fast_mode
FAIL: Doctest: crds.tests.test_bad_files.dt_bad_rules_jwst_getreferences_warning
FAIL: Doctest: crds.tests.test_certify.certify_recursive
FAIL: Doctest: crds.tests.test_certify.certify_table_comparison_context
FAIL: Doctest: crds.tests.test_heavy_client.dt_getreferences_ignore_cache
FAIL: Doctest: crds.tests.test_list.dt_list_cached_references
FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test
FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test
XXXX TODO: Enable the mode=None test case and resolve the ensuing test failures in other modules.
>> locate_dir('wfi', None) # doctest: +ELLIPSIS
'.../references/roman/wfi'
>>> locate_dir('wfi', 'instrument') # doctest: +ELLIPSIS
'.../references/roman/wfi'
>>> locate_dir('wfi', 'flat') # doctest: +ELLIPSIS
'.../references/roman'
>>> locate_dir('wfi', 'other') # doctest: +ELLIPSIS
Traceback (most recent call last):
...
AssertionError: Invalid CRDS cache subdirectory mode = 'other'
"""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="roman")
else:
config.check_crds_ref_subdir_mode(mode)
crds_refpath = config.get_crds_refpath("roman")
if mode == "instrument": # use simple names inside CRDS cache.
rootdir = os.path.join(crds_refpath, instrument.lower())
if not os.path.exists(rootdir):
if config.writable_cache_or_verbose(
"Skipping making instrument directory link for",
repr(instrument)):
utils.ensure_dir_exists(rootdir + "/locate_dir.fits")
elif mode == "flat": # use original flat cache structure, all instruments in same directory.
rootdir = crds_refpath
else:
raise ValueError("Unhandled reference file location mode " +
repr(mode))
return rootdir
def locate_file(refname, mode=None):
"""Given a valid reffilename in CDBS or CRDS format, return a cache path for the file.
The aspect of this which is complicated is determining instrument and an instrument
specific sub-directory for it based on the filename alone, not the file contents.
"""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="jwst")
if mode == "instrument":
instrument = utils.file_to_instrument(refname)
rootdir = locate_dir(instrument, mode)
elif mode == "flat":
rootdir = config.get_crds_refpath("jwst")
else:
raise ValueError("Unhandled reference file location mode " + repr(mode))
return os.path.join(rootdir, os.path.basename(refname))
def locate_dir(instrument, mode=None):
"""Locate the instrument specific directory for a reference file."""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="jwst")
else:
config.check_crds_ref_subdir_mode(mode)
crds_refpath = config.get_crds_refpath("jwst")
if mode == "instrument": # use simple names inside CRDS cache.
rootdir = os.path.join(crds_refpath, instrument.lower())
if not os.path.exists(rootdir):
if config.writable_cache_or_verbose("Skipping making instrument directory link for", repr(instrument)):
utils.ensure_dir_exists(rootdir + "/locate_dir.fits")
elif mode == "flat": # use original flat cache structure, all instruments in same directory.
rootdir = crds_refpath
else:
raise ValueError("Unhandled reference file location mode " + repr(mode))
return rootdir
def list_config(self):
"""Print out configuration info about the current environment and server."""
info = config.get_crds_env_vars()
real_paths = config.get_crds_actual_paths(self.observatory)
server = self.server_info
current_server_url = api.get_crds_server()
cache_subdir_mode = config.get_crds_ref_subdir_mode(self.observatory)
pyinfo = _get_python_info()
_print_dict("CRDS Environment", info)
_print_dict(
"CRDS Client Config", {
"server_url":
current_server_url,
"cache_subdir_mode":
cache_subdir_mode,
"readonly_cache":
self.readonly_cache,
"effective_context":
heavy_client.get_context_name(self.observatory),
"crds":
repr(crds),
"version":
heavy_client.version_info()
})
_print_dict("CRDS Actual Paths", real_paths)
_print_dict("CRDS Server Info", server, [
"observatory", "status", "connected", "operational_context",
"last_synced", "reference_url", "mapping_url", "effective_mode"
])
download_uris = dict(
config_uri=os.path.dirname(api.get_flex_uri("server_config")),
pickle_uri=os.path.dirname(api.get_flex_uri("xyz.pmap.pkl")),
mapping_uri=os.path.dirname(api.get_flex_uri("xyz.pmap")),
reference_uri=os.path.dirname(api.get_flex_uri("xyz.fits")),
)
_print_dict("Actual Download URIs", download_uris)
if self.observatory == "hst":
cal_vars = {
var: os.environ[var]
for var in os.environ
if len(var) == 4 and var.lower().endswith("ref")
}
_print_dict("Calibration Environment", cal_vars)
_print_dict("Python Environment", pyinfo)
def locate_file(refname, mode=None):
"""Given a valid reffilename in CDBS or CRDS format, return a cache path for the file.
The aspect of this which is complicated is determining instrument and an instrument
specific sub-directory for it based on the filename alone, not the file contents.
The mode=None test case is disabled because it mysteriously causes these tests to
fail when running the runtests script:
ERROR: test_throughput_lookup_generation (crds.tests.test_synphot_lookup_generator.TestSynphotLookupGenerator)
FAIL: Doctest: crds.tests.test_bad_files.dt_bad_references_fast_mode
FAIL: Doctest: crds.tests.test_bad_files.dt_bad_rules_jwst_getreferences_warning
FAIL: Doctest: crds.tests.test_certify.certify_recursive
FAIL: Doctest: crds.tests.test_certify.certify_table_comparison_context
FAIL: Doctest: crds.tests.test_heavy_client.dt_getreferences_ignore_cache
FAIL: Doctest: crds.tests.test_list.dt_list_cached_references
FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test
FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test
XXXX TODO: Enable the mode=None test case and resolve the ensuing test failures in other modules.
>> locate_file('tests/data/roman_wfi16_f158_flat_small.asdf', None) # doctest: +ELLIPSIS
'.../references/roman/wfi/roman_wfi16_f158_flat_small.asdf'
>>> locate_file('tests/data/roman_wfi16_f158_flat_small.asdf', 'instrument') # doctest: +ELLIPSIS
'.../references/roman/wfi/roman_wfi16_f158_flat_small.asdf'
>>> locate_file('tests/data/roman_wfi16_f158_flat_small.asdf', 'flat') # doctest: +ELLIPSIS
'.../references/roman/roman_wfi16_f158_flat_small.asdf'
>>> locate_file('tests/data/roman_wfi16_f158_flat_small.asdf', 'other') # doctest: +ELLIPSIS
Traceback (most recent call last):
...
ValueError: Unhandled reference file location mode 'other'
"""
if mode is None:
mode = config.get_crds_ref_subdir_mode(observatory="roman")
if mode == "instrument":
instrument = utils.file_to_instrument(refname)
rootdir = locate_dir(instrument, mode)
elif mode == "flat":
rootdir = config.get_crds_refpath("roman")
else:
raise ValueError("Unhandled reference file location mode " +
repr(mode))
return os.path.join(rootdir, os.path.basename(refname))
