def get_config_info(observatory):
"""Get the operational context and server s/w version from (in order of priority):
1. The server.
2. The cache from a prior server access.
3. The basic CRDS installation.
Return ConfigInfo
"""
try:
info = ConfigInfo(api.get_server_info())
info.status = "server"
info.connected = True
log.verbose("Connected to server at", srepr(api.get_crds_server()))
if info.effective_mode != "remote":
if not config.writable_cache_or_verbose("Using cached configuration and default context."):
info = load_server_info(observatory)
info.status = "cache"
info.connected = True
log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced))
except CrdsError as exc:
if "serverless" not in api.get_crds_server():
log.verbose_warning("Couldn't contact CRDS server:", srepr(api.get_crds_server()), ":", str(exc))
info = load_server_info(observatory)
info.status = "cache"
info.connected = False
log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced))
return info
def get_config_info(observatory):
"""Get the operational context and server s/w version from (in order of priority):
1. The server.
2. The cache from a prior server access.
3. The basic CRDS installation.
Return ConfigInfo
"""
try:
info = ConfigInfo(api.get_server_info())
if info.effective_mode != "remote":
if not config.writable_cache_or_verbose("Using cached configuration and default context."):
info = load_server_info(observatory)
info.status = "cache"
info.connected = True
log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced))
except CrdsError as exc:
if "serverless" not in api.get_crds_server():
log.verbose_warning("Couldn't contact CRDS server:", srepr(api.get_crds_server()), ":", str(exc))
info = load_server_info(observatory)
info.status = "cache"
info.connected = False
log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced))
if info.observatory != observatory:
raise CrdsConfigError(
"CRDS server at", repr(api.get_crds_server()),
"is inconsistent with observatory", repr(observatory) + ".",
"You may be configured for the wrong project. "
"Check CRDS_SERVER_URL and CRDS_OBSERVATORY "
"environment settings. See https://jwst-crds.stsci.edu/docs/cmdline_bestrefs/ (JWST) "
"or https://hst-crds.stsci.edu/docs/cmdline_bestrefs/ (HST) for information on configuring CRDS.")
return info
def get_config_info(observatory):
"""Get the operational context and server s/w version from (in order of priority):
1. The server.
2. The cache from a prior server access.
3. The basic CRDS installation.
Return ConfigInfo
"""
try:
info = ConfigInfo(api.get_server_info())
info.status = "server"
info.connected = True
log.verbose("Connected to server at", srepr(api.get_crds_server()))
if info.effective_mode != "remote":
if not config.writable_cache_or_verbose(
"Using cached configuration and default context."):
info = load_server_info(observatory)
info.status = "cache"
info.connected = True
log.verbose(
"Using CACHED CRDS reference assignment rules last updated on",
repr(info.last_synced))
except CrdsError as exc:
if "serverless" not in api.get_crds_server():
log.verbose_warning("Couldn't contact CRDS server:",
srepr(api.get_crds_server()), ":", str(exc))
info = load_server_info(observatory)
info.status = "cache"
info.connected = False
log.verbose(
"Using CACHED CRDS reference assignment rules last updated on",
repr(info.last_synced))
return info
def list_config(self):
"""Print out configuration info about the current environment and server."""
info = config.get_crds_env_vars()
real_paths = config.get_crds_actual_paths(self.observatory)
server = self.server_info
current_server_url = api.get_crds_server()
cache_subdir_mode = config.get_crds_ref_subdir_mode(self.observatory)
pyinfo = _get_python_info()
_print_dict("CRDS Environment", info)
_print_dict("CRDS Client Config", {
"server_url" : current_server_url,
"cache_subdir_mode": cache_subdir_mode,
"readonly_cache": self.readonly_cache,
"effective_context": heavy_client.get_context_name(self.observatory),
"crds" : repr(crds),
"version": heavy_client.version_info()
})
_print_dict("CRDS Actual Paths", real_paths)
_print_dict("CRDS Server Info", server,
["observatory", "status", "connected", "operational_context", "last_synced",
"reference_url", "mapping_url", "effective_mode"])
if self.observatory == "hst":
cal_vars = { var : os.environ[var] for var in os.environ
if len(var) == 4 and var.lower().endswith("ref") }
_print_dict("Calibration Environment", cal_vars)
_print_dict("Python Environment", pyinfo)
def list_config(self):
"""Print out configuration info about the current environment and server."""
info = config.get_crds_env_vars()
real_paths = config.get_crds_actual_paths(self.observatory)
server = self.server_info
current_server_url = api.get_crds_server()
cache_subdir_mode = config.get_crds_ref_subdir_mode(self.observatory)
pyinfo = _get_python_info()
_print_dict("CRDS Environment", info)
_print_dict("CRDS Client Config", {
"server_url" : current_server_url,
"cache_subdir_mode": cache_subdir_mode,
"readonly_cache": self.readonly_cache,
"effective_context": heavy_client.get_processing_mode(self.observatory)[1],
"crds" : repr(crds),
"version": heavy_client.version_info()
})
_print_dict("CRDS Actual Paths", real_paths)
_print_dict("CRDS Server Info", server,
["observatory", "status", "connected", "operational_context", "last_synced",
"reference_url", "mapping_url", "effective_mode"])
if self.observatory == "hst":
cal_vars = { var : os.environ[var] for var in os.environ
if len(var) == 4 and var.lower().endswith("ref") }
_print_dict("Calibration Environment", cal_vars)
_print_dict("Python Environment", pyinfo)
def require_server_connection(self):
"""Check a *required* server connection and ERROR/exit if offline."""
try:
if not self.server_info.connected:
raise RuntimeError("Required server connection unavailable.")
except Exception as exc:
self.fatal_error("Failed connecting to CRDS server at CRDS_SERVER_URL =",
repr(api.get_crds_server()), "::", str(exc))
def require_server_connection(self):
"""Check a *required* server connection and ERROR/exit if offline."""
try:
if not self.server_info.connected:
raise RuntimeError("Required server connection unavailable.")
except Exception as exc:
self.fatal_error("Failed connecting to CRDS server at CRDS_SERVER_URL =",
repr(api.get_crds_server()), "::", str(exc))
def _initial_recommendations(name,
parameters,
reftypes=None,
context=None,
ignore_cache=False,
observatory="jwst",
fast=False):
"""shared logic for getreferences() and getrecommendations()."""
if not fast:
log.verbose("=" * 120)
log.verbose(name + "() CRDS version: ", version_info())
log.verbose(name + "() server:", api.get_crds_server())
log.verbose(name + "() observatory:", observatory)
log.verbose(name + "() parameters:\n",
log.PP(parameters),
verbosity=65)
log.verbose(name + "() reftypes:", reftypes)
log.verbose(name + "() context:", repr(context))
log.verbose(name + "() ignore_cache:", ignore_cache)
for var in os.environ:
if var.upper().startswith("CRDS"):
log.verbose(var, "=", repr(os.environ[var]))
check_observatory(observatory)
check_parameters(parameters)
check_reftypes(reftypes)
check_context(context)
mode, final_context = get_processing_mode(observatory, context)
log.verbose("Final effective context is", repr(final_context))
if mode == "local":
log.verbose("Computing best references locally.")
bestrefs = local_bestrefs(parameters,
reftypes=reftypes,
context=final_context,
ignore_cache=ignore_cache)
else:
log.verbose("Computing best references remotely.")
bestrefs = api.get_best_references(final_context,
parameters,
reftypes=reftypes)
if not fast:
# nominally fast=True (this is skipped) in crds.bestrefs, used for HST and reprocessing
# because bestrefs sreprocessing determinations for two contexts which run on 100's of
# thousands of datasets and should only report bad files once and only when arising from
# the new vs. the old context.
update_config_info(observatory)
log.verbose(name + "() results:\n", log.PP(bestrefs), verbosity=65)
instrument = utils.header_to_instrument(parameters)
warn_bad_context(observatory, final_context, instrument)
warn_bad_references(observatory, bestrefs)
return final_context, bestrefs
def _initial_recommendations(
name, parameters, reftypes=None, context=None, ignore_cache=False, observatory="jwst", fast=False):
"""shared logic for getreferences() and getrecommendations()."""
if not fast:
log.verbose("="*120)
log.verbose(name + "() CRDS version: ", version_info())
log.verbose(name + "() server:", api.get_crds_server())
log.verbose(name + "() observatory:", observatory)
log.verbose(name + "() parameters:\n", log.PP(parameters), verbosity=65)
log.verbose(name + "() reftypes:", reftypes)
log.verbose(name + "() context:", repr(context))
log.verbose(name + "() ignore_cache:", ignore_cache)
# for var in os.environ:
# if var.upper().startswith("CRDS"):
# log.verbose(var, "=", repr(os.environ[var]))
log.verbose("CRDS_PATH =", os.environ.get("CRDS_PATH", "UNDEFINED"))
log.verbose("CRDS_SERVER_URL =", os.environ.get("CRDS_SERVER_URL", "UNDEFINED"))
check_observatory(observatory)
parameters = check_parameters(parameters)
check_reftypes(reftypes)
check_context(context)
mode, final_context = get_processing_mode(observatory, context)
log.verbose("Final effective context is", repr(final_context))
if mode == "local":
log.verbose("Computing best references locally.")
bestrefs = local_bestrefs(
parameters, reftypes=reftypes, context=final_context, ignore_cache=ignore_cache)
else:
log.verbose("Computing best references remotely.")
bestrefs = api.get_best_references(final_context, parameters, reftypes=reftypes)
if not fast:
# nominally fast=True (this is skipped) in crds.bestrefs, used for HST and reprocessing
# because bestrefs sreprocessing determinations for two contexts which run on 100's of
# thousands of datasets and should only report bad files once and only when arising from
# the new vs. the old context.
update_config_info(observatory)
log.verbose(name + "() results:\n", log.PP(bestrefs), verbosity=65)
instrument = utils.header_to_instrument(parameters)
warn_bad_context(observatory, final_context, instrument)
warn_bad_references(observatory, bestrefs)
return final_context, bestrefs
def determine_source_ids(self):
"""Return the dataset ids for all instruments."""
server = api.get_crds_server()
source_ids = []
for instrument in self.instruments:
since_date = self.datasets_since(instrument)
if since_date:
log.info("Dumping dataset parameters for", repr(instrument), "from CRDS server at", repr(server),
"since", repr(since_date))
else:
log.info("Dumping dataset parameters for", repr(instrument), "from CRDS server at", repr(server))
instr_ids = api.get_dataset_ids(self.context, instrument, self.datasets_since(instrument))
log.info("Downloaded ", len(instr_ids), "dataset ids for", repr(instrument), "since", repr(since_date))
source_ids.extend(instr_ids)
return sorted(source_ids) # sort is needed to match generic __iter__() sort. assumes instruments don't shuffle
def __init__(self, context, datasets, datasets_since):
""""Contact the CRDS server and get headers for the list of `datasets` ids with respect to `context`."""
super(DatasetHeaderGenerator, self).__init__(context, datasets, datasets_since)
server = api.get_crds_server()
log.info("Dumping dataset parameters from CRDS server at", repr(server), "for", repr(datasets))
self.headers = api.get_dataset_headers_by_id(context, datasets)
log.info("Dumped", len(self.headers), "of", len(datasets), "datasets from CRDS server at", repr(server))
# every command line id should correspond to 1 or more headers
for source in self.sources:
if self.matching_two_part_id(source) not in self.headers.keys():
log.warning("Dataset", repr(source), "isn't represented by downloaded parameters.")
# Process according to downloaded 2-part ids, not command line ids.
self.sources = sorted(self.headers.keys())
def get_config_info(observatory):
"""Get the operational context and server s/w version from (in order of priority):
1. The server.
2. The cache from a prior server access.
3. The basic CRDS installation.
Return ConfigInfo
"""
try:
info = ConfigInfo(api.get_server_info())
info.status = "server"
info.connected = True
log.verbose("Connected to server at", srepr(api.get_crds_server()))
if info.effective_mode != "remote":
if not config.writable_cache_or_verbose("Using cached configuration and default context."):
info = load_server_info(observatory)
info.status = "cache"
info.connected = True
log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced))
except CrdsError as exc:
if "serverless" not in api.get_crds_server():
log.verbose_warning("Couldn't contact CRDS server:", srepr(api.get_crds_server()), ":", str(exc))
info = load_server_info(observatory)
info.status = "cache"
info.connected = False
log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced))
if info.observatory != observatory:
raise CrdsConfigError(
"CRDS server at", repr(api.get_crds_server()),
"is inconsistent with observatory", repr(observatory) + ".",
"You may be configured for the wrong project. "
"Check CRDS_SERVER_URL and CRDS_OBSERVATORY "
"environment settings. See https://jwst-crds.stsci.edu/docs/cmdline_bestrefs/ (JWST) "
"or https://hst-crds.stsci.edu/docs/cmdline_bestrefs/ (HST) for information on configuring CRDS.")
return info
def list_status(self):
"""Print out *basic* configuration info about the current environment and server."""
info = config.get_crds_env_vars()
real_paths = config.get_crds_actual_paths(self.observatory)
server = self.server_info
current_server_url = api.get_crds_server()
cache_subdir_mode = config.get_crds_ref_subdir_mode(self.observatory)
pyinfo = _get_python_info()
status = OrderedDict(
[("CRDS_PATH", info.get("CRDS_PATH", "undefined")),
("CRDS_SERVER_URL", info.get("CRDS_SERVER_URL", "undefined")),
("CRDS_MODE", info["CRDS_MODE"]),
("Readonly Cache", self.readonly_cache),
("Effective Context", heavy_client.get_processing_mode(self.observatory)[1]),
("Last Synced", server.last_synced),
("CRDS Version", heavy_client.version_info()),
("Python Version", pyinfo["Python Version"]),
("Python Executable", pyinfo["Python Executable"]),
])
_print_dict(None, status)
def list_status(self):
"""Print out *basic* configuration info about the current environment and server."""
info = config.get_crds_env_vars()
real_paths = config.get_crds_actual_paths(self.observatory)
server = self.server_info
current_server_url = api.get_crds_server()
cache_subdir_mode = config.get_crds_ref_subdir_mode(self.observatory)
pyinfo = _get_python_info()
status = OrderedDict(
[("CRDS_PATH", info.get("CRDS_PATH", "undefined")),
("CRDS_SERVER_URL", info.get("CRDS_SERVER_URL", "undefined")),
("CRDS_MODE", info["CRDS_MODE"]),
("Readonly Cache", self.readonly_cache),
("Effective Context", heavy_client.get_processing_mode(self.observatory)[1]),
("Last Synced", server.last_synced),
("CRDS Version", heavy_client.version_info()),
("Python Version", pyinfo["Python Version"]),
("Python Executable", pyinfo["Python Executable"]),
])
_print_dict(None, status)
