def get_config_info(observatory): """Get the operational context and server s/w version from (in order of priority): 1. The server. 2. The cache from a prior server access. 3. The basic CRDS installation. Return ConfigInfo """ try: info = ConfigInfo(api.get_server_info()) info.status = "server" info.connected = True log.verbose("Connected to server at", srepr(api.get_crds_server())) if info.effective_mode != "remote": if not config.writable_cache_or_verbose("Using cached configuration and default context."): info = load_server_info(observatory) info.status = "cache" info.connected = True log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced)) except CrdsError as exc: if "serverless" not in api.get_crds_server(): log.verbose_warning("Couldn't contact CRDS server:", srepr(api.get_crds_server()), ":", str(exc)) info = load_server_info(observatory) info.status = "cache" info.connected = False log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced)) return info
def get_config_info(observatory): """Get the operational context and server s/w version from (in order of priority): 1. The server. 2. The cache from a prior server access. 3. The basic CRDS installation. Return ConfigInfo """ try: info = ConfigInfo(api.get_server_info()) if info.effective_mode != "remote": if not config.writable_cache_or_verbose("Using cached configuration and default context."): info = load_server_info(observatory) info.status = "cache" info.connected = True log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced)) except CrdsError as exc: if "serverless" not in api.get_crds_server(): log.verbose_warning("Couldn't contact CRDS server:", srepr(api.get_crds_server()), ":", str(exc)) info = load_server_info(observatory) info.status = "cache" info.connected = False log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced)) if info.observatory != observatory: raise CrdsConfigError( "CRDS server at", repr(api.get_crds_server()), "is inconsistent with observatory", repr(observatory) + ".", "You may be configured for the wrong project. " "Check CRDS_SERVER_URL and CRDS_OBSERVATORY " "environment settings. See https://jwst-crds.stsci.edu/docs/cmdline_bestrefs/ (JWST) " "or https://hst-crds.stsci.edu/docs/cmdline_bestrefs/ (HST) for information on configuring CRDS.") return info
def get_config_info(observatory): """Get the operational context and server s/w version from (in order of priority): 1. The server. 2. The cache from a prior server access. 3. The basic CRDS installation. Return ConfigInfo """ try: info = ConfigInfo(api.get_server_info()) info.status = "server" info.connected = True log.verbose("Connected to server at", srepr(api.get_crds_server())) if info.effective_mode != "remote": if not config.writable_cache_or_verbose( "Using cached configuration and default context."): info = load_server_info(observatory) info.status = "cache" info.connected = True log.verbose( "Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced)) except CrdsError as exc: if "serverless" not in api.get_crds_server(): log.verbose_warning("Couldn't contact CRDS server:", srepr(api.get_crds_server()), ":", str(exc)) info = load_server_info(observatory) info.status = "cache" info.connected = False log.verbose( "Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced)) return info
def list_config(self): """Print out configuration info about the current environment and server.""" info = config.get_crds_env_vars() real_paths = config.get_crds_actual_paths(self.observatory) server = self.server_info current_server_url = api.get_crds_server() cache_subdir_mode = config.get_crds_ref_subdir_mode(self.observatory) pyinfo = _get_python_info() _print_dict("CRDS Environment", info) _print_dict("CRDS Client Config", { "server_url" : current_server_url, "cache_subdir_mode": cache_subdir_mode, "readonly_cache": self.readonly_cache, "effective_context": heavy_client.get_context_name(self.observatory), "crds" : repr(crds), "version": heavy_client.version_info() }) _print_dict("CRDS Actual Paths", real_paths) _print_dict("CRDS Server Info", server, ["observatory", "status", "connected", "operational_context", "last_synced", "reference_url", "mapping_url", "effective_mode"]) if self.observatory == "hst": cal_vars = { var : os.environ[var] for var in os.environ if len(var) == 4 and var.lower().endswith("ref") } _print_dict("Calibration Environment", cal_vars) _print_dict("Python Environment", pyinfo)
def list_config(self): """Print out configuration info about the current environment and server.""" info = config.get_crds_env_vars() real_paths = config.get_crds_actual_paths(self.observatory) server = self.server_info current_server_url = api.get_crds_server() cache_subdir_mode = config.get_crds_ref_subdir_mode(self.observatory) pyinfo = _get_python_info() _print_dict("CRDS Environment", info) _print_dict("CRDS Client Config", { "server_url" : current_server_url, "cache_subdir_mode": cache_subdir_mode, "readonly_cache": self.readonly_cache, "effective_context": heavy_client.get_processing_mode(self.observatory)[1], "crds" : repr(crds), "version": heavy_client.version_info() }) _print_dict("CRDS Actual Paths", real_paths) _print_dict("CRDS Server Info", server, ["observatory", "status", "connected", "operational_context", "last_synced", "reference_url", "mapping_url", "effective_mode"]) if self.observatory == "hst": cal_vars = { var : os.environ[var] for var in os.environ if len(var) == 4 and var.lower().endswith("ref") } _print_dict("Calibration Environment", cal_vars) _print_dict("Python Environment", pyinfo)
def require_server_connection(self): """Check a *required* server connection and ERROR/exit if offline.""" try: if not self.server_info.connected: raise RuntimeError("Required server connection unavailable.") except Exception as exc: self.fatal_error("Failed connecting to CRDS server at CRDS_SERVER_URL =", repr(api.get_crds_server()), "::", str(exc))
def _initial_recommendations(name, parameters, reftypes=None, context=None, ignore_cache=False, observatory="jwst", fast=False): """shared logic for getreferences() and getrecommendations().""" if not fast: log.verbose("=" * 120) log.verbose(name + "() CRDS version: ", version_info()) log.verbose(name + "() server:", api.get_crds_server()) log.verbose(name + "() observatory:", observatory) log.verbose(name + "() parameters:\n", log.PP(parameters), verbosity=65) log.verbose(name + "() reftypes:", reftypes) log.verbose(name + "() context:", repr(context)) log.verbose(name + "() ignore_cache:", ignore_cache) for var in os.environ: if var.upper().startswith("CRDS"): log.verbose(var, "=", repr(os.environ[var])) check_observatory(observatory) check_parameters(parameters) check_reftypes(reftypes) check_context(context) mode, final_context = get_processing_mode(observatory, context) log.verbose("Final effective context is", repr(final_context)) if mode == "local": log.verbose("Computing best references locally.") bestrefs = local_bestrefs(parameters, reftypes=reftypes, context=final_context, ignore_cache=ignore_cache) else: log.verbose("Computing best references remotely.") bestrefs = api.get_best_references(final_context, parameters, reftypes=reftypes) if not fast: # nominally fast=True (this is skipped) in crds.bestrefs, used for HST and reprocessing # because bestrefs sreprocessing determinations for two contexts which run on 100's of # thousands of datasets and should only report bad files once and only when arising from # the new vs. the old context. update_config_info(observatory) log.verbose(name + "() results:\n", log.PP(bestrefs), verbosity=65) instrument = utils.header_to_instrument(parameters) warn_bad_context(observatory, final_context, instrument) warn_bad_references(observatory, bestrefs) return final_context, bestrefs
def _initial_recommendations( name, parameters, reftypes=None, context=None, ignore_cache=False, observatory="jwst", fast=False): """shared logic for getreferences() and getrecommendations().""" if not fast: log.verbose("="*120) log.verbose(name + "() CRDS version: ", version_info()) log.verbose(name + "() server:", api.get_crds_server()) log.verbose(name + "() observatory:", observatory) log.verbose(name + "() parameters:\n", log.PP(parameters), verbosity=65) log.verbose(name + "() reftypes:", reftypes) log.verbose(name + "() context:", repr(context)) log.verbose(name + "() ignore_cache:", ignore_cache) # for var in os.environ: # if var.upper().startswith("CRDS"): # log.verbose(var, "=", repr(os.environ[var])) log.verbose("CRDS_PATH =", os.environ.get("CRDS_PATH", "UNDEFINED")) log.verbose("CRDS_SERVER_URL =", os.environ.get("CRDS_SERVER_URL", "UNDEFINED")) check_observatory(observatory) parameters = check_parameters(parameters) check_reftypes(reftypes) check_context(context) mode, final_context = get_processing_mode(observatory, context) log.verbose("Final effective context is", repr(final_context)) if mode == "local": log.verbose("Computing best references locally.") bestrefs = local_bestrefs( parameters, reftypes=reftypes, context=final_context, ignore_cache=ignore_cache) else: log.verbose("Computing best references remotely.") bestrefs = api.get_best_references(final_context, parameters, reftypes=reftypes) if not fast: # nominally fast=True (this is skipped) in crds.bestrefs, used for HST and reprocessing # because bestrefs sreprocessing determinations for two contexts which run on 100's of # thousands of datasets and should only report bad files once and only when arising from # the new vs. the old context. update_config_info(observatory) log.verbose(name + "() results:\n", log.PP(bestrefs), verbosity=65) instrument = utils.header_to_instrument(parameters) warn_bad_context(observatory, final_context, instrument) warn_bad_references(observatory, bestrefs) return final_context, bestrefs
def determine_source_ids(self): """Return the dataset ids for all instruments.""" server = api.get_crds_server() source_ids = [] for instrument in self.instruments: since_date = self.datasets_since(instrument) if since_date: log.info("Dumping dataset parameters for", repr(instrument), "from CRDS server at", repr(server), "since", repr(since_date)) else: log.info("Dumping dataset parameters for", repr(instrument), "from CRDS server at", repr(server)) instr_ids = api.get_dataset_ids(self.context, instrument, self.datasets_since(instrument)) log.info("Downloaded ", len(instr_ids), "dataset ids for", repr(instrument), "since", repr(since_date)) source_ids.extend(instr_ids) return sorted(source_ids) # sort is needed to match generic __iter__() sort. assumes instruments don't shuffle
def __init__(self, context, datasets, datasets_since): """"Contact the CRDS server and get headers for the list of `datasets` ids with respect to `context`.""" super(DatasetHeaderGenerator, self).__init__(context, datasets, datasets_since) server = api.get_crds_server() log.info("Dumping dataset parameters from CRDS server at", repr(server), "for", repr(datasets)) self.headers = api.get_dataset_headers_by_id(context, datasets) log.info("Dumped", len(self.headers), "of", len(datasets), "datasets from CRDS server at", repr(server)) # every command line id should correspond to 1 or more headers for source in self.sources: if self.matching_two_part_id(source) not in self.headers.keys(): log.warning("Dataset", repr(source), "isn't represented by downloaded parameters.") # Process according to downloaded 2-part ids, not command line ids. self.sources = sorted(self.headers.keys())
def get_config_info(observatory): """Get the operational context and server s/w version from (in order of priority): 1. The server. 2. The cache from a prior server access. 3. The basic CRDS installation. Return ConfigInfo """ try: info = ConfigInfo(api.get_server_info()) info.status = "server" info.connected = True log.verbose("Connected to server at", srepr(api.get_crds_server())) if info.effective_mode != "remote": if not config.writable_cache_or_verbose("Using cached configuration and default context."): info = load_server_info(observatory) info.status = "cache" info.connected = True log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced)) except CrdsError as exc: if "serverless" not in api.get_crds_server(): log.verbose_warning("Couldn't contact CRDS server:", srepr(api.get_crds_server()), ":", str(exc)) info = load_server_info(observatory) info.status = "cache" info.connected = False log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced)) if info.observatory != observatory: raise CrdsConfigError( "CRDS server at", repr(api.get_crds_server()), "is inconsistent with observatory", repr(observatory) + ".", "You may be configured for the wrong project. " "Check CRDS_SERVER_URL and CRDS_OBSERVATORY " "environment settings. See https://jwst-crds.stsci.edu/docs/cmdline_bestrefs/ (JWST) " "or https://hst-crds.stsci.edu/docs/cmdline_bestrefs/ (HST) for information on configuring CRDS.") return info
def list_status(self): """Print out *basic* configuration info about the current environment and server.""" info = config.get_crds_env_vars() real_paths = config.get_crds_actual_paths(self.observatory) server = self.server_info current_server_url = api.get_crds_server() cache_subdir_mode = config.get_crds_ref_subdir_mode(self.observatory) pyinfo = _get_python_info() status = OrderedDict( [("CRDS_PATH", info.get("CRDS_PATH", "undefined")), ("CRDS_SERVER_URL", info.get("CRDS_SERVER_URL", "undefined")), ("CRDS_MODE", info["CRDS_MODE"]), ("Readonly Cache", self.readonly_cache), ("Effective Context", heavy_client.get_processing_mode(self.observatory)[1]), ("Last Synced", server.last_synced), ("CRDS Version", heavy_client.version_info()), ("Python Version", pyinfo["Python Version"]), ("Python Executable", pyinfo["Python Executable"]), ]) _print_dict(None, status)