def organize_references(self, new_mode): """Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`. This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument. """ old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True) old_mode = config.get_crds_ref_subdir_mode(self.observatory) log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode)) config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory) new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change. for refpath in old_refpaths: with log.error_on_exception("Failed relocating:", repr(refpath)): desired_loc = rmap.locate_file(os.path.basename(refpath), observatory=self.observatory) if desired_loc != refpath: if os.path.exists(desired_loc): if not self.args.organize_delete_junk: log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath)) continue utils.remove(desired_loc, observatory=self.observatory) if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)): log.info("Relocating", repr(refpath), "to", repr(desired_loc)) shutil.move(refpath, desired_loc) else: if old_mode != new_mode: log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode)) else: log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath)) if new_mode == "flat" and old_mode == "instrument": log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.") for instrument in self.locator.INSTRUMENTS: self.remove_dir(instrument)
def organize_references(self, new_mode): """Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`. This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument. """ old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True) old_mode = config.get_crds_ref_subdir_mode(self.observatory) log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode)) config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory) new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change. for refpath in old_refpaths: with log.error_on_exception("Failed relocating:", repr(refpath)): desired_loc = config.locate_file(os.path.basename(refpath), observatory=self.observatory) if desired_loc != refpath: if os.path.exists(desired_loc): if not self.args.organize_delete_junk: log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath)) continue utils.remove(desired_loc, observatory=self.observatory) if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)): log.info("Relocating", repr(refpath), "to", repr(desired_loc)) shutil.move(refpath, desired_loc) else: if old_mode != new_mode: log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode)) else: log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath)) if new_mode == "flat" and old_mode == "instrument": log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.") for instrument in self.locator.INSTRUMENTS: self.remove_dir(instrument)
def list_config(self): """Print out configuration info about the current environment and server.""" info = config.get_crds_env_vars() real_paths = config.get_crds_actual_paths(self.observatory) server = self.server_info current_server_url = api.get_crds_server() cache_subdir_mode = config.get_crds_ref_subdir_mode(self.observatory) pyinfo = _get_python_info() _print_dict("CRDS Environment", info) _print_dict("CRDS Client Config", { "server_url" : current_server_url, "cache_subdir_mode": cache_subdir_mode, "readonly_cache": self.readonly_cache, "effective_context": heavy_client.get_context_name(self.observatory), "crds" : repr(crds), "version": heavy_client.version_info() }) _print_dict("CRDS Actual Paths", real_paths) _print_dict("CRDS Server Info", server, ["observatory", "status", "connected", "operational_context", "last_synced", "reference_url", "mapping_url", "effective_mode"]) if self.observatory == "hst": cal_vars = { var : os.environ[var] for var in os.environ if len(var) == 4 and var.lower().endswith("ref") } _print_dict("Calibration Environment", cal_vars) _print_dict("Python Environment", pyinfo)
def locate_dir(instrument, mode=None): """Locate the instrument specific directory for a reference file.""" if mode is None: mode = config.get_crds_ref_subdir_mode(observatory="tobs") else: config.check_crds_ref_subdir_mode(mode) crds_refpath = config.get_crds_refpath("tobs") prefix = get_env_prefix(instrument) if mode == "legacy": # Locate cached files at the appropriate CDBS-style iref$ locations try: rootdir = os.environ[prefix] except KeyError: try: rootdir = os.environ[prefix[:-1]] except KeyError as exc: raise KeyError("Reference location not defined for", repr(instrument), ". Did you configure", repr(prefix) + "?") from exc elif mode == "instrument": # use simple names inside CRDS cache. rootdir = os.path.join(crds_refpath, instrument) refdir = os.path.join(crds_refpath, prefix[:-1]) if not os.path.exists(refdir): if config.writable_cache_or_verbose( "Skipping making instrument directory link for", repr(instrument)): log.verbose("Creating legacy cache link", repr(refdir), "-->", repr(rootdir)) utils.ensure_dir_exists(rootdir + "/locate_dir.fits") os.symlink(rootdir, refdir) elif mode == "flat": # use original flat cache structure, all instruments in same directory. rootdir = crds_refpath else: raise ValueError("Unhandled reference file location mode " + repr(mode)) return rootdir
def locate_dir(instrument, mode=None): """Locate the instrument specific directory for a reference file.""" if mode is None: mode = config.get_crds_ref_subdir_mode(observatory="hst") else: config.check_crds_ref_subdir_mode(mode) crds_refpath = config.get_crds_refpath("hst") prefix = get_env_prefix(instrument) if mode == "legacy": # Locate cached files at the appropriate CDBS-style iref$ locations try: rootdir = os.environ[prefix] except KeyError: try: rootdir = os.environ[prefix[:-1]] except KeyError: raise KeyError("Reference location not defined for " + repr(instrument) + ". Did you configure " + repr(prefix) + "?") elif mode == "instrument" and instrument != "synphot": # use simple names inside CRDS cache. rootdir = os.path.join(crds_refpath, instrument) refdir = os.path.join(crds_refpath, prefix[:-1]) if not os.path.exists(refdir): if config.writable_cache_or_verbose("Skipping making instrument directory link for", repr(instrument)): log.verbose("Creating legacy cache link", repr(refdir), "-->", repr(rootdir)) with log.verbose_warning_on_exception("Failed creating legacy symlink:", refdir, "-->", rootdir): utils.ensure_dir_exists(rootdir + "/locate_dir.fits") os.symlink(rootdir, refdir) elif mode == "instrument" and instrument == "synphot": rootdir = os.path.join(crds_refpath, instrument) elif mode == "flat": # use original flat cache structure, all instruments in same directory. rootdir = crds_refpath else: raise ValueError("Unhandled reference file location mode " + repr(mode)) return rootdir
def locate_dir(instrument, mode=None): """Locate the instrument specific directory for a reference file. The mode=None test case is disabled because it mysteriously causes these tests to fail when running the runtests script: ERROR: test_throughput_lookup_generation (crds.tests.test_synphot_lookup_generator.TestSynphotLookupGenerator) FAIL: Doctest: crds.tests.test_bad_files.dt_bad_references_fast_mode FAIL: Doctest: crds.tests.test_bad_files.dt_bad_rules_jwst_getreferences_warning FAIL: Doctest: crds.tests.test_certify.certify_recursive FAIL: Doctest: crds.tests.test_certify.certify_table_comparison_context FAIL: Doctest: crds.tests.test_heavy_client.dt_getreferences_ignore_cache FAIL: Doctest: crds.tests.test_list.dt_list_cached_references FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test XXXX TODO: Enable the mode=None test case and resolve the ensuing test failures in other modules. >> locate_dir('wfi', None) # doctest: +ELLIPSIS '.../references/roman/wfi' >>> locate_dir('wfi', 'instrument') # doctest: +ELLIPSIS '.../references/roman/wfi' >>> locate_dir('wfi', 'flat') # doctest: +ELLIPSIS '.../references/roman' >>> locate_dir('wfi', 'other') # doctest: +ELLIPSIS Traceback (most recent call last): ... AssertionError: Invalid CRDS cache subdirectory mode = 'other' """ if mode is None: mode = config.get_crds_ref_subdir_mode(observatory="roman") else: config.check_crds_ref_subdir_mode(mode) crds_refpath = config.get_crds_refpath("roman") if mode == "instrument": # use simple names inside CRDS cache. rootdir = os.path.join(crds_refpath, instrument.lower()) if not os.path.exists(rootdir): if config.writable_cache_or_verbose( "Skipping making instrument directory link for", repr(instrument)): utils.ensure_dir_exists(rootdir + "/locate_dir.fits") elif mode == "flat": # use original flat cache structure, all instruments in same directory. rootdir = crds_refpath else: raise ValueError("Unhandled reference file location mode " + repr(mode)) return rootdir
def locate_file(refname, mode=None): """Given a valid reffilename in CDBS or CRDS format, return a cache path for the file. The aspect of this which is complicated is determining instrument and an instrument specific sub-directory for it based on the filename alone, not the file contents. """ if mode is None: mode = config.get_crds_ref_subdir_mode(observatory="jwst") if mode == "instrument": instrument = utils.file_to_instrument(refname) rootdir = locate_dir(instrument, mode) elif mode == "flat": rootdir = config.get_crds_refpath("jwst") else: raise ValueError("Unhandled reference file location mode " + repr(mode)) return os.path.join(rootdir, os.path.basename(refname))
def locate_dir(instrument, mode=None): """Locate the instrument specific directory for a reference file.""" if mode is None: mode = config.get_crds_ref_subdir_mode(observatory="jwst") else: config.check_crds_ref_subdir_mode(mode) crds_refpath = config.get_crds_refpath("jwst") if mode == "instrument": # use simple names inside CRDS cache. rootdir = os.path.join(crds_refpath, instrument.lower()) if not os.path.exists(rootdir): if config.writable_cache_or_verbose("Skipping making instrument directory link for", repr(instrument)): utils.ensure_dir_exists(rootdir + "/locate_dir.fits") elif mode == "flat": # use original flat cache structure, all instruments in same directory. rootdir = crds_refpath else: raise ValueError("Unhandled reference file location mode " + repr(mode)) return rootdir
def list_config(self): """Print out configuration info about the current environment and server.""" info = config.get_crds_env_vars() real_paths = config.get_crds_actual_paths(self.observatory) server = self.server_info current_server_url = api.get_crds_server() cache_subdir_mode = config.get_crds_ref_subdir_mode(self.observatory) pyinfo = _get_python_info() _print_dict("CRDS Environment", info) _print_dict( "CRDS Client Config", { "server_url": current_server_url, "cache_subdir_mode": cache_subdir_mode, "readonly_cache": self.readonly_cache, "effective_context": heavy_client.get_context_name(self.observatory), "crds": repr(crds), "version": heavy_client.version_info() }) _print_dict("CRDS Actual Paths", real_paths) _print_dict("CRDS Server Info", server, [ "observatory", "status", "connected", "operational_context", "last_synced", "reference_url", "mapping_url", "effective_mode" ]) download_uris = dict( config_uri=os.path.dirname(api.get_flex_uri("server_config")), pickle_uri=os.path.dirname(api.get_flex_uri("xyz.pmap.pkl")), mapping_uri=os.path.dirname(api.get_flex_uri("xyz.pmap")), reference_uri=os.path.dirname(api.get_flex_uri("xyz.fits")), ) _print_dict("Actual Download URIs", download_uris) if self.observatory == "hst": cal_vars = { var: os.environ[var] for var in os.environ if len(var) == 4 and var.lower().endswith("ref") } _print_dict("Calibration Environment", cal_vars) _print_dict("Python Environment", pyinfo)
def locate_file(refname, mode=None): """Given a valid reffilename in CDBS or CRDS format, return a cache path for the file. The aspect of this which is complicated is determining instrument and an instrument specific sub-directory for it based on the filename alone, not the file contents. The mode=None test case is disabled because it mysteriously causes these tests to fail when running the runtests script: ERROR: test_throughput_lookup_generation (crds.tests.test_synphot_lookup_generator.TestSynphotLookupGenerator) FAIL: Doctest: crds.tests.test_bad_files.dt_bad_references_fast_mode FAIL: Doctest: crds.tests.test_bad_files.dt_bad_rules_jwst_getreferences_warning FAIL: Doctest: crds.tests.test_certify.certify_recursive FAIL: Doctest: crds.tests.test_certify.certify_table_comparison_context FAIL: Doctest: crds.tests.test_heavy_client.dt_getreferences_ignore_cache FAIL: Doctest: crds.tests.test_list.dt_list_cached_references FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test XXXX TODO: Enable the mode=None test case and resolve the ensuing test failures in other modules. >> locate_file('tests/data/roman_wfi16_f158_flat_small.asdf', None) # doctest: +ELLIPSIS '.../references/roman/wfi/roman_wfi16_f158_flat_small.asdf' >>> locate_file('tests/data/roman_wfi16_f158_flat_small.asdf', 'instrument') # doctest: +ELLIPSIS '.../references/roman/wfi/roman_wfi16_f158_flat_small.asdf' >>> locate_file('tests/data/roman_wfi16_f158_flat_small.asdf', 'flat') # doctest: +ELLIPSIS '.../references/roman/roman_wfi16_f158_flat_small.asdf' >>> locate_file('tests/data/roman_wfi16_f158_flat_small.asdf', 'other') # doctest: +ELLIPSIS Traceback (most recent call last): ... ValueError: Unhandled reference file location mode 'other' """ if mode is None: mode = config.get_crds_ref_subdir_mode(observatory="roman") if mode == "instrument": instrument = utils.file_to_instrument(refname) rootdir = locate_dir(instrument, mode) elif mode == "flat": rootdir = config.get_crds_refpath("roman") else: raise ValueError("Unhandled reference file location mode " + repr(mode)) return os.path.join(rootdir, os.path.basename(refname))