def locate_dir(instrument, mode=None): """Locate the instrument specific directory for a reference file.""" if mode is None: mode = config.get_crds_ref_subdir_mode(observatory="tobs") else: config.check_crds_ref_subdir_mode(mode) crds_refpath = config.get_crds_refpath("tobs") prefix = get_env_prefix(instrument) if mode == "legacy": # Locate cached files at the appropriate CDBS-style iref$ locations try: rootdir = os.environ[prefix] except KeyError: try: rootdir = os.environ[prefix[:-1]] except KeyError as exc: raise KeyError("Reference location not defined for", repr(instrument), ". Did you configure", repr(prefix) + "?") from exc elif mode == "instrument": # use simple names inside CRDS cache. rootdir = os.path.join(crds_refpath, instrument) refdir = os.path.join(crds_refpath, prefix[:-1]) if not os.path.exists(refdir): if config.writable_cache_or_verbose( "Skipping making instrument directory link for", repr(instrument)): log.verbose("Creating legacy cache link", repr(refdir), "-->", repr(rootdir)) utils.ensure_dir_exists(rootdir + "/locate_dir.fits") os.symlink(rootdir, refdir) elif mode == "flat": # use original flat cache structure, all instruments in same directory. rootdir = crds_refpath else: raise ValueError("Unhandled reference file location mode " + repr(mode)) return rootdir
def locate_dir(instrument, mode=None): """Locate the instrument specific directory for a reference file.""" if mode is None: mode = config.get_crds_ref_subdir_mode(observatory="hst") else: config.check_crds_ref_subdir_mode(mode) crds_refpath = config.get_crds_refpath("hst") prefix = get_env_prefix(instrument) if mode == "legacy": # Locate cached files at the appropriate CDBS-style iref$ locations try: rootdir = os.environ[prefix] except KeyError: try: rootdir = os.environ[prefix[:-1]] except KeyError: raise KeyError("Reference location not defined for " + repr(instrument) + ". Did you configure " + repr(prefix) + "?") elif mode == "instrument" and instrument != "synphot": # use simple names inside CRDS cache. rootdir = os.path.join(crds_refpath, instrument) refdir = os.path.join(crds_refpath, prefix[:-1]) if not os.path.exists(refdir): if config.writable_cache_or_verbose("Skipping making instrument directory link for", repr(instrument)): log.verbose("Creating legacy cache link", repr(refdir), "-->", repr(rootdir)) with log.verbose_warning_on_exception("Failed creating legacy symlink:", refdir, "-->", rootdir): utils.ensure_dir_exists(rootdir + "/locate_dir.fits") os.symlink(rootdir, refdir) elif mode == "instrument" and instrument == "synphot": rootdir = os.path.join(crds_refpath, instrument) elif mode == "flat": # use original flat cache structure, all instruments in same directory. rootdir = crds_refpath else: raise ValueError("Unhandled reference file location mode " + repr(mode)) return rootdir
def _write_last_processed(self, hist): """Write down the history tuple of the last context processed.""" log.verbose("Saving last processed:", repr(hist)) log.verbose("Storing last processed state at", repr(self.last_processed_path)) utils.ensure_dir_exists(self.last_processed_path) with open(self.last_processed_path, "w+") as last: last.write(str(hist))
def get_sqlite_db(observatory): """Download the CRDS database as a SQLite database.""" assert not config.get_cache_readonly(), "Readonly cache, updating the SQLite database cannot be done.""" encoded_compressed_data = S.get_sqlite_db(observatory) data = zlib.decompress(base64.b64decode(encoded_compressed_data)) path = config.get_sqlite3_db_path(observatory) utils.ensure_dir_exists(path) with open(path, "wb+") as db_out: db_out.write(data) return path
def get_sqlite_db(observatory): """Download the CRDS database as a SQLite database.""" assert not config.get_cache_readonly(), "Readonly cache, updating the SQLite database cannot be done.""" encoded_compressed_data = S.get_sqlite_db(observatory) data = zlib.decompress(base64.b64decode(encoded_compressed_data)) path = config.get_sqlite3_db_path(observatory) utils.ensure_dir_exists(path) with open(path, "wb+") as db_out: db_out.write(data) return path
def locate_dir(instrument, mode=None): """Locate the instrument specific directory for a reference file. The mode=None test case is disabled because it mysteriously causes these tests to fail when running the runtests script: ERROR: test_throughput_lookup_generation (crds.tests.test_synphot_lookup_generator.TestSynphotLookupGenerator) FAIL: Doctest: crds.tests.test_bad_files.dt_bad_references_fast_mode FAIL: Doctest: crds.tests.test_bad_files.dt_bad_rules_jwst_getreferences_warning FAIL: Doctest: crds.tests.test_certify.certify_recursive FAIL: Doctest: crds.tests.test_certify.certify_table_comparison_context FAIL: Doctest: crds.tests.test_heavy_client.dt_getreferences_ignore_cache FAIL: Doctest: crds.tests.test_list.dt_list_cached_references FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test FAIL: Doctest: crds.tests.test_synphot_hst.dt_synphot_core_integration_test XXXX TODO: Enable the mode=None test case and resolve the ensuing test failures in other modules. >> locate_dir('wfi', None) # doctest: +ELLIPSIS '.../references/roman/wfi' >>> locate_dir('wfi', 'instrument') # doctest: +ELLIPSIS '.../references/roman/wfi' >>> locate_dir('wfi', 'flat') # doctest: +ELLIPSIS '.../references/roman' >>> locate_dir('wfi', 'other') # doctest: +ELLIPSIS Traceback (most recent call last): ... AssertionError: Invalid CRDS cache subdirectory mode = 'other' """ if mode is None: mode = config.get_crds_ref_subdir_mode(observatory="roman") else: config.check_crds_ref_subdir_mode(mode) crds_refpath = config.get_crds_refpath("roman") if mode == "instrument": # use simple names inside CRDS cache. rootdir = os.path.join(crds_refpath, instrument.lower()) if not os.path.exists(rootdir): if config.writable_cache_or_verbose( "Skipping making instrument directory link for", repr(instrument)): utils.ensure_dir_exists(rootdir + "/locate_dir.fits") elif mode == "flat": # use original flat cache structure, all instruments in same directory. rootdir = crds_refpath else: raise ValueError("Unhandled reference file location mode " + repr(mode)) return rootdir
def locate_dir(instrument, mode=None): """Locate the instrument specific directory for a reference file.""" if mode is None: mode = config.get_crds_ref_subdir_mode(observatory="jwst") else: config.check_crds_ref_subdir_mode(mode) crds_refpath = config.get_crds_refpath("jwst") if mode == "instrument": # use simple names inside CRDS cache. rootdir = os.path.join(crds_refpath, instrument.lower()) if not os.path.exists(rootdir): if config.writable_cache_or_verbose("Skipping making instrument directory link for", repr(instrument)): utils.ensure_dir_exists(rootdir + "/locate_dir.fits") elif mode == "flat": # use original flat cache structure, all instruments in same directory. rootdir = crds_refpath else: raise ValueError("Unhandled reference file location mode " + repr(mode)) return rootdir
def download(self, name, localpath): """Download a single file.""" # This code is complicated by the desire to blow away failed downloads. For the specific # case of KeyboardInterrupt, the file needs to be blown away, but the interrupt should not # be re-characterized so it is still un-trapped elsewhere under normal idioms which try *not* # to trap KeyboardInterrupt. assert not config.get_cache_readonly(), "Readonly cache, cannot download files " + repr(name) try: utils.ensure_dir_exists(localpath) return proxy.apply_with_retries(self.download_core, name, localpath) except Exception as exc: self.remove_file(localpath) raise CrdsDownloadError( "Error fetching data for", srepr(name), "at CRDS server", srepr(get_crds_server()), "with mode", srepr(config.get_download_mode()), ":", str(exc)) from exc except: # mainly for control-c, catch it and throw it. self.remove_file(localpath) raise
def download(self, name, localpath): """Download a single file.""" # This code is complicated by the desire to blow away failed downloads. For the specific # case of KeyboardInterrupt, the file needs to be blown away, but the interrupt should not # be re-characterized so it is still un-trapped elsewhere under normal idioms which try *not* # to trap KeyboardInterrupt. assert not config.get_cache_readonly(), "Readonly cache, cannot download files " + repr(name) try: utils.ensure_dir_exists(localpath) return proxy.apply_with_retries(self.download_core, name, localpath) except Exception as exc: self.remove_file(localpath) raise CrdsDownloadError( "Error fetching data for", srepr(name), "at CRDS server", srepr(get_crds_server()), "with mode", srepr(config.get_download_mode()), ":", str(exc)) from exc except: # mainly for control-c, catch it and throw it. self.remove_file(localpath) raise
def cross_link_cdbs_paths(self, syn_name_map): """Hard link files from the downloaded CRDS cache structure to the classic CDBS-style directory locations specified by `syn_name_map`. On OS-X and Linux this creates files which are referenced from two different paths on the file system. Deleting one path or the other leaves behind a normal file referenced from only one location. syn_name_map dict { syn_basename : pysyn_filepath, ... } returns None """ for reference in syn_name_map: with log.error_on_exception( "Failed linking", repr(reference), "to CDBS directory."): crds_filepath = os.path.abspath(self.imap.locate_file(reference)) cdbs_filepath = syn_name_map[reference] utils.ensure_dir_exists(cdbs_filepath) if not os.path.exists(cdbs_filepath): os.link(crds_filepath, cdbs_filepath) log.verbose("Linked", cdbs_filepath, "-->", crds_filepath) else: log.verbose("Skipped existing", cdbs_filepath)
def cross_link_cdbs_paths(self, syn_name_map): """Hard link files from the downloaded CRDS cache structure to the classic CDBS-style directory locations specified by `syn_name_map`. On OS-X and Linux this creates files which are referenced from two different paths on the file system. Deleting one path or the other leaves behind a normal file referenced from only one location. syn_name_map dict { syn_basename : pysyn_filepath, ... } returns None """ for reference in syn_name_map: with log.error_on_exception( "Failed linking", repr(reference), "to CDBS directory."): crds_filepath = os.path.abspath(self.imap.locate_file(reference)) cdbs_filepath = syn_name_map[reference] utils.ensure_dir_exists(cdbs_filepath) if not os.path.exists(cdbs_filepath): os.link(crds_filepath, cdbs_filepath) log.verbose("Linked", cdbs_filepath, "-->", crds_filepath) else: log.verbose("Skipped existing", cdbs_filepath)