def get_local_files(self, names): """Given a list of basename `mapping_names` which are pertinent to the given `pipeline_context`, cache the mappings locally where they can be used by CRDS. """ if isinstance(names, dict): names = names.values() localpaths = {} # Add in GEIS format "conjugate" data files, .rmaps specify only .rXh names2 = names[:] for refname in names2: if re.match(r"\w+\.r[0-9]h$", refname): names.append(refname[:-1]+"d") downloads = [] for name in names: localpath = self.locate(name) if name.lower() in ["n/a", "undefined"]: continue if not os.path.exists(localpath): downloads.append(name) elif self.ignore_cache: utils.remove(localpath, observatory=self.observatory) downloads.append(name) utils.remove(localpath, observatory=self.observatory) localpaths[name] = localpath if downloads: n_bytes = self.download_files(downloads, localpaths) else: log.verbose("Skipping download for cached files", sorted(names), verbosity=60) n_bytes = 0 return localpaths, len(downloads), n_bytes
def organize_references(self, new_mode): """Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`. This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument. """ old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True) old_mode = config.get_crds_ref_subdir_mode(self.observatory) log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode)) config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory) new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change. for refpath in old_refpaths: with log.error_on_exception("Failed relocating:", repr(refpath)): desired_loc = config.locate_file(os.path.basename(refpath), observatory=self.observatory) if desired_loc != refpath: if os.path.exists(desired_loc): if not self.args.organize_delete_junk: log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath)) continue utils.remove(desired_loc, observatory=self.observatory) if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)): log.info("Relocating", repr(refpath), "to", repr(desired_loc)) shutil.move(refpath, desired_loc) else: if old_mode != new_mode: log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode)) else: log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath)) if new_mode == "flat" and old_mode == "instrument": log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.") for instrument in self.locator.INSTRUMENTS: self.remove_dir(instrument)
def get_local_files(self, names): """Given a list of basename `mapping_names` which are pertinent to the given `pipeline_context`, cache the mappings locally where they can be used by CRDS. """ if isinstance(names, dict): names = names.values() localpaths = {} # Add in GEIS format "conjugate" data files, .rmaps specify only .rXh names2 = names[:] for refname in names2: if re.match(r"\w+\.r[0-9]h$", refname): names.append(refname[:-1]+"d") downloads = [] for name in names: localpath = self.locate(name) if name.lower() in ["n/a", "undefined"]: continue if not os.path.exists(localpath): downloads.append(name) elif self.ignore_cache: utils.remove(localpath, observatory=self.observatory) downloads.append(name) utils.remove(localpath, observatory=self.observatory) localpaths[name] = localpath if downloads: n_bytes = self.download_files(downloads, localpaths) else: log.verbose("Skipping download for cached files", sorted(names), verbosity=60) n_bytes = 0 return localpaths, len(downloads), n_bytes
def organize_references(self, new_mode): """Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`. This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument. """ old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True) old_mode = config.get_crds_ref_subdir_mode(self.observatory) log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode)) config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory) new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change. for refpath in old_refpaths: with log.error_on_exception("Failed relocating:", repr(refpath)): desired_loc = rmap.locate_file(os.path.basename(refpath), observatory=self.observatory) if desired_loc != refpath: if os.path.exists(desired_loc): if not self.args.organize_delete_junk: log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath)) continue utils.remove(desired_loc, observatory=self.observatory) if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)): log.info("Relocating", repr(refpath), "to", repr(desired_loc)) shutil.move(refpath, desired_loc) else: if old_mode != new_mode: log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode)) else: log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath)) if new_mode == "flat" and old_mode == "instrument": log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.") for instrument in self.locator.INSTRUMENTS: self.remove_dir(instrument)
def error_and_repair(self, file, *args, **keys): """Issue an error message and repair `file` if requested by command line args.""" log.error(*args, **keys) if self.args.repair_files: if config.writable_cache_or_info("Skipping remove and re-download of", repr(file)): log.info("Repairing file", repr(file)) utils.remove(file, observatory=self.observatory) self.dump_files(self.default_context, [file])
def error_and_repair(self, file, *args, **keys): """Issue an error message and repair `file` if requested by command line args.""" log.error(*args, **keys) if self.args.repair_files: if config.writable_cache_or_info("Skipping remove and re-download of", repr(file)): log.info("Repairing file", repr(file)) utils.remove(file, observatory=self.observatory) self.dump_files(self.default_context, [file])
def clear_pickles(self): """Remove all pickles.""" log.info( "Removing all context pickles. Use --save-pickles to recreate for specified contexts." ) for path in rmap.list_pickles("*.pmap", self.observatory, full_path=True): if os.path.exists(path): utils.remove(path, self.observatory)
def remove_dir(self, instrument): """Remove an instrument cache directory and any associated legacy link.""" if config.writable_cache_or_info("Skipping remove instrument", repr(instrument), "directory."): crds_refpath = config.get_crds_refpath(self.observatory) prefix = self.locator.get_env_prefix(instrument) rootdir = os.path.join(crds_refpath, instrument) refdir = os.path.join(crds_refpath, prefix[:-1]) if len(glob.glob(os.path.join(rootdir, "*"))): log.info("Residual files in '{}'. Not removing.".format(rootdir)) return if os.path.exists(refdir): # skip crds:// vs. oref utils.remove(refdir, observatory=self.observatory) utils.remove(rootdir, observatory=self.observatory)
def remove_dir(self, instrument): """Remove an instrument cache directory and any associated legacy link.""" if config.writable_cache_or_info("Skipping remove instrument", repr(instrument), "directory."): crds_refpath = config.get_crds_refpath(self.observatory) prefix = self.locator.get_env_prefix(instrument) rootdir = os.path.join(crds_refpath, instrument) refdir = os.path.join(crds_refpath, prefix[:-1]) if len(glob.glob(os.path.join(rootdir, "*"))): log.info("Residual files in '{}'. Not removing.".format(rootdir)) return if os.path.exists(refdir): # skip crds:// vs. oref utils.remove(refdir, observatory=self.observatory) utils.remove(rootdir, observatory=self.observatory)
def remove_files(self, files, kind): """Remove the list of `files` basenames which are converted to fully specified CRDS paths using the locator module associated with context. """ if not files: log.verbose("No " + kind + "s to remove.") files2 = set(files) for filename in files: if re.match(r"\w+\.r[0-9]h", filename): files2.add(filename[:-1] + "d") for filename in files: with log.error_on_exception("Failed purging", kind, repr(filename)): where = rmap.locate_file(filename, self.observatory) utils.remove(where, observatory=self.observatory)
def remove_files(self, files, kind): """Remove the list of `files` basenames which are converted to fully specified CRDS paths using the locator module associated with context. """ if not files: log.verbose("No " + kind + "s to remove.") files2 = set(files) for filename in files: if re.match(r"\w+\.r[0-9]h", filename): files2.add(filename[:-1] + "d") for filename in files: with log.error_on_exception("Failed purging", kind, repr(filename)): where = config.locate_file(filename, self.observatory) utils.remove(where, observatory=self.observatory)
def clear_pickles(self): """Remove all pickles.""" log.info("Removing all context pickles. Use --save-pickles to recreate for specified contexts.") for path in rmap.list_pickles("*.pmap", self.observatory, full_path=True): if os.path.exists(path): utils.remove(path, self.observatory)