def get_local_files(self, names):
"""Given a list of basename `mapping_names` which are pertinent to the
given `pipeline_context`, cache the mappings locally where they can
be used by CRDS.
"""
if isinstance(names, dict):
names = names.values()
localpaths = {}
# Add in GEIS format "conjugate" data files, .rmaps specify only .rXh
names2 = names[:]
for refname in names2:
if re.match(r"\w+\.r[0-9]h$", refname):
names.append(refname[:-1]+"d")
downloads = []
for name in names:
localpath = self.locate(name)
if name.lower() in ["n/a", "undefined"]:
continue
if not os.path.exists(localpath):
downloads.append(name)
elif self.ignore_cache:
utils.remove(localpath, observatory=self.observatory)
downloads.append(name)
utils.remove(localpath, observatory=self.observatory)
localpaths[name] = localpath
if downloads:
n_bytes = self.download_files(downloads, localpaths)
else:
log.verbose("Skipping download for cached files", sorted(names), verbosity=60)
n_bytes = 0
return localpaths, len(downloads), n_bytes
def organize_references(self, new_mode):
"""Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`.
This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument.
"""
old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True)
old_mode = config.get_crds_ref_subdir_mode(self.observatory)
log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode))
config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory)
new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change.
for refpath in old_refpaths:
with log.error_on_exception("Failed relocating:", repr(refpath)):
desired_loc = config.locate_file(os.path.basename(refpath), observatory=self.observatory)
if desired_loc != refpath:
if os.path.exists(desired_loc):
if not self.args.organize_delete_junk:
log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath))
continue
utils.remove(desired_loc, observatory=self.observatory)
if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)):
log.info("Relocating", repr(refpath), "to", repr(desired_loc))
shutil.move(refpath, desired_loc)
else:
if old_mode != new_mode:
log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode))
else:
log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath))
if new_mode == "flat" and old_mode == "instrument":
log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.")
for instrument in self.locator.INSTRUMENTS:
self.remove_dir(instrument)
def get_local_files(self, names):
"""Given a list of basename `mapping_names` which are pertinent to the
given `pipeline_context`, cache the mappings locally where they can
be used by CRDS.
"""
if isinstance(names, dict):
names = names.values()
localpaths = {}
# Add in GEIS format "conjugate" data files, .rmaps specify only .rXh
names2 = names[:]
for refname in names2:
if re.match(r"\w+\.r[0-9]h$", refname):
names.append(refname[:-1]+"d")
downloads = []
for name in names:
localpath = self.locate(name)
if name.lower() in ["n/a", "undefined"]:
continue
if not os.path.exists(localpath):
downloads.append(name)
elif self.ignore_cache:
utils.remove(localpath, observatory=self.observatory)
downloads.append(name)
utils.remove(localpath, observatory=self.observatory)
localpaths[name] = localpath
if downloads:
n_bytes = self.download_files(downloads, localpaths)
else:
log.verbose("Skipping download for cached files", sorted(names), verbosity=60)
n_bytes = 0
return localpaths, len(downloads), n_bytes
def organize_references(self, new_mode):
"""Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`.
This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument.
"""
old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True)
old_mode = config.get_crds_ref_subdir_mode(self.observatory)
log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode))
config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory)
new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change.
for refpath in old_refpaths:
with log.error_on_exception("Failed relocating:", repr(refpath)):
desired_loc = rmap.locate_file(os.path.basename(refpath), observatory=self.observatory)
if desired_loc != refpath:
if os.path.exists(desired_loc):
if not self.args.organize_delete_junk:
log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath))
continue
utils.remove(desired_loc, observatory=self.observatory)
if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)):
log.info("Relocating", repr(refpath), "to", repr(desired_loc))
shutil.move(refpath, desired_loc)
else:
if old_mode != new_mode:
log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode))
else:
log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath))
if new_mode == "flat" and old_mode == "instrument":
log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.")
for instrument in self.locator.INSTRUMENTS:
self.remove_dir(instrument)
def error_and_repair(self, file, *args, **keys):
"""Issue an error message and repair `file` if requested by command line args."""
log.error(*args, **keys)
if self.args.repair_files:
if config.writable_cache_or_info("Skipping remove and re-download of", repr(file)):
log.info("Repairing file", repr(file))
utils.remove(file, observatory=self.observatory)
self.dump_files(self.default_context, [file])
def error_and_repair(self, file, *args, **keys):
"""Issue an error message and repair `file` if requested by command line args."""
log.error(*args, **keys)
if self.args.repair_files:
if config.writable_cache_or_info("Skipping remove and re-download of", repr(file)):
log.info("Repairing file", repr(file))
utils.remove(file, observatory=self.observatory)
self.dump_files(self.default_context, [file])
def clear_pickles(self):
"""Remove all pickles."""
log.info(
"Removing all context pickles. Use --save-pickles to recreate for specified contexts."
)
for path in rmap.list_pickles("*.pmap",
self.observatory,
full_path=True):
if os.path.exists(path):
utils.remove(path, self.observatory)
def remove_dir(self, instrument):
"""Remove an instrument cache directory and any associated legacy link."""
if config.writable_cache_or_info("Skipping remove instrument", repr(instrument), "directory."):
crds_refpath = config.get_crds_refpath(self.observatory)
prefix = self.locator.get_env_prefix(instrument)
rootdir = os.path.join(crds_refpath, instrument)
refdir = os.path.join(crds_refpath, prefix[:-1])
if len(glob.glob(os.path.join(rootdir, "*"))):
log.info("Residual files in '{}'. Not removing.".format(rootdir))
return
if os.path.exists(refdir): # skip crds:// vs. oref
utils.remove(refdir, observatory=self.observatory)
utils.remove(rootdir, observatory=self.observatory)
def remove_dir(self, instrument):
"""Remove an instrument cache directory and any associated legacy link."""
if config.writable_cache_or_info("Skipping remove instrument", repr(instrument), "directory."):
crds_refpath = config.get_crds_refpath(self.observatory)
prefix = self.locator.get_env_prefix(instrument)
rootdir = os.path.join(crds_refpath, instrument)
refdir = os.path.join(crds_refpath, prefix[:-1])
if len(glob.glob(os.path.join(rootdir, "*"))):
log.info("Residual files in '{}'. Not removing.".format(rootdir))
return
if os.path.exists(refdir): # skip crds:// vs. oref
utils.remove(refdir, observatory=self.observatory)
utils.remove(rootdir, observatory=self.observatory)
def remove_files(self, files, kind):
"""Remove the list of `files` basenames which are converted to fully
specified CRDS paths using the locator module associated with context.
"""
if not files:
log.verbose("No " + kind + "s to remove.")
files2 = set(files)
for filename in files:
if re.match(r"\w+\.r[0-9]h", filename):
files2.add(filename[:-1] + "d")
for filename in files:
with log.error_on_exception("Failed purging", kind, repr(filename)):
where = rmap.locate_file(filename, self.observatory)
utils.remove(where, observatory=self.observatory)
def remove_files(self, files, kind):
"""Remove the list of `files` basenames which are converted to fully
specified CRDS paths using the locator module associated with context.
"""
if not files:
log.verbose("No " + kind + "s to remove.")
files2 = set(files)
for filename in files:
if re.match(r"\w+\.r[0-9]h", filename):
files2.add(filename[:-1] + "d")
for filename in files:
with log.error_on_exception("Failed purging", kind, repr(filename)):
where = config.locate_file(filename, self.observatory)
utils.remove(where, observatory=self.observatory)
def clear_pickles(self):
"""Remove all pickles."""
log.info("Removing all context pickles. Use --save-pickles to recreate for specified contexts.")
for path in rmap.list_pickles("*.pmap", self.observatory, full_path=True):
if os.path.exists(path):
utils.remove(path, self.observatory)