def test_purge_mappings(self): self.run_script("crds.sync --contexts hst_cos_deadtab.rmap --fetch-references") self.run_script("crds.sync --organize=flat") r = crds.get_cached_mapping("hst_cos_deadtab.rmap") self.assertEqual(r.reference_names(), ["s7g1700gl_dead.fits", "s7g1700ql_dead.fits"]) self.assertEqual(rmap.list_references("*", "hst"), ["s7g1700gl_dead.fits", "s7g1700ql_dead.fits"]) self.assert_crds_exists("s7g1700gl_dead.fits") self.assert_crds_exists("s7g1700ql_dead.fits") self.run_script("crds.sync --contexts hst_acs_imphttab.rmap --fetch-references --purge-mappings --purge-references") self.assertEqual(rmap.list_references("*", "hst"), ['w3m1716tj_imp.fits', 'w3m17170j_imp.fits', 'w3m17171j_imp.fits']) self.assertEqual(rmap.list_mappings("*", "hst"), ['hst_acs_imphttab.rmap'])
def test_rmap_list_references(self): os.environ["CRDS_REFPATH_SINGLE"] = self.data_dir config.CRDS_REF_SUBDIR_MODE = "flat" self.assertEqual(rmap.list_references("*.r1h", "hst"), [ 'dbu1405fu.r1h', 'dbu1405iu.r1h', 'e1b09593u.r1h', 'e1b09594u.r1h', 'valid.r1h' ])
def organize_references(self, new_mode): """Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`. This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument. """ old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True) old_mode = config.get_crds_ref_subdir_mode(self.observatory) log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode)) config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory) new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change. for refpath in old_refpaths: with log.error_on_exception("Failed relocating:", repr(refpath)): desired_loc = rmap.locate_file(os.path.basename(refpath), observatory=self.observatory) if desired_loc != refpath: if os.path.exists(desired_loc): if not self.args.organize_delete_junk: log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath)) continue utils.remove(desired_loc, observatory=self.observatory) if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)): log.info("Relocating", repr(refpath), "to", repr(desired_loc)) shutil.move(refpath, desired_loc) else: if old_mode != new_mode: log.warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode)) else: log.warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath)) if new_mode == "flat" and old_mode == "instrument": log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.") for instrument in self.locator.INSTRUMENTS: self.remove_dir(instrument)
def organize_references(self, new_mode): """Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`. This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument. """ old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True) old_mode = config.get_crds_ref_subdir_mode(self.observatory) log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode)) config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory) new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change. for refpath in old_refpaths: with log.error_on_exception("Failed relocating:", repr(refpath)): desired_loc = rmap.locate_file(os.path.basename(refpath), observatory=self.observatory) if desired_loc != refpath: if os.path.exists(desired_loc): if not self.args.organize_delete_junk: log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath)) continue utils.remove(desired_loc, observatory=self.observatory) if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)): log.info("Relocating", repr(refpath), "to", repr(desired_loc)) shutil.move(refpath, desired_loc) else: if old_mode != new_mode: log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode)) else: log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath)) if new_mode == "flat" and old_mode == "instrument": log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.") for instrument in self.locator.INSTRUMENTS: self.remove_dir(instrument)
def purge_references(self, keep=None): """Remove all references not references under pmaps `self.contexts`.""" purge_refs = set(rmap.list_references("*", self.observatory)) if keep is None: keep = set(self.get_context_references()) else: keep = set(keep) self.remove_files(sorted(purge_refs - keep), "reference")
def purge_references(self, keep=None): """Remove all references not references under pmaps `self.contexts`.""" purge_refs = set(rmap.list_references("*", self.observatory)) if keep is None: keep = set(self.get_context_references()) else: keep = set(keep) self.remove_files(sorted(purge_refs - keep), "reference")
def fetch_references(self, references): """Gets all references required to support `only_contexts`. Removes all references from the CRDS reference cache which are not required for `only_contexts`. """ if not self.contexts: return if self.args.readonly_cache: already_have = set(rmap.list_references("*", self.observatory)) fetched = [ x for x in sorted(set(references)-set(already_have)) if not x.startswith("NOT FOUND") ] if fetched: log.info("READONLY CACHE would fetch references:", repr(fetched)) with log.info_on_exception("Reference size information not available."): info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"]) total_bytes = api.get_total_bytes(info_map) log.info("READONLY CACHE would download", len(fetched), "references totaling", utils.human_format_number(total_bytes).strip(), "bytes.") else: self.dump_files(self.contexts[0], references)
def fetch_references(self, references): """Gets all references required to support `only_contexts`. Removes all references from the CRDS reference cache which are not required for `only_contexts`. """ if not self.contexts: return if self.args.readonly_cache: already_have = set(rmap.list_references("*", self.observatory)) fetched = [ x for x in sorted(set(references)-set(already_have)) if not x.startswith("NOT FOUND") ] if fetched: log.info("READONLY CACHE would fetch references:", repr(fetched)) with log.info_on_exception("Reference size information not available."): info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"]) total_bytes = api.get_total_bytes(info_map) log.info("READONLY CACHE would download", len(fetched), "references totaling", utils.human_format_number(total_bytes).strip(), "bytes.") else: self.dump_files(self.contexts[0], references)
def test_rmap_list_references(self): os.environ["CRDS_REFPATH_SINGLE"] = self.data_dir config.CRDS_REF_SUBDIR_MODE = "flat" self.assertEqual(rmap.list_references("*.r1h", "hst"), ['dbu1405fu.r1h', 'dbu1405iu.r1h', 'e1b09593u.r1h', 'e1b09594u.r1h', 'valid.r1h'])
def list_cached_references(self): """List the reference paths in the local cache.""" _print_list(rmap.list_references("*", self.observatory, full_path=self.args.full_path))
def check_naming_consistency(checked_instrument=None, exhaustive_mapping_check=False): """Dev function to compare the properties returned by name decomposition to the properties determined by file contents and make sure they're the same. Also checks rmap membership. >> from crds.tests import test_config >> old_config = test_config.setup() >> check_naming_consistency("acs") >> check_naming_consistency("cos") >> check_naming_consistency("nicmos") >> check_naming_consistency("stis") >> check_naming_consistency("wfc3") >> check_naming_consistency("wfpc2") >> test_config.cleanup(old_config) """ from crds import certify for ref in rmap.list_references("*", observatory="hst", full_path=True): with log.error_on_exception("Failed processing:", repr(ref)): _path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path(ref) if checked_instrument is not None and instrument != checked_instrument: continue if data_file.is_geis_data(ref): if os.path.exists(data_file.get_conjugate(ref)): continue else: log.warning("No GEIS header for", repr(ref)) log.verbose("Processing:", instrument, filekind, ref) _path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header(ref) if instrument != instrument2: log.error("Inconsistent instruments", repr(instrument), "vs.", repr(instrument2), "for", repr(ref)) if filekind != filekind2: log.error("Inconsistent filekinds", repr(filekind), "vs.", repr(filekind2), "for", repr(ref)) for pmap_name in reversed(sorted(rmap.list_mappings("*.pmap", observatory="hst"))): pmap = crds.get_cached_mapping(pmap_name) r = certify.find_governing_rmap(pmap_name, ref) if not r: continue if r.instrument != instrument: log.error("Rmap instrument", repr(r.instrument), "inconsistent with name derived instrument", repr(instrument), "for", repr(ref), "in", repr(pmap_name)) if r.filekind != filekind: log.error("Rmap filekind", repr(r.filekind), "inconsistent with name derived filekind", repr(filekind), "for", repr(ref), "in", repr(pmap_name)) if r.instrument != instrument2: log.error("Rmap instrument", repr(r.instrument), "inconsistent with content derived instrument", repr(instrument2), "for", repr(ref), "in", repr(pmap_name)) if r.filekind != filekind2: log.error("Rmap filekind", repr(r.filekind), "inconsistent with content derived filekind", repr(filekind2), "for", repr(ref), "in", repr(pmap_name)) if not exhaustive_mapping_check: break else: log.error("Orphan reference", repr(ref), "not found under any context.")
def check_naming_consistency(checked_instrument=None, exhaustive_mapping_check=False): """Dev function to compare the properties returned by name decomposition to the properties determined by file contents and make sure they're the same. Also checks rmap membership. >> from crds.tests import test_config >> old_config = test_config.setup() >> check_naming_consistency("acs") >> check_naming_consistency("cos") >> check_naming_consistency("nicmos") >> check_naming_consistency("stis") >> check_naming_consistency("wfc3") >> check_naming_consistency("wfpc2") >> test_config.cleanup(old_config) """ from crds import certify for ref in rmap.list_references("*", observatory="hst", full_path=True): with log.error_on_exception("Failed processing:", repr(ref)): _path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path( ref) if checked_instrument is not None and instrument != checked_instrument: continue if data_file.is_geis_data(ref): if os.path.exists(data_file.get_conjugate(ref)): continue else: log.warning("No GEIS header for", repr(ref)) log.verbose("Processing:", instrument, filekind, ref) _path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header( ref) if instrument != instrument2: log.error("Inconsistent instruments", repr(instrument), "vs.", repr(instrument2), "for", repr(ref)) if filekind != filekind2: log.error("Inconsistent filekinds", repr(filekind), "vs.", repr(filekind2), "for", repr(ref)) for pmap_name in reversed( sorted(rmap.list_mappings("*.pmap", observatory="hst"))): pmap = crds.get_cached_mapping(pmap_name) r = certify.find_governing_rmap(pmap_name, ref) if not r: continue if r.instrument != instrument: log.error("Rmap instrument", repr(r.instrument), "inconsistent with name derived instrument", repr(instrument), "for", repr(ref), "in", repr(pmap_name)) if r.filekind != filekind: log.error("Rmap filekind", repr(r.filekind), "inconsistent with name derived filekind", repr(filekind), "for", repr(ref), "in", repr(pmap_name)) if r.instrument != instrument2: log.error("Rmap instrument", repr(r.instrument), "inconsistent with content derived instrument", repr(instrument2), "for", repr(ref), "in", repr(pmap_name)) if r.filekind != filekind2: log.error("Rmap filekind", repr(r.filekind), "inconsistent with content derived filekind", repr(filekind2), "for", repr(ref), "in", repr(pmap_name)) if not exhaustive_mapping_check: break else: log.error("Orphan reference", repr(ref), "not found under any context.")