def purge_mappings(self): """Remove all mappings not under pmaps `self.contexts`.""" # rmap.list_mappings lists all mappings in the *local* cache. # in contrast, client.list_mappings globs available mappings in the server cache. purge_maps = set(rmap.list_mappings('*.[pir]map', self.observatory)) keep = set(self.get_context_mappings()) self.remove_files(sorted(purge_maps-keep), "mapping")
def test_rmap_list_mappings(self): os.environ["CRDS_MAPPATH_SINGLE"] = self.data_dir self.assertEqual(rmap.list_mappings("*.imap", "hst"), [ 'hst_acs.imap', 'hst_acs_0001.imap', 'hst_acs_0002.imap', 'hst_acs_9999.imap', 'hst_cos.imap', 'hst_nicmos.imap', 'hst_stis.imap', 'hst_wfc3.imap', 'hst_wfpc2.imap', 'jwst_fgs_na.imap', 'jwst_miri_omit.imap', 'jwst_niriss_na_omit.imap' ])
def all_filekinds(): filekinds = set() for context in rmap.list_mappings("*.pmap", "jwst"): p = crds.get_cached_mapping(context) for i in p.selections.values(): for r in i.selections.values(): if r.filekind not in filekinds: log.info("Adding", repr(r.filekind), "from", repr(context)) filekinds.add(r.filekind) return sorted(filekinds)
def test_purge_mappings(self): self.run_script("crds.sync --contexts hst_cos_deadtab.rmap --fetch-references") self.run_script("crds.sync --organize=flat") r = crds.get_cached_mapping("hst_cos_deadtab.rmap") self.assertEqual(r.reference_names(), ["s7g1700gl_dead.fits", "s7g1700ql_dead.fits"]) self.assertEqual(rmap.list_references("*", "hst"), ["s7g1700gl_dead.fits", "s7g1700ql_dead.fits"]) self.assert_crds_exists("s7g1700gl_dead.fits") self.assert_crds_exists("s7g1700ql_dead.fits") self.run_script("crds.sync --contexts hst_acs_imphttab.rmap --fetch-references --purge-mappings --purge-references") self.assertEqual(rmap.list_references("*", "hst"), ['w3m1716tj_imp.fits', 'w3m17170j_imp.fits', 'w3m17171j_imp.fits']) self.assertEqual(rmap.list_mappings("*", "hst"), ['hst_acs_imphttab.rmap'])
def get_updated_files(context1, context2): """Return the sorted list of files names which are in `context2` (or any intermediate context) but not in `context1`. context2 > context1. """ extension1 = os.path.splitext(context1)[1] extension2 = os.path.splitext(context2)[1] assert extension1 == extension2, "Only compare mappings of same type/extension." old_map = rmap.get_cached_mapping(context1) old_files = set(old_map.mapping_names() + old_map.reference_names()) all_mappings = rmap.list_mappings("*"+extension1, old_map.observatory) updated = set() context1, context2 = os.path.basename(context1), os.path.basename(context2) for new in all_mappings: new = os.path.basename(new) if context1 < new <= context2: new_map = rmap.get_cached_mapping(new) updated |= set(new_map.mapping_names() + new_map.reference_names()) return sorted(list(updated - old_files))
def get_updated_files(context1, context2): """Return the sorted list of files names which are in `context2` (or any intermediate context) but not in `context1`. context2 > context1. """ extension1 = os.path.splitext(context1)[1] extension2 = os.path.splitext(context2)[1] assert extension1 == extension2, "Only compare mappings of same type/extension." old_map = crds.get_cached_mapping(context1) old_files = set(old_map.mapping_names() + old_map.reference_names()) all_mappings = rmap.list_mappings("*" + extension1, old_map.observatory) updated = set() context1, context2 = os.path.basename(context1), os.path.basename(context2) for new in all_mappings: new = os.path.basename(new) if context1 < new <= context2: new_map = crds.get_cached_mapping(new) updated |= set(new_map.mapping_names() + new_map.reference_names()) return sorted(list(updated - old_files))
def test_rmap_list_mappings(self): os.environ["CRDS_MAPPATH_SINGLE"] = self.data_dir self.assertEqual(rmap.list_mappings("*.imap", "hst"), [ 'hst_acs.imap', 'hst_acs_0001.imap', 'hst_acs_0002.imap', 'hst_acs_9999.imap', 'hst_cos.imap', 'hst_nicmos.imap', 'hst_stis.imap', 'hst_wfc3.imap', 'hst_wfpc2.imap', 'jwst_fgs_na.imap', 'jwst_miri_omit.imap', 'jwst_niriss_na_omit.imap'])
def gen_all_specs(): for context in reversed(rmap.list_mappings("*.pmap", "jwst")): gen_specs(context) check_specs()
def list_cached_mappings(self): """List the mapping paths in the local cache.""" _print_list(rmap.list_mappings("*.*map", self.observatory, full_path=self.args.full_path))
def check_naming_consistency(checked_instrument=None, exhaustive_mapping_check=False): """Dev function to compare the properties returned by name decomposition to the properties determined by file contents and make sure they're the same. Also checks rmap membership. >> from crds.tests import test_config >> old_config = test_config.setup() >> check_naming_consistency("acs") >> check_naming_consistency("cos") >> check_naming_consistency("nicmos") >> check_naming_consistency("stis") >> check_naming_consistency("wfc3") >> check_naming_consistency("wfpc2") >> test_config.cleanup(old_config) """ from crds import certify for ref in rmap.list_references("*", observatory="hst", full_path=True): with log.error_on_exception("Failed processing:", repr(ref)): _path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path(ref) if checked_instrument is not None and instrument != checked_instrument: continue if data_file.is_geis_data(ref): if os.path.exists(data_file.get_conjugate(ref)): continue else: log.warning("No GEIS header for", repr(ref)) log.verbose("Processing:", instrument, filekind, ref) _path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header(ref) if instrument != instrument2: log.error("Inconsistent instruments", repr(instrument), "vs.", repr(instrument2), "for", repr(ref)) if filekind != filekind2: log.error("Inconsistent filekinds", repr(filekind), "vs.", repr(filekind2), "for", repr(ref)) for pmap_name in reversed(sorted(rmap.list_mappings("*.pmap", observatory="hst"))): pmap = crds.get_cached_mapping(pmap_name) r = certify.find_governing_rmap(pmap_name, ref) if not r: continue if r.instrument != instrument: log.error("Rmap instrument", repr(r.instrument), "inconsistent with name derived instrument", repr(instrument), "for", repr(ref), "in", repr(pmap_name)) if r.filekind != filekind: log.error("Rmap filekind", repr(r.filekind), "inconsistent with name derived filekind", repr(filekind), "for", repr(ref), "in", repr(pmap_name)) if r.instrument != instrument2: log.error("Rmap instrument", repr(r.instrument), "inconsistent with content derived instrument", repr(instrument2), "for", repr(ref), "in", repr(pmap_name)) if r.filekind != filekind2: log.error("Rmap filekind", repr(r.filekind), "inconsistent with content derived filekind", repr(filekind2), "for", repr(ref), "in", repr(pmap_name)) if not exhaustive_mapping_check: break else: log.error("Orphan reference", repr(ref), "not found under any context.")
def check_naming_consistency(checked_instrument=None, exhaustive_mapping_check=False): """Dev function to compare the properties returned by name decomposition to the properties determined by file contents and make sure they're the same. Also checks rmap membership. >> from crds.tests import test_config >> old_config = test_config.setup() >> check_naming_consistency("acs") >> check_naming_consistency("cos") >> check_naming_consistency("nicmos") >> check_naming_consistency("stis") >> check_naming_consistency("wfc3") >> check_naming_consistency("wfpc2") >> test_config.cleanup(old_config) """ from crds import certify for ref in rmap.list_references("*", observatory="hst", full_path=True): with log.error_on_exception("Failed processing:", repr(ref)): _path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path( ref) if checked_instrument is not None and instrument != checked_instrument: continue if data_file.is_geis_data(ref): if os.path.exists(data_file.get_conjugate(ref)): continue else: log.warning("No GEIS header for", repr(ref)) log.verbose("Processing:", instrument, filekind, ref) _path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header( ref) if instrument != instrument2: log.error("Inconsistent instruments", repr(instrument), "vs.", repr(instrument2), "for", repr(ref)) if filekind != filekind2: log.error("Inconsistent filekinds", repr(filekind), "vs.", repr(filekind2), "for", repr(ref)) for pmap_name in reversed( sorted(rmap.list_mappings("*.pmap", observatory="hst"))): pmap = crds.get_cached_mapping(pmap_name) r = certify.find_governing_rmap(pmap_name, ref) if not r: continue if r.instrument != instrument: log.error("Rmap instrument", repr(r.instrument), "inconsistent with name derived instrument", repr(instrument), "for", repr(ref), "in", repr(pmap_name)) if r.filekind != filekind: log.error("Rmap filekind", repr(r.filekind), "inconsistent with name derived filekind", repr(filekind), "for", repr(ref), "in", repr(pmap_name)) if r.instrument != instrument2: log.error("Rmap instrument", repr(r.instrument), "inconsistent with content derived instrument", repr(instrument2), "for", repr(ref), "in", repr(pmap_name)) if r.filekind != filekind2: log.error("Rmap filekind", repr(r.filekind), "inconsistent with content derived filekind", repr(filekind2), "for", repr(ref), "in", repr(pmap_name)) if not exhaustive_mapping_check: break else: log.error("Orphan reference", repr(ref), "not found under any context.")