def print_affected_modes(self): """Print out all the affected mode tuples associated with the differences.""" assert rmap.is_mapping(self.old_file) and rmap.is_mapping(self.new_file), \ "for --print-affected-modes both files must be mappings." modes = mapping_affected_modes(self.old_file, self.new_file, self.args.include_header_diffs) for affected in modes: print(format_affected_mode(affected)) return 1 if modes else 0
def __init__(self, *args, **keys): super(MappingDifferencer, self).__init__(*args, **keys) assert rmap.is_mapping(self.old_file), \ "File " + repr(self.old_file) + " is not a CRDS mapping." assert rmap.is_mapping(self.new_file), \ "File " + repr(self.new_file) + " is not a CRDS mapping." assert os.path.splitext(self.old_file)[-1] == os.path.splitext(self.new_file)[-1], \ "Files " + repr(self.old_file) + " and " + repr(self.new_file) + \ " are not the same kind of CRDS mapping: .pmap, .imap, .rmap"
def print_affected_modes(self): """Print out all the affected mode tuples associated with the differences.""" assert rmap.is_mapping(self.old_file) and rmap.is_mapping(self.new_file), \ "for --print-affected-modes both files must be mappings." modes = mapping_affected_modes(self.old_file, self.new_file, self.args.include_header_diffs) for affected in modes: print(format_affected_mode(affected)) return 1 if modes else 0
def __init__(self, *args, **keys): super(MappingDifferencer, self).__init__(*args, **keys) assert rmap.is_mapping(self.old_file), \ "File " + repr(self.old_file) + " is not a CRDS mapping." assert rmap.is_mapping(self.new_file), \ "File " + repr(self.new_file) + " is not a CRDS mapping." assert os.path.splitext(self.old_file)[-1] == os.path.splitext(self.new_file)[-1], \ "Files " + repr(self.old_file) + " and " + repr(self.new_file) + \ " are not the same kind of CRDS mapping: .pmap, .imap, .rmap"
def print_all_new_files(self): """Print the names of all files which are in `new_file` (or any intermediary context) but not in `old_file`. new_file > old_file. Both new_file and old_file are similar mappings. """ updated = get_updated_files(self.old_file, self.new_file) for mapping in updated: if rmap.is_mapping(mapping): print(mapping, self.instrument_filekind(mapping)) for reference in updated: if not rmap.is_mapping(reference): print(reference, self.instrument_filekind(reference)) return 1 if updated else 0
def print_all_new_files(self): """Print the names of all files which are in `new_file` (or any intermediary context) but not in `old_file`. new_file > old_file. Both new_file and old_file are similar mappings. """ updated = get_updated_files(self.old_file, self.new_file) for mapping in updated: if rmap.is_mapping(mapping): print(mapping, self.instrument_filekind(mapping)) for reference in updated: if not rmap.is_mapping(reference): print(reference, self.instrument_filekind(reference)) return 1 if updated else 0
def get_affected(self): """Examine the diffs between `old_pmap` and `new_pmap` and return sorted lists of affected instruments and types. Returns { affected_instrument : { affected_type, ... } } """ instrs = defaultdict(set) diffs = self.mapping_diffs() diffs = remove_boring(diffs) for diff in diffs: for step in diff: # Walking down the diff steps 1-by-1 eventually hits an rmap comparison which # will define both instrument and type. pmaps and imaps leave at least one blank. if len(step) == 2 and rmap.is_mapping(step[0]): instrument, filekind = utils.get_file_properties( self.observatory, step[0]) # This is inefficient since diff doesn't vary by step, but set logic cleans up the redundancy # New rmaps imply reprocessing the entire type. elif isinstance(diff[-1],str) and diff[-1].startswith(("added","deleted")) and \ diff[-1].endswith(".rmap'"): rmap_name = diff[-1].split()[-1].replace("'", "") rmapping = rmap.fetch_mapping(rmap_name, ignore_checksum=True) instrument, filekind = rmapping.instrument, rmapping.filekind if instrument.strip() and filekind.strip(): if filekind not in instrs[instrument]: log.verbose("Affected", (instrument, filekind), "based on diff", diff, verbosity=20) instrs[instrument].add(filekind) return {key: list(val) for (key, val) in instrs.items()}
def get_affected(self): """Examine the diffs between `old_pmap` and `new_pmap` and return sorted lists of affected instruments and types. Returns { affected_instrument : { affected_type, ... } } """ instrs = defaultdict(set) diffs = self.mapping_diffs() diffs = remove_boring(diffs) for diff in diffs: for step in diff: # Walking down the diff steps 1-by-1 eventually hits an rmap comparison which # will define both instrument and type. pmaps and imaps leave at least one blank. if len(step) == 2 and rmap.is_mapping(step[0]): instrument, filekind = utils.get_file_properties(self.observatory, step[0]) # This is inefficient since diff doesn't vary by step, but set logic cleans up the redundancy # New rmaps imply reprocessing the entire type. elif isinstance(diff[-1],str) and diff[-1].startswith(("added","deleted")) and \ diff[-1].endswith(".rmap'"): rmap_name = diff[-1].split()[-1].replace("'","") rmapping = rmap.fetch_mapping(rmap_name, ignore_checksum=True) instrument, filekind = rmapping.instrument, rmapping.filekind if instrument.strip() and filekind.strip(): if filekind not in instrs[instrument]: log.verbose("Affected", (instrument, filekind), "based on diff", diff, verbosity=20) instrs[instrument].add(filekind) return { key:list(val) for (key, val) in instrs.items() }
def get_file_properties(filename): """Figure out (instrument, filekind, serial) based on `filename` which should be a mapping or FITS reference file. >> get_file_properties("./hst_acs_biasfile_0001.rmap") ('acs', 'biasfile') >> get_file_properties("./hst_acs_biasfile_0001.pmap") Traceback (most recent call last): ... AssertionError: Invalid .pmap filename './hst_acs_biasfile_0001.pmap' >> get_file_properties("test_data/s7g1700gl_dead.fits") """ if rmap.is_mapping(filename): try: return decompose_newstyle_name(filename)[2:4] except Exception: # NOTE: load_mapping more conservative than fetch_mapping used in properties_from_mapping mapping = rmap.load_mapping(filename) return mapping.instrument, mapping.filekind elif config.is_reference(filename): result = get_reference_properties(filename)[2:4] else: try: result = properties_inside_mapping(filename) except Exception as exc: result = get_reference_properties(filename)[2:4] assert result[0] in INSTRUMENTS+[""], "Bad instrument " + \ repr(result[0]) + " in filename " + repr(filename) assert result[1] in FILEKINDS+[""], "Bad filekind " + \ repr(result[1]) + " in filename " + repr(filename) return result
def get_file_properties(filename): """Figure out (instrument, filekind, serial) based on `filename` which should be a mapping or FITS reference file. >> get_file_properties("./hst_acs_biasfile_0001.rmap") ('acs', 'biasfile') >> get_file_properties("./hst_acs_biasfile_0001.pmap") Traceback (most recent call last): ... AssertionError: Invalid .pmap filename './hst_acs_biasfile_0001.pmap' >> get_file_properties("test_data/s7g1700gl_dead.fits") """ if rmap.is_mapping(filename): return decompose_newstyle_name(filename)[2:4] elif REF_EXT_RE.search(filename): result = get_reference_properties(filename)[2:4] else: try: result = properties_inside_mapping(filename) except Exception as exc: result = get_reference_properties(filename)[2:4] assert result[0] in INSTRUMENTS+[""], "Bad instrument " + \ repr(result[0]) + " in filename " + repr(filename) assert result[1] in FILEKINDS+[""], "Bad filekind " + \ repr(result[1]) + " in filename " + repr(filename) return result
def mapping_pairs(differences): """Return the sorted list of all mapping tuples found in differences.""" pairs = set() for diff in differences: for pair in diff: if len(pair) == 2 and rmap.is_mapping(pair[0]): pairs.add(pair) return sorted(pairs)
def mapping_pairs(differences): """Return the sorted list of all mapping tuples found in differences.""" pairs = set() for diff in differences: for pair in diff: if len(pair) == 2 and rmap.is_mapping(pair[0]): pairs.add(pair) return sorted(pairs)
def _find_diff_str(self, diff_str): """Return True IFF `diff_str` is in some rmap diff.""" diffs = self.mapping_diffs() diffs = remove_boring(diffs) for diff in diffs: for step in diff: if len(step) == 2 and rmap.is_mapping(step[0]): if diff_str in diff_action(diff): log.verbose("Found", repr(diff_str), "diff between", repr(step[0:1])) return True return False
def remove_checksum(file_): """Remove checksums from `file_`.""" log.info("Removing checksum for", repr(file_)) if config.is_reference(file_): data_file.remove_checksum(file_) elif rmap.is_mapping(file_): raise exceptions.CrdsError("Mapping checksums cannot be removed for:", repr(file_)) else: raise exceptions.CrdsError( "File", repr(file_), "does not appear to be a CRDS reference or mapping file.")
def _find_diff_str(self, diff_str): """Return True IFF `diff_str` is in some rmap diff.""" diffs = self.mapping_diffs() diffs = remove_boring(diffs) for diff in diffs: for step in diff: if len(step) == 2 and rmap.is_mapping(step[0]): if diff_str in diff_action(diff): log.verbose("Found", repr(diff_str), "diff between", repr(step[0:1])) return True return False
def print_new_files(self): """Print the references or mappings which are in the second (new) context and not the firtst (old) context. """ if not rmap.is_mapping(self.old_file) or not rmap.is_mapping(self.new_file): log.error("--print-new-files really only works for mapping differences.") return -1 old = crds.get_pickled_mapping(self.old_file) # reviewed new = crds.get_pickled_mapping(self.new_file) # reviewed old_mappings = set(old.mapping_names()) new_mappings = set(new.mapping_names()) old_references = set(old.reference_names()) new_references = set(new.reference_names()) status = 0 for name in sorted(new_mappings - old_mappings): print(name) status = 1 for name in sorted(new_references - old_references): print(name) status = 1 return status
def add_checksum(file_): """Add checksums to file_.""" log.info("Adding checksum for", repr(file_)) if config.is_reference(file_): with log.error_on_exception("Failed updating checksum for", repr(file_)): data_file.add_checksum(file_) elif rmap.is_mapping(file_): update_mapping_checksum(file_) else: raise exceptions.CrdsError( "File", repr(file_), "does not appear to be a CRDS reference or mapping file.")
def print_new_files(self): """Print the references or mappings which are in the second (new) context and not the firtst (old) context. """ if not rmap.is_mapping(self.old_file) or not rmap.is_mapping( self.new_file): log.error( "--print-new-files really only works for mapping differences.") return -1 old = crds.get_pickled_mapping(self.old_file) # reviewed new = crds.get_pickled_mapping(self.new_file) # reviewed old_mappings = set(old.mapping_names()) new_mappings = set(new.mapping_names()) old_references = set(old.reference_names()) new_references = set(new.reference_names()) status = 0 for name in sorted(new_mappings - old_mappings): print(name) status = 1 for name in sorted(new_references - old_references): print(name) status = 1 return status
def verify_checksum(file_): """Verify checksums in `file_`.""" log.info("Verifying checksum for", repr(file_)) if config.is_reference(file_): data_file.verify_checksum(file_) elif rmap.is_mapping(file_): if config.CRDS_IGNORE_MAPPING_CHECKSUM.get(): log.warning( "Mapping checksums are disabled by config.CRDS_IGNORE_MAPPING_CHECKSM." ) rmap.load_mapping(file_) else: raise exceptions.CrdsError( "File", repr(file_), "does not appear to be a CRDS reference or mapping file.")