def mapping_diffs(self): """Return the logical differences between CRDS mappings named `old_file` and `new_file`. IFF include_header_diffs, include differences in mapping headers. Some are "boring", e.g. sha1sum or name. IFF recurse_added_deleted, include difference tuples for all nested adds and deletes whenever a higher level mapping is added or deleted. Else, only include the higher level mapping, not contained files. """ old_map = rmap.fetch_mapping(self.locate_file(self.old_file), ignore_checksum=True, path=self.mappings_cache1) new_map = rmap.fetch_mapping(self.locate_file(self.new_file), ignore_checksum=True, path=self.mappings_cache2) differences = old_map.difference(new_map, include_header_diffs=self.include_header_diffs, recurse_added_deleted=self.recurse_added_deleted) return differences
def properties_inside_mapping(filename): """Load `filename`s mapping header to discover and return (instrument, filekind). >>> from crds.tests import test_config >>> old_config = test_config.setup() >>> properties_inside_mapping("hst.pmap") ('', '') >>> properties_inside_mapping("hst_acs.imap") ('acs', '') >>> properties_inside_mapping("hst_acs_darkfile.rmap") ('acs', 'darkfile') >>> test_config.cleanup(old_config) """ loaded = rmap.fetch_mapping(filename) if loaded.mapping == "pipeline": result = "", "" elif loaded.mapping == "instrument": result = loaded.instrument, "" else: result = loaded.instrument, loaded.filekind return result
def properties_inside_mapping(filename): """Load `filename`s mapping header to discover and return (instrument, filekind). >>> from crds.tests import test_config >>> old_config = test_config.setup() >>> properties_inside_mapping("hst.pmap") ('', '') >>> properties_inside_mapping("hst_acs.imap") ('acs', '') >>> properties_inside_mapping("hst_acs_darkfile.rmap") ('acs', 'darkfile') >>> test_config.cleanup(old_config) """ loaded = rmap.fetch_mapping(filename) if loaded.mapping == "pipeline": result = "", "" elif loaded.mapping == "instrument": result = loaded.instrument, "" else: result = loaded.instrument, loaded.filekind return result
def get_affected(self): """Examine the diffs between `old_pmap` and `new_pmap` and return sorted lists of affected instruments and types. Returns { affected_instrument : { affected_type, ... } } """ instrs = defaultdict(set) diffs = self.mapping_diffs() diffs = remove_boring(diffs) for diff in diffs: for step in diff: # Walking down the diff steps 1-by-1 eventually hits an rmap comparison which # will define both instrument and type. pmaps and imaps leave at least one blank. if len(step) == 2 and rmap.is_mapping(step[0]): instrument, filekind = utils.get_file_properties(self.observatory, step[0]) # This is inefficient since diff doesn't vary by step, but set logic cleans up the redundancy # New rmaps imply reprocessing the entire type. elif isinstance(diff[-1],str) and diff[-1].startswith(("added","deleted")) and \ diff[-1].endswith(".rmap'"): rmap_name = diff[-1].split()[-1].replace("'","") rmapping = rmap.fetch_mapping(rmap_name, ignore_checksum=True) instrument, filekind = rmapping.instrument, rmapping.filekind if instrument.strip() and filekind.strip(): if filekind not in instrs[instrument]: log.verbose("Affected", (instrument, filekind), "based on diff", diff, verbosity=20) instrs[instrument].add(filekind) return { key:list(val) for (key, val) in instrs.items() }
def get_affected(self): """Examine the diffs between `old_pmap` and `new_pmap` and return sorted lists of affected instruments and types. Returns { affected_instrument : { affected_type, ... } } """ instrs = defaultdict(set) diffs = self.mapping_diffs() diffs = remove_boring(diffs) for diff in diffs: for step in diff: # Walking down the diff steps 1-by-1 eventually hits an rmap comparison which # will define both instrument and type. pmaps and imaps leave at least one blank. if len(step) == 2 and rmap.is_mapping(step[0]): instrument, filekind = utils.get_file_properties( self.observatory, step[0]) # This is inefficient since diff doesn't vary by step, but set logic cleans up the redundancy # New rmaps imply reprocessing the entire type. elif isinstance(diff[-1],str) and diff[-1].startswith(("added","deleted")) and \ diff[-1].endswith(".rmap'"): rmap_name = diff[-1].split()[-1].replace("'", "") rmapping = rmap.fetch_mapping(rmap_name, ignore_checksum=True) instrument, filekind = rmapping.instrument, rmapping.filekind if instrument.strip() and filekind.strip(): if filekind not in instrs[instrument]: log.verbose("Affected", (instrument, filekind), "based on diff", diff, verbosity=20) instrs[instrument].add(filekind) return {key: list(val) for (key, val) in instrs.items()}
def properties_inside_mapping(filename): """Load `filename`s mapping header to discover and return (instrument, filekind). """ map = rmap.fetch_mapping(filename) if map.filekind == "PIPELINE": result = "", "" elif map.filekind == "INSTRUMENT": result = map.instrument, "" else: result = map.instrument, map.filekind return result
def properties_inside_mapping(filename): """Load `filename`s mapping header to discover and return (instrument, filekind). """ map = rmap.fetch_mapping(filename) if map.filekind == "PIPELINE": result = "", "" elif map.filekind == "INSTRUMENT": result = map.instrument, "" else: result = map.instrument, map.filekind return result
def mapping_diffs(self): """Return the logical differences between CRDS mappings named `old_file` and `new_file`. IFF include_header_diffs, include differences in mapping headers. Some are "boring", e.g. sha1sum or name. IFF recurse_added_deleted, include difference tuples for all nested adds and deletes whenever a higher level mapping is added or deleted. Else, only include the higher level mapping, not contained files. """ old_map = rmap.fetch_mapping(self.locate_file(self.old_file), ignore_checksum=True, path=self.mappings_cache1) new_map = rmap.fetch_mapping(self.locate_file(self.new_file), ignore_checksum=True, path=self.mappings_cache2) differences = old_map.difference( new_map, include_header_diffs=self.include_header_diffs, recurse_added_deleted=self.recurse_added_deleted) return differences
def update_derivation(new_path, old_basename=None): """Set the 'derived_from' and 'name' header fields of `new_path`. This function works for all Mapping classes: pmap, imap, and rmap. """ new = rmap.fetch_mapping(new_path) if old_basename is None: # assume new is a copy of old, with old's name in header derived_from = new.name else: derived_from = old_basename new.header["derived_from"] = str(derived_from) new.header["name"] = str(os.path.basename(new_path)) new.write(new_path) return str(derived_from)
def update_derivation(new_path, old_basename=None): """Set the 'derived_from' and 'name' header fields of `new_path`. This function works for all Mapping classes: pmap, imap, and rmap. """ new = rmap.fetch_mapping(new_path) if old_basename is None: # assume new is a copy of old, with old's name in header derived_from = new.name else: derived_from = old_basename new.header["derived_from"] = str(derived_from) new.header["name"] = str(os.path.basename(new_path)) new.write(new_path) return str(derived_from)
def rmap_delete_references(old_rmap, new_rmap, deleted_references): """Given the full path of starting rmap `old_rmap`, modify it by deleting all files in `deleted_references` and write out the result to `new_rmap`. If no actions are performed, don't write out `new_rmap`. Return new ReferenceMapping named `new_rmap` """ new = old = rmap.fetch_mapping(old_rmap, ignore_checksum=True) for reference in deleted_references: log.info("Deleting", repr(reference), "from", repr(new.name)) new = new.delete(reference) new.header["derived_from"] = old.basename log.verbose("Writing", repr(new_rmap)) new.write(new_rmap) formatted = new.format() for reference in deleted_references: reference = os.path.basename(reference) assert reference not in formatted, \ "Rules update failure. Deleted" + repr(reference) + " still appears in new rmap." return new
def rmap_delete_references(old_rmap, new_rmap, deleted_references): """Given the full path of starting rmap `old_rmap`, modify it by deleting all files in `deleted_references` and write out the result to `new_rmap`. If no actions are performed, don't write out `new_rmap`. Return new ReferenceMapping named `new_rmap` """ new = old = rmap.fetch_mapping(old_rmap, ignore_checksum=True) for reference in deleted_references: log.info("Deleting", repr(reference), "from", repr(new.name)) new = new.delete(reference) new.header["derived_from"] = old.basename log.verbose("Writing", repr(new_rmap)) new.write(new_rmap) formatted = new.format() for reference in deleted_references: reference = os.path.basename(reference) assert reference not in formatted, \ "Rules update failure. Deleted" + repr(reference) + " still appears in new rmap." return new
def rmap_insert_references(old_rmap, new_rmap, inserted_references): """Given the full path of starting rmap `old_rmap`, modify it by inserting or replacing all files in `inserted_references` and write out the result to `new_rmap`. If no actions are performed, don't write out `new_rmap`. Return new ReferenceMapping named `new_rmap` """ new = old = rmap.fetch_mapping(old_rmap, ignore_checksum=True) for reference in inserted_references: log.info("Inserting", os.path.basename(reference), "into", repr(new.name)) new = new.insert_reference(reference) new.header["derived_from"] = old.basename log.verbose("Writing", repr(new_rmap)) new.write(new_rmap) formatted = new.format() for reference in inserted_references: reference = os.path.basename(reference) assert reference in formatted, \ "Rules update failure. " + repr(reference) + " does not appear in new rmap." \ " May be identical match with other submitted references." return new
def rmap_insert_references(old_rmap, new_rmap, inserted_references): """Given the full path of starting rmap `old_rmap`, modify it by inserting or replacing all files in `inserted_references` and write out the result to `new_rmap`. If no actions are performed, don't write out `new_rmap`. Return new ReferenceMapping named `new_rmap` """ new = old = rmap.fetch_mapping(old_rmap, ignore_checksum=True) for reference in inserted_references: log.info("Inserting", os.path.basename(reference), "into", repr(new.name)) new = new.insert_reference(reference) new.header["derived_from"] = old.basename log.verbose("Writing", repr(new_rmap)) new.write(new_rmap) formatted = new.format() for reference in inserted_references: reference = os.path.basename(reference) assert reference in formatted, \ "Rules update failure. " + repr(reference) + " does not appear in new rmap." \ " May be identical match with other submitted references." return new