def mapping_diffs(self):
"""Return the logical differences between CRDS mappings named `old_file`
and `new_file`.
IFF include_header_diffs, include differences in mapping headers. Some are "boring", e.g. sha1sum or name.
IFF recurse_added_deleted, include difference tuples for all nested adds and deletes whenever a higher level
mapping is added or deleted. Else, only include the higher level mapping, not contained files.
"""
old_map = rmap.fetch_mapping(self.locate_file(self.old_file), ignore_checksum=True, path=self.mappings_cache1)
new_map = rmap.fetch_mapping(self.locate_file(self.new_file), ignore_checksum=True, path=self.mappings_cache2)
differences = old_map.difference(new_map, include_header_diffs=self.include_header_diffs,
recurse_added_deleted=self.recurse_added_deleted)
return differences
def properties_inside_mapping(filename):
"""Load `filename`s mapping header to discover and
return (instrument, filekind).
>>> from crds.tests import test_config
>>> old_config = test_config.setup()
>>> properties_inside_mapping("hst.pmap")
('', '')
>>> properties_inside_mapping("hst_acs.imap")
('acs', '')
>>> properties_inside_mapping("hst_acs_darkfile.rmap")
('acs', 'darkfile')
>>> test_config.cleanup(old_config)
"""
loaded = rmap.fetch_mapping(filename)
if loaded.mapping == "pipeline":
result = "", ""
elif loaded.mapping == "instrument":
result = loaded.instrument, ""
else:
result = loaded.instrument, loaded.filekind
return result
def properties_inside_mapping(filename):
"""Load `filename`s mapping header to discover and
return (instrument, filekind).
>>> from crds.tests import test_config
>>> old_config = test_config.setup()
>>> properties_inside_mapping("hst.pmap")
('', '')
>>> properties_inside_mapping("hst_acs.imap")
('acs', '')
>>> properties_inside_mapping("hst_acs_darkfile.rmap")
('acs', 'darkfile')
>>> test_config.cleanup(old_config)
"""
loaded = rmap.fetch_mapping(filename)
if loaded.mapping == "pipeline":
result = "", ""
elif loaded.mapping == "instrument":
result = loaded.instrument, ""
else:
result = loaded.instrument, loaded.filekind
return result
def get_affected(self):
"""Examine the diffs between `old_pmap` and `new_pmap` and return sorted lists of affected instruments and types.
Returns { affected_instrument : { affected_type, ... } }
"""
instrs = defaultdict(set)
diffs = self.mapping_diffs()
diffs = remove_boring(diffs)
for diff in diffs:
for step in diff:
# Walking down the diff steps 1-by-1 eventually hits an rmap comparison which
# will define both instrument and type. pmaps and imaps leave at least one blank.
if len(step) == 2 and rmap.is_mapping(step[0]):
instrument, filekind = utils.get_file_properties(self.observatory, step[0])
# This is inefficient since diff doesn't vary by step, but set logic cleans up the redundancy
# New rmaps imply reprocessing the entire type.
elif isinstance(diff[-1],str) and diff[-1].startswith(("added","deleted")) and \
diff[-1].endswith(".rmap'"):
rmap_name = diff[-1].split()[-1].replace("'","")
rmapping = rmap.fetch_mapping(rmap_name, ignore_checksum=True)
instrument, filekind = rmapping.instrument, rmapping.filekind
if instrument.strip() and filekind.strip():
if filekind not in instrs[instrument]:
log.verbose("Affected", (instrument, filekind), "based on diff", diff, verbosity=20)
instrs[instrument].add(filekind)
return { key:list(val) for (key, val) in instrs.items() }
def get_affected(self):
"""Examine the diffs between `old_pmap` and `new_pmap` and return sorted lists of affected instruments and types.
Returns { affected_instrument : { affected_type, ... } }
"""
instrs = defaultdict(set)
diffs = self.mapping_diffs()
diffs = remove_boring(diffs)
for diff in diffs:
for step in diff:
# Walking down the diff steps 1-by-1 eventually hits an rmap comparison which
# will define both instrument and type. pmaps and imaps leave at least one blank.
if len(step) == 2 and rmap.is_mapping(step[0]):
instrument, filekind = utils.get_file_properties(
self.observatory, step[0])
# This is inefficient since diff doesn't vary by step, but set logic cleans up the redundancy
# New rmaps imply reprocessing the entire type.
elif isinstance(diff[-1],str) and diff[-1].startswith(("added","deleted")) and \
diff[-1].endswith(".rmap'"):
rmap_name = diff[-1].split()[-1].replace("'", "")
rmapping = rmap.fetch_mapping(rmap_name,
ignore_checksum=True)
instrument, filekind = rmapping.instrument, rmapping.filekind
if instrument.strip() and filekind.strip():
if filekind not in instrs[instrument]:
log.verbose("Affected", (instrument, filekind),
"based on diff",
diff,
verbosity=20)
instrs[instrument].add(filekind)
return {key: list(val) for (key, val) in instrs.items()}
def properties_inside_mapping(filename):
"""Load `filename`s mapping header to discover and
return (instrument, filekind).
"""
map = rmap.fetch_mapping(filename)
if map.filekind == "PIPELINE":
result = "", ""
elif map.filekind == "INSTRUMENT":
result = map.instrument, ""
else:
result = map.instrument, map.filekind
return result
def properties_inside_mapping(filename):
"""Load `filename`s mapping header to discover and
return (instrument, filekind).
"""
map = rmap.fetch_mapping(filename)
if map.filekind == "PIPELINE":
result = "", ""
elif map.filekind == "INSTRUMENT":
result = map.instrument, ""
else:
result = map.instrument, map.filekind
return result
def mapping_diffs(self):
"""Return the logical differences between CRDS mappings named `old_file`
and `new_file`.
IFF include_header_diffs, include differences in mapping headers. Some are "boring", e.g. sha1sum or name.
IFF recurse_added_deleted, include difference tuples for all nested adds and deletes whenever a higher level
mapping is added or deleted. Else, only include the higher level mapping, not contained files.
"""
old_map = rmap.fetch_mapping(self.locate_file(self.old_file),
ignore_checksum=True,
path=self.mappings_cache1)
new_map = rmap.fetch_mapping(self.locate_file(self.new_file),
ignore_checksum=True,
path=self.mappings_cache2)
differences = old_map.difference(
new_map,
include_header_diffs=self.include_header_diffs,
recurse_added_deleted=self.recurse_added_deleted)
return differences
def update_derivation(new_path, old_basename=None):
"""Set the 'derived_from' and 'name' header fields of `new_path`.
This function works for all Mapping classes: pmap, imap, and rmap.
"""
new = rmap.fetch_mapping(new_path)
if old_basename is None: # assume new is a copy of old, with old's name in header
derived_from = new.name
else:
derived_from = old_basename
new.header["derived_from"] = str(derived_from)
new.header["name"] = str(os.path.basename(new_path))
new.write(new_path)
return str(derived_from)
def update_derivation(new_path, old_basename=None):
"""Set the 'derived_from' and 'name' header fields of `new_path`.
This function works for all Mapping classes: pmap, imap, and rmap.
"""
new = rmap.fetch_mapping(new_path)
if old_basename is None: # assume new is a copy of old, with old's name in header
derived_from = new.name
else:
derived_from = old_basename
new.header["derived_from"] = str(derived_from)
new.header["name"] = str(os.path.basename(new_path))
new.write(new_path)
return str(derived_from)
def rmap_delete_references(old_rmap, new_rmap, deleted_references):
"""Given the full path of starting rmap `old_rmap`, modify it by deleting
all files in `deleted_references` and write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.fetch_mapping(old_rmap, ignore_checksum=True)
for reference in deleted_references:
log.info("Deleting", repr(reference), "from", repr(new.name))
new = new.delete(reference)
new.header["derived_from"] = old.basename
log.verbose("Writing", repr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for reference in deleted_references:
reference = os.path.basename(reference)
assert reference not in formatted, \
"Rules update failure. Deleted" + repr(reference) + " still appears in new rmap."
return new
def rmap_delete_references(old_rmap, new_rmap, deleted_references):
"""Given the full path of starting rmap `old_rmap`, modify it by deleting
all files in `deleted_references` and write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.fetch_mapping(old_rmap, ignore_checksum=True)
for reference in deleted_references:
log.info("Deleting", repr(reference), "from", repr(new.name))
new = new.delete(reference)
new.header["derived_from"] = old.basename
log.verbose("Writing", repr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for reference in deleted_references:
reference = os.path.basename(reference)
assert reference not in formatted, \
"Rules update failure. Deleted" + repr(reference) + " still appears in new rmap."
return new
def rmap_insert_references(old_rmap, new_rmap, inserted_references):
"""Given the full path of starting rmap `old_rmap`, modify it by inserting
or replacing all files in `inserted_references` and write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.fetch_mapping(old_rmap, ignore_checksum=True)
for reference in inserted_references:
log.info("Inserting", os.path.basename(reference), "into", repr(new.name))
new = new.insert_reference(reference)
new.header["derived_from"] = old.basename
log.verbose("Writing", repr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for reference in inserted_references:
reference = os.path.basename(reference)
assert reference in formatted, \
"Rules update failure. " + repr(reference) + " does not appear in new rmap." \
" May be identical match with other submitted references."
return new
def rmap_insert_references(old_rmap, new_rmap, inserted_references):
"""Given the full path of starting rmap `old_rmap`, modify it by inserting
or replacing all files in `inserted_references` and write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.fetch_mapping(old_rmap, ignore_checksum=True)
for reference in inserted_references:
log.info("Inserting", os.path.basename(reference), "into",
repr(new.name))
new = new.insert_reference(reference)
new.header["derived_from"] = old.basename
log.verbose("Writing", repr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for reference in inserted_references:
reference = os.path.basename(reference)
assert reference in formatted, \
"Rules update failure. " + repr(reference) + " does not appear in new rmap." \
" May be identical match with other submitted references."
return new