def print_affected_modes(self):
"""Print out all the affected mode tuples associated with the differences."""
assert rmap.is_mapping(self.old_file) and rmap.is_mapping(self.new_file), \
"for --print-affected-modes both files must be mappings."
modes = mapping_affected_modes(self.old_file, self.new_file, self.args.include_header_diffs)
for affected in modes:
print(format_affected_mode(affected))
return 1 if modes else 0
def __init__(self, *args, **keys):
super(MappingDifferencer, self).__init__(*args, **keys)
assert rmap.is_mapping(self.old_file), \
"File " + repr(self.old_file) + " is not a CRDS mapping."
assert rmap.is_mapping(self.new_file), \
"File " + repr(self.new_file) + " is not a CRDS mapping."
assert os.path.splitext(self.old_file)[-1] == os.path.splitext(self.new_file)[-1], \
"Files " + repr(self.old_file) + " and " + repr(self.new_file) + \
" are not the same kind of CRDS mapping: .pmap, .imap, .rmap"
def print_affected_modes(self):
"""Print out all the affected mode tuples associated with the differences."""
assert rmap.is_mapping(self.old_file) and rmap.is_mapping(self.new_file), \
"for --print-affected-modes both files must be mappings."
modes = mapping_affected_modes(self.old_file, self.new_file,
self.args.include_header_diffs)
for affected in modes:
print(format_affected_mode(affected))
return 1 if modes else 0
def __init__(self, *args, **keys):
super(MappingDifferencer, self).__init__(*args, **keys)
assert rmap.is_mapping(self.old_file), \
"File " + repr(self.old_file) + " is not a CRDS mapping."
assert rmap.is_mapping(self.new_file), \
"File " + repr(self.new_file) + " is not a CRDS mapping."
assert os.path.splitext(self.old_file)[-1] == os.path.splitext(self.new_file)[-1], \
"Files " + repr(self.old_file) + " and " + repr(self.new_file) + \
" are not the same kind of CRDS mapping: .pmap, .imap, .rmap"
def print_all_new_files(self):
"""Print the names of all files which are in `new_file` (or any intermediary context) but not
in `old_file`. new_file > old_file. Both new_file and old_file are similar mappings.
"""
updated = get_updated_files(self.old_file, self.new_file)
for mapping in updated:
if rmap.is_mapping(mapping):
print(mapping, self.instrument_filekind(mapping))
for reference in updated:
if not rmap.is_mapping(reference):
print(reference, self.instrument_filekind(reference))
return 1 if updated else 0
def print_all_new_files(self):
"""Print the names of all files which are in `new_file` (or any intermediary context) but not
in `old_file`. new_file > old_file. Both new_file and old_file are similar mappings.
"""
updated = get_updated_files(self.old_file, self.new_file)
for mapping in updated:
if rmap.is_mapping(mapping):
print(mapping, self.instrument_filekind(mapping))
for reference in updated:
if not rmap.is_mapping(reference):
print(reference, self.instrument_filekind(reference))
return 1 if updated else 0
def get_affected(self):
"""Examine the diffs between `old_pmap` and `new_pmap` and return sorted lists of affected instruments and types.
Returns { affected_instrument : { affected_type, ... } }
"""
instrs = defaultdict(set)
diffs = self.mapping_diffs()
diffs = remove_boring(diffs)
for diff in diffs:
for step in diff:
# Walking down the diff steps 1-by-1 eventually hits an rmap comparison which
# will define both instrument and type. pmaps and imaps leave at least one blank.
if len(step) == 2 and rmap.is_mapping(step[0]):
instrument, filekind = utils.get_file_properties(
self.observatory, step[0])
# This is inefficient since diff doesn't vary by step, but set logic cleans up the redundancy
# New rmaps imply reprocessing the entire type.
elif isinstance(diff[-1],str) and diff[-1].startswith(("added","deleted")) and \
diff[-1].endswith(".rmap'"):
rmap_name = diff[-1].split()[-1].replace("'", "")
rmapping = rmap.fetch_mapping(rmap_name,
ignore_checksum=True)
instrument, filekind = rmapping.instrument, rmapping.filekind
if instrument.strip() and filekind.strip():
if filekind not in instrs[instrument]:
log.verbose("Affected", (instrument, filekind),
"based on diff",
diff,
verbosity=20)
instrs[instrument].add(filekind)
return {key: list(val) for (key, val) in instrs.items()}
def get_affected(self):
"""Examine the diffs between `old_pmap` and `new_pmap` and return sorted lists of affected instruments and types.
Returns { affected_instrument : { affected_type, ... } }
"""
instrs = defaultdict(set)
diffs = self.mapping_diffs()
diffs = remove_boring(diffs)
for diff in diffs:
for step in diff:
# Walking down the diff steps 1-by-1 eventually hits an rmap comparison which
# will define both instrument and type. pmaps and imaps leave at least one blank.
if len(step) == 2 and rmap.is_mapping(step[0]):
instrument, filekind = utils.get_file_properties(self.observatory, step[0])
# This is inefficient since diff doesn't vary by step, but set logic cleans up the redundancy
# New rmaps imply reprocessing the entire type.
elif isinstance(diff[-1],str) and diff[-1].startswith(("added","deleted")) and \
diff[-1].endswith(".rmap'"):
rmap_name = diff[-1].split()[-1].replace("'","")
rmapping = rmap.fetch_mapping(rmap_name, ignore_checksum=True)
instrument, filekind = rmapping.instrument, rmapping.filekind
if instrument.strip() and filekind.strip():
if filekind not in instrs[instrument]:
log.verbose("Affected", (instrument, filekind), "based on diff", diff, verbosity=20)
instrs[instrument].add(filekind)
return { key:list(val) for (key, val) in instrs.items() }
def get_file_properties(filename):
"""Figure out (instrument, filekind, serial) based on `filename` which
should be a mapping or FITS reference file.
>> get_file_properties("./hst_acs_biasfile_0001.rmap")
('acs', 'biasfile')
>> get_file_properties("./hst_acs_biasfile_0001.pmap")
Traceback (most recent call last):
...
AssertionError: Invalid .pmap filename './hst_acs_biasfile_0001.pmap'
>> get_file_properties("test_data/s7g1700gl_dead.fits")
"""
if rmap.is_mapping(filename):
try:
return decompose_newstyle_name(filename)[2:4]
except Exception:
# NOTE: load_mapping more conservative than fetch_mapping used in properties_from_mapping
mapping = rmap.load_mapping(filename)
return mapping.instrument, mapping.filekind
elif config.is_reference(filename):
result = get_reference_properties(filename)[2:4]
else:
try:
result = properties_inside_mapping(filename)
except Exception as exc:
result = get_reference_properties(filename)[2:4]
assert result[0] in INSTRUMENTS+[""], "Bad instrument " + \
repr(result[0]) + " in filename " + repr(filename)
assert result[1] in FILEKINDS+[""], "Bad filekind " + \
repr(result[1]) + " in filename " + repr(filename)
return result
def get_file_properties(filename):
"""Figure out (instrument, filekind, serial) based on `filename` which
should be a mapping or FITS reference file.
>> get_file_properties("./hst_acs_biasfile_0001.rmap")
('acs', 'biasfile')
>> get_file_properties("./hst_acs_biasfile_0001.pmap")
Traceback (most recent call last):
...
AssertionError: Invalid .pmap filename './hst_acs_biasfile_0001.pmap'
>> get_file_properties("test_data/s7g1700gl_dead.fits")
"""
if rmap.is_mapping(filename):
return decompose_newstyle_name(filename)[2:4]
elif REF_EXT_RE.search(filename):
result = get_reference_properties(filename)[2:4]
else:
try:
result = properties_inside_mapping(filename)
except Exception as exc:
result = get_reference_properties(filename)[2:4]
assert result[0] in INSTRUMENTS+[""], "Bad instrument " + \
repr(result[0]) + " in filename " + repr(filename)
assert result[1] in FILEKINDS+[""], "Bad filekind " + \
repr(result[1]) + " in filename " + repr(filename)
return result
def mapping_pairs(differences):
"""Return the sorted list of all mapping tuples found in differences."""
pairs = set()
for diff in differences:
for pair in diff:
if len(pair) == 2 and rmap.is_mapping(pair[0]):
pairs.add(pair)
return sorted(pairs)
def mapping_pairs(differences):
"""Return the sorted list of all mapping tuples found in differences."""
pairs = set()
for diff in differences:
for pair in diff:
if len(pair) == 2 and rmap.is_mapping(pair[0]):
pairs.add(pair)
return sorted(pairs)
def _find_diff_str(self, diff_str):
"""Return True IFF `diff_str` is in some rmap diff."""
diffs = self.mapping_diffs()
diffs = remove_boring(diffs)
for diff in diffs:
for step in diff:
if len(step) == 2 and rmap.is_mapping(step[0]):
if diff_str in diff_action(diff):
log.verbose("Found", repr(diff_str), "diff between", repr(step[0:1]))
return True
return False
def remove_checksum(file_):
"""Remove checksums from `file_`."""
log.info("Removing checksum for", repr(file_))
if config.is_reference(file_):
data_file.remove_checksum(file_)
elif rmap.is_mapping(file_):
raise exceptions.CrdsError("Mapping checksums cannot be removed for:",
repr(file_))
else:
raise exceptions.CrdsError(
"File", repr(file_),
"does not appear to be a CRDS reference or mapping file.")
def _find_diff_str(self, diff_str):
"""Return True IFF `diff_str` is in some rmap diff."""
diffs = self.mapping_diffs()
diffs = remove_boring(diffs)
for diff in diffs:
for step in diff:
if len(step) == 2 and rmap.is_mapping(step[0]):
if diff_str in diff_action(diff):
log.verbose("Found", repr(diff_str), "diff between",
repr(step[0:1]))
return True
return False
def print_new_files(self):
"""Print the references or mappings which are in the second (new) context and not
the firtst (old) context.
"""
if not rmap.is_mapping(self.old_file) or not rmap.is_mapping(self.new_file):
log.error("--print-new-files really only works for mapping differences.")
return -1
old = crds.get_pickled_mapping(self.old_file) # reviewed
new = crds.get_pickled_mapping(self.new_file) # reviewed
old_mappings = set(old.mapping_names())
new_mappings = set(new.mapping_names())
old_references = set(old.reference_names())
new_references = set(new.reference_names())
status = 0
for name in sorted(new_mappings - old_mappings):
print(name)
status = 1
for name in sorted(new_references - old_references):
print(name)
status = 1
return status
def add_checksum(file_):
"""Add checksums to file_."""
log.info("Adding checksum for", repr(file_))
if config.is_reference(file_):
with log.error_on_exception("Failed updating checksum for",
repr(file_)):
data_file.add_checksum(file_)
elif rmap.is_mapping(file_):
update_mapping_checksum(file_)
else:
raise exceptions.CrdsError(
"File", repr(file_),
"does not appear to be a CRDS reference or mapping file.")
def print_new_files(self):
"""Print the references or mappings which are in the second (new) context and not
the firtst (old) context.
"""
if not rmap.is_mapping(self.old_file) or not rmap.is_mapping(
self.new_file):
log.error(
"--print-new-files really only works for mapping differences.")
return -1
old = crds.get_pickled_mapping(self.old_file) # reviewed
new = crds.get_pickled_mapping(self.new_file) # reviewed
old_mappings = set(old.mapping_names())
new_mappings = set(new.mapping_names())
old_references = set(old.reference_names())
new_references = set(new.reference_names())
status = 0
for name in sorted(new_mappings - old_mappings):
print(name)
status = 1
for name in sorted(new_references - old_references):
print(name)
status = 1
return status
def verify_checksum(file_):
"""Verify checksums in `file_`."""
log.info("Verifying checksum for", repr(file_))
if config.is_reference(file_):
data_file.verify_checksum(file_)
elif rmap.is_mapping(file_):
if config.CRDS_IGNORE_MAPPING_CHECKSUM.get():
log.warning(
"Mapping checksums are disabled by config.CRDS_IGNORE_MAPPING_CHECKSM."
)
rmap.load_mapping(file_)
else:
raise exceptions.CrdsError(
"File", repr(file_),
"does not appear to be a CRDS reference or mapping file.")
