def load_fingerprint(self, kind):
"""Load the content of the fingerprint from disc.
:param kind: the primitive name
:type kind: str
:return: Returns a Fingerprint object (the content of the fingerprint
file or an empty Fingerprint when the fingerprint is invalid
or does not exist).
:rtype: Fingerprint
"""
fingerprint_file = self.fingerprint_filename(kind)
result = None
if os.path.exists(fingerprint_file):
try:
result = Fingerprint.load_from_file(fingerprint_file)
except Exception as e:
logger.warning(e)
# Invalid fingerprint
logger.warning('invalid fingerprint, discarding it')
result = None
if not isinstance(result, Fingerprint):
# The fingerprint file did not exist or was invalid
# returns an empty fingerprint
result = Fingerprint()
return result
def test_corrupted_fingerprint(setup_sbx):
"""Test the case where a fingerprint somehow got corrupted."""
actions = DAG()
actions.add_vertex('1')
actions.add_vertex('2', predecessors=['1'])
actions.add_vertex('3')
actions.add_vertex('4', predecessors=['2', '3'])
actions.add_vertex('5', predecessors=['4'])
actions.add_vertex('6')
# Now, execute the plan a first time; everything should be run
# and finish succesfullly.
r1 = FingerprintWalk(actions)
for uid in ('1', '2', '3', '4', '5', '6'):
job = r1.saved_jobs[uid]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert r1.job_status == {'1': ReturnValue.success,
'2': ReturnValue.success,
'3': ReturnValue.success,
'4': ReturnValue.success,
'5': ReturnValue.success,
'6': ReturnValue.success}
assert r1.requeued == {}
# Now, corrupt the fingerprint of node '3', and then rerun
# the scheduler... We expect the following:
# - The scheduler does _not_ crash ;-)
# - The fingerprint of node '3' gets discarded, and as a result
# it should be re-run again.
# - Since nothing changed in node "3"'s predecessors, the end
# result for node '3' should be the same, which means
# its fingerprint should be the same as before the corruption.
# As a result of that, nodes '4' and '5', which directly
# or indirectly depend on node '3', do not need to be rerun.
with open(r1.fingerprint_filename('3'), 'w') as f:
f.write('{')
r2 = FingerprintWalk(actions)
job = r2.saved_jobs['3']
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
for uid in ('1', '2', '4', '5', '6'):
job = r2.saved_jobs[uid]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
# Verify also that the fingerprint corruption is gone.
f3 = Fingerprint.load_from_file(r2.fingerprint_filename('3'))
assert isinstance(f3, Fingerprint)
def load_previous_fingerprint(self, uid):
# In dry-run mode, the fingerprints on file are let untouched,
# so they might be out of date compared to this job's status
# as part of this dry run. So, if we have already computed
# the fingerprint before, use that.
if self.dry_run_mode and uid in self.new_fingerprints:
return self.new_fingerprints[uid]
filename = self.fingerprint_filename(uid)
if os.path.exists(filename):
return Fingerprint.load_from_file(filename)
else:
return None
def test_fingerprint_save_and_load():
# Create a directory where to store our fingerprints, allowing us
# to use any fingerprint name without potentially colliding with
# other files used by this testcase.
os.mkdir("fingerprints")
def fingerprint_path(filename):
return os.path.join("fingerprints", filename)
# Save and then load a minimal fingerprint...
f_min = Fingerprint()
f_min_filename = fingerprint_path("f_min")
assert not os.path.exists(f_min_filename)
f_min.save_to_file(f_min_filename)
f_min_restored = Fingerprint.load_from_file(f_min_filename)
assert f_min_restored == f_min
assert str(f_min_restored) == str(f_min)
assert f_min_restored.checksum() == f_min.checksum()
# Save and then load a fingerprint with more data than the minimum.
f2 = Fingerprint()
f2.add("job1", "job1sha1")
f2.add("job2", "sha1job2")
f2_filename = fingerprint_path("f2")
assert not os.path.exists(f2_filename)
f2.save_to_file(f2_filename)
f2_restored = Fingerprint.load_from_file(f2_filename)
assert f2_restored == f2
assert str(f2_restored) == str(f2)
assert f2_restored.checksum() == f2.checksum()
# Trying to load from a file with invalid contents (bad JSON)
f_bad_filename = fingerprint_path("f_bad_JSON")
with open(f_bad_filename, "w") as f:
f.write("yello{")
f3 = Fingerprint.load_from_file(f_bad_filename)
assert f3 is None
# Trying to load from a file which contains valid data, but
# is not an dictionary, and therefore clearly not something
# that comes from a fingerprint...
f_not_filename = fingerprint_path("not_a_fingerprint")
with open(f_not_filename, "w") as f:
json.dump([1, 2, 3], f)
f4 = Fingerprint.load_from_file(f_not_filename)
assert f4 is None
# Try to load from a file which is missing one of the mandatory
# elements.
for key in ("fingerprint_version", "elements"):
f_key_missing_filename = fingerprint_path("no_%s_key")
# To create the bad file without assuming too much in this test
# how the fingerprint is saved to file, we save a valid fingerprint
# to file, load that file back, remove the key, and then save the
# truncated data again.
f2.save_to_file(f_key_missing_filename)
with open(f_key_missing_filename) as f:
data = json.load(f)
del data[key]
with open(f_key_missing_filename, "w") as f:
json.dump(data, f)
f5 = Fingerprint.load_from_file(f_key_missing_filename)
assert f5 is None
# Try loading a fingerprint whose version number is not recognized
# (typically, and old fingerprint version that we no longer support).
#
# To create the file without assuming too much in this test how
# fingerprint are saved to file, we start with a valid fingerprint
# that we saved to a file, load that file back, adjust the version
# number, and then replace the good fingeprint in that file by
# the modified one.
f_bad_version = fingerprint_path("bad_version")
f2.save_to_file(f_bad_version)
with open(f_bad_version) as f:
data = json.load(f)
data["fingerprint_version"] = "1.0"
data["elements"]["fingerprint_version"] = "1.0"
with open(f_bad_version, "w") as f:
json.dump(data, f)
f = Fingerprint.load_from_file(f_bad_version)
assert f is None
def load_previous_fingerprint(self, uid):
"""See Walk.load_previous_fingerprint."""
return Fingerprint.load_from_file(self.fingerprint_filename(uid))