def load_fingerprint(self, kind):
"""Load the content of the fingerprint from disc.
:param kind: the primitive name
:type kind: str
:return: Returns a Fingerprint object (the content of the fingerprint
file or an empty Fingerprint when the fingerprint is invalid
or does not exist).
:rtype: Fingerprint
"""
fingerprint_file = self.fingerprint_filename(kind)
result = None
if os.path.exists(fingerprint_file):
try:
result = Fingerprint.load_from_file(fingerprint_file)
except Exception as e:
logger.warning(e)
# Invalid fingerprint
logger.warning('invalid fingerprint, discarding it')
result = None
if not isinstance(result, Fingerprint):
# The fingerprint file did not exist or was invalid
# returns an empty fingerprint
result = Fingerprint()
return result
def test_fingerprint_version():
"""Changing the FINGERPRINT_VERSION modify the fingerprint's checksum."""
import e3.fingerprint
f1 = Fingerprint()
e3.fingerprint.FINGERPRINT_VERSION = "0.0"
f2 = Fingerprint()
assert f1 != f2
assert f1.checksum() != f2.checksum()
f3 = Fingerprint()
assert f2 == f3
assert f2.checksum() == f3.checksum()
def test_corrupted_fingerprint(setup_sbx):
"""Test the case where a fingerprint somehow got corrupted."""
actions = DAG()
actions.add_vertex('1')
actions.add_vertex('2', predecessors=['1'])
actions.add_vertex('3')
actions.add_vertex('4', predecessors=['2', '3'])
actions.add_vertex('5', predecessors=['4'])
actions.add_vertex('6')
# Now, execute the plan a first time; everything should be run
# and finish succesfullly.
r1 = FingerprintWalk(actions)
for uid in ('1', '2', '3', '4', '5', '6'):
job = r1.saved_jobs[uid]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert r1.job_status == {'1': ReturnValue.success,
'2': ReturnValue.success,
'3': ReturnValue.success,
'4': ReturnValue.success,
'5': ReturnValue.success,
'6': ReturnValue.success}
assert r1.requeued == {}
# Now, corrupt the fingerprint of node '3', and then rerun
# the scheduler... We expect the following:
# - The scheduler does _not_ crash ;-)
# - The fingerprint of node '3' gets discarded, and as a result
# it should be re-run again.
# - Since nothing changed in node "3"'s predecessors, the end
# result for node '3' should be the same, which means
# its fingerprint should be the same as before the corruption.
# As a result of that, nodes '4' and '5', which directly
# or indirectly depend on node '3', do not need to be rerun.
with open(r1.fingerprint_filename('3'), 'w') as f:
f.write('{')
r2 = FingerprintWalk(actions)
job = r2.saved_jobs['3']
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
for uid in ('1', '2', '4', '5', '6'):
job = r2.saved_jobs[uid]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
# Verify also that the fingerprint corruption is gone.
f3 = Fingerprint.load_from_file(r2.fingerprint_filename('3'))
assert isinstance(f3, Fingerprint)
def create_fingerprint(first, second, third):
"""Create a fingerprint with 3 elements added in the given order.
The arguments, first, second, and third should be either
1, 2, or 3 (integers), and are meant as indices to (key, value)
pairs in the idx_to_entry_map below, indicating which part
of the fingerprint gets added in what order.
"""
idx_to_entry_map = {1: ("foo", "1"), 2: ("bar", "2"), 3: ("baz", "3")}
f = Fingerprint()
f.add(*idx_to_entry_map[first])
f.add(*idx_to_entry_map[second])
f.add(*idx_to_entry_map[third])
return f
def test_unexpected_exception_during_fingerprint_load():
"""Check that we catch all exceptions raised during fingerprint load.
"""
bs = BuildSpace(root_dir=os.getcwd(), primitive='build')
bs.create()
fingerprint_filename = bs.fingerprint_filename(kind='build')
# Scenario: The fingerprint file is actually not a file...
mkdir(fingerprint_filename)
fp = bs.load_fingerprint(kind='build')
assert fp == Fingerprint()
os.rmdir(fingerprint_filename)
# Scenario: The fingerprint file is not readable (lack of permissions
# in this case).
#
# Note that we do not have an easy way to remove read permission
# to a file when on Windows, so we simply avoid that test when
# on that platform. This test exercises the same code as in
# the previous scenario, so this is not a big loss.
if sys.platform != 'win32':
ref_fp = Fingerprint()
ref_fp.add('key1', 'val1')
bs.save_fingerprint(kind='build', fingerprint=ref_fp)
os.chmod(fingerprint_filename, 0)
fp = bs.load_fingerprint(kind='build')
assert fp == Fingerprint()
os.chmod(fingerprint_filename, 0o600)
os.remove(fingerprint_filename)
def load_previous_fingerprint(self, uid):
# In dry-run mode, the fingerprints on file are let untouched,
# so they might be out of date compared to this job's status
# as part of this dry run. So, if we have already computed
# the fingerprint before, use that.
if self.dry_run_mode and uid in self.new_fingerprints:
return self.new_fingerprints[uid]
filename = self.fingerprint_filename(uid)
if os.path.exists(filename):
return Fingerprint.load_from_file(filename)
else:
return None
def compute_new_fingerprint(self, uid, data):
if 'no_fingerprint' in uid:
return None
f = Fingerprint()
for pred_uid in self.actions.get_predecessors(uid):
pred_fingerprint = self.new_fingerprints[pred_uid]
if pred_fingerprint is not None:
f.add('pred:%s' % pred_uid, pred_fingerprint.checksum())
if os.path.exists(source_fullpath(uid)):
f.add_file(source_fullpath(uid))
return f
def compute_fingerprint(self, uid, data, is_prediction=False):
if "fingerprint_after_job" in uid and is_prediction:
return None
if "no_fingerprint" in uid:
return None
f = Fingerprint()
for pred_uid in self.actions.get_predecessors(uid):
pred_fingerprint = self.new_fingerprints[pred_uid]
if pred_fingerprint is not None:
f.add("pred:%s" % pred_uid, pred_fingerprint.checksum())
if os.path.exists(source_fullpath(uid)):
f.add_file(source_fullpath(uid))
return f
def update_status(self,
kind,
status=ReturnValue.failure,
fingerprint=None):
"""Update meta information on disk.
:param kind: the primitive name
:type kind: str
:param status: the last action return code
:type status: ReturnValue
:param fingerprint: the fingerprint
:type fingerprint: Fingerprint
"""
if fingerprint is None:
fingerprint = Fingerprint()
self.save_fingerprint(kind, fingerprint)
self.save_last_status(kind, status)
if kind == 'build':
self.update_status('install', status, fingerprint)
def compute_fingerprint(self, uid, data, is_prediction=False):
"""See Walk.compute_fingerprint."""
if is_prediction and self.force:
# Force a rebuild
return None
# All fingerprints start with the same minimum amount
# of data: The fingerprint of all the predecessors.
f = Fingerprint()
for pred_uid in self.actions.get_predecessors(uid):
if self.new_fingerprints[pred_uid] is None:
logging.debug(
"Returning no fingerprint for %s"
" (predecessor %s has no fingerprint)",
uid,
pred_uid,
)
return None
else:
f.add(pred_uid, self.new_fingerprints[pred_uid].checksum())
if isinstance(data, Checkout):
if is_prediction:
# We cannot predict the fingerprint of a checkout without
# performing it, thus checkout will always be executed.
return None
m = CheckoutManager(name=data.repo_name,
working_dir=self.sandbox.vcs_dir)
with open(m.metadata_file) as fd:
content = json.load(fd)
f.add(data.repo_name + ".url", content["url"])
f.add(data.repo_name + ".commit", content["new_commit"])
elif isinstance(data, (CreateSource, Build)):
add_anod_files_to_fingerprint(data.anod_instance, f)
elif isinstance(data, InstallSource):
add_anod_files_to_fingerprint(data.spec, f)
return f
def test_fingerprint():
"""Check dump/restore of fingerprint."""
bs = BuildSpace(root_dir=os.getcwd(), primitive='build')
bs.create()
fp = Fingerprint()
fp.add('foo', 'bar')
bs.save_fingerprint(kind='build', fingerprint=fp)
loaded_fp = bs.load_fingerprint(kind='build')
assert loaded_fp == fp
# Now make sure that load_fingerprint does not fail when the bumped
# value is corrupted
with open(bs.fingerprint_filename('build'), 'w') as f:
f.write('[')
assert bs.load_fingerprint(kind='build') == Fingerprint()
# Try updating the fingerprint
fp.add('key', 'value')
bs.update_status(kind='build', status=ReturnValue.notready, fingerprint=fp)
assert bs.get_last_status(kind='build')[0] == ReturnValue.notready
loaded_fp = bs.load_fingerprint(kind='build')
assert loaded_fp == fp
# Updating build fingerprint also update install fingerprint
loaded_fp = bs.load_fingerprint(kind='install')
assert loaded_fp == fp
# Now update install status
bs.update_status(kind='install', status=ReturnValue.success)
assert bs.get_last_status(kind='install')[0] == ReturnValue.success
# build status should not be modified
assert bs.get_last_status(kind='build')[0] == ReturnValue.notready
def add_anod_files_to_fingerprint(anod_instance: Anod,
fingerprint: Fingerprint) -> None:
"""Add the Anod's spec and yaml files to the given fingerprint.
:param anod_instance: an Anod instance.
:type anod_instance: Anod
:param fingerprint: The fingerprint to update.
:type fingerprint: e3.fingerprint.Fingerprint.
"""
anod_specs = [
c.name for c in anod_instance.__class__.__mro__
if c.__name__ != "Anod" and "Anod" in (sc.__name__ for sc in c.__mro__)
]
for spec_name in anod_specs:
fingerprint.add_file(
os.path.join(anod_instance.spec_dir, spec_name + ".anod"))
for yaml_name in anod_instance.data_files:
fingerprint.add_file(
os.path.join(anod_instance.spec_dir, yaml_name + ".yaml"))
deps = getattr(anod_instance, "%s_deps" % anod_instance.kind, ())
for dep in deps:
if isinstance(dep, anod_instance.BuildVar):
fingerprint.add(dep.name, dep.value)
def test_fingerprint_save_and_load():
# Create a directory where to store our fingerprints, allowing us
# to use any fingerprint name without potentially colliding with
# other files used by this testcase.
os.mkdir("fingerprints")
def fingerprint_path(filename):
return os.path.join("fingerprints", filename)
# Save and then load a minimal fingerprint...
f_min = Fingerprint()
f_min_filename = fingerprint_path("f_min")
assert not os.path.exists(f_min_filename)
f_min.save_to_file(f_min_filename)
f_min_restored = Fingerprint.load_from_file(f_min_filename)
assert f_min_restored == f_min
assert str(f_min_restored) == str(f_min)
assert f_min_restored.checksum() == f_min.checksum()
# Save and then load a fingerprint with more data than the minimum.
f2 = Fingerprint()
f2.add("job1", "job1sha1")
f2.add("job2", "sha1job2")
f2_filename = fingerprint_path("f2")
assert not os.path.exists(f2_filename)
f2.save_to_file(f2_filename)
f2_restored = Fingerprint.load_from_file(f2_filename)
assert f2_restored == f2
assert str(f2_restored) == str(f2)
assert f2_restored.checksum() == f2.checksum()
# Trying to load from a file with invalid contents (bad JSON)
f_bad_filename = fingerprint_path("f_bad_JSON")
with open(f_bad_filename, "w") as f:
f.write("yello{")
f3 = Fingerprint.load_from_file(f_bad_filename)
assert f3 is None
# Trying to load from a file which contains valid data, but
# is not an dictionary, and therefore clearly not something
# that comes from a fingerprint...
f_not_filename = fingerprint_path("not_a_fingerprint")
with open(f_not_filename, "w") as f:
json.dump([1, 2, 3], f)
f4 = Fingerprint.load_from_file(f_not_filename)
assert f4 is None
# Try to load from a file which is missing one of the mandatory
# elements.
for key in ("fingerprint_version", "elements"):
f_key_missing_filename = fingerprint_path("no_%s_key")
# To create the bad file without assuming too much in this test
# how the fingerprint is saved to file, we save a valid fingerprint
# to file, load that file back, remove the key, and then save the
# truncated data again.
f2.save_to_file(f_key_missing_filename)
with open(f_key_missing_filename) as f:
data = json.load(f)
del data[key]
with open(f_key_missing_filename, "w") as f:
json.dump(data, f)
f5 = Fingerprint.load_from_file(f_key_missing_filename)
assert f5 is None
# Try loading a fingerprint whose version number is not recognized
# (typically, and old fingerprint version that we no longer support).
#
# To create the file without assuming too much in this test how
# fingerprint are saved to file, we start with a valid fingerprint
# that we saved to a file, load that file back, adjust the version
# number, and then replace the good fingeprint in that file by
# the modified one.
f_bad_version = fingerprint_path("bad_version")
f2.save_to_file(f_bad_version)
with open(f_bad_version) as f:
data = json.load(f)
data["fingerprint_version"] = "1.0"
data["elements"]["fingerprint_version"] = "1.0"
with open(f_bad_version, "w") as f:
json.dump(data, f)
f = Fingerprint.load_from_file(f_bad_version)
assert f is None
def test_fingerprint_eq():
"""Check fingerprint __eq__ function."""
f1 = Fingerprint()
f1.add("1", "1")
assert f1 != 1
f2 = Fingerprint()
f2.add("1", "1")
f2.add("2", "2")
assert f1 != f2
assert f1.checksum() != f2.checksum()
assert f1.compare_to(f1) is None
def test_invalid_fingerprint():
"""A fingerprint value should be hashable."""
f1 = Fingerprint()
with pytest.raises(E3Error):
f1.add("invalid", {})
def test_fingerprint():
f1 = Fingerprint()
f1.add("foo", "2")
f2 = Fingerprint()
f2.add("foo", "4")
f12_diff = f2.compare_to(f1)
assert f12_diff["new"] == set()
assert f12_diff["updated"] == {"foo"}
assert f12_diff["obsolete"] == set()
f3 = Fingerprint()
f3.add_file(__file__)
f23_diff = f3.compare_to(f2)
assert f23_diff["new"] == {"foo"}
assert f23_diff["updated"] == set()
assert f23_diff["obsolete"] == {os.path.abspath(__file__)}
assert f1.checksum() != f2.checksum() != f3.checksum()
assert Env().build.os.version in str(f3)
f4 = Fingerprint()
f4.add_file(__file__)
assert f4 == f3
f5 = Fingerprint()
with pytest.raises(E3Error) as err:
f5.add("f4", f4)
assert "f4 should be a string" in str(err.value)
f6 = Fingerprint()
f6.add("unicode", "6")
assert len(f6.checksum()) == 64
def test_fingerprint():
f1 = Fingerprint()
f1.add('foo', '2')
f2 = Fingerprint()
f2.add('foo', '4')
f12_diff = f2.compare_to(f1)
assert f12_diff['new'] == set([])
assert f12_diff['updated'] == {'foo'}
assert f12_diff['obsolete'] == set([])
f3 = Fingerprint()
f3.add_file(__file__)
f23_diff = f3.compare_to(f2)
assert f23_diff['new'] == {'foo'}
assert f23_diff['updated'] == set([])
assert f23_diff['obsolete'] == {os.path.basename(__file__)}
assert f1.checksum() != f2.checksum() != f3.checksum()
assert Env().build.os.version in str(f3)
f4 = Fingerprint()
f4.add_file(__file__)
assert f4 == f3
f5 = Fingerprint()
with pytest.raises(E3Error) as err:
f5.add('f4', f4)
assert 'f4 should be a string' in str(err.value)
f6 = Fingerprint()
f6.add('unicode', u'6')
assert len(f6.checksum()) == 64
def load_previous_fingerprint(self, uid):
"""See Walk.load_previous_fingerprint."""
return Fingerprint.load_from_file(self.fingerprint_filename(uid))
