def test_unexpected_exception_during_fingerprint_load():
"""Check that we catch all exceptions raised during fingerprint load.
"""
bs = BuildSpace(root_dir=os.getcwd(), primitive='build')
bs.create()
fingerprint_filename = bs.fingerprint_filename(kind='build')
# Scenario: The fingerprint file is actually not a file...
mkdir(fingerprint_filename)
fp = bs.load_fingerprint(kind='build')
assert fp == Fingerprint()
os.rmdir(fingerprint_filename)
# Scenario: The fingerprint file is not readable (lack of permissions
# in this case).
#
# Note that we do not have an easy way to remove read permission
# to a file when on Windows, so we simply avoid that test when
# on that platform. This test exercises the same code as in
# the previous scenario, so this is not a big loss.
if sys.platform != 'win32':
ref_fp = Fingerprint()
ref_fp.add('key1', 'val1')
bs.save_fingerprint(kind='build', fingerprint=ref_fp)
os.chmod(fingerprint_filename, 0)
fp = bs.load_fingerprint(kind='build')
assert fp == Fingerprint()
os.chmod(fingerprint_filename, 0o600)
os.remove(fingerprint_filename)
def compute_new_fingerprint(self, uid, data):
if 'no_fingerprint' in uid:
return None
f = Fingerprint()
for pred_uid in self.actions.get_predecessors(uid):
pred_fingerprint = self.new_fingerprints[pred_uid]
if pred_fingerprint is not None:
f.add('pred:%s' % pred_uid, pred_fingerprint.checksum())
if os.path.exists(source_fullpath(uid)):
f.add_file(source_fullpath(uid))
return f
def test_fingerprint_eq():
"""Check fingerprint __eq__ function."""
f1 = Fingerprint()
f1.add("1", "1")
assert f1 != 1
f2 = Fingerprint()
f2.add("1", "1")
f2.add("2", "2")
assert f1 != f2
assert f1.checksum() != f2.checksum()
assert f1.compare_to(f1) is None
def compute_fingerprint(self, uid, data, is_prediction=False):
if "fingerprint_after_job" in uid and is_prediction:
return None
if "no_fingerprint" in uid:
return None
f = Fingerprint()
for pred_uid in self.actions.get_predecessors(uid):
pred_fingerprint = self.new_fingerprints[pred_uid]
if pred_fingerprint is not None:
f.add("pred:%s" % pred_uid, pred_fingerprint.checksum())
if os.path.exists(source_fullpath(uid)):
f.add_file(source_fullpath(uid))
return f
def test_fingerprint():
"""Check dump/restore of fingerprint."""
bs = BuildSpace(root_dir=os.getcwd(), primitive='build')
bs.create()
fp = Fingerprint()
fp.add('foo', 'bar')
bs.save_fingerprint(kind='build', fingerprint=fp)
loaded_fp = bs.load_fingerprint(kind='build')
assert loaded_fp == fp
# Now make sure that load_fingerprint does not fail when the bumped
# value is corrupted
with open(bs.fingerprint_filename('build'), 'w') as f:
f.write('[')
assert bs.load_fingerprint(kind='build') == Fingerprint()
# Try updating the fingerprint
fp.add('key', 'value')
bs.update_status(kind='build', status=ReturnValue.notready, fingerprint=fp)
assert bs.get_last_status(kind='build')[0] == ReturnValue.notready
loaded_fp = bs.load_fingerprint(kind='build')
assert loaded_fp == fp
# Updating build fingerprint also update install fingerprint
loaded_fp = bs.load_fingerprint(kind='install')
assert loaded_fp == fp
# Now update install status
bs.update_status(kind='install', status=ReturnValue.success)
assert bs.get_last_status(kind='install')[0] == ReturnValue.success
# build status should not be modified
assert bs.get_last_status(kind='build')[0] == ReturnValue.notready
def add_anod_files_to_fingerprint(anod_instance: Anod,
fingerprint: Fingerprint) -> None:
"""Add the Anod's spec and yaml files to the given fingerprint.
:param anod_instance: an Anod instance.
:type anod_instance: Anod
:param fingerprint: The fingerprint to update.
:type fingerprint: e3.fingerprint.Fingerprint.
"""
anod_specs = [
c.name for c in anod_instance.__class__.__mro__
if c.__name__ != "Anod" and "Anod" in (sc.__name__ for sc in c.__mro__)
]
for spec_name in anod_specs:
fingerprint.add_file(
os.path.join(anod_instance.spec_dir, spec_name + ".anod"))
for yaml_name in anod_instance.data_files:
fingerprint.add_file(
os.path.join(anod_instance.spec_dir, yaml_name + ".yaml"))
deps = getattr(anod_instance, "%s_deps" % anod_instance.kind, ())
for dep in deps:
if isinstance(dep, anod_instance.BuildVar):
fingerprint.add(dep.name, dep.value)
def create_fingerprint(first, second, third):
"""Create a fingerprint with 3 elements added in the given order.
The arguments, first, second, and third should be either
1, 2, or 3 (integers), and are meant as indices to (key, value)
pairs in the idx_to_entry_map below, indicating which part
of the fingerprint gets added in what order.
"""
idx_to_entry_map = {1: ("foo", "1"), 2: ("bar", "2"), 3: ("baz", "3")}
f = Fingerprint()
f.add(*idx_to_entry_map[first])
f.add(*idx_to_entry_map[second])
f.add(*idx_to_entry_map[third])
return f
def test_fingerprint():
f1 = Fingerprint()
f1.add("foo", "2")
f2 = Fingerprint()
f2.add("foo", "4")
f12_diff = f2.compare_to(f1)
assert f12_diff["new"] == set()
assert f12_diff["updated"] == {"foo"}
assert f12_diff["obsolete"] == set()
f3 = Fingerprint()
f3.add_file(__file__)
f23_diff = f3.compare_to(f2)
assert f23_diff["new"] == {"foo"}
assert f23_diff["updated"] == set()
assert f23_diff["obsolete"] == {os.path.abspath(__file__)}
assert f1.checksum() != f2.checksum() != f3.checksum()
assert Env().build.os.version in str(f3)
f4 = Fingerprint()
f4.add_file(__file__)
assert f4 == f3
f5 = Fingerprint()
with pytest.raises(E3Error) as err:
f5.add("f4", f4)
assert "f4 should be a string" in str(err.value)
f6 = Fingerprint()
f6.add("unicode", "6")
assert len(f6.checksum()) == 64
def test_fingerprint():
f1 = Fingerprint()
f1.add('foo', '2')
f2 = Fingerprint()
f2.add('foo', '4')
f12_diff = f2.compare_to(f1)
assert f12_diff['new'] == set([])
assert f12_diff['updated'] == {'foo'}
assert f12_diff['obsolete'] == set([])
f3 = Fingerprint()
f3.add_file(__file__)
f23_diff = f3.compare_to(f2)
assert f23_diff['new'] == {'foo'}
assert f23_diff['updated'] == set([])
assert f23_diff['obsolete'] == {os.path.basename(__file__)}
assert f1.checksum() != f2.checksum() != f3.checksum()
assert Env().build.os.version in str(f3)
f4 = Fingerprint()
f4.add_file(__file__)
assert f4 == f3
f5 = Fingerprint()
with pytest.raises(E3Error) as err:
f5.add('f4', f4)
assert 'f4 should be a string' in str(err.value)
f6 = Fingerprint()
f6.add('unicode', u'6')
assert len(f6.checksum()) == 64
def compute_fingerprint(self, uid, data, is_prediction=False):
"""See Walk.compute_fingerprint."""
if is_prediction and self.force:
# Force a rebuild
return None
# All fingerprints start with the same minimum amount
# of data: The fingerprint of all the predecessors.
f = Fingerprint()
for pred_uid in self.actions.get_predecessors(uid):
if self.new_fingerprints[pred_uid] is None:
logging.debug(
"Returning no fingerprint for %s"
" (predecessor %s has no fingerprint)",
uid,
pred_uid,
)
return None
else:
f.add(pred_uid, self.new_fingerprints[pred_uid].checksum())
if isinstance(data, Checkout):
if is_prediction:
# We cannot predict the fingerprint of a checkout without
# performing it, thus checkout will always be executed.
return None
m = CheckoutManager(name=data.repo_name,
working_dir=self.sandbox.vcs_dir)
with open(m.metadata_file) as fd:
content = json.load(fd)
f.add(data.repo_name + ".url", content["url"])
f.add(data.repo_name + ".commit", content["new_commit"])
elif isinstance(data, (CreateSource, Build)):
add_anod_files_to_fingerprint(data.anod_instance, f)
elif isinstance(data, InstallSource):
add_anod_files_to_fingerprint(data.spec, f)
return f
def test_fingerprint_save_and_load():
# Create a directory where to store our fingerprints, allowing us
# to use any fingerprint name without potentially colliding with
# other files used by this testcase.
os.mkdir("fingerprints")
def fingerprint_path(filename):
return os.path.join("fingerprints", filename)
# Save and then load a minimal fingerprint...
f_min = Fingerprint()
f_min_filename = fingerprint_path("f_min")
assert not os.path.exists(f_min_filename)
f_min.save_to_file(f_min_filename)
f_min_restored = Fingerprint.load_from_file(f_min_filename)
assert f_min_restored == f_min
assert str(f_min_restored) == str(f_min)
assert f_min_restored.checksum() == f_min.checksum()
# Save and then load a fingerprint with more data than the minimum.
f2 = Fingerprint()
f2.add("job1", "job1sha1")
f2.add("job2", "sha1job2")
f2_filename = fingerprint_path("f2")
assert not os.path.exists(f2_filename)
f2.save_to_file(f2_filename)
f2_restored = Fingerprint.load_from_file(f2_filename)
assert f2_restored == f2
assert str(f2_restored) == str(f2)
assert f2_restored.checksum() == f2.checksum()
# Trying to load from a file with invalid contents (bad JSON)
f_bad_filename = fingerprint_path("f_bad_JSON")
with open(f_bad_filename, "w") as f:
f.write("yello{")
f3 = Fingerprint.load_from_file(f_bad_filename)
assert f3 is None
# Trying to load from a file which contains valid data, but
# is not an dictionary, and therefore clearly not something
# that comes from a fingerprint...
f_not_filename = fingerprint_path("not_a_fingerprint")
with open(f_not_filename, "w") as f:
json.dump([1, 2, 3], f)
f4 = Fingerprint.load_from_file(f_not_filename)
assert f4 is None
# Try to load from a file which is missing one of the mandatory
# elements.
for key in ("fingerprint_version", "elements"):
f_key_missing_filename = fingerprint_path("no_%s_key")
# To create the bad file without assuming too much in this test
# how the fingerprint is saved to file, we save a valid fingerprint
# to file, load that file back, remove the key, and then save the
# truncated data again.
f2.save_to_file(f_key_missing_filename)
with open(f_key_missing_filename) as f:
data = json.load(f)
del data[key]
with open(f_key_missing_filename, "w") as f:
json.dump(data, f)
f5 = Fingerprint.load_from_file(f_key_missing_filename)
assert f5 is None
# Try loading a fingerprint whose version number is not recognized
# (typically, and old fingerprint version that we no longer support).
#
# To create the file without assuming too much in this test how
# fingerprint are saved to file, we start with a valid fingerprint
# that we saved to a file, load that file back, adjust the version
# number, and then replace the good fingeprint in that file by
# the modified one.
f_bad_version = fingerprint_path("bad_version")
f2.save_to_file(f_bad_version)
with open(f_bad_version) as f:
data = json.load(f)
data["fingerprint_version"] = "1.0"
data["elements"]["fingerprint_version"] = "1.0"
with open(f_bad_version, "w") as f:
json.dump(data, f)
f = Fingerprint.load_from_file(f_bad_version)
assert f is None
def test_invalid_fingerprint():
"""A fingerprint value should be hashable."""
f1 = Fingerprint()
with pytest.raises(E3Error):
f1.add("invalid", {})