def load_fingerprint(self, kind): """Load the content of the fingerprint from disc. :param kind: the primitive name :type kind: str :return: Returns a Fingerprint object (the content of the fingerprint file or an empty Fingerprint when the fingerprint is invalid or does not exist). :rtype: Fingerprint """ fingerprint_file = self.fingerprint_filename(kind) result = None if os.path.exists(fingerprint_file): try: result = Fingerprint.load_from_file(fingerprint_file) except Exception as e: logger.warning(e) # Invalid fingerprint logger.warning('invalid fingerprint, discarding it') result = None if not isinstance(result, Fingerprint): # The fingerprint file did not exist or was invalid # returns an empty fingerprint result = Fingerprint() return result
def test_fingerprint_version(): """Changing the FINGERPRINT_VERSION modify the fingerprint's checksum.""" import e3.fingerprint f1 = Fingerprint() e3.fingerprint.FINGERPRINT_VERSION = "0.0" f2 = Fingerprint() assert f1 != f2 assert f1.checksum() != f2.checksum() f3 = Fingerprint() assert f2 == f3 assert f2.checksum() == f3.checksum()
def test_corrupted_fingerprint(setup_sbx): """Test the case where a fingerprint somehow got corrupted.""" actions = DAG() actions.add_vertex('1') actions.add_vertex('2', predecessors=['1']) actions.add_vertex('3') actions.add_vertex('4', predecessors=['2', '3']) actions.add_vertex('5', predecessors=['4']) actions.add_vertex('6') # Now, execute the plan a first time; everything should be run # and finish succesfullly. r1 = FingerprintWalk(actions) for uid in ('1', '2', '3', '4', '5', '6'): job = r1.saved_jobs[uid] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue.success assert r1.job_status == {'1': ReturnValue.success, '2': ReturnValue.success, '3': ReturnValue.success, '4': ReturnValue.success, '5': ReturnValue.success, '6': ReturnValue.success} assert r1.requeued == {} # Now, corrupt the fingerprint of node '3', and then rerun # the scheduler... We expect the following: # - The scheduler does _not_ crash ;-) # - The fingerprint of node '3' gets discarded, and as a result # it should be re-run again. # - Since nothing changed in node "3"'s predecessors, the end # result for node '3' should be the same, which means # its fingerprint should be the same as before the corruption. # As a result of that, nodes '4' and '5', which directly # or indirectly depend on node '3', do not need to be rerun. with open(r1.fingerprint_filename('3'), 'w') as f: f.write('{') r2 = FingerprintWalk(actions) job = r2.saved_jobs['3'] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue.success for uid in ('1', '2', '4', '5', '6'): job = r2.saved_jobs[uid] assert isinstance(job, EmptyJob) assert job.should_skip is True assert job.status == ReturnValue.skip # Verify also that the fingerprint corruption is gone. f3 = Fingerprint.load_from_file(r2.fingerprint_filename('3')) assert isinstance(f3, Fingerprint)
def create_fingerprint(first, second, third): """Create a fingerprint with 3 elements added in the given order. The arguments, first, second, and third should be either 1, 2, or 3 (integers), and are meant as indices to (key, value) pairs in the idx_to_entry_map below, indicating which part of the fingerprint gets added in what order. """ idx_to_entry_map = {1: ("foo", "1"), 2: ("bar", "2"), 3: ("baz", "3")} f = Fingerprint() f.add(*idx_to_entry_map[first]) f.add(*idx_to_entry_map[second]) f.add(*idx_to_entry_map[third]) return f
def test_unexpected_exception_during_fingerprint_load(): """Check that we catch all exceptions raised during fingerprint load. """ bs = BuildSpace(root_dir=os.getcwd(), primitive='build') bs.create() fingerprint_filename = bs.fingerprint_filename(kind='build') # Scenario: The fingerprint file is actually not a file... mkdir(fingerprint_filename) fp = bs.load_fingerprint(kind='build') assert fp == Fingerprint() os.rmdir(fingerprint_filename) # Scenario: The fingerprint file is not readable (lack of permissions # in this case). # # Note that we do not have an easy way to remove read permission # to a file when on Windows, so we simply avoid that test when # on that platform. This test exercises the same code as in # the previous scenario, so this is not a big loss. if sys.platform != 'win32': ref_fp = Fingerprint() ref_fp.add('key1', 'val1') bs.save_fingerprint(kind='build', fingerprint=ref_fp) os.chmod(fingerprint_filename, 0) fp = bs.load_fingerprint(kind='build') assert fp == Fingerprint() os.chmod(fingerprint_filename, 0o600) os.remove(fingerprint_filename)
def load_previous_fingerprint(self, uid): # In dry-run mode, the fingerprints on file are let untouched, # so they might be out of date compared to this job's status # as part of this dry run. So, if we have already computed # the fingerprint before, use that. if self.dry_run_mode and uid in self.new_fingerprints: return self.new_fingerprints[uid] filename = self.fingerprint_filename(uid) if os.path.exists(filename): return Fingerprint.load_from_file(filename) else: return None
def compute_new_fingerprint(self, uid, data): if 'no_fingerprint' in uid: return None f = Fingerprint() for pred_uid in self.actions.get_predecessors(uid): pred_fingerprint = self.new_fingerprints[pred_uid] if pred_fingerprint is not None: f.add('pred:%s' % pred_uid, pred_fingerprint.checksum()) if os.path.exists(source_fullpath(uid)): f.add_file(source_fullpath(uid)) return f
def compute_fingerprint(self, uid, data, is_prediction=False): if "fingerprint_after_job" in uid and is_prediction: return None if "no_fingerprint" in uid: return None f = Fingerprint() for pred_uid in self.actions.get_predecessors(uid): pred_fingerprint = self.new_fingerprints[pred_uid] if pred_fingerprint is not None: f.add("pred:%s" % pred_uid, pred_fingerprint.checksum()) if os.path.exists(source_fullpath(uid)): f.add_file(source_fullpath(uid)) return f
def update_status(self, kind, status=ReturnValue.failure, fingerprint=None): """Update meta information on disk. :param kind: the primitive name :type kind: str :param status: the last action return code :type status: ReturnValue :param fingerprint: the fingerprint :type fingerprint: Fingerprint """ if fingerprint is None: fingerprint = Fingerprint() self.save_fingerprint(kind, fingerprint) self.save_last_status(kind, status) if kind == 'build': self.update_status('install', status, fingerprint)
def compute_fingerprint(self, uid, data, is_prediction=False): """See Walk.compute_fingerprint.""" if is_prediction and self.force: # Force a rebuild return None # All fingerprints start with the same minimum amount # of data: The fingerprint of all the predecessors. f = Fingerprint() for pred_uid in self.actions.get_predecessors(uid): if self.new_fingerprints[pred_uid] is None: logging.debug( "Returning no fingerprint for %s" " (predecessor %s has no fingerprint)", uid, pred_uid, ) return None else: f.add(pred_uid, self.new_fingerprints[pred_uid].checksum()) if isinstance(data, Checkout): if is_prediction: # We cannot predict the fingerprint of a checkout without # performing it, thus checkout will always be executed. return None m = CheckoutManager(name=data.repo_name, working_dir=self.sandbox.vcs_dir) with open(m.metadata_file) as fd: content = json.load(fd) f.add(data.repo_name + ".url", content["url"]) f.add(data.repo_name + ".commit", content["new_commit"]) elif isinstance(data, (CreateSource, Build)): add_anod_files_to_fingerprint(data.anod_instance, f) elif isinstance(data, InstallSource): add_anod_files_to_fingerprint(data.spec, f) return f
def test_fingerprint(): """Check dump/restore of fingerprint.""" bs = BuildSpace(root_dir=os.getcwd(), primitive='build') bs.create() fp = Fingerprint() fp.add('foo', 'bar') bs.save_fingerprint(kind='build', fingerprint=fp) loaded_fp = bs.load_fingerprint(kind='build') assert loaded_fp == fp # Now make sure that load_fingerprint does not fail when the bumped # value is corrupted with open(bs.fingerprint_filename('build'), 'w') as f: f.write('[') assert bs.load_fingerprint(kind='build') == Fingerprint() # Try updating the fingerprint fp.add('key', 'value') bs.update_status(kind='build', status=ReturnValue.notready, fingerprint=fp) assert bs.get_last_status(kind='build')[0] == ReturnValue.notready loaded_fp = bs.load_fingerprint(kind='build') assert loaded_fp == fp # Updating build fingerprint also update install fingerprint loaded_fp = bs.load_fingerprint(kind='install') assert loaded_fp == fp # Now update install status bs.update_status(kind='install', status=ReturnValue.success) assert bs.get_last_status(kind='install')[0] == ReturnValue.success # build status should not be modified assert bs.get_last_status(kind='build')[0] == ReturnValue.notready
def add_anod_files_to_fingerprint(anod_instance: Anod, fingerprint: Fingerprint) -> None: """Add the Anod's spec and yaml files to the given fingerprint. :param anod_instance: an Anod instance. :type anod_instance: Anod :param fingerprint: The fingerprint to update. :type fingerprint: e3.fingerprint.Fingerprint. """ anod_specs = [ c.name for c in anod_instance.__class__.__mro__ if c.__name__ != "Anod" and "Anod" in (sc.__name__ for sc in c.__mro__) ] for spec_name in anod_specs: fingerprint.add_file( os.path.join(anod_instance.spec_dir, spec_name + ".anod")) for yaml_name in anod_instance.data_files: fingerprint.add_file( os.path.join(anod_instance.spec_dir, yaml_name + ".yaml")) deps = getattr(anod_instance, "%s_deps" % anod_instance.kind, ()) for dep in deps: if isinstance(dep, anod_instance.BuildVar): fingerprint.add(dep.name, dep.value)
def test_fingerprint_save_and_load(): # Create a directory where to store our fingerprints, allowing us # to use any fingerprint name without potentially colliding with # other files used by this testcase. os.mkdir("fingerprints") def fingerprint_path(filename): return os.path.join("fingerprints", filename) # Save and then load a minimal fingerprint... f_min = Fingerprint() f_min_filename = fingerprint_path("f_min") assert not os.path.exists(f_min_filename) f_min.save_to_file(f_min_filename) f_min_restored = Fingerprint.load_from_file(f_min_filename) assert f_min_restored == f_min assert str(f_min_restored) == str(f_min) assert f_min_restored.checksum() == f_min.checksum() # Save and then load a fingerprint with more data than the minimum. f2 = Fingerprint() f2.add("job1", "job1sha1") f2.add("job2", "sha1job2") f2_filename = fingerprint_path("f2") assert not os.path.exists(f2_filename) f2.save_to_file(f2_filename) f2_restored = Fingerprint.load_from_file(f2_filename) assert f2_restored == f2 assert str(f2_restored) == str(f2) assert f2_restored.checksum() == f2.checksum() # Trying to load from a file with invalid contents (bad JSON) f_bad_filename = fingerprint_path("f_bad_JSON") with open(f_bad_filename, "w") as f: f.write("yello{") f3 = Fingerprint.load_from_file(f_bad_filename) assert f3 is None # Trying to load from a file which contains valid data, but # is not an dictionary, and therefore clearly not something # that comes from a fingerprint... f_not_filename = fingerprint_path("not_a_fingerprint") with open(f_not_filename, "w") as f: json.dump([1, 2, 3], f) f4 = Fingerprint.load_from_file(f_not_filename) assert f4 is None # Try to load from a file which is missing one of the mandatory # elements. for key in ("fingerprint_version", "elements"): f_key_missing_filename = fingerprint_path("no_%s_key") # To create the bad file without assuming too much in this test # how the fingerprint is saved to file, we save a valid fingerprint # to file, load that file back, remove the key, and then save the # truncated data again. f2.save_to_file(f_key_missing_filename) with open(f_key_missing_filename) as f: data = json.load(f) del data[key] with open(f_key_missing_filename, "w") as f: json.dump(data, f) f5 = Fingerprint.load_from_file(f_key_missing_filename) assert f5 is None # Try loading a fingerprint whose version number is not recognized # (typically, and old fingerprint version that we no longer support). # # To create the file without assuming too much in this test how # fingerprint are saved to file, we start with a valid fingerprint # that we saved to a file, load that file back, adjust the version # number, and then replace the good fingeprint in that file by # the modified one. f_bad_version = fingerprint_path("bad_version") f2.save_to_file(f_bad_version) with open(f_bad_version) as f: data = json.load(f) data["fingerprint_version"] = "1.0" data["elements"]["fingerprint_version"] = "1.0" with open(f_bad_version, "w") as f: json.dump(data, f) f = Fingerprint.load_from_file(f_bad_version) assert f is None
def test_fingerprint_eq(): """Check fingerprint __eq__ function.""" f1 = Fingerprint() f1.add("1", "1") assert f1 != 1 f2 = Fingerprint() f2.add("1", "1") f2.add("2", "2") assert f1 != f2 assert f1.checksum() != f2.checksum() assert f1.compare_to(f1) is None
def test_invalid_fingerprint(): """A fingerprint value should be hashable.""" f1 = Fingerprint() with pytest.raises(E3Error): f1.add("invalid", {})
def test_fingerprint(): f1 = Fingerprint() f1.add("foo", "2") f2 = Fingerprint() f2.add("foo", "4") f12_diff = f2.compare_to(f1) assert f12_diff["new"] == set() assert f12_diff["updated"] == {"foo"} assert f12_diff["obsolete"] == set() f3 = Fingerprint() f3.add_file(__file__) f23_diff = f3.compare_to(f2) assert f23_diff["new"] == {"foo"} assert f23_diff["updated"] == set() assert f23_diff["obsolete"] == {os.path.abspath(__file__)} assert f1.checksum() != f2.checksum() != f3.checksum() assert Env().build.os.version in str(f3) f4 = Fingerprint() f4.add_file(__file__) assert f4 == f3 f5 = Fingerprint() with pytest.raises(E3Error) as err: f5.add("f4", f4) assert "f4 should be a string" in str(err.value) f6 = Fingerprint() f6.add("unicode", "6") assert len(f6.checksum()) == 64
def test_fingerprint(): f1 = Fingerprint() f1.add('foo', '2') f2 = Fingerprint() f2.add('foo', '4') f12_diff = f2.compare_to(f1) assert f12_diff['new'] == set([]) assert f12_diff['updated'] == {'foo'} assert f12_diff['obsolete'] == set([]) f3 = Fingerprint() f3.add_file(__file__) f23_diff = f3.compare_to(f2) assert f23_diff['new'] == {'foo'} assert f23_diff['updated'] == set([]) assert f23_diff['obsolete'] == {os.path.basename(__file__)} assert f1.checksum() != f2.checksum() != f3.checksum() assert Env().build.os.version in str(f3) f4 = Fingerprint() f4.add_file(__file__) assert f4 == f3 f5 = Fingerprint() with pytest.raises(E3Error) as err: f5.add('f4', f4) assert 'f4 should be a string' in str(err.value) f6 = Fingerprint() f6.add('unicode', u'6') assert len(f6.checksum()) == 64
def load_previous_fingerprint(self, uid): """See Walk.load_previous_fingerprint.""" return Fingerprint.load_from_file(self.fingerprint_filename(uid))