def __init__(self, data): self.basehash = {} self.taskhash = {} self.unihash = {} self.taskdeps = {} self.runtaskdeps = {} self.file_checksum_values = {} self.taints = {} self.gendeps = {} self.lookupcache = {} self.setscenetasks = set() self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS") or "").split()) self.taskhash_ignore_tasks = None self.init_rundepcheck(data) checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") if checksum_cache_file: self.checksum_cache = FileChecksumCache() self.checksum_cache.init_cache(data, checksum_cache_file) else: self.checksum_cache = None self.unihash_cache = bb.cache.SimpleCache("3") self.unitaskhashes = self.unihash_cache.init_cache( data, "bb_unihashes.dat", {}) self.localdirsexclude = ( data.getVar("BB_SIGNATURE_LOCAL_DIRS_EXCLUDE") or "CVS .bzr .git .hg .osc .p4 .repo .svn").split() self.tidtopn = {}
def __init__(self, data): self.basehash = {} self.taskhash = {} self.taskdeps = {} self.runtaskdeps = {} self.file_checksum_values = {} self.taints = {} self.gendeps = {} self.lookupcache = {} self.pkgnameextract = re.compile("(?P<fn>.*)\..*") self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split()) self.taskwhitelist = None self.init_rundepcheck(data) checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE", True) if checksum_cache_file: self.checksum_cache = FileChecksumCache() self.checksum_cache.init_cache(data, checksum_cache_file) else: self.checksum_cache = None
def __init__(self, data): self.basehash = {} self.taskhash = {} self.taskdeps = {} self.runtaskdeps = {} self.file_checksum_values = {} self.taints = {} self.gendeps = {} self.lookupcache = {} self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) self.taskwhitelist = None self.init_rundepcheck(data) checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") if checksum_cache_file: self.checksum_cache = FileChecksumCache() self.checksum_cache.init_cache(data, checksum_cache_file) else: self.checksum_cache = None self.unihash_cache = bb.cache.SimpleCache("1") self.unitaskhashes = self.unihash_cache.init_cache(data, "bb_unihashes.dat", {})
def __init__(self, data): self.basehash = {} self.taskhash = {} self.taskdeps = {} self.runtaskdeps = {} self.file_checksum_values = {} self.taints = {} self.gendeps = {} self.lookupcache = {} self.pkgnameextract = re.compile("(?P<fn>.*)\..*") self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) self.taskwhitelist = None self.init_rundepcheck(data) checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") if checksum_cache_file: self.checksum_cache = FileChecksumCache() self.checksum_cache.init_cache(data, checksum_cache_file) else: self.checksum_cache = None
class SignatureGeneratorBasic(SignatureGenerator): """ """ name = "basic" def __init__(self, data): self.basehash = {} self.taskhash = {} self.unihash = {} self.taskdeps = {} self.runtaskdeps = {} self.file_checksum_values = {} self.taints = {} self.gendeps = {} self.lookupcache = {} self.setscenetasks = set() self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS") or "").split()) self.taskhash_ignore_tasks = None self.init_rundepcheck(data) checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") if checksum_cache_file: self.checksum_cache = FileChecksumCache() self.checksum_cache.init_cache(data, checksum_cache_file) else: self.checksum_cache = None self.unihash_cache = bb.cache.SimpleCache("3") self.unitaskhashes = self.unihash_cache.init_cache( data, "bb_unihashes.dat", {}) self.localdirsexclude = ( data.getVar("BB_SIGNATURE_LOCAL_DIRS_EXCLUDE") or "CVS .bzr .git .hg .osc .p4 .repo .svn").split() self.tidtopn = {} def init_rundepcheck(self, data): self.taskhash_ignore_tasks = data.getVar( "BB_TASKHASH_IGNORE_TASKS") or None if self.taskhash_ignore_tasks: self.twl = re.compile(self.taskhash_ignore_tasks) else: self.twl = None def _build_data(self, fn, d): ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') tasklist, gendeps, lookupcache = bb.data.generate_dependencies( d, self.basehash_ignore_vars) taskdeps, basehash = bb.data.generate_dependency_hash( tasklist, gendeps, lookupcache, self.basehash_ignore_vars, fn) for task in tasklist: tid = fn + ":" + task if not ignore_mismatch and tid in self.basehash and self.basehash[ tid] != basehash[tid]: bb.error( "When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (tid, self.basehash[tid], basehash[tid])) bb.error("The following commands may help:") cmd = "$ bitbake %s -c%s" % (d.getVar('PN'), task) # Make sure sigdata is dumped before run printdiff bb.error("%s -Snone" % cmd) bb.error("Then:") bb.error("%s -Sprintdiff\n" % cmd) self.basehash[tid] = basehash[tid] self.taskdeps[fn] = taskdeps self.gendeps[fn] = gendeps self.lookupcache[fn] = lookupcache return taskdeps def set_setscene_tasks(self, setscene_tasks): self.setscenetasks = set(setscene_tasks) def finalise(self, fn, d, variant): mc = d.getVar("__BBMULTICONFIG", False) or "" if variant or mc: fn = bb.cache.realfn2virtual(fn, variant, mc) try: taskdeps = self._build_data(fn, d) except bb.parse.SkipRecipe: raise except: bb.warn("Error during finalise of %s" % fn) raise #Slow but can be useful for debugging mismatched basehashes #for task in self.taskdeps[fn]: # self.dump_sigtask(fn, task, d.getVar("STAMP"), False) for task in taskdeps: d.setVar("BB_BASEHASH:task-%s" % task, self.basehash[fn + ":" + task]) def postparsing_clean_cache(self): # # After parsing we can remove some things from memory to reduce our memory footprint # self.gendeps = {} self.lookupcache = {} self.taskdeps = {} def rundep_check(self, fn, recipename, task, dep, depname, dataCaches): # Return True if we should keep the dependency, False to drop it # We only manipulate the dependencies for packages not in the ignore # list if self.twl and not self.twl.search(recipename): # then process the actual dependencies if self.twl.search(depname): return False return True def read_taint(self, fn, task, stampbase): taint = None try: with open(stampbase + '.' + task + '.taint', 'r') as taintf: taint = taintf.read() except IOError: pass return taint def prep_taskhash(self, tid, deps, dataCaches): (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) self.basehash[tid] = dataCaches[mc].basetaskhash[tid] self.runtaskdeps[tid] = [] self.file_checksum_values[tid] = [] recipename = dataCaches[mc].pkg_fn[fn] self.tidtopn[tid] = recipename for dep in sorted(deps, key=clean_basepath): (depmc, _, _, depmcfn) = bb.runqueue.split_tid_mcfn(dep) depname = dataCaches[depmc].pkg_fn[depmcfn] if not self.supports_multiconfig_datacaches and mc != depmc: # If the signature generator doesn't understand multiconfig # data caches, any dependency not in the same multiconfig must # be skipped for backward compatibility continue if not self.rundep_check(fn, recipename, task, dep, depname, dataCaches): continue if dep not in self.taskhash: bb.fatal( "%s is not in taskhash, caller isn't calling in dependency order?" % dep) self.runtaskdeps[tid].append(dep) if task in dataCaches[mc].file_checksums[fn]: if self.checksum_cache: checksums = self.checksum_cache.get_checksums( dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude) else: checksums = bb.fetch2.get_file_checksums( dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude) for (f, cs) in checksums: self.file_checksum_values[tid].append((f, cs)) taskdep = dataCaches[mc].task_deps[fn] if 'nostamp' in taskdep and task in taskdep['nostamp']: # Nostamp tasks need an implicit taint so that they force any dependent tasks to run if tid in self.taints and self.taints[tid].startswith("nostamp:"): # Don't reset taint value upon every call pass else: import uuid taint = str(uuid.uuid4()) self.taints[tid] = "nostamp:" + taint taint = self.read_taint(fn, task, dataCaches[mc].stamp[fn]) if taint: self.taints[tid] = taint logger.warning("%s is tainted from a forced run" % tid) return def get_taskhash(self, tid, deps, dataCaches): data = self.basehash[tid] for dep in self.runtaskdeps[tid]: data = data + self.get_unihash(dep) for (f, cs) in self.file_checksum_values[tid]: if cs: if "/./" in f: data = data + "./" + f.split("/./")[1] data = data + cs if tid in self.taints: if self.taints[tid].startswith("nostamp:"): data = data + self.taints[tid][8:] else: data = data + self.taints[tid] h = hashlib.sha256(data.encode("utf-8")).hexdigest() self.taskhash[tid] = h #d.setVar("BB_TASKHASH:task-%s" % task, taskhash[task]) return h def writeout_file_checksum_cache(self): """Write/update the file checksum cache onto disk""" if self.checksum_cache: self.checksum_cache.save_extras() self.checksum_cache.save_merge() else: bb.fetch2.fetcher_parse_save() bb.fetch2.fetcher_parse_done() def save_unitaskhashes(self): self.unihash_cache.save(self.unitaskhashes) def dump_sigtask(self, fn, task, stampbase, runtime): tid = fn + ":" + task referencestamp = stampbase if isinstance(runtime, str) and runtime.startswith("customfile"): sigfile = stampbase referencestamp = runtime[11:] elif runtime and tid in self.taskhash: sigfile = stampbase + "." + task + ".sigdata" + "." + self.get_unihash( tid) else: sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[ tid] with bb.utils.umask(0o002): bb.utils.mkdirhier(os.path.dirname(sigfile)) data = {} data['task'] = task data['basehash_ignore_vars'] = self.basehash_ignore_vars data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks data['taskdeps'] = self.taskdeps[fn][task] data['basehash'] = self.basehash[tid] data['gendeps'] = {} data['varvals'] = {} data['varvals'][task] = self.lookupcache[fn][task] for dep in self.taskdeps[fn][task]: if dep in self.basehash_ignore_vars: continue data['gendeps'][dep] = self.gendeps[fn][dep] data['varvals'][dep] = self.lookupcache[fn][dep] if runtime and tid in self.taskhash: data['runtaskdeps'] = self.runtaskdeps[tid] data['file_checksum_values'] = [] for f, cs in self.file_checksum_values[tid]: if "/./" in f: data['file_checksum_values'].append( ("./" + f.split("/./")[1], cs)) else: data['file_checksum_values'].append( (os.path.basename(f), cs)) data['runtaskhashes'] = {} for dep in data['runtaskdeps']: data['runtaskhashes'][dep] = self.get_unihash(dep) data['taskhash'] = self.taskhash[tid] data['unihash'] = self.get_unihash(tid) taint = self.read_taint(fn, task, referencestamp) if taint: data['taint'] = taint if runtime and tid in self.taints: if 'nostamp:' in self.taints[tid]: data['taint'] = self.taints[tid] computed_basehash = calc_basehash(data) if computed_basehash != self.basehash[tid]: bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[tid], tid)) if runtime and tid in self.taskhash: computed_taskhash = calc_taskhash(data) if computed_taskhash != self.taskhash[tid]: bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[tid], tid)) sigfile = sigfile.replace(self.taskhash[tid], computed_taskhash) fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") try: with bb.compress.zstd.open(fd, "wt", encoding="utf-8", num_threads=1) as f: json.dump(data, f, sort_keys=True, separators=(",", ":"), cls=SetEncoder) f.flush() os.chmod(tmpfile, 0o664) bb.utils.rename(tmpfile, sigfile) except (OSError, IOError) as err: try: os.unlink(tmpfile) except OSError: pass raise err def dump_sigfn(self, fn, dataCaches, options): if fn in self.taskdeps: for task in self.taskdeps[fn]: tid = fn + ":" + task mc = bb.runqueue.mc_from_tid(tid) if tid not in self.taskhash: continue if dataCaches[mc].basetaskhash[tid] != self.basehash[tid]: bb.error( "Bitbake's cached basehash does not match the one we just generated (%s)!" % tid) bb.error( "The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[tid], self.basehash[tid])) self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
class SignatureGeneratorBasic(SignatureGenerator): """ """ name = "basic" def __init__(self, data): self.basehash = {} self.taskhash = {} self.taskdeps = {} self.runtaskdeps = {} self.file_checksum_values = {} self.taints = {} self.gendeps = {} self.lookupcache = {} self.pkgnameextract = re.compile("(?P<fn>.*)\..*") self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) self.taskwhitelist = None self.init_rundepcheck(data) checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") if checksum_cache_file: self.checksum_cache = FileChecksumCache() self.checksum_cache.init_cache(data, checksum_cache_file) else: self.checksum_cache = None def init_rundepcheck(self, data): self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None if self.taskwhitelist: self.twl = re.compile(self.taskwhitelist) else: self.twl = None def _build_data(self, fn, d): ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) taskdeps, basehash = bb.data.generate_dependency_hash( tasklist, gendeps, lookupcache, self.basewhitelist, fn) for task in tasklist: k = fn + "." + task if not ignore_mismatch and k in self.basehash and self.basehash[ k] != basehash[k]: bb.error( "When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], basehash[k])) self.basehash[k] = basehash[k] self.taskdeps[fn] = taskdeps self.gendeps[fn] = gendeps self.lookupcache[fn] = lookupcache return taskdeps def finalise(self, fn, d, variant): mc = d.getVar("__BBMULTICONFIG", False) or "" if variant or mc: fn = bb.cache.realfn2virtual(fn, variant, mc) try: taskdeps = self._build_data(fn, d) except bb.parse.SkipRecipe: raise except: bb.warn("Error during finalise of %s" % fn) raise #Slow but can be useful for debugging mismatched basehashes #for task in self.taskdeps[fn]: # self.dump_sigtask(fn, task, d.getVar("STAMP"), False) for task in taskdeps: d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) def rundep_check(self, fn, recipename, task, dep, depname, dataCache): # Return True if we should keep the dependency, False to drop it # We only manipulate the dependencies for packages not in the whitelist if self.twl and not self.twl.search(recipename): # then process the actual dependencies if self.twl.search(depname): return False return True def read_taint(self, fn, task, stampbase): taint = None try: with open(stampbase + '.' + task + '.taint', 'r') as taintf: taint = taintf.read() except IOError: pass return taint def get_taskhash(self, fn, task, deps, dataCache): mc = '' if fn.startswith('multiconfig:'): mc = fn.split(':')[1] k = fn + "." + task data = dataCache.basetaskhash[k] self.basehash[k] = data self.runtaskdeps[k] = [] self.file_checksum_values[k] = [] recipename = dataCache.pkg_fn[fn] for dep in sorted(deps, key=clean_basepath): pkgname = self.pkgnameextract.search(dep).group('fn') if mc: depmc = pkgname.split(':')[1] if mc != depmc: continue if dep.startswith("multiconfig:") and not mc: continue depname = dataCache.pkg_fn[pkgname] if not self.rundep_check(fn, recipename, task, dep, depname, dataCache): continue if dep not in self.taskhash: bb.fatal( "%s is not in taskhash, caller isn't calling in dependency order?" % dep) data = data + self.taskhash[dep] self.runtaskdeps[k].append(dep) if task in dataCache.file_checksums[fn]: if self.checksum_cache: checksums = self.checksum_cache.get_checksums( dataCache.file_checksums[fn][task], recipename) else: checksums = bb.fetch2.get_file_checksums( dataCache.file_checksums[fn][task], recipename) for (f, cs) in checksums: self.file_checksum_values[k].append((f, cs)) if cs: data = data + cs taskdep = dataCache.task_deps[fn] if 'nostamp' in taskdep and task in taskdep['nostamp']: # Nostamp tasks need an implicit taint so that they force any dependent tasks to run import uuid taint = str(uuid.uuid4()) data = data + taint self.taints[k] = "nostamp:" + taint taint = self.read_taint(fn, task, dataCache.stamp[fn]) if taint: data = data + taint self.taints[k] = taint logger.warning("%s is tainted from a forced run" % k) h = hashlib.md5(data.encode("utf-8")).hexdigest() self.taskhash[k] = h #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) return h def writeout_file_checksum_cache(self): """Write/update the file checksum cache onto disk""" if self.checksum_cache: self.checksum_cache.save_extras() self.checksum_cache.save_merge() else: bb.fetch2.fetcher_parse_save() bb.fetch2.fetcher_parse_done() def dump_sigtask(self, fn, task, stampbase, runtime): k = fn + "." + task referencestamp = stampbase if isinstance(runtime, str) and runtime.startswith("customfile"): sigfile = stampbase referencestamp = runtime[11:] elif runtime and k in self.taskhash: sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[ k] else: sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[ k] bb.utils.mkdirhier(os.path.dirname(sigfile)) data = {} data['task'] = task data['basewhitelist'] = self.basewhitelist data['taskwhitelist'] = self.taskwhitelist data['taskdeps'] = self.taskdeps[fn][task] data['basehash'] = self.basehash[k] data['gendeps'] = {} data['varvals'] = {} data['varvals'][task] = self.lookupcache[fn][task] for dep in self.taskdeps[fn][task]: if dep in self.basewhitelist: continue data['gendeps'][dep] = self.gendeps[fn][dep] data['varvals'][dep] = self.lookupcache[fn][dep] if runtime and k in self.taskhash: data['runtaskdeps'] = self.runtaskdeps[k] data['file_checksum_values'] = [ (os.path.basename(f), cs) for f, cs in self.file_checksum_values[k] ] data['runtaskhashes'] = {} for dep in data['runtaskdeps']: data['runtaskhashes'][dep] = self.taskhash[dep] data['taskhash'] = self.taskhash[k] taint = self.read_taint(fn, task, referencestamp) if taint: data['taint'] = taint if runtime and k in self.taints: if 'nostamp:' in self.taints[k]: data['taint'] = self.taints[k] computed_basehash = calc_basehash(data) if computed_basehash != self.basehash[k]: bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k)) if runtime and k in self.taskhash: computed_taskhash = calc_taskhash(data) if computed_taskhash != self.taskhash[k]: bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k)) sigfile = sigfile.replace(self.taskhash[k], computed_taskhash) fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") try: with os.fdopen(fd, "wb") as stream: p = pickle.dump(data, stream, -1) stream.flush() os.chmod(tmpfile, 0o664) os.rename(tmpfile, sigfile) except (OSError, IOError) as err: try: os.unlink(tmpfile) except OSError: pass raise err def dump_sigfn(self, fn, dataCaches, options): if fn in self.taskdeps: for task in self.taskdeps[fn]: tid = fn + ":" + task (mc, _, _) = bb.runqueue.split_tid(tid) k = fn + "." + task if k not in self.taskhash: continue if dataCaches[mc].basetaskhash[k] != self.basehash[k]: bb.error( "Bitbake's cached basehash does not match the one we just generated (%s)!" % k) bb.error( "The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[k], self.basehash[k])) self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
class SignatureGeneratorBasic(SignatureGenerator): """ """ name = "basic" def __init__(self, data): self.basehash = {} self.taskhash = {} self.taskdeps = {} self.runtaskdeps = {} self.file_checksum_values = {} self.taints = {} self.gendeps = {} self.lookupcache = {} self.pkgnameextract = re.compile("(?P<fn>.*)\..*") self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) self.taskwhitelist = None self.init_rundepcheck(data) checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") if checksum_cache_file: self.checksum_cache = FileChecksumCache() self.checksum_cache.init_cache(data, checksum_cache_file) else: self.checksum_cache = None def init_rundepcheck(self, data): self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None if self.taskwhitelist: self.twl = re.compile(self.taskwhitelist) else: self.twl = None def _build_data(self, fn, d): ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) taskdeps = {} basehash = {} for task in tasklist: data = lookupcache[task] if data is None: bb.error("Task %s from %s seems to be empty?!" % (task, fn)) data = '' gendeps[task] -= self.basewhitelist newdeps = gendeps[task] seen = set() while newdeps: nextdeps = newdeps seen |= nextdeps newdeps = set() for dep in nextdeps: if dep in self.basewhitelist: continue gendeps[dep] -= self.basewhitelist newdeps |= gendeps[dep] newdeps -= seen alldeps = sorted(seen) for dep in alldeps: data = data + dep var = lookupcache[dep] if var is not None: data = data + str(var) datahash = hashlib.md5(data.encode("utf-8")).hexdigest() k = fn + "." + task if not ignore_mismatch and k in self.basehash and self.basehash[k] != datahash: bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], datahash)) self.basehash[k] = datahash taskdeps[task] = alldeps self.taskdeps[fn] = taskdeps self.gendeps[fn] = gendeps self.lookupcache[fn] = lookupcache return taskdeps def finalise(self, fn, d, variant): mc = d.getVar("__BBMULTICONFIG", False) or "" if variant or mc: fn = bb.cache.realfn2virtual(fn, variant, mc) try: taskdeps = self._build_data(fn, d) except: bb.warn("Error during finalise of %s" % fn) raise #Slow but can be useful for debugging mismatched basehashes #for task in self.taskdeps[fn]: # self.dump_sigtask(fn, task, d.getVar("STAMP"), False) for task in taskdeps: d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) def rundep_check(self, fn, recipename, task, dep, depname, dataCache): # Return True if we should keep the dependency, False to drop it # We only manipulate the dependencies for packages not in the whitelist if self.twl and not self.twl.search(recipename): # then process the actual dependencies if self.twl.search(depname): return False return True def read_taint(self, fn, task, stampbase): taint = None try: with open(stampbase + '.' + task + '.taint', 'r') as taintf: taint = taintf.read() except IOError: pass return taint def get_taskhash(self, fn, task, deps, dataCache): k = fn + "." + task data = dataCache.basetaskhash[k] self.basehash[k] = data self.runtaskdeps[k] = [] self.file_checksum_values[k] = [] recipename = dataCache.pkg_fn[fn] for dep in sorted(deps, key=clean_basepath): depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')] if not self.rundep_check(fn, recipename, task, dep, depname, dataCache): continue if dep not in self.taskhash: bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep) data = data + self.taskhash[dep] self.runtaskdeps[k].append(dep) if task in dataCache.file_checksums[fn]: if self.checksum_cache: checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename) else: checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) for (f,cs) in checksums: self.file_checksum_values[k].append((f,cs)) if cs: data = data + cs taskdep = dataCache.task_deps[fn] if 'nostamp' in taskdep and task in taskdep['nostamp']: # Nostamp tasks need an implicit taint so that they force any dependent tasks to run import uuid taint = str(uuid.uuid4()) data = data + taint self.taints[k] = "nostamp:" + taint taint = self.read_taint(fn, task, dataCache.stamp[fn]) if taint: data = data + taint self.taints[k] = taint logger.warning("%s is tainted from a forced run" % k) h = hashlib.md5(data.encode("utf-8")).hexdigest() self.taskhash[k] = h #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) return h def writeout_file_checksum_cache(self): """Write/update the file checksum cache onto disk""" if self.checksum_cache: self.checksum_cache.save_extras() self.checksum_cache.save_merge() else: bb.fetch2.fetcher_parse_save() bb.fetch2.fetcher_parse_done() def dump_sigtask(self, fn, task, stampbase, runtime): k = fn + "." + task referencestamp = stampbase if isinstance(runtime, str) and runtime.startswith("customfile"): sigfile = stampbase referencestamp = runtime[11:] elif runtime and k in self.taskhash: sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k] else: sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k] bb.utils.mkdirhier(os.path.dirname(sigfile)) data = {} data['task'] = task data['basewhitelist'] = self.basewhitelist data['taskwhitelist'] = self.taskwhitelist data['taskdeps'] = self.taskdeps[fn][task] data['basehash'] = self.basehash[k] data['gendeps'] = {} data['varvals'] = {} data['varvals'][task] = self.lookupcache[fn][task] for dep in self.taskdeps[fn][task]: if dep in self.basewhitelist: continue data['gendeps'][dep] = self.gendeps[fn][dep] data['varvals'][dep] = self.lookupcache[fn][dep] if runtime and k in self.taskhash: data['runtaskdeps'] = self.runtaskdeps[k] data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k]] data['runtaskhashes'] = {} for dep in data['runtaskdeps']: data['runtaskhashes'][dep] = self.taskhash[dep] data['taskhash'] = self.taskhash[k] taint = self.read_taint(fn, task, referencestamp) if taint: data['taint'] = taint if runtime and k in self.taints: if 'nostamp:' in self.taints[k]: data['taint'] = self.taints[k] computed_basehash = calc_basehash(data) if computed_basehash != self.basehash[k]: bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k)) if runtime and k in self.taskhash: computed_taskhash = calc_taskhash(data) if computed_taskhash != self.taskhash[k]: bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k)) sigfile = sigfile.replace(self.taskhash[k], computed_taskhash) fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") try: with os.fdopen(fd, "wb") as stream: p = pickle.dump(data, stream, -1) stream.flush() os.chmod(tmpfile, 0o664) os.rename(tmpfile, sigfile) except (OSError, IOError) as err: try: os.unlink(tmpfile) except OSError: pass raise err def dump_sigfn(self, fn, dataCaches, options): if fn in self.taskdeps: for task in self.taskdeps[fn]: tid = fn + ":" + task (mc, _, _) = bb.runqueue.split_tid(tid) k = fn + "." + task if k not in self.taskhash: continue if dataCaches[mc].basetaskhash[k] != self.basehash[k]: bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k) bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[k], self.basehash[k])) self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
class SignatureGeneratorBasic(SignatureGenerator): """ """ name = "basic" def __init__(self, data): self.basehash = {} self.taskhash = {} self.taskdeps = {} self.runtaskdeps = {} self.file_checksum_values = {} self.taints = {} self.gendeps = {} self.lookupcache = {} self.pkgnameextract = re.compile("(?P<fn>.*)\..*") self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split()) self.taskwhitelist = None self.init_rundepcheck(data) checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE", True) if checksum_cache_file: self.checksum_cache = FileChecksumCache() self.checksum_cache.init_cache(data, checksum_cache_file) else: self.checksum_cache = None def init_rundepcheck(self, data): self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None if self.taskwhitelist: self.twl = re.compile(self.taskwhitelist) else: self.twl = None def _build_data(self, fn, d): tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) taskdeps = {} basehash = {} for task in tasklist: data = lookupcache[task] if data is None: bb.error("Task %s from %s seems to be empty?!" % (task, fn)) data = '' gendeps[task] -= self.basewhitelist newdeps = gendeps[task] seen = set() while newdeps: nextdeps = newdeps seen |= nextdeps newdeps = set() for dep in nextdeps: if dep in self.basewhitelist: continue gendeps[dep] -= self.basewhitelist newdeps |= gendeps[dep] newdeps -= seen alldeps = sorted(seen) for dep in alldeps: data = data + dep var = lookupcache[dep] if var is not None: data = data + str(var) self.basehash[fn + "." + task] = hashlib.md5(data).hexdigest() taskdeps[task] = alldeps self.taskdeps[fn] = taskdeps self.gendeps[fn] = gendeps self.lookupcache[fn] = lookupcache return taskdeps def finalise(self, fn, d, variant): if variant: fn = "virtual:" + variant + ":" + fn try: taskdeps = self._build_data(fn, d) except: bb.warn("Error during finalise of %s" % fn) raise #Slow but can be useful for debugging mismatched basehashes #for task in self.taskdeps[fn]: # self.dump_sigtask(fn, task, d.getVar("STAMP", True), False) for task in taskdeps: d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) def rundep_check(self, fn, recipename, task, dep, depname, dataCache): # Return True if we should keep the dependency, False to drop it # We only manipulate the dependencies for packages not in the whitelist if self.twl and not self.twl.search(recipename): # then process the actual dependencies if self.twl.search(depname): return False return True def read_taint(self, fn, task, stampbase): taint = None try: with open(stampbase + '.' + task + '.taint', 'r') as taintf: taint = taintf.read() except IOError: pass return taint def get_taskhash(self, fn, task, deps, dataCache): k = fn + "." + task data = dataCache.basetaskhash[k] self.runtaskdeps[k] = [] self.file_checksum_values[k] = {} recipename = dataCache.pkg_fn[fn] for dep in sorted(deps, key=clean_basepath): depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group( 'fn')] if not self.rundep_check(fn, recipename, task, dep, depname, dataCache): continue if dep not in self.taskhash: bb.fatal( "%s is not in taskhash, caller isn't calling in dependency order?", dep) data = data + self.taskhash[dep] self.runtaskdeps[k].append(dep) if task in dataCache.file_checksums[fn]: if self.checksum_cache: checksums = self.checksum_cache.get_checksums( dataCache.file_checksums[fn][task], recipename) else: checksums = bb.fetch2.get_file_checksums( dataCache.file_checksums[fn][task], recipename) for (f, cs) in checksums: self.file_checksum_values[k][f] = cs if cs: data = data + cs taskdep = dataCache.task_deps[fn] if 'nostamp' in taskdep and task in taskdep['nostamp']: # Nostamp tasks need an implicit taint so that they force any dependent tasks to run import uuid taint = str(uuid.uuid4()) data = data + taint self.taints[k] = "nostamp:" + taint taint = self.read_taint(fn, task, dataCache.stamp[fn]) if taint: data = data + taint self.taints[k] = taint logger.warn("%s is tainted from a forced run" % k) h = hashlib.md5(data).hexdigest() self.taskhash[k] = h #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) return h def writeout_file_checksum_cache(self): """Write/update the file checksum cache onto disk""" if self.checksum_cache: self.checksum_cache.save_extras() self.checksum_cache.save_merge() else: bb.fetch2.fetcher_parse_save() bb.fetch2.fetcher_parse_done() def dump_sigtask(self, fn, task, stampbase, runtime): k = fn + "." + task if runtime == "customfile": sigfile = stampbase elif runtime and k in self.taskhash: sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[ k] else: sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[ k] bb.utils.mkdirhier(os.path.dirname(sigfile)) data = {} data['basewhitelist'] = self.basewhitelist data['taskwhitelist'] = self.taskwhitelist data['taskdeps'] = self.taskdeps[fn][task] data['basehash'] = self.basehash[k] data['gendeps'] = {} data['varvals'] = {} data['varvals'][task] = self.lookupcache[fn][task] for dep in self.taskdeps[fn][task]: if dep in self.basewhitelist: continue data['gendeps'][dep] = self.gendeps[fn][dep] data['varvals'][dep] = self.lookupcache[fn][dep] if runtime and k in self.taskhash: data['runtaskdeps'] = self.runtaskdeps[k] data['file_checksum_values'] = [ (os.path.basename(f), cs) for f, cs in self.file_checksum_values[k].items() ] data['runtaskhashes'] = {} for dep in data['runtaskdeps']: data['runtaskhashes'][dep] = self.taskhash[dep] taint = self.read_taint(fn, task, stampbase) if taint: data['taint'] = taint if runtime and k in self.taints: if 'nostamp:' in self.taints[k]: data['taint'] = self.taints[k] fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") try: with os.fdopen(fd, "wb") as stream: p = pickle.dump(data, stream, -1) stream.flush() os.chmod(tmpfile, 0664) os.rename(tmpfile, sigfile) except (OSError, IOError) as err: try: os.unlink(tmpfile) except OSError: pass raise err def dump_sigs(self, dataCache, options): for fn in self.taskdeps: for task in self.taskdeps[fn]: k = fn + "." + task if k not in self.taskhash: continue if dataCache.basetaskhash[k] != self.basehash[k]: bb.error( "Bitbake's cached basehash does not match the one we just generated (%s)!" % k) bb.error("The mismatched hashes were %s and %s" % (dataCache.basetaskhash[k], self.basehash[k])) self.dump_sigtask(fn, task, dataCache.stamp[fn], True)