Exemplo n.º 1
0
    def __init__(self, data):
        self.basehash = {}
        self.taskhash = {}
        self.unihash = {}
        self.taskdeps = {}
        self.runtaskdeps = {}
        self.file_checksum_values = {}
        self.taints = {}
        self.gendeps = {}
        self.lookupcache = {}
        self.setscenetasks = set()
        self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS")
                                         or "").split())
        self.taskhash_ignore_tasks = None
        self.init_rundepcheck(data)
        checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
        if checksum_cache_file:
            self.checksum_cache = FileChecksumCache()
            self.checksum_cache.init_cache(data, checksum_cache_file)
        else:
            self.checksum_cache = None

        self.unihash_cache = bb.cache.SimpleCache("3")
        self.unitaskhashes = self.unihash_cache.init_cache(
            data, "bb_unihashes.dat", {})
        self.localdirsexclude = (
            data.getVar("BB_SIGNATURE_LOCAL_DIRS_EXCLUDE")
            or "CVS .bzr .git .hg .osc .p4 .repo .svn").split()
        self.tidtopn = {}
Exemplo n.º 2
0
 def __init__(self, data):
     self.basehash = {}
     self.taskhash = {}
     self.taskdeps = {}
     self.runtaskdeps = {}
     self.file_checksum_values = {}
     self.taints = {}
     self.gendeps = {}
     self.lookupcache = {}
     self.pkgnameextract = re.compile("(?P<fn>.*)\..*")
     self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split())
     self.taskwhitelist = None
     self.init_rundepcheck(data)
     checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE", True)
     if checksum_cache_file:
         self.checksum_cache = FileChecksumCache()
         self.checksum_cache.init_cache(data, checksum_cache_file)
     else:
         self.checksum_cache = None
Exemplo n.º 3
0
    def __init__(self, data):
        self.basehash = {}
        self.taskhash = {}
        self.taskdeps = {}
        self.runtaskdeps = {}
        self.file_checksum_values = {}
        self.taints = {}
        self.gendeps = {}
        self.lookupcache = {}
        self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
        self.taskwhitelist = None
        self.init_rundepcheck(data)
        checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
        if checksum_cache_file:
            self.checksum_cache = FileChecksumCache()
            self.checksum_cache.init_cache(data, checksum_cache_file)
        else:
            self.checksum_cache = None

        self.unihash_cache = bb.cache.SimpleCache("1")
        self.unitaskhashes = self.unihash_cache.init_cache(data, "bb_unihashes.dat", {})
Exemplo n.º 4
0
Arquivo: siggen.py Projeto: kacf/poky
 def __init__(self, data):
     self.basehash = {}
     self.taskhash = {}
     self.taskdeps = {}
     self.runtaskdeps = {}
     self.file_checksum_values = {}
     self.taints = {}
     self.gendeps = {}
     self.lookupcache = {}
     self.pkgnameextract = re.compile("(?P<fn>.*)\..*")
     self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
     self.taskwhitelist = None
     self.init_rundepcheck(data)
     checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
     if checksum_cache_file:
         self.checksum_cache = FileChecksumCache()
         self.checksum_cache.init_cache(data, checksum_cache_file)
     else:
         self.checksum_cache = None
Exemplo n.º 5
0
class SignatureGeneratorBasic(SignatureGenerator):
    """
    """
    name = "basic"

    def __init__(self, data):
        self.basehash = {}
        self.taskhash = {}
        self.unihash = {}
        self.taskdeps = {}
        self.runtaskdeps = {}
        self.file_checksum_values = {}
        self.taints = {}
        self.gendeps = {}
        self.lookupcache = {}
        self.setscenetasks = set()
        self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS")
                                         or "").split())
        self.taskhash_ignore_tasks = None
        self.init_rundepcheck(data)
        checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
        if checksum_cache_file:
            self.checksum_cache = FileChecksumCache()
            self.checksum_cache.init_cache(data, checksum_cache_file)
        else:
            self.checksum_cache = None

        self.unihash_cache = bb.cache.SimpleCache("3")
        self.unitaskhashes = self.unihash_cache.init_cache(
            data, "bb_unihashes.dat", {})
        self.localdirsexclude = (
            data.getVar("BB_SIGNATURE_LOCAL_DIRS_EXCLUDE")
            or "CVS .bzr .git .hg .osc .p4 .repo .svn").split()
        self.tidtopn = {}

    def init_rundepcheck(self, data):
        self.taskhash_ignore_tasks = data.getVar(
            "BB_TASKHASH_IGNORE_TASKS") or None
        if self.taskhash_ignore_tasks:
            self.twl = re.compile(self.taskhash_ignore_tasks)
        else:
            self.twl = None

    def _build_data(self, fn, d):

        ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(
            d, self.basehash_ignore_vars)

        taskdeps, basehash = bb.data.generate_dependency_hash(
            tasklist, gendeps, lookupcache, self.basehash_ignore_vars, fn)

        for task in tasklist:
            tid = fn + ":" + task
            if not ignore_mismatch and tid in self.basehash and self.basehash[
                    tid] != basehash[tid]:
                bb.error(
                    "When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed."
                    % (tid, self.basehash[tid], basehash[tid]))
                bb.error("The following commands may help:")
                cmd = "$ bitbake %s -c%s" % (d.getVar('PN'), task)
                # Make sure sigdata is dumped before run printdiff
                bb.error("%s -Snone" % cmd)
                bb.error("Then:")
                bb.error("%s -Sprintdiff\n" % cmd)
            self.basehash[tid] = basehash[tid]

        self.taskdeps[fn] = taskdeps
        self.gendeps[fn] = gendeps
        self.lookupcache[fn] = lookupcache

        return taskdeps

    def set_setscene_tasks(self, setscene_tasks):
        self.setscenetasks = set(setscene_tasks)

    def finalise(self, fn, d, variant):

        mc = d.getVar("__BBMULTICONFIG", False) or ""
        if variant or mc:
            fn = bb.cache.realfn2virtual(fn, variant, mc)

        try:
            taskdeps = self._build_data(fn, d)
        except bb.parse.SkipRecipe:
            raise
        except:
            bb.warn("Error during finalise of %s" % fn)
            raise

        #Slow but can be useful for debugging mismatched basehashes
        #for task in self.taskdeps[fn]:
        #    self.dump_sigtask(fn, task, d.getVar("STAMP"), False)

        for task in taskdeps:
            d.setVar("BB_BASEHASH:task-%s" % task,
                     self.basehash[fn + ":" + task])

    def postparsing_clean_cache(self):
        #
        # After parsing we can remove some things from memory to reduce our memory footprint
        #
        self.gendeps = {}
        self.lookupcache = {}
        self.taskdeps = {}

    def rundep_check(self, fn, recipename, task, dep, depname, dataCaches):
        # Return True if we should keep the dependency, False to drop it
        # We only manipulate the dependencies for packages not in the ignore
        # list
        if self.twl and not self.twl.search(recipename):
            # then process the actual dependencies
            if self.twl.search(depname):
                return False
        return True

    def read_taint(self, fn, task, stampbase):
        taint = None
        try:
            with open(stampbase + '.' + task + '.taint', 'r') as taintf:
                taint = taintf.read()
        except IOError:
            pass
        return taint

    def prep_taskhash(self, tid, deps, dataCaches):

        (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)

        self.basehash[tid] = dataCaches[mc].basetaskhash[tid]
        self.runtaskdeps[tid] = []
        self.file_checksum_values[tid] = []
        recipename = dataCaches[mc].pkg_fn[fn]

        self.tidtopn[tid] = recipename

        for dep in sorted(deps, key=clean_basepath):
            (depmc, _, _, depmcfn) = bb.runqueue.split_tid_mcfn(dep)
            depname = dataCaches[depmc].pkg_fn[depmcfn]
            if not self.supports_multiconfig_datacaches and mc != depmc:
                # If the signature generator doesn't understand multiconfig
                # data caches, any dependency not in the same multiconfig must
                # be skipped for backward compatibility
                continue
            if not self.rundep_check(fn, recipename, task, dep, depname,
                                     dataCaches):
                continue
            if dep not in self.taskhash:
                bb.fatal(
                    "%s is not in taskhash, caller isn't calling in dependency order?"
                    % dep)
            self.runtaskdeps[tid].append(dep)

        if task in dataCaches[mc].file_checksums[fn]:
            if self.checksum_cache:
                checksums = self.checksum_cache.get_checksums(
                    dataCaches[mc].file_checksums[fn][task], recipename,
                    self.localdirsexclude)
            else:
                checksums = bb.fetch2.get_file_checksums(
                    dataCaches[mc].file_checksums[fn][task], recipename,
                    self.localdirsexclude)
            for (f, cs) in checksums:
                self.file_checksum_values[tid].append((f, cs))

        taskdep = dataCaches[mc].task_deps[fn]
        if 'nostamp' in taskdep and task in taskdep['nostamp']:
            # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
            if tid in self.taints and self.taints[tid].startswith("nostamp:"):
                # Don't reset taint value upon every call
                pass
            else:
                import uuid
                taint = str(uuid.uuid4())
                self.taints[tid] = "nostamp:" + taint

        taint = self.read_taint(fn, task, dataCaches[mc].stamp[fn])
        if taint:
            self.taints[tid] = taint
            logger.warning("%s is tainted from a forced run" % tid)

        return

    def get_taskhash(self, tid, deps, dataCaches):

        data = self.basehash[tid]
        for dep in self.runtaskdeps[tid]:
            data = data + self.get_unihash(dep)

        for (f, cs) in self.file_checksum_values[tid]:
            if cs:
                if "/./" in f:
                    data = data + "./" + f.split("/./")[1]
                data = data + cs

        if tid in self.taints:
            if self.taints[tid].startswith("nostamp:"):
                data = data + self.taints[tid][8:]
            else:
                data = data + self.taints[tid]

        h = hashlib.sha256(data.encode("utf-8")).hexdigest()
        self.taskhash[tid] = h
        #d.setVar("BB_TASKHASH:task-%s" % task, taskhash[task])
        return h

    def writeout_file_checksum_cache(self):
        """Write/update the file checksum cache onto disk"""
        if self.checksum_cache:
            self.checksum_cache.save_extras()
            self.checksum_cache.save_merge()
        else:
            bb.fetch2.fetcher_parse_save()
            bb.fetch2.fetcher_parse_done()

    def save_unitaskhashes(self):
        self.unihash_cache.save(self.unitaskhashes)

    def dump_sigtask(self, fn, task, stampbase, runtime):

        tid = fn + ":" + task
        referencestamp = stampbase
        if isinstance(runtime, str) and runtime.startswith("customfile"):
            sigfile = stampbase
            referencestamp = runtime[11:]
        elif runtime and tid in self.taskhash:
            sigfile = stampbase + "." + task + ".sigdata" + "." + self.get_unihash(
                tid)
        else:
            sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[
                tid]

        with bb.utils.umask(0o002):
            bb.utils.mkdirhier(os.path.dirname(sigfile))

        data = {}
        data['task'] = task
        data['basehash_ignore_vars'] = self.basehash_ignore_vars
        data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks
        data['taskdeps'] = self.taskdeps[fn][task]
        data['basehash'] = self.basehash[tid]
        data['gendeps'] = {}
        data['varvals'] = {}
        data['varvals'][task] = self.lookupcache[fn][task]
        for dep in self.taskdeps[fn][task]:
            if dep in self.basehash_ignore_vars:
                continue
            data['gendeps'][dep] = self.gendeps[fn][dep]
            data['varvals'][dep] = self.lookupcache[fn][dep]

        if runtime and tid in self.taskhash:
            data['runtaskdeps'] = self.runtaskdeps[tid]
            data['file_checksum_values'] = []
            for f, cs in self.file_checksum_values[tid]:
                if "/./" in f:
                    data['file_checksum_values'].append(
                        ("./" + f.split("/./")[1], cs))
                else:
                    data['file_checksum_values'].append(
                        (os.path.basename(f), cs))
            data['runtaskhashes'] = {}
            for dep in data['runtaskdeps']:
                data['runtaskhashes'][dep] = self.get_unihash(dep)
            data['taskhash'] = self.taskhash[tid]
            data['unihash'] = self.get_unihash(tid)

        taint = self.read_taint(fn, task, referencestamp)
        if taint:
            data['taint'] = taint

        if runtime and tid in self.taints:
            if 'nostamp:' in self.taints[tid]:
                data['taint'] = self.taints[tid]

        computed_basehash = calc_basehash(data)
        if computed_basehash != self.basehash[tid]:
            bb.error("Basehash mismatch %s versus %s for %s" %
                     (computed_basehash, self.basehash[tid], tid))
        if runtime and tid in self.taskhash:
            computed_taskhash = calc_taskhash(data)
            if computed_taskhash != self.taskhash[tid]:
                bb.error("Taskhash mismatch %s versus %s for %s" %
                         (computed_taskhash, self.taskhash[tid], tid))
                sigfile = sigfile.replace(self.taskhash[tid],
                                          computed_taskhash)

        fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile),
                                       prefix="sigtask.")
        try:
            with bb.compress.zstd.open(fd,
                                       "wt",
                                       encoding="utf-8",
                                       num_threads=1) as f:
                json.dump(data,
                          f,
                          sort_keys=True,
                          separators=(",", ":"),
                          cls=SetEncoder)
                f.flush()
            os.chmod(tmpfile, 0o664)
            bb.utils.rename(tmpfile, sigfile)
        except (OSError, IOError) as err:
            try:
                os.unlink(tmpfile)
            except OSError:
                pass
            raise err

    def dump_sigfn(self, fn, dataCaches, options):
        if fn in self.taskdeps:
            for task in self.taskdeps[fn]:
                tid = fn + ":" + task
                mc = bb.runqueue.mc_from_tid(tid)
                if tid not in self.taskhash:
                    continue
                if dataCaches[mc].basetaskhash[tid] != self.basehash[tid]:
                    bb.error(
                        "Bitbake's cached basehash does not match the one we just generated (%s)!"
                        % tid)
                    bb.error(
                        "The mismatched hashes were %s and %s" %
                        (dataCaches[mc].basetaskhash[tid], self.basehash[tid]))
                self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
Exemplo n.º 6
0
class SignatureGeneratorBasic(SignatureGenerator):
    """
    """
    name = "basic"

    def __init__(self, data):
        self.basehash = {}
        self.taskhash = {}
        self.taskdeps = {}
        self.runtaskdeps = {}
        self.file_checksum_values = {}
        self.taints = {}
        self.gendeps = {}
        self.lookupcache = {}
        self.pkgnameextract = re.compile("(?P<fn>.*)\..*")
        self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST")
                                  or "").split())
        self.taskwhitelist = None
        self.init_rundepcheck(data)
        checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
        if checksum_cache_file:
            self.checksum_cache = FileChecksumCache()
            self.checksum_cache.init_cache(data, checksum_cache_file)
        else:
            self.checksum_cache = None

    def init_rundepcheck(self, data):
        self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
        if self.taskwhitelist:
            self.twl = re.compile(self.taskwhitelist)
        else:
            self.twl = None

    def _build_data(self, fn, d):

        ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)

        taskdeps, basehash = bb.data.generate_dependency_hash(
            tasklist, gendeps, lookupcache, self.basewhitelist, fn)

        for task in tasklist:
            k = fn + "." + task
            if not ignore_mismatch and k in self.basehash and self.basehash[
                    k] != basehash[k]:
                bb.error(
                    "When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed."
                    % (k, self.basehash[k], basehash[k]))
            self.basehash[k] = basehash[k]

        self.taskdeps[fn] = taskdeps
        self.gendeps[fn] = gendeps
        self.lookupcache[fn] = lookupcache

        return taskdeps

    def finalise(self, fn, d, variant):

        mc = d.getVar("__BBMULTICONFIG", False) or ""
        if variant or mc:
            fn = bb.cache.realfn2virtual(fn, variant, mc)

        try:
            taskdeps = self._build_data(fn, d)
        except bb.parse.SkipRecipe:
            raise
        except:
            bb.warn("Error during finalise of %s" % fn)
            raise

        #Slow but can be useful for debugging mismatched basehashes
        #for task in self.taskdeps[fn]:
        #    self.dump_sigtask(fn, task, d.getVar("STAMP"), False)

        for task in taskdeps:
            d.setVar("BB_BASEHASH_task-%s" % task,
                     self.basehash[fn + "." + task])

    def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
        # Return True if we should keep the dependency, False to drop it
        # We only manipulate the dependencies for packages not in the whitelist
        if self.twl and not self.twl.search(recipename):
            # then process the actual dependencies
            if self.twl.search(depname):
                return False
        return True

    def read_taint(self, fn, task, stampbase):
        taint = None
        try:
            with open(stampbase + '.' + task + '.taint', 'r') as taintf:
                taint = taintf.read()
        except IOError:
            pass
        return taint

    def get_taskhash(self, fn, task, deps, dataCache):

        mc = ''
        if fn.startswith('multiconfig:'):
            mc = fn.split(':')[1]
        k = fn + "." + task

        data = dataCache.basetaskhash[k]
        self.basehash[k] = data
        self.runtaskdeps[k] = []
        self.file_checksum_values[k] = []
        recipename = dataCache.pkg_fn[fn]
        for dep in sorted(deps, key=clean_basepath):
            pkgname = self.pkgnameextract.search(dep).group('fn')
            if mc:
                depmc = pkgname.split(':')[1]
                if mc != depmc:
                    continue
            if dep.startswith("multiconfig:") and not mc:
                continue
            depname = dataCache.pkg_fn[pkgname]
            if not self.rundep_check(fn, recipename, task, dep, depname,
                                     dataCache):
                continue
            if dep not in self.taskhash:
                bb.fatal(
                    "%s is not in taskhash, caller isn't calling in dependency order?"
                    % dep)
            data = data + self.taskhash[dep]
            self.runtaskdeps[k].append(dep)

        if task in dataCache.file_checksums[fn]:
            if self.checksum_cache:
                checksums = self.checksum_cache.get_checksums(
                    dataCache.file_checksums[fn][task], recipename)
            else:
                checksums = bb.fetch2.get_file_checksums(
                    dataCache.file_checksums[fn][task], recipename)
            for (f, cs) in checksums:
                self.file_checksum_values[k].append((f, cs))
                if cs:
                    data = data + cs

        taskdep = dataCache.task_deps[fn]
        if 'nostamp' in taskdep and task in taskdep['nostamp']:
            # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
            import uuid
            taint = str(uuid.uuid4())
            data = data + taint
            self.taints[k] = "nostamp:" + taint

        taint = self.read_taint(fn, task, dataCache.stamp[fn])
        if taint:
            data = data + taint
            self.taints[k] = taint
            logger.warning("%s is tainted from a forced run" % k)

        h = hashlib.md5(data.encode("utf-8")).hexdigest()
        self.taskhash[k] = h
        #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
        return h

    def writeout_file_checksum_cache(self):
        """Write/update the file checksum cache onto disk"""
        if self.checksum_cache:
            self.checksum_cache.save_extras()
            self.checksum_cache.save_merge()
        else:
            bb.fetch2.fetcher_parse_save()
            bb.fetch2.fetcher_parse_done()

    def dump_sigtask(self, fn, task, stampbase, runtime):

        k = fn + "." + task
        referencestamp = stampbase
        if isinstance(runtime, str) and runtime.startswith("customfile"):
            sigfile = stampbase
            referencestamp = runtime[11:]
        elif runtime and k in self.taskhash:
            sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[
                k]
        else:
            sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[
                k]

        bb.utils.mkdirhier(os.path.dirname(sigfile))

        data = {}
        data['task'] = task
        data['basewhitelist'] = self.basewhitelist
        data['taskwhitelist'] = self.taskwhitelist
        data['taskdeps'] = self.taskdeps[fn][task]
        data['basehash'] = self.basehash[k]
        data['gendeps'] = {}
        data['varvals'] = {}
        data['varvals'][task] = self.lookupcache[fn][task]
        for dep in self.taskdeps[fn][task]:
            if dep in self.basewhitelist:
                continue
            data['gendeps'][dep] = self.gendeps[fn][dep]
            data['varvals'][dep] = self.lookupcache[fn][dep]

        if runtime and k in self.taskhash:
            data['runtaskdeps'] = self.runtaskdeps[k]
            data['file_checksum_values'] = [
                (os.path.basename(f), cs)
                for f, cs in self.file_checksum_values[k]
            ]
            data['runtaskhashes'] = {}
            for dep in data['runtaskdeps']:
                data['runtaskhashes'][dep] = self.taskhash[dep]
            data['taskhash'] = self.taskhash[k]

        taint = self.read_taint(fn, task, referencestamp)
        if taint:
            data['taint'] = taint

        if runtime and k in self.taints:
            if 'nostamp:' in self.taints[k]:
                data['taint'] = self.taints[k]

        computed_basehash = calc_basehash(data)
        if computed_basehash != self.basehash[k]:
            bb.error("Basehash mismatch %s versus %s for %s" %
                     (computed_basehash, self.basehash[k], k))
        if runtime and k in self.taskhash:
            computed_taskhash = calc_taskhash(data)
            if computed_taskhash != self.taskhash[k]:
                bb.error("Taskhash mismatch %s versus %s for %s" %
                         (computed_taskhash, self.taskhash[k], k))
                sigfile = sigfile.replace(self.taskhash[k], computed_taskhash)

        fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile),
                                       prefix="sigtask.")
        try:
            with os.fdopen(fd, "wb") as stream:
                p = pickle.dump(data, stream, -1)
                stream.flush()
            os.chmod(tmpfile, 0o664)
            os.rename(tmpfile, sigfile)
        except (OSError, IOError) as err:
            try:
                os.unlink(tmpfile)
            except OSError:
                pass
            raise err

    def dump_sigfn(self, fn, dataCaches, options):
        if fn in self.taskdeps:
            for task in self.taskdeps[fn]:
                tid = fn + ":" + task
                (mc, _, _) = bb.runqueue.split_tid(tid)
                k = fn + "." + task
                if k not in self.taskhash:
                    continue
                if dataCaches[mc].basetaskhash[k] != self.basehash[k]:
                    bb.error(
                        "Bitbake's cached basehash does not match the one we just generated (%s)!"
                        % k)
                    bb.error(
                        "The mismatched hashes were %s and %s" %
                        (dataCaches[mc].basetaskhash[k], self.basehash[k]))
                self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
Exemplo n.º 7
0
Arquivo: siggen.py Projeto: kacf/poky
class SignatureGeneratorBasic(SignatureGenerator):
    """
    """
    name = "basic"

    def __init__(self, data):
        self.basehash = {}
        self.taskhash = {}
        self.taskdeps = {}
        self.runtaskdeps = {}
        self.file_checksum_values = {}
        self.taints = {}
        self.gendeps = {}
        self.lookupcache = {}
        self.pkgnameextract = re.compile("(?P<fn>.*)\..*")
        self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
        self.taskwhitelist = None
        self.init_rundepcheck(data)
        checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
        if checksum_cache_file:
            self.checksum_cache = FileChecksumCache()
            self.checksum_cache.init_cache(data, checksum_cache_file)
        else:
            self.checksum_cache = None

    def init_rundepcheck(self, data):
        self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
        if self.taskwhitelist:
            self.twl = re.compile(self.taskwhitelist)
        else:
            self.twl = None

    def _build_data(self, fn, d):

        ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)

        taskdeps = {}
        basehash = {}

        for task in tasklist:
            data = lookupcache[task]

            if data is None:
                bb.error("Task %s from %s seems to be empty?!" % (task, fn))
                data = ''

            gendeps[task] -= self.basewhitelist
            newdeps = gendeps[task]
            seen = set()
            while newdeps:
                nextdeps = newdeps
                seen |= nextdeps
                newdeps = set()
                for dep in nextdeps:
                    if dep in self.basewhitelist:
                        continue
                    gendeps[dep] -= self.basewhitelist
                    newdeps |= gendeps[dep]
                newdeps -= seen

            alldeps = sorted(seen)
            for dep in alldeps:
                data = data + dep
                var = lookupcache[dep]
                if var is not None:
                    data = data + str(var)
            datahash = hashlib.md5(data.encode("utf-8")).hexdigest()
            k = fn + "." + task
            if not ignore_mismatch and k in self.basehash and self.basehash[k] != datahash:
                bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], datahash))
            self.basehash[k] = datahash
            taskdeps[task] = alldeps

        self.taskdeps[fn] = taskdeps
        self.gendeps[fn] = gendeps
        self.lookupcache[fn] = lookupcache

        return taskdeps

    def finalise(self, fn, d, variant):

        mc = d.getVar("__BBMULTICONFIG", False) or ""
        if variant or mc:
            fn = bb.cache.realfn2virtual(fn, variant, mc)

        try:
            taskdeps = self._build_data(fn, d)
        except:
            bb.warn("Error during finalise of %s" % fn)
            raise

        #Slow but can be useful for debugging mismatched basehashes
        #for task in self.taskdeps[fn]:
        #    self.dump_sigtask(fn, task, d.getVar("STAMP"), False)

        for task in taskdeps:
            d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task])

    def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
        # Return True if we should keep the dependency, False to drop it
        # We only manipulate the dependencies for packages not in the whitelist
        if self.twl and not self.twl.search(recipename):
            # then process the actual dependencies
            if self.twl.search(depname):
                return False
        return True

    def read_taint(self, fn, task, stampbase):
        taint = None
        try:
            with open(stampbase + '.' + task + '.taint', 'r') as taintf:
                taint = taintf.read()
        except IOError:
            pass
        return taint

    def get_taskhash(self, fn, task, deps, dataCache):
        k = fn + "." + task
        data = dataCache.basetaskhash[k]
        self.basehash[k] = data
        self.runtaskdeps[k] = []
        self.file_checksum_values[k] = []
        recipename = dataCache.pkg_fn[fn]

        for dep in sorted(deps, key=clean_basepath):
            depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')]
            if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
                continue
            if dep not in self.taskhash:
                bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep)
            data = data + self.taskhash[dep]
            self.runtaskdeps[k].append(dep)

        if task in dataCache.file_checksums[fn]:
            if self.checksum_cache:
                checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename)
            else:
                checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
            for (f,cs) in checksums:
                self.file_checksum_values[k].append((f,cs))
                if cs:
                    data = data + cs

        taskdep = dataCache.task_deps[fn]
        if 'nostamp' in taskdep and task in taskdep['nostamp']:
            # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
            import uuid
            taint = str(uuid.uuid4())
            data = data + taint
            self.taints[k] = "nostamp:" + taint

        taint = self.read_taint(fn, task, dataCache.stamp[fn])
        if taint:
            data = data + taint
            self.taints[k] = taint
            logger.warning("%s is tainted from a forced run" % k)

        h = hashlib.md5(data.encode("utf-8")).hexdigest()
        self.taskhash[k] = h
        #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
        return h

    def writeout_file_checksum_cache(self):
        """Write/update the file checksum cache onto disk"""
        if self.checksum_cache:
            self.checksum_cache.save_extras()
            self.checksum_cache.save_merge()
        else:
            bb.fetch2.fetcher_parse_save()
            bb.fetch2.fetcher_parse_done()

    def dump_sigtask(self, fn, task, stampbase, runtime):

        k = fn + "." + task
        referencestamp = stampbase
        if isinstance(runtime, str) and runtime.startswith("customfile"):
            sigfile = stampbase
            referencestamp = runtime[11:]
        elif runtime and k in self.taskhash:
            sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k]
        else:
            sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k]

        bb.utils.mkdirhier(os.path.dirname(sigfile))

        data = {}
        data['task'] = task
        data['basewhitelist'] = self.basewhitelist
        data['taskwhitelist'] = self.taskwhitelist
        data['taskdeps'] = self.taskdeps[fn][task]
        data['basehash'] = self.basehash[k]
        data['gendeps'] = {}
        data['varvals'] = {}
        data['varvals'][task] = self.lookupcache[fn][task]
        for dep in self.taskdeps[fn][task]:
            if dep in self.basewhitelist:
                continue
            data['gendeps'][dep] = self.gendeps[fn][dep]
            data['varvals'][dep] = self.lookupcache[fn][dep]

        if runtime and k in self.taskhash:
            data['runtaskdeps'] = self.runtaskdeps[k]
            data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k]]
            data['runtaskhashes'] = {}
            for dep in data['runtaskdeps']:
                data['runtaskhashes'][dep] = self.taskhash[dep]
            data['taskhash'] = self.taskhash[k]

        taint = self.read_taint(fn, task, referencestamp)
        if taint:
            data['taint'] = taint

        if runtime and k in self.taints:
            if 'nostamp:' in self.taints[k]:
                data['taint'] = self.taints[k]

        computed_basehash = calc_basehash(data)
        if computed_basehash != self.basehash[k]:
            bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k))
        if runtime and k in self.taskhash:
            computed_taskhash = calc_taskhash(data)
            if computed_taskhash != self.taskhash[k]:
                bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k))
                sigfile = sigfile.replace(self.taskhash[k], computed_taskhash)

        fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
        try:
            with os.fdopen(fd, "wb") as stream:
                p = pickle.dump(data, stream, -1)
                stream.flush()
            os.chmod(tmpfile, 0o664)
            os.rename(tmpfile, sigfile)
        except (OSError, IOError) as err:
            try:
                os.unlink(tmpfile)
            except OSError:
                pass
            raise err

    def dump_sigfn(self, fn, dataCaches, options):
        if fn in self.taskdeps:
            for task in self.taskdeps[fn]:
                tid = fn + ":" + task
                (mc, _, _) = bb.runqueue.split_tid(tid)
                k = fn + "." + task
                if k not in self.taskhash:
                    continue
                if dataCaches[mc].basetaskhash[k] != self.basehash[k]:
                    bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k)
                    bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[k], self.basehash[k]))
                self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
Exemplo n.º 8
0
class SignatureGeneratorBasic(SignatureGenerator):
    """
    """
    name = "basic"

    def __init__(self, data):
        self.basehash = {}
        self.taskhash = {}
        self.taskdeps = {}
        self.runtaskdeps = {}
        self.file_checksum_values = {}
        self.taints = {}
        self.gendeps = {}
        self.lookupcache = {}
        self.pkgnameextract = re.compile("(?P<fn>.*)\..*")
        self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True)
                                  or "").split())
        self.taskwhitelist = None
        self.init_rundepcheck(data)
        checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE", True)
        if checksum_cache_file:
            self.checksum_cache = FileChecksumCache()
            self.checksum_cache.init_cache(data, checksum_cache_file)
        else:
            self.checksum_cache = None

    def init_rundepcheck(self, data):
        self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None
        if self.taskwhitelist:
            self.twl = re.compile(self.taskwhitelist)
        else:
            self.twl = None

    def _build_data(self, fn, d):

        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)

        taskdeps = {}
        basehash = {}

        for task in tasklist:
            data = lookupcache[task]

            if data is None:
                bb.error("Task %s from %s seems to be empty?!" % (task, fn))
                data = ''

            gendeps[task] -= self.basewhitelist
            newdeps = gendeps[task]
            seen = set()
            while newdeps:
                nextdeps = newdeps
                seen |= nextdeps
                newdeps = set()
                for dep in nextdeps:
                    if dep in self.basewhitelist:
                        continue
                    gendeps[dep] -= self.basewhitelist
                    newdeps |= gendeps[dep]
                newdeps -= seen

            alldeps = sorted(seen)
            for dep in alldeps:
                data = data + dep
                var = lookupcache[dep]
                if var is not None:
                    data = data + str(var)
            self.basehash[fn + "." + task] = hashlib.md5(data).hexdigest()
            taskdeps[task] = alldeps

        self.taskdeps[fn] = taskdeps
        self.gendeps[fn] = gendeps
        self.lookupcache[fn] = lookupcache

        return taskdeps

    def finalise(self, fn, d, variant):

        if variant:
            fn = "virtual:" + variant + ":" + fn

        try:
            taskdeps = self._build_data(fn, d)
        except:
            bb.warn("Error during finalise of %s" % fn)
            raise

        #Slow but can be useful for debugging mismatched basehashes
        #for task in self.taskdeps[fn]:
        #    self.dump_sigtask(fn, task, d.getVar("STAMP", True), False)

        for task in taskdeps:
            d.setVar("BB_BASEHASH_task-%s" % task,
                     self.basehash[fn + "." + task])

    def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
        # Return True if we should keep the dependency, False to drop it
        # We only manipulate the dependencies for packages not in the whitelist
        if self.twl and not self.twl.search(recipename):
            # then process the actual dependencies
            if self.twl.search(depname):
                return False
        return True

    def read_taint(self, fn, task, stampbase):
        taint = None
        try:
            with open(stampbase + '.' + task + '.taint', 'r') as taintf:
                taint = taintf.read()
        except IOError:
            pass
        return taint

    def get_taskhash(self, fn, task, deps, dataCache):
        k = fn + "." + task
        data = dataCache.basetaskhash[k]
        self.runtaskdeps[k] = []
        self.file_checksum_values[k] = {}
        recipename = dataCache.pkg_fn[fn]
        for dep in sorted(deps, key=clean_basepath):
            depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group(
                'fn')]
            if not self.rundep_check(fn, recipename, task, dep, depname,
                                     dataCache):
                continue
            if dep not in self.taskhash:
                bb.fatal(
                    "%s is not in taskhash, caller isn't calling in dependency order?",
                    dep)
            data = data + self.taskhash[dep]
            self.runtaskdeps[k].append(dep)

        if task in dataCache.file_checksums[fn]:
            if self.checksum_cache:
                checksums = self.checksum_cache.get_checksums(
                    dataCache.file_checksums[fn][task], recipename)
            else:
                checksums = bb.fetch2.get_file_checksums(
                    dataCache.file_checksums[fn][task], recipename)
            for (f, cs) in checksums:
                self.file_checksum_values[k][f] = cs
                if cs:
                    data = data + cs

        taskdep = dataCache.task_deps[fn]
        if 'nostamp' in taskdep and task in taskdep['nostamp']:
            # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
            import uuid
            taint = str(uuid.uuid4())
            data = data + taint
            self.taints[k] = "nostamp:" + taint

        taint = self.read_taint(fn, task, dataCache.stamp[fn])
        if taint:
            data = data + taint
            self.taints[k] = taint
            logger.warn("%s is tainted from a forced run" % k)

        h = hashlib.md5(data).hexdigest()
        self.taskhash[k] = h
        #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
        return h

    def writeout_file_checksum_cache(self):
        """Write/update the file checksum cache onto disk"""
        if self.checksum_cache:
            self.checksum_cache.save_extras()
            self.checksum_cache.save_merge()
        else:
            bb.fetch2.fetcher_parse_save()
            bb.fetch2.fetcher_parse_done()

    def dump_sigtask(self, fn, task, stampbase, runtime):
        k = fn + "." + task
        if runtime == "customfile":
            sigfile = stampbase
        elif runtime and k in self.taskhash:
            sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[
                k]
        else:
            sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[
                k]

        bb.utils.mkdirhier(os.path.dirname(sigfile))

        data = {}
        data['basewhitelist'] = self.basewhitelist
        data['taskwhitelist'] = self.taskwhitelist
        data['taskdeps'] = self.taskdeps[fn][task]
        data['basehash'] = self.basehash[k]
        data['gendeps'] = {}
        data['varvals'] = {}
        data['varvals'][task] = self.lookupcache[fn][task]
        for dep in self.taskdeps[fn][task]:
            if dep in self.basewhitelist:
                continue
            data['gendeps'][dep] = self.gendeps[fn][dep]
            data['varvals'][dep] = self.lookupcache[fn][dep]

        if runtime and k in self.taskhash:
            data['runtaskdeps'] = self.runtaskdeps[k]
            data['file_checksum_values'] = [
                (os.path.basename(f), cs)
                for f, cs in self.file_checksum_values[k].items()
            ]
            data['runtaskhashes'] = {}
            for dep in data['runtaskdeps']:
                data['runtaskhashes'][dep] = self.taskhash[dep]

        taint = self.read_taint(fn, task, stampbase)
        if taint:
            data['taint'] = taint

        if runtime and k in self.taints:
            if 'nostamp:' in self.taints[k]:
                data['taint'] = self.taints[k]

        fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile),
                                       prefix="sigtask.")
        try:
            with os.fdopen(fd, "wb") as stream:
                p = pickle.dump(data, stream, -1)
                stream.flush()
            os.chmod(tmpfile, 0664)
            os.rename(tmpfile, sigfile)
        except (OSError, IOError) as err:
            try:
                os.unlink(tmpfile)
            except OSError:
                pass
            raise err

    def dump_sigs(self, dataCache, options):
        for fn in self.taskdeps:
            for task in self.taskdeps[fn]:
                k = fn + "." + task
                if k not in self.taskhash:
                    continue
                if dataCache.basetaskhash[k] != self.basehash[k]:
                    bb.error(
                        "Bitbake's cached basehash does not match the one we just generated (%s)!"
                        % k)
                    bb.error("The mismatched hashes were %s and %s" %
                             (dataCache.basetaskhash[k], self.basehash[k]))
                self.dump_sigtask(fn, task, dataCache.stamp[fn], True)