Exemple #1
0
def build(f, any_errors, should_build, add_dep_to=None, delegate=None, re_do=True):
    if f.dolock():
        if f.check_deadlocks():
            err("%s: recursive dependency, breaking deadlock\n", f.printable_name())
            any_errors[0] += 1
            any_errors[1] += 1
        else:
            jwack.get_token(f)
            f.dolock().waitlock()
            if any_errors[0] and not vars.KEEP_GOING:
                return False
            f.refresh()
            debug3('think about building %r\n', f.name)
            dirty = should_build(f)
            while dirty and dirty != deps.DIRTY:
                # FIXME: bring back the old (targetname) notation in the output
                #  when we need to do this.  And add comments.
                for t2 in dirty:
                    if not build(t2, any_errors, should_build, delegate, re_do):
                        return False
                jwack.wait_all()
                dirty = should_build(f)
            if dirty:
                job = BuildJob(f, any_errors, add_dep_to, delegate, re_do)
                add_dep_to = None
                job.schedule_job()
            else:
                f.dolock().unlock()
    if add_dep_to:
        f.refresh()
        add_dep_to.add_dep(f)
    return True
Exemple #2
0
 def add_dep(self, mode, dep):
     src = File(name=dep)
     debug3('add-dep: "%s" < %s "%s"\n' % (self.name, mode, src.name))
     assert(self.id != src.id)
     _write("insert or replace into Deps "
            "    (target, mode, source, delete_me) values (?,?,?,?)",
            [self.id, mode, src.id, False])
Exemple #3
0
    def build(self):
        debug3('running build job for %r\n', self.target.name)

        (dodir, dofile, basedir, basename, ext) = (
            self.dodir, self.dofile, self.dobasedir, self.dobasename, self.doext)

        # this will run in the dofile's directory, so use only basenames here
        if vars.OLD_ARGS:
            arg1 = basename  # target name (no extension)
            arg2 = ext       # extension (if any), including leading dot
        else:
            arg1 = basename + ext  # target name (including extension)
            arg2 = basename        # target name (without extension)
        argv = ['sh', '-e',
                dofile,
                arg1,
                arg2,
                # temp output file name
                os.path.relpath(self.tmpname_arg3, dodir),
                ]
        if vars.VERBOSE: argv[1] += 'v'
        if vars.XTRACE: argv[1] += 'x'
        if vars.VERBOSE or vars.XTRACE: log_e('\n')

        firstline = open(os.path.join(dodir, dofile)).readline().strip()
        if firstline.startswith('#!.../'):
            _, _, interp_argv = firstline.partition("/")
            interp_argv = interp_argv.split(' ')
            interpreter = _find_interpreter(self.dodir, interp_argv[0])
            if not interpreter:
                err('%s unable to find interpreter %s.\n', self.dofile, interp_argv[0])
                os._exit(208)
            self.target.add_dep(state.File(interpreter))
            argv[0:2] = [interpreter] + interp_argv[1:]
        elif firstline.startswith('#!/'):
            argv[0:2] = firstline[2:].split(' ')
        log('%s\n', self.target.printable_name())
        log_cmd("redo", self.target.name + "\n")

        try:
            dn = dodir
            os.environ['REDO_PWD'] = os.path.join(vars.PWD, dn)
            os.environ['REDO_TARGET'] = basename + ext
            os.environ['REDO_DEPTH'] = vars.DEPTH + '  '
            if dn:
                os.chdir(dn)
            l = logger.Logger(self.log_fd, self.tmp_sout_fd)
            l.fork()
            os.close(self.tmp_sout_fd)
            close_on_exec(1, False)
            if vars.VERBOSE or vars.XTRACE: log_e('* %s\n' % ' '.join(argv))
            os.execvp(argv[0], argv)
        except:
            import traceback
            sys.stderr.write(traceback.format_exc())
            err('internal exception - see above\n')
            raise
        finally:
            # returns only if there's an exception (exec in other case)
            os._exit(127)
Exemple #4
0
 def _add(self, line):
     depsname = self.tmpfilename('deps2')
     debug3('_add(%s) to %r\n', line, depsname)
     #assert os.path.exists(depsname)
     line = str(line)
     assert('\n' not in line)
     with open(depsname, 'a') as f:
         f.write(line + '\n')
Exemple #5
0
 def add_dep(self, mode, dep):
     src = File(name=dep)
     debug3('add-dep: "%s" < %s "%s"\n' % (self.name, mode, src.name))
     assert (self.id != src.id)
     _write(
         "insert or replace into Deps "
         "    (target, mode, source, delete_me) values (?,?,?,?)",
         [self.id, mode, src.id, False])
Exemple #6
0
 def build_starting(self):
     """Call this when you're about to start building this target."""
     if vars.TARGET:
         with open(self.tmpfilename('parent'), "w") as f:
             f.write(os.path.relpath(vars.TARGET, self.dir))
     depsname = self.tmpfilename('deps2')
     debug3('build starting: %r\n', depsname)
     unlink(depsname)
Exemple #7
0
 def build_done(self, exitcode):
     """Call this when you're done building this target."""
     depsname = self.tmpfilename('deps2')
     debug3('build ending: %r\n', depsname)
     self._add(self.read_stamp(runid=vars.RUNID).stamp)
     self._add(exitcode)
     os.utime(depsname, (vars.RUNID, vars.RUNID))
     os.rename(depsname, self.tmpfilename('deps'))
     unlink(self.tmpfilename('parent'))
Exemple #8
0
    def prepare(self):
        assert self.target.dolock().owned == state.LOCK_EX
        self.target.build_starting()
        self.before_t = _try_stat(self.target.name)

        newstamp = self.target.read_stamp()
        if newstamp.is_override_or_missing(self.target):
            if newstamp.is_missing():
                # was marked generated, but is now deleted
                debug3('oldstamp=%r newstamp=%r\n', self.target.stamp, newstamp)
                self.target.forget()
                self.target.refresh()
            elif vars.OVERWRITE:
                warn('%s: you modified it; overwrite\n', self.target.printable_name())
            else:
                warn('%s: you modified it; skipping\n', self.target.printable_name())
                return 0
        if self.target.exists_not_dir() and not self.target.is_generated:
            # an existing source file that was not generated by us.
            # This step is mentioned by djb in his notes.
            # For example, a rule called default.c.do could be used to try
            # to produce hello.c, but we don't want that to happen if
            # hello.c was created in advance by the end user.
            if vars.OVERWRITE:
                warn('%s: exists and not marked as generated; overwrite.\n',
                     self.target.printable_name())
            else:
                warn('%s: exists and not marked as generated; not redoing.\n',
                     self.target.printable_name())
                debug2('-- static (%r)\n', self.target.name)
                return 0

        (self.dodir, self.dofile, self.dobasedir, self.dobasename, self.doext) = _find_do_file(self.target)
        if not self.dofile:
            if newstamp.is_missing():
                err('no rule to make %r\n', self.target.name)
                return 1
            else:
                self.target.forget()
                debug2('-- forget (%r)\n', self.target.name)
                return 0  # no longer a generated target, but exists, so ok

        self.outdir = self._mkoutdir()
        # name connected to stdout
        self.tmpname_sout = self.target.tmpfilename('out.tmp')
        # name provided as $3
        self.tmpname_arg3 = os.path.join(self.outdir, self.target.basename())
        # name for the log file
        unlink(self.tmpname_sout)
        unlink(self.tmpname_arg3)
        self.log_fd = logger.open_log(self.target, truncate=True)
        self.tmp_sout_fd = os.open(self.tmpname_sout, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666)
        close_on_exec(self.tmp_sout_fd, True)
        self.tmp_sout_f = os.fdopen(self.tmp_sout_fd, 'w+')

        return None
Exemple #9
0
 def build_starting(self):
     """Call this when you're about to start building this target."""
     if vars.TARGET:
         with open(self.tmpfilename('parent'), "w") as f:
             f.write(os.path.relpath(vars.TARGET, self.dir))
     depsname = self.tmpfilename('deps2')
     debug3('build starting: %r\n', depsname)
     unlink(depsname)
     with open(depsname, 'a') as f:
         f.write(DEPSFILE_TAG + '\n')
         st = os.fstat(f.fileno())
         f.write('%d %d\n' % (st.st_dev, st.st_ino))
Exemple #10
0
    def add_dep(self, file):
        """Mark the given File() object as a dependency of this target.

        The filesystem file it refers to may or may not exist.  If it doesn't
        exist, creating the file is considered a "modified" event and will
        result in this target being rebuilt.
        """
        if file.name == ALWAYS:
            relname = file.name
        else:
            relname = os.path.relpath(file.name, self.dir)
        debug3('add-dep: %r < %r %r\n', self.name, file.stamp, relname)
        assert('\n' not in file.name)
        assert(isinstance(file.stamp, Stamp))
        self._add('%s %s' % (file.stamp.csum_or_stamp(), relname))
Exemple #11
0
    def _find_uncovered_dn(self, generator, break_early=False):
        is_covered = self.nsec3_chain.covers
        while True:
            for ptlabel,dn_hash in self._prehash_iter:
                if not is_covered(dn_hash):
                    dn = name.DomainName(name.Label(ptlabel), *self.zone.labels)
                    owner_b32 = util.base32_ext_hex_encode( dn_hash).lower()
                    hashed_dn = name.DomainName( name.Label(owner_b32), *self.zone.labels)
                    log.debug3('found uncovered dn: ', str(dn), '; hashed: ', str(hashed_dn))
                    return dn,dn_hash

            self.stats['tested_hashes'] += len(self._prehash_list)
            hashes, label_counter_state = self._hash_queues.next().recv()
            if self._label_counter_state < label_counter_state:
                self._label_counter_state = label_counter_state
            self._prehash_list = hashes
            self._prehash_iter = iter(hashes)
            log.update()
            if break_early:
                return None,None
Exemple #12
0
 def forget(self):
     """Turn a 'target' file back into a 'source' file."""
     debug3('forget(%s)\n', self.name)
     unlink(self.tmpfilename('deps'))
Exemple #13
0
def isdirty(f, depth, expect_stamp):
    assert(isinstance(expect_stamp, state.Stamp))
    debug('%s?%s\n', depth, f.name)

    debug3('%sexpect: %r\n', depth, expect_stamp)
    debug3('%sold:    %r\n', depth, f.stamp)

    if not f.is_generated and expect_stamp.is_none() and f.exists():
        debug('%s-- CLEAN (static)\n', depth)
        return CLEAN
    if f.exitcode:
        debug('%s-- DIRTY (failed last time)\n', depth)
        return DIRTY
    if not expect_stamp.is_missing() and f.stamp.is_missing() and not f.stamp.runid():
        debug('%s-- DIRTY (never built)\n', depth)
        return DIRTY
    if f.stamp.is_old():
        debug('%s-- DIRTY (from old redo)\n', depth)
        return DIRTY
    if not f.stamp or f.stamp.is_none():
        debug('%s-- DIRTY (no stamp)\n', depth)
        return DIRTY

    newstamp = f.read_stamp()

    debug3('%snew:    %r\n', depth, newstamp)

    if newstamp.is_override_or_missing(f) and not newstamp.is_missing():
        if vars.OVERWRITE:
            debug('%s-- DIRTY (override)\n', depth)
            return DIRTY
        else:
            debug('%s-- CLEAN (override)\n', depth)
            return CLEAN

    if newstamp.is_stamp_dirty(f):
        if newstamp.is_missing():
            debug('%s-- DIRTY (missing)\n', depth)
        else:
            debug('%s-- DIRTY (mtime)\n', depth)
        return [f] if f.stamp.is_csum() else DIRTY

    must_build = []
    for stamp2, f2 in f.deps:
        dirty = CLEAN

        if f2 == state.ALWAYS:
            if f.stamp_mtime >= vars.RUNID:
                # has already been checked during this session
                debug('%s-- CLEAN (always, checked)\n', depth)
            else:
                debug('%s-- DIRTY (always)\n', depth)
                dirty = DIRTY
        else:
            f2 = state.File(f2, f.dir)
            sub = isdirty(f2, depth = depth + '  ',
                          expect_stamp = stamp2)
            if sub:
                debug('%s-- DIRTY (sub)\n', depth)
                dirty = sub

        if not f.stamp.is_csum():
            # f is a "normal" target: dirty f2 means f is instantly dirty
            if dirty:
                # if dirty==DIRTY, this means f is definitely dirty.
                # if dirty==[...], it's a list of the uncertain children.
                return dirty
        else:
            # f is "checksummable": dirty f2 means f needs to redo,
            # but f might turn out to be clean after that (ie. our parent
            # might not be dirty).
            if dirty == DIRTY:
                # f2 is definitely dirty, so f definitely needs to
                # redo.  However, after that, f might turn out to be
                # unchanged.
                return [f]
            elif isinstance(dirty, list):
                # our child f2 might be dirty, but it's not sure yet.  It's
                # given us a list of targets we have to redo in order to
                # be sure.
                must_build += dirty

    if must_build:
        # f is *maybe* dirty because at least one of its children is maybe
        # dirty.  must_build has accumulated a list of "topmost" uncertain
        # objects in the tree.  If we build all those, we can then
        # redo-ifchange f and it won't have any uncertainty next time.
        return must_build

    if expect_stamp.is_dirty(f):
        # This must be after we checked the children. Before, we didn't knew
        # if the current target was dirty or not
        debug('%s-- DIRTY (parent)\n', depth)
        return DIRTY

    # if we get here, it's because the target is clean
    debug2('%s-- CLEAN (dropped off)\n', depth)
    return CLEAN
Exemple #14
0
 def refresh(self):
     if self.name == ALWAYS:
         self.stamp_mtime = str(vars.RUNID)
         self.exitcode = 0
         self.deps = []
         self.is_generated = True
         self.stamp = Stamp(str(vars.RUNID))
         return
     assert(not self.name.startswith('/'))
     try:
         # read the state file
         f = open(self.tmpfilename('deps'))
     except IOError:
         try:
             # okay, check for the file itself
             st = os.stat(self.name)
         except OSError:
             # it doesn't exist at all yet
             self.stamp_mtime = 0  # no stamp file
             self.exitcode = 0
             self.deps = []
             self.stamp = Stamp(STAMP_MISSING)
             self.runid = None
             self.is_generated = True
         else:
             # it's a source file (without a .deps file)
             self.stamp_mtime = 0  # no stamp file
             self.exitcode = 0
             self.deps = []
             self.is_generated = False
             self.stamp = self.read_stamp(st=st)
             self.runid = self.stamp.runid()
     else:
         # it's a target (with a .deps file)
         st = os.fstat(f.fileno())
         lines = f.read().strip().split('\n')
         version = None
         device  = None
         inode   = None
         try:
             version = lines.pop(0)
             device, inode = [int(i) for i in lines.pop(0).split(" ")]
         except: pass
         if version != DEPSFILE_TAG or device != st.st_dev or inode != st.st_ino:
             # It is an old .deps file, consider it missing
             self.stamp_mtime = 0  # no stamp file
             self.exitcode = 0
             self.deps = []
             self.stamp = Stamp(STAMP_OLD)
             self.runid = None
             self.is_generated = True
         else:
             # Read .deps file
             self.stamp_mtime = int(st.st_mtime)
             self.exitcode = int(lines.pop(-1))
             self.is_generated = True
             self.stamp = Stamp(lines.pop(-1))
             if not self.exists() and not self.stamp.is_missing():
                 #debug3('deleted %s (mark as missing)\n', self.name)
                 #self.stamp.stamp = STAMP_MISSING
                 debug3('deleted %s (delete .deps and rescan)\n', self.name)
                 unlink(self.tmpfilename('deps'))
                 return self.refresh()
             self.runid = self.stamp.runid()
             self.deps = [line.split(' ', 1) for line in lines]
             # if the next line fails, it means that the .dep file is not
             # correctly formatted
             while self.deps and self.deps[-1][1] == '.':
                 # a line added by redo-stamp
                 self.stamp.csum = self.deps.pop(-1)[0]
             for i in range(len(self.deps)):
                 self.deps[i][0] = Stamp(auto_detect=self.deps[i][0])