Example #1
0
def build(f, any_errors, should_build, add_dep_to=None, delegate=None, re_do=True):
    if f.dolock():
        if f.check_deadlocks():
            err("%s: recursive dependency, breaking deadlock\n", f.printable_name())
            any_errors[0] += 1
            any_errors[1] += 1
        else:
            jwack.get_token(f)
            f.dolock().waitlock()
            if any_errors[0] and not vars.KEEP_GOING:
                return False
            f.refresh()
            debug3('think about building %r\n', f.name)
            dirty = should_build(f)
            while dirty and dirty != deps.DIRTY:
                # FIXME: bring back the old (targetname) notation in the output
                #  when we need to do this.  And add comments.
                for t2 in dirty:
                    if not build(t2, any_errors, should_build, delegate, re_do):
                        return False
                jwack.wait_all()
                dirty = should_build(f)
            if dirty:
                job = BuildJob(f, any_errors, add_dep_to, delegate, re_do)
                add_dep_to = None
                job.schedule_job()
            else:
                f.dolock().unlock()
    if add_dep_to:
        f.refresh()
        add_dep_to.add_dep(f)
    return True
Example #2
0
def main(targets, should_build=(lambda f: deps.DIRTY), parent=None, delegate=None, re_do=True):
    any_errors = [0, 0]
    if vars.SHUFFLE:
        import random

        random.shuffle(targets)

    if delegate:
        debug("delegated: %s\n", delegate)

    try:
        for t in targets:
            f = state.File(name=t)
            if not build(f, any_errors, should_build, add_dep_to=parent, delegate=delegate, re_do=re_do):
                break
        jwack.wait_all()
    finally:
        jwack.force_return_tokens()

    if any_errors[1] == 1:
        return any_errors[0]
    elif any_errors[0]:
        return 1
    else:
        return 0
Example #3
0
def main(targets, buildfunc):
    retcode = [0]  # a list so that it can be reassigned from done()
    if vars.SHUFFLE:
        random.shuffle(targets)

    locked = []

    def done(t, rv):
        if rv:
            err('%s: exit code was %r\n' % (t, rv))
            retcode[0] = 1

    for i in range(len(targets)):
        t = targets[i]
        if os.path.exists('%s/all.do' % t):
            # t is a directory, but it has a default target
            targets[i] = '%s/all' % t
    
    for t in targets:
        jwack.get_token(t)
        lock = state.Lock(t)
        lock.trylock()
        if not lock.owned:
            log('%s (locked...)\n' % relpath(t, vars.STARTDIR))
            locked.append(t)
        else:
            jwack.start_job(t, lock,
                            lambda: buildfunc(t), lambda t,rv: done(t,rv))
    
    while locked or jwack.running():
        jwack.wait_all()
        if locked:
            t = locked.pop(0)
            lock = state.Lock(t)
            while not lock.owned:
                lock.wait()
                lock.trylock()
            assert(lock.owned)
            relp = relpath(t, vars.STARTDIR)
            log('%s (...unlocked!)\n' % relp)
            if state.stamped(t) == None:
                err('%s: failed in another thread\n' % relp)
                retcode[0] = 2
                lock.unlock()
            else:
                jwack.start_job(t, lock, 
                                lambda: buildfunc(t), lambda t,rv: done(t,rv))
    return retcode[0]
Example #4
0
def main(targets, shouldbuildfunc):
    retcode = [0]  # a list so that it can be reassigned from done()
    if vars.SHUFFLE:
        import random
        random.shuffle(targets)

    locked = []

    def done(t, rv):
        if rv:
            retcode[0] = 1

    # In the first cycle, we just build as much as we can without worrying
    # about any lock contention.  If someone else has it locked, we move on.
    seen = {}
    for t in targets:
        if t in seen:
            continue
        seen[t] = 1
        if not jwack.has_token():
            state.commit()
        jwack.get_token(t)
        if retcode[0] and not vars.KEEP_GOING:
            break
        if not state.check_sane():
            err('.redo directory disappeared; cannot continue.\n')
            retcode[0] = 205
            break
        f = state.File(name=t)
        lock = state.Lock(f.id)
        if vars.UNLOCKED:
            lock.owned = True
        else:
            lock.trylock()
        if not lock.owned:
            if vars.DEBUG_LOCKS:
                log('%s (locked...)\n' % _nice(t))
            locked.append((f.id,t))
        else:
            BuildJob(t, f, lock, shouldbuildfunc, done).start()

    del lock

    # Now we've built all the "easy" ones.  Go back and just wait on the
    # remaining ones one by one.  There's no reason to do it any more
    # efficiently, because if these targets were previously locked, that
    # means someone else was building them; thus, we probably won't need to
    # do anything.  The only exception is if we're invoked as redo instead
    # of redo-ifchange; then we have to redo it even if someone else already
    # did.  But that should be rare.
    while locked or jwack.running():
        state.commit()
        jwack.wait_all()
        # at this point, we don't have any children holding any tokens, so
        # it's okay to block below.
        if retcode[0] and not vars.KEEP_GOING:
            break
        if locked:
            if not state.check_sane():
                err('.redo directory disappeared; cannot continue.\n')
                retcode[0] = 205
                break
            fid,t = locked.pop(0)
            lock = state.Lock(fid)
            lock.trylock()
            while not lock.owned:
                if vars.DEBUG_LOCKS:
                    warn('%s (WAITING)\n' % _nice(t))
                # this sequence looks a little silly, but the idea is to
                # give up our personal token while we wait for the lock to
                # be released; but we should never run get_token() while
                # holding a lock, or we could cause deadlocks.
                jwack.release_mine()
                lock.waitlock()
                lock.unlock()
                jwack.get_token(t)
                lock.trylock()
            assert(lock.owned)
            if vars.DEBUG_LOCKS:
                log('%s (...unlocked!)\n' % _nice(t))
            if state.File(name=t).is_failed():
                err('%s: failed in another thread\n' % _nice(t))
                retcode[0] = 2
                lock.unlock()
            else:
                BuildJob(t, state.File(id=fid), lock,
                         shouldbuildfunc, done).start()
    state.commit()
    return retcode[0]
Example #5
0
def main(targets, shouldbuildfunc):
    retcode = [0]  # a list so that it can be reassigned from done()
    if vars.SHUFFLE:
        import random
        random.shuffle(targets)

    locked = []

    def done(t, rv):
        if rv:
            retcode[0] = 1

    # In the first cycle, we just build as much as we can without worrying
    # about any lock contention.  If someone else has it locked, we move on.
    seen = {}
    lock = None
    for t in targets:
        if not t:
            err('cannot build the empty target ("").\n')
            retcode[0] = 204
            break
        assert (state.is_flushed())
        if t in seen:
            continue
        seen[t] = 1
        if not jwack.has_token():
            state.commit()
        jwack.get_token(t)
        if retcode[0] and not vars.KEEP_GOING:
            break
        if not state.check_sane():
            err('.redo directory disappeared; cannot continue.\n')
            retcode[0] = 205
            break
        f = state.File(name=t)
        lock = state.Lock(f.id)
        if vars.UNLOCKED:
            lock.owned = True
        else:
            lock.trylock()
        if not lock.owned:
            if vars.DEBUG_LOCKS:
                log('%s (locked...)\n' % _nice(t))
            locked.append((f.id, t))
        else:
            # We had to create f before we had a lock, because we need f.id
            # to make the lock.  But someone may have updated the state
            # between then and now.
            # FIXME: separate obtaining the fid from creating the File.
            # FIXME: maybe integrate locking into the File object?
            f.refresh()
            BuildJob(t, f, lock, shouldbuildfunc, done).start()
        state.commit()
        assert (state.is_flushed())
        lock = None

    del lock

    # Now we've built all the "easy" ones.  Go back and just wait on the
    # remaining ones one by one.  There's no reason to do it any more
    # efficiently, because if these targets were previously locked, that
    # means someone else was building them; thus, we probably won't need to
    # do anything.  The only exception is if we're invoked as redo instead
    # of redo-ifchange; then we have to redo it even if someone else already
    # did.  But that should be rare.
    while locked or jwack.running():
        state.commit()
        jwack.wait_all()
        # at this point, we don't have any children holding any tokens, so
        # it's okay to block below.
        if retcode[0] and not vars.KEEP_GOING:
            break
        if locked:
            if not state.check_sane():
                err('.redo directory disappeared; cannot continue.\n')
                retcode[0] = 205
                break
            fid, t = locked.pop(0)
            lock = state.Lock(fid)
            backoff = 0.01
            lock.trylock()
            while not lock.owned:
                # Don't spin with 100% CPU while we fight for the lock.
                import random
                time.sleep(random.random() * min(backoff, 1.0))
                backoff *= 2
                if vars.DEBUG_LOCKS:
                    warn('%s (WAITING)\n' % _nice(t))
                # this sequence looks a little silly, but the idea is to
                # give up our personal token while we wait for the lock to
                # be released; but we should never run get_token() while
                # holding a lock, or we could cause deadlocks.
                jwack.release_mine()
                try:
                    lock.waitlock()
                except state.CyclicDependencyError:
                    err('cyclic dependency while building %s\n' % _nice(t))
                    jwack.get_token(t)
                    retcode[0] = 208
                    return retcode[0]
                lock.unlock()
                jwack.get_token(t)
                lock.trylock()
            assert (lock.owned)
            if vars.DEBUG_LOCKS:
                log('%s (...unlocked!)\n' % _nice(t))
            if state.File(name=t).is_failed():
                err('%s: failed in another thread\n' % _nice(t))
                retcode[0] = 2
                lock.unlock()
            else:
                BuildJob(t, state.File(id=fid), lock, shouldbuildfunc,
                         done).start()
            lock = None
    state.commit()
    return retcode[0]