def build(f, any_errors, should_build, add_dep_to=None, delegate=None, re_do=True):
if f.dolock():
if f.check_deadlocks():
err("%s: recursive dependency, breaking deadlock\n", f.printable_name())
any_errors[0] += 1
any_errors[1] += 1
else:
jwack.get_token(f)
f.dolock().waitlock()
if any_errors[0] and not vars.KEEP_GOING:
return False
f.refresh()
debug3('think about building %r\n', f.name)
dirty = should_build(f)
while dirty and dirty != deps.DIRTY:
# FIXME: bring back the old (targetname) notation in the output
# when we need to do this. And add comments.
for t2 in dirty:
if not build(t2, any_errors, should_build, delegate, re_do):
return False
jwack.wait_all()
dirty = should_build(f)
if dirty:
job = BuildJob(f, any_errors, add_dep_to, delegate, re_do)
add_dep_to = None
job.schedule_job()
else:
f.dolock().unlock()
if add_dep_to:
f.refresh()
add_dep_to.add_dep(f)
return True
def main(targets, should_build=(lambda f: deps.DIRTY), parent=None, delegate=None, re_do=True):
any_errors = [0, 0]
if vars.SHUFFLE:
import random
random.shuffle(targets)
if delegate:
debug("delegated: %s\n", delegate)
try:
for t in targets:
f = state.File(name=t)
if not build(f, any_errors, should_build, add_dep_to=parent, delegate=delegate, re_do=re_do):
break
jwack.wait_all()
finally:
jwack.force_return_tokens()
if any_errors[1] == 1:
return any_errors[0]
elif any_errors[0]:
return 1
else:
return 0
def main(targets, buildfunc):
retcode = [0] # a list so that it can be reassigned from done()
if vars.SHUFFLE:
random.shuffle(targets)
locked = []
def done(t, rv):
if rv:
err('%s: exit code was %r\n' % (t, rv))
retcode[0] = 1
for i in range(len(targets)):
t = targets[i]
if os.path.exists('%s/all.do' % t):
# t is a directory, but it has a default target
targets[i] = '%s/all' % t
for t in targets:
jwack.get_token(t)
lock = state.Lock(t)
lock.trylock()
if not lock.owned:
log('%s (locked...)\n' % relpath(t, vars.STARTDIR))
locked.append(t)
else:
jwack.start_job(t, lock,
lambda: buildfunc(t), lambda t,rv: done(t,rv))
while locked or jwack.running():
jwack.wait_all()
if locked:
t = locked.pop(0)
lock = state.Lock(t)
while not lock.owned:
lock.wait()
lock.trylock()
assert(lock.owned)
relp = relpath(t, vars.STARTDIR)
log('%s (...unlocked!)\n' % relp)
if state.stamped(t) == None:
err('%s: failed in another thread\n' % relp)
retcode[0] = 2
lock.unlock()
else:
jwack.start_job(t, lock,
lambda: buildfunc(t), lambda t,rv: done(t,rv))
return retcode[0]
def main(targets, shouldbuildfunc):
retcode = [0] # a list so that it can be reassigned from done()
if vars.SHUFFLE:
import random
random.shuffle(targets)
locked = []
def done(t, rv):
if rv:
retcode[0] = 1
# In the first cycle, we just build as much as we can without worrying
# about any lock contention. If someone else has it locked, we move on.
seen = {}
for t in targets:
if t in seen:
continue
seen[t] = 1
if not jwack.has_token():
state.commit()
jwack.get_token(t)
if retcode[0] and not vars.KEEP_GOING:
break
if not state.check_sane():
err('.redo directory disappeared; cannot continue.\n')
retcode[0] = 205
break
f = state.File(name=t)
lock = state.Lock(f.id)
if vars.UNLOCKED:
lock.owned = True
else:
lock.trylock()
if not lock.owned:
if vars.DEBUG_LOCKS:
log('%s (locked...)\n' % _nice(t))
locked.append((f.id,t))
else:
BuildJob(t, f, lock, shouldbuildfunc, done).start()
del lock
# Now we've built all the "easy" ones. Go back and just wait on the
# remaining ones one by one. There's no reason to do it any more
# efficiently, because if these targets were previously locked, that
# means someone else was building them; thus, we probably won't need to
# do anything. The only exception is if we're invoked as redo instead
# of redo-ifchange; then we have to redo it even if someone else already
# did. But that should be rare.
while locked or jwack.running():
state.commit()
jwack.wait_all()
# at this point, we don't have any children holding any tokens, so
# it's okay to block below.
if retcode[0] and not vars.KEEP_GOING:
break
if locked:
if not state.check_sane():
err('.redo directory disappeared; cannot continue.\n')
retcode[0] = 205
break
fid,t = locked.pop(0)
lock = state.Lock(fid)
lock.trylock()
while not lock.owned:
if vars.DEBUG_LOCKS:
warn('%s (WAITING)\n' % _nice(t))
# this sequence looks a little silly, but the idea is to
# give up our personal token while we wait for the lock to
# be released; but we should never run get_token() while
# holding a lock, or we could cause deadlocks.
jwack.release_mine()
lock.waitlock()
lock.unlock()
jwack.get_token(t)
lock.trylock()
assert(lock.owned)
if vars.DEBUG_LOCKS:
log('%s (...unlocked!)\n' % _nice(t))
if state.File(name=t).is_failed():
err('%s: failed in another thread\n' % _nice(t))
retcode[0] = 2
lock.unlock()
else:
BuildJob(t, state.File(id=fid), lock,
shouldbuildfunc, done).start()
state.commit()
return retcode[0]
def main(targets, shouldbuildfunc):
retcode = [0] # a list so that it can be reassigned from done()
if vars.SHUFFLE:
import random
random.shuffle(targets)
locked = []
def done(t, rv):
if rv:
retcode[0] = 1
# In the first cycle, we just build as much as we can without worrying
# about any lock contention. If someone else has it locked, we move on.
seen = {}
lock = None
for t in targets:
if not t:
err('cannot build the empty target ("").\n')
retcode[0] = 204
break
assert (state.is_flushed())
if t in seen:
continue
seen[t] = 1
if not jwack.has_token():
state.commit()
jwack.get_token(t)
if retcode[0] and not vars.KEEP_GOING:
break
if not state.check_sane():
err('.redo directory disappeared; cannot continue.\n')
retcode[0] = 205
break
f = state.File(name=t)
lock = state.Lock(f.id)
if vars.UNLOCKED:
lock.owned = True
else:
lock.trylock()
if not lock.owned:
if vars.DEBUG_LOCKS:
log('%s (locked...)\n' % _nice(t))
locked.append((f.id, t))
else:
# We had to create f before we had a lock, because we need f.id
# to make the lock. But someone may have updated the state
# between then and now.
# FIXME: separate obtaining the fid from creating the File.
# FIXME: maybe integrate locking into the File object?
f.refresh()
BuildJob(t, f, lock, shouldbuildfunc, done).start()
state.commit()
assert (state.is_flushed())
lock = None
del lock
# Now we've built all the "easy" ones. Go back and just wait on the
# remaining ones one by one. There's no reason to do it any more
# efficiently, because if these targets were previously locked, that
# means someone else was building them; thus, we probably won't need to
# do anything. The only exception is if we're invoked as redo instead
# of redo-ifchange; then we have to redo it even if someone else already
# did. But that should be rare.
while locked or jwack.running():
state.commit()
jwack.wait_all()
# at this point, we don't have any children holding any tokens, so
# it's okay to block below.
if retcode[0] and not vars.KEEP_GOING:
break
if locked:
if not state.check_sane():
err('.redo directory disappeared; cannot continue.\n')
retcode[0] = 205
break
fid, t = locked.pop(0)
lock = state.Lock(fid)
backoff = 0.01
lock.trylock()
while not lock.owned:
# Don't spin with 100% CPU while we fight for the lock.
import random
time.sleep(random.random() * min(backoff, 1.0))
backoff *= 2
if vars.DEBUG_LOCKS:
warn('%s (WAITING)\n' % _nice(t))
# this sequence looks a little silly, but the idea is to
# give up our personal token while we wait for the lock to
# be released; but we should never run get_token() while
# holding a lock, or we could cause deadlocks.
jwack.release_mine()
try:
lock.waitlock()
except state.CyclicDependencyError:
err('cyclic dependency while building %s\n' % _nice(t))
jwack.get_token(t)
retcode[0] = 208
return retcode[0]
lock.unlock()
jwack.get_token(t)
lock.trylock()
assert (lock.owned)
if vars.DEBUG_LOCKS:
log('%s (...unlocked!)\n' % _nice(t))
if state.File(name=t).is_failed():
err('%s: failed in another thread\n' % _nice(t))
retcode[0] = 2
lock.unlock()
else:
BuildJob(t, state.File(id=fid), lock, shouldbuildfunc,
done).start()
lock = None
state.commit()
return retcode[0]