def should_build(t):
f = state.File(name=t)
if f.is_failed():
raise builder.ImmediateReturn(32)
dirty = deps.isdirty(f, depth = '', max_changed = vars.RUNID,
already_checked=[])
return f.is_generated, dirty==[f] and deps.DIRTY or dirty
def should_build(t):
f = state.File(name=t)
if f.is_failed():
raise builder.ImmediateReturn(32)
status = deps.isdirty(f, depth='', max_changed=vars.RUNID)
if status == [f]:
return deps.DIRTY
else:
# FIXME: This is really confusing!
# The status can be any one of deps.CLEAN or deps.DIRTY or
# a list of targets to build with redo-unlocked.
return status
os.environ['REDO_COLOR'] = '0'
import vars_init
vars_init.init(targets)
import vars, state, builder, jwack
from logs import warn, err
try:
if vars_init.is_toplevel:
builder.start_stdin_log_reader(status=opt.status, details=opt.details,
pretty=opt.pretty, color=opt.color,
debug_locks=opt.debug_locks, debug_pids=opt.debug_pids)
for t in targets:
if os.path.exists(t):
f = state.File(name=t)
if not f.is_generated:
warn('%s: exists and not marked as generated; not redoing.\n'
% f.nicename())
state.rollback()
j = atoi(opt.jobs or 1)
if j < 1 or j > 1000:
err('invalid --jobs value: %r\n' % opt.jobs)
jwack.setup(j)
try:
assert(state.is_flushed())
retcode = builder.main(targets, lambda t: (True, True))
assert(state.is_flushed())
finally:
try:
#!/usr/bin/env python
import sys, os
import state
from log import err
if len(sys.argv[1:]) < 2:
err('%s: at least 2 arguments expected.\n' % sys.argv[0])
sys.exit(1)
target = sys.argv[1]
deps = sys.argv[2:]
for d in deps:
assert(d != target)
me = state.File(name=target)
# Build the known dependencies of our primary target. This *does* require
# grabbing locks.
os.environ['REDO_NO_OOB'] = '1'
argv = ['redo-ifchange'] + deps
rv = os.spawnvp(os.P_WAIT, argv[0], argv)
if rv:
sys.exit(rv)
# We know our caller already owns the lock on target, so we don't have to
# acquire another one; tell redo-ifchange about that. Also, REDO_NO_OOB
# persists from up above, because we don't want to do OOB now either.
# (Actually it's most important for the primary target, since it's the one
# who initiated the OOB in the first place.)
os.environ['REDO_UNLOCKED'] = '1'
from log import debug, debug2, err
def should_build(t):
f = state.File(name=t)
if f.is_failed():
raise builder.ImmediateReturn(32)
dirty = deps.isdirty(f, depth = '', max_changed = vars.RUNID)
return dirty==[f] and deps.DIRTY or dirty
rv = 202
try:
if vars.TARGET and not vars.UNLOCKED:
me = os.path.join(vars.STARTDIR,
os.path.join(vars.PWD, vars.TARGET))
f = state.File(name=me)
debug2('TARGET: %r %r %r\n' % (vars.STARTDIR, vars.PWD, vars.TARGET))
else:
f = me = None
debug2('redo-ifchange: not adding depends.\n')
try:
targets = sys.argv[1:]
if f:
for t in targets:
f.add_dep('m', t)
f.save()
rv = builder.main(targets, should_build)
finally:
jwack.force_return_tokens()
except KeyboardInterrupt:
sys.exit(200)
def _start_do(self):
assert (self.lock.owned)
t = self.t
sf = self.sf
newstamp = sf.read_stamp()
if (sf.is_generated and newstamp != state.STAMP_MISSING
and (sf.stamp != newstamp or sf.is_override)):
state.warn_override(_nice(t))
if not sf.is_override:
warn('%s - old: %r\n' % (_nice(t), sf.stamp))
warn('%s - new: %r\n' % (_nice(t), newstamp))
sf.set_override()
sf.set_checked()
sf.save()
return self._after2(0)
if (os.path.exists(t) and not os.path.isdir(t + '/.')
and not sf.is_generated):
# an existing source file that was not generated by us.
# This step is mentioned by djb in his notes.
# For example, a rule called default.c.do could be used to try
# to produce hello.c, but we don't want that to happen if
# hello.c was created by the end user.
debug2("-- static (%r)\n" % t)
sf.set_static()
sf.save()
return self._after2(0)
sf.zap_deps1()
(dodir, dofile, basedir, basename, ext) = paths.find_do_file(sf)
if not dofile:
if os.path.exists(t):
sf.set_static()
sf.save()
return self._after2(0)
else:
err('no rule to make %r\n' % t)
return self._after2(1)
unlink(self.tmpname1)
unlink(self.tmpname2)
ffd = os.open(self.tmpname1, os.O_CREAT | os.O_RDWR | os.O_EXCL, 0666)
close_on_exec(ffd, True)
self.f = os.fdopen(ffd, 'w+')
# this will run in the dofile's directory, so use only basenames here
arg1 = basename + ext # target name (including extension)
arg2 = basename # target name (without extension)
argv = [
'sh',
'-e',
dofile,
arg1,
arg2,
# temp output file name
state.relpath(os.path.abspath(self.tmpname2), dodir),
]
if vars.VERBOSE: argv[1] += 'v'
if vars.XTRACE: argv[1] += 'x'
if vars.VERBOSE or vars.XTRACE: log_('\n')
firstline = open(os.path.join(dodir, dofile)).readline().strip()
if firstline.startswith('#!/'):
argv[0:2] = firstline[2:].split(' ')
log('%s\n' % _nice(t))
self.dodir = dodir
self.basename = basename
self.ext = ext
self.argv = argv
sf.is_generated = True
sf.save()
dof = state.File(name=os.path.join(dodir, dofile))
dof.set_static()
dof.save()
state.commit()
jwack.start_job(t, self._do_subproc, self._after)
def main(targets, shouldbuildfunc):
retcode = [0] # a list so that it can be reassigned from done()
if vars.SHUFFLE:
import random
random.shuffle(targets)
locked = []
def done(t, rv):
if rv:
retcode[0] = 1
# In the first cycle, we just build as much as we can without worrying
# about any lock contention. If someone else has it locked, we move on.
seen = {}
lock = None
for t in targets:
if not t:
err('cannot build the empty target ("").\n')
retcode[0] = 204
break
assert (state.is_flushed())
if t in seen:
continue
seen[t] = 1
if not jwack.has_token():
state.commit()
jwack.get_token(t)
if retcode[0] and not vars.KEEP_GOING:
break
if not state.check_sane():
err('.redo directory disappeared; cannot continue.\n')
retcode[0] = 205
break
f = state.File(name=t)
lock = state.Lock(f.id)
if vars.UNLOCKED:
lock.owned = True
else:
lock.trylock()
if not lock.owned:
if vars.DEBUG_LOCKS:
log('%s (locked...)\n' % _nice(t))
locked.append((f.id, t))
else:
# We had to create f before we had a lock, because we need f.id
# to make the lock. But someone may have updated the state
# between then and now.
# FIXME: separate obtaining the fid from creating the File.
# FIXME: maybe integrate locking into the File object?
f.refresh()
BuildJob(t, f, lock, shouldbuildfunc, done).start()
state.commit()
assert (state.is_flushed())
lock = None
del lock
# Now we've built all the "easy" ones. Go back and just wait on the
# remaining ones one by one. There's no reason to do it any more
# efficiently, because if these targets were previously locked, that
# means someone else was building them; thus, we probably won't need to
# do anything. The only exception is if we're invoked as redo instead
# of redo-ifchange; then we have to redo it even if someone else already
# did. But that should be rare.
while locked or jwack.running():
state.commit()
jwack.wait_all()
# at this point, we don't have any children holding any tokens, so
# it's okay to block below.
if retcode[0] and not vars.KEEP_GOING:
break
if locked:
if not state.check_sane():
err('.redo directory disappeared; cannot continue.\n')
retcode[0] = 205
break
fid, t = locked.pop(0)
lock = state.Lock(fid)
backoff = 0.01
lock.trylock()
while not lock.owned:
# Don't spin with 100% CPU while we fight for the lock.
import random
time.sleep(random.random() * min(backoff, 1.0))
backoff *= 2
if vars.DEBUG_LOCKS:
warn('%s (WAITING)\n' % _nice(t))
# this sequence looks a little silly, but the idea is to
# give up our personal token while we wait for the lock to
# be released; but we should never run get_token() while
# holding a lock, or we could cause deadlocks.
jwack.release_mine()
try:
lock.waitlock()
except state.CyclicDependencyError:
err('cyclic dependency while building %s\n' % _nice(t))
jwack.get_token(t)
retcode[0] = 208
return retcode[0]
lock.unlock()
jwack.get_token(t)
lock.trylock()
assert (lock.owned)
if vars.DEBUG_LOCKS:
log('%s (...unlocked!)\n' % _nice(t))
if state.File(name=t).is_failed():
err('%s: failed in another thread\n' % _nice(t))
retcode[0] = 2
lock.unlock()
else:
BuildJob(t, state.File(id=fid), lock, shouldbuildfunc,
done).start()
lock = None
state.commit()
return retcode[0]
def main(targets, shouldbuildfunc):
retcode = [0] # a list so that it can be reassigned from done()
if vars.SHUFFLE:
import random
random.shuffle(targets)
locked = []
def done(t, rv):
if rv:
retcode[0] = 1
# In the first cycle, we just build as much as we can without worrying
# about any lock contention. If someone else has it locked, we move on.
seen = {}
lock = None
for t in targets:
if t in seen:
continue
seen[t] = 1
if not jobs.has_token():
state.commit()
jobs.get_token(t)
if retcode[0] and not vars.KEEP_GOING:
break
if not state.check_sane():
err('.redo directory disappeared; cannot continue.\n')
retcode[0] = 205
break
f = state.File(name=t)
lock = state.Lock(f.id)
if vars.UNLOCKED:
lock.owned = True
else:
lock.trylock()
if not lock.owned:
if vars.DEBUG_LOCKS:
log('%s (locked...)\n' % _nice(t))
locked.append((f.id, t))
else:
BuildJob(t, f, lock, shouldbuildfunc, done).start()
del lock
# Now we've built all the "easy" ones. Go back and just wait on the
# remaining ones one by one. There's no reason to do it any more
# efficiently, because if these targets were previously locked, that
# means someone else was building them; thus, we probably won't need to
# do anything. The only exception is if we're invoked as redo instead
# of redo-ifchange; then we have to redo it even if someone else already
# did. But that should be rare.
while locked or jobs.running():
state.commit()
jobs.wait_all()
# at this point, we don't have any children holding any tokens, so
# it's okay to block below.
if retcode[0] and not vars.KEEP_GOING:
break
if locked:
if not state.check_sane():
err('.redo directory disappeared; cannot continue.\n')
retcode[0] = 205
break
fid, t = locked.pop(0)
target_list = targets_seen.get()
nice_t = _nice(t)
if nice_t in target_list:
# Target locked by parent: cyclic dependence
err('encountered a dependence cycle:\n')
_print_cycle(target_list, nice_t)
retcode[0] = 209
break
lock = state.Lock(fid)
lock.trylock()
while not lock.owned:
if vars.DEBUG_LOCKS:
warn('%s (WAITING)\n' % _nice(t))
# this sequence looks a little silly, but the idea is to
# give up our personal token while we wait for the lock to
# be released; but we should never run get_token() while
# holding a lock, or we could cause deadlocks.
jobs.put_token()
lock.waitlock()
lock.unlock()
jobs.get_token(t)
lock.trylock()
assert (lock.owned)
if vars.DEBUG_LOCKS:
log('%s (...unlocked!)\n' % _nice(t))
if state.File(name=t).is_failed():
err('%s: failed in another thread\n' % _nice(t))
retcode[0] = 2
lock.unlock()
else:
BuildJob(t, state.File(id=fid), lock, shouldbuildfunc,
done).start()
state.commit()
return retcode[0]
def catlog(t):
global total_lines, status
if t in already:
return
if t != '-':
depth.append(t)
_fix_depth()
already.add(t)
mydir = os.path.dirname(t)
if t == '-':
f = sys.stdin
fid = None
loglock = None
logname = None
else:
try:
sf = state.File(name=t, allow_add=False)
except KeyError:
sys.stderr.write('redo-log: [%s] %r: not known to redo.\n' % (
os.getcwd(),
t,
))
sys.exit(24)
fid = sf.id
del sf
state.rollback()
logname = state.logname(fid)
loglock = state.Lock(fid + state.LOG_LOCK_MAGIC)
loglock.waitlock(shared=True)
f = None
delay = 0.01
was_locked = is_locked(fid)
line_head = ''
width = _tty_width()
while 1:
if not f:
try:
f = open(logname)
except IOError, e:
if e.errno == errno.ENOENT:
# ignore files without logs
pass
else:
raise
if f:
# Note: normally includes trailing \n.
# In 'follow' mode, might get a line with no trailing \n
# (eg. when ./configure is halfway through a test), which we
# deal with below.
line = f.readline()
else:
line = None
if not line and (not opt.follow or not was_locked):
# file not locked, and no new lines: done
break
if not line:
was_locked = is_locked(fid)
if opt.follow:
# Don't display status line for extremely short-lived runs
if opt.status and time.time() - start_time > 1.0:
width = _tty_width()
head = 'redo %s ' % ('{:,}'.format(total_lines))
tail = ''
for n in reversed(depth):
remain = width - len(head) - len(tail)
# always leave room for a final '... ' prefix
if remain < len(n) + 4 + 1 or remain <= 4:
if len(n) < 6 or remain < 6 + 1 + 4:
tail = '... %s' % tail
else:
start = len(n) - (remain - 3 - 1)
tail = '...%s %s' % (n[start:], tail)
break
elif n != '-':
tail = n + ' ' + tail
status = head + tail
if len(status) > width:
sys.stderr.write('\nOVERSIZE STATUS (%d):\n%r\n' %
(len(status), status))
assert (len(status) <= width)
sys.stdout.flush()
sys.stderr.write('\r%-*.*s\r' % (width, width, status))
time.sleep(min(delay, 1.0))
delay += 0.01
continue
total_lines += 1
delay = 0.01
if not line.endswith('\n'):
line_head += line
continue
if line_head:
line = line_head + line
line_head = ''
if status:
sys.stdout.flush()
sys.stderr.write('\r%-*.*s\r' % (width, width, ''))
status = None
g = re.match(REDO_LINE_RE, line)
if g:
# FIXME: print prefix if @@REDO is not at start of line.
# logs.PrettyLog does it, but only if we actually call .write().
words, text = g.groups()
kind, pid, when = words.split(':')[0:3]
pid = atoi(pid)
relname = _rel(topdir, mydir, text)
fixname = os.path.normpath(os.path.join(mydir, text))
if kind == 'unchanged':
if opt.unchanged:
if opt.debug_locks:
logs.meta(kind, relname, pid=pid)
elif fixname not in already:
logs.meta('do', relname, pid=pid)
if opt.recursive:
if loglock: loglock.unlock()
catlog(os.path.join(mydir, text))
if loglock: loglock.waitlock(shared=True)
already.add(fixname)
elif kind in ('do', 'waiting', 'locked', 'unlocked'):
if opt.debug_locks:
logs.meta(kind, relname, pid=pid)
logs.write(line.rstrip())
elif fixname not in already:
logs.meta('do', relname, pid=pid)
if opt.recursive:
assert text
if loglock: loglock.unlock()
catlog(os.path.join(mydir, text))
if loglock: loglock.waitlock(shared=True)
already.add(fixname)
elif kind == 'done':
rv, name = text.split(' ', 1)
logs.meta(kind, rv + ' ' + _rel(topdir, mydir, name))
else:
logs.write(line.rstrip())
else:
if opt.details:
logs.write(line.rstrip())
#!/usr/bin/env python2
import sys, os
import vars, state
from log import err
try:
me = os.path.join(vars.STARTDIR,
os.path.join(vars.PWD, vars.TARGET))
f = state.File(name=me)
f.add_dep('m', state.ALWAYS)
always = state.File(name=state.ALWAYS)
always.stamp = state.STAMP_MISSING
always.set_changed()
always.save()
state.commit()
except KeyboardInterrupt:
sys.exit(200)