def _start_unlocked(self, dirty):
# out-of-band redo of some sub-objects. This happens when we're not
# quite sure if t needs to be built or not (because some children
# look dirty, but might turn out to be clean thanks to checksums).
# We have to call redo-unlocked to figure it all out.
#
# Note: redo-unlocked will handle all the updating of sf, so we
# don't have to do it here, nor call _after1. However, we have to
# hold onto the lock because otherwise we would introduce a race
# condition; that's why it's called redo-unlocked, because it doesn't
# grab a lock.
argv = ['redo-unlocked', self.sf.name] + [d.name for d in dirty]
log('(%s)\n' % _nice(self.t))
state.commit()
def run():
os.chdir(vars.BASE)
os.environ['REDO_DEPTH'] = vars.DEPTH + ' '
signal.signal(signal.SIGPIPE,
signal.SIG_DFL) # python ignores SIGPIPE
os.execvp(argv[0], argv)
assert (0)
# returns only if there's an exception
def after(t, rv):
return self._after2(rv)
jwack.start_job(self.t, run, after)
def _start_unlocked(self, dirty):
# out-of-band redo of some sub-objects. This happens when we're not
# quite sure if t needs to be built or not (because some children
# look dirty, but might turn out to be clean thanks to checksums).
# We have to call redo-unlocked to figure it all out.
#
# Note: redo-unlocked will handle all the updating of sf, so we
# don't have to do it here, nor call _after1. However, we have to
# hold onto the lock because otherwise we would introduce a race
# condition; that's why it's called redo-unlocked, because it doesn't
# grab a lock.
argv = ['redo-unlocked', self.sf.name] + [d.name for d in dirty]
log('(%s)\n' % _nice(self.t))
state.commit()
def run():
os.chdir(vars.BASE)
os.environ['REDO_DEPTH'] = vars.DEPTH + ' '
os.execvp(argv[0], argv)
assert (0)
# returns only if there's an exception
def after(t, rv):
return self._after2(rv)
jwack.start_job(self.t, run, after)
def schedule_job(self):
assert self.target.dolock().owned == state.LOCK_EX
rv = self.prepare()
if rv != None:
self.result[0] += rv
self.result[1] += 1
else:
jwack.start_job(self.target, self.build, self.done)
def main(targets, buildfunc):
retcode = [0] # a list so that it can be reassigned from done()
if vars.SHUFFLE:
random.shuffle(targets)
locked = []
def done(t, rv):
if rv:
err('%s: exit code was %r\n' % (t, rv))
retcode[0] = 1
for i in range(len(targets)):
t = targets[i]
if os.path.exists('%s/all.do' % t):
# t is a directory, but it has a default target
targets[i] = '%s/all' % t
for t in targets:
jwack.get_token(t)
lock = state.Lock(t)
lock.trylock()
if not lock.owned:
log('%s (locked...)\n' % relpath(t, vars.STARTDIR))
locked.append(t)
else:
jwack.start_job(t, lock,
lambda: buildfunc(t), lambda t,rv: done(t,rv))
while locked or jwack.running():
jwack.wait_all()
if locked:
t = locked.pop(0)
lock = state.Lock(t)
while not lock.owned:
lock.wait()
lock.trylock()
assert(lock.owned)
relp = relpath(t, vars.STARTDIR)
log('%s (...unlocked!)\n' % relp)
if state.stamped(t) == None:
err('%s: failed in another thread\n' % relp)
retcode[0] = 2
lock.unlock()
else:
jwack.start_job(t, lock,
lambda: buildfunc(t), lambda t,rv: done(t,rv))
return retcode[0]
def _start_oob(self, dirty):
# out-of-band redo of some sub-objects. This happens when we're not
# quite sure if t needs to be built or not (because some children look
# dirty, but might turn out to be clean thanks to checksums). We have
# to call redo-oob to figure it all out.
#
# Note: redo-oob will handle all the updating of sf, so we don't have
# to do it here, nor call _after1.
argv = ['redo-oob', self.sf.name] + [d.name for d in dirty]
log('(%s)\n' % _nice(self.t))
state.commit()
def run():
os.chdir(vars.BASE)
os.environ['REDO_DEPTH'] = vars.DEPTH + ' '
os.execvp(argv[0], argv)
assert(0)
# returns only if there's an exception
def after(t, rv):
return self._after2(rv)
jwack.start_job(self.t, run, after)
def _start_do(self):
assert(self.lock.owned)
t = self.t
sf = self.sf
newstamp = sf.read_stamp()
if (sf.is_generated and
not sf.failed_runid and
newstamp != state.STAMP_MISSING and
(sf.stamp != newstamp or sf.is_override)):
state.warn_override(_nice(t))
sf.set_override()
sf.set_checked()
sf.save()
return self._after2(0)
if (os.path.exists(t) and not os.path.exists(t + '/.')
and not sf.is_generated):
# an existing source file that was not generated by us.
# This step is mentioned by djb in his notes.
# For example, a rule called default.c.do could be used to try
# to produce hello.c, but we don't want that to happen if
# hello.c was created by the end user.
# FIXME: always refuse to redo any file that was modified outside
# of redo? That would make it easy for someone to override a
# file temporarily, and could be undone by deleting the file.
debug2("-- static (%r)\n" % t)
sf.set_static()
sf.save()
return self._after2(0)
sf.zap_deps1()
(dodir, dofile, basedir, basename, ext) = _find_do_file(sf)
if not dofile:
if os.path.exists(t):
sf.set_static()
sf.save()
return self._after2(0)
else:
err('no rule to make %r\n' % t)
return self._after2(1)
unlink(self.tmpname1)
unlink(self.tmpname2)
ffd = os.open(self.tmpname1, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666)
close_on_exec(ffd, True)
self.f = os.fdopen(ffd, 'w+')
# this will run in the dofile's directory, so use only basenames here
argv = ['sh', '-e',
dofile,
basename, # target name (no extension)
ext, # extension (if any), including leading dot
os.path.join(basedir, os.path.basename(self.tmpname2)) # temp output file name
]
if vars.VERBOSE: argv[1] += 'v'
if vars.XTRACE: argv[1] += 'x'
if vars.VERBOSE or vars.XTRACE: log_('\n')
firstline = open(os.path.join(dodir, dofile)).readline().strip()
if firstline.startswith('#!/'):
argv[0:2] = firstline[2:].split(' ')
log('%s\n' % _nice(t))
self.dodir = dodir
self.basename = basename
self.ext = ext
self.argv = argv
sf.is_generated = True
sf.save()
dof = state.File(name=os.path.join(dodir, dofile))
dof.set_static()
dof.save()
state.commit()
jwack.start_job(t, self._do_subproc, self._after)
def _start_do(self):
assert (self.lock.owned)
t = self.t
sf = self.sf
newstamp = sf.read_stamp()
if (sf.is_generated and newstamp != state.STAMP_MISSING
and (sf.stamp != newstamp or sf.is_override)):
state.warn_override(_nice(t))
if not sf.is_override:
warn('%s - old: %r\n' % (_nice(t), sf.stamp))
warn('%s - new: %r\n' % (_nice(t), newstamp))
sf.set_override()
sf.set_checked()
sf.save()
return self._after2(0)
if (os.path.exists(t) and not os.path.isdir(t + '/.')
and not sf.is_generated):
# an existing source file that was not generated by us.
# This step is mentioned by djb in his notes.
# For example, a rule called default.c.do could be used to try
# to produce hello.c, but we don't want that to happen if
# hello.c was created by the end user.
debug2("-- static (%r)\n" % t)
sf.set_static()
sf.save()
return self._after2(0)
sf.zap_deps1()
(dodir, dofile, basedir, basename, ext) = paths.find_do_file(sf)
if not dofile:
if os.path.exists(t):
sf.set_static()
sf.save()
return self._after2(0)
else:
err('no rule to make %r\n' % t)
return self._after2(1)
unlink(self.tmpname1)
unlink(self.tmpname2)
ffd = os.open(self.tmpname1, os.O_CREAT | os.O_RDWR | os.O_EXCL, 0666)
close_on_exec(ffd, True)
self.f = os.fdopen(ffd, 'w+')
# this will run in the dofile's directory, so use only basenames here
arg1 = basename + ext # target name (including extension)
arg2 = basename # target name (without extension)
argv = [
'sh',
'-e',
dofile,
arg1,
arg2,
# temp output file name
state.relpath(os.path.abspath(self.tmpname2), dodir),
]
if vars.VERBOSE: argv[1] += 'v'
if vars.XTRACE: argv[1] += 'x'
if vars.VERBOSE or vars.XTRACE: log_('\n')
firstline = open(os.path.join(dodir, dofile)).readline().strip()
if firstline.startswith('#!/'):
argv[0:2] = firstline[2:].split(' ')
log('%s\n' % _nice(t))
self.dodir = dodir
self.basename = basename
self.ext = ext
self.argv = argv
sf.is_generated = True
sf.save()
dof = state.File(name=os.path.join(dodir, dofile))
dof.set_static()
dof.save()
state.commit()
jwack.start_job(t, self._do_subproc, self._after)
