def _start_unlocked(self, dirty): # out-of-band redo of some sub-objects. This happens when we're not # quite sure if t needs to be built or not (because some children # look dirty, but might turn out to be clean thanks to checksums). # We have to call redo-unlocked to figure it all out. # # Note: redo-unlocked will handle all the updating of sf, so we # don't have to do it here, nor call _after1. However, we have to # hold onto the lock because otherwise we would introduce a race # condition; that's why it's called redo-unlocked, because it doesn't # grab a lock. argv = ['redo-unlocked', self.sf.name] + [d.name for d in dirty] log('(%s)\n' % _nice(self.t)) state.commit() def run(): os.chdir(vars.BASE) os.environ['REDO_DEPTH'] = vars.DEPTH + ' ' signal.signal(signal.SIGPIPE, signal.SIG_DFL) # python ignores SIGPIPE os.execvp(argv[0], argv) assert (0) # returns only if there's an exception def after(t, rv): return self._after2(rv) jwack.start_job(self.t, run, after)
def _start_unlocked(self, dirty): # out-of-band redo of some sub-objects. This happens when we're not # quite sure if t needs to be built or not (because some children # look dirty, but might turn out to be clean thanks to checksums). # We have to call redo-unlocked to figure it all out. # # Note: redo-unlocked will handle all the updating of sf, so we # don't have to do it here, nor call _after1. However, we have to # hold onto the lock because otherwise we would introduce a race # condition; that's why it's called redo-unlocked, because it doesn't # grab a lock. argv = ['redo-unlocked', self.sf.name] + [d.name for d in dirty] log('(%s)\n' % _nice(self.t)) state.commit() def run(): os.chdir(vars.BASE) os.environ['REDO_DEPTH'] = vars.DEPTH + ' ' os.execvp(argv[0], argv) assert (0) # returns only if there's an exception def after(t, rv): return self._after2(rv) jwack.start_job(self.t, run, after)
def schedule_job(self): assert self.target.dolock().owned == state.LOCK_EX rv = self.prepare() if rv != None: self.result[0] += rv self.result[1] += 1 else: jwack.start_job(self.target, self.build, self.done)
def main(targets, buildfunc): retcode = [0] # a list so that it can be reassigned from done() if vars.SHUFFLE: random.shuffle(targets) locked = [] def done(t, rv): if rv: err('%s: exit code was %r\n' % (t, rv)) retcode[0] = 1 for i in range(len(targets)): t = targets[i] if os.path.exists('%s/all.do' % t): # t is a directory, but it has a default target targets[i] = '%s/all' % t for t in targets: jwack.get_token(t) lock = state.Lock(t) lock.trylock() if not lock.owned: log('%s (locked...)\n' % relpath(t, vars.STARTDIR)) locked.append(t) else: jwack.start_job(t, lock, lambda: buildfunc(t), lambda t,rv: done(t,rv)) while locked or jwack.running(): jwack.wait_all() if locked: t = locked.pop(0) lock = state.Lock(t) while not lock.owned: lock.wait() lock.trylock() assert(lock.owned) relp = relpath(t, vars.STARTDIR) log('%s (...unlocked!)\n' % relp) if state.stamped(t) == None: err('%s: failed in another thread\n' % relp) retcode[0] = 2 lock.unlock() else: jwack.start_job(t, lock, lambda: buildfunc(t), lambda t,rv: done(t,rv)) return retcode[0]
def _start_oob(self, dirty): # out-of-band redo of some sub-objects. This happens when we're not # quite sure if t needs to be built or not (because some children look # dirty, but might turn out to be clean thanks to checksums). We have # to call redo-oob to figure it all out. # # Note: redo-oob will handle all the updating of sf, so we don't have # to do it here, nor call _after1. argv = ['redo-oob', self.sf.name] + [d.name for d in dirty] log('(%s)\n' % _nice(self.t)) state.commit() def run(): os.chdir(vars.BASE) os.environ['REDO_DEPTH'] = vars.DEPTH + ' ' os.execvp(argv[0], argv) assert(0) # returns only if there's an exception def after(t, rv): return self._after2(rv) jwack.start_job(self.t, run, after)
def _start_do(self): assert(self.lock.owned) t = self.t sf = self.sf newstamp = sf.read_stamp() if (sf.is_generated and not sf.failed_runid and newstamp != state.STAMP_MISSING and (sf.stamp != newstamp or sf.is_override)): state.warn_override(_nice(t)) sf.set_override() sf.set_checked() sf.save() return self._after2(0) if (os.path.exists(t) and not os.path.exists(t + '/.') and not sf.is_generated): # an existing source file that was not generated by us. # This step is mentioned by djb in his notes. # For example, a rule called default.c.do could be used to try # to produce hello.c, but we don't want that to happen if # hello.c was created by the end user. # FIXME: always refuse to redo any file that was modified outside # of redo? That would make it easy for someone to override a # file temporarily, and could be undone by deleting the file. debug2("-- static (%r)\n" % t) sf.set_static() sf.save() return self._after2(0) sf.zap_deps1() (dodir, dofile, basedir, basename, ext) = _find_do_file(sf) if not dofile: if os.path.exists(t): sf.set_static() sf.save() return self._after2(0) else: err('no rule to make %r\n' % t) return self._after2(1) unlink(self.tmpname1) unlink(self.tmpname2) ffd = os.open(self.tmpname1, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666) close_on_exec(ffd, True) self.f = os.fdopen(ffd, 'w+') # this will run in the dofile's directory, so use only basenames here argv = ['sh', '-e', dofile, basename, # target name (no extension) ext, # extension (if any), including leading dot os.path.join(basedir, os.path.basename(self.tmpname2)) # temp output file name ] if vars.VERBOSE: argv[1] += 'v' if vars.XTRACE: argv[1] += 'x' if vars.VERBOSE or vars.XTRACE: log_('\n') firstline = open(os.path.join(dodir, dofile)).readline().strip() if firstline.startswith('#!/'): argv[0:2] = firstline[2:].split(' ') log('%s\n' % _nice(t)) self.dodir = dodir self.basename = basename self.ext = ext self.argv = argv sf.is_generated = True sf.save() dof = state.File(name=os.path.join(dodir, dofile)) dof.set_static() dof.save() state.commit() jwack.start_job(t, self._do_subproc, self._after)
def _start_do(self): assert (self.lock.owned) t = self.t sf = self.sf newstamp = sf.read_stamp() if (sf.is_generated and newstamp != state.STAMP_MISSING and (sf.stamp != newstamp or sf.is_override)): state.warn_override(_nice(t)) if not sf.is_override: warn('%s - old: %r\n' % (_nice(t), sf.stamp)) warn('%s - new: %r\n' % (_nice(t), newstamp)) sf.set_override() sf.set_checked() sf.save() return self._after2(0) if (os.path.exists(t) and not os.path.isdir(t + '/.') and not sf.is_generated): # an existing source file that was not generated by us. # This step is mentioned by djb in his notes. # For example, a rule called default.c.do could be used to try # to produce hello.c, but we don't want that to happen if # hello.c was created by the end user. debug2("-- static (%r)\n" % t) sf.set_static() sf.save() return self._after2(0) sf.zap_deps1() (dodir, dofile, basedir, basename, ext) = paths.find_do_file(sf) if not dofile: if os.path.exists(t): sf.set_static() sf.save() return self._after2(0) else: err('no rule to make %r\n' % t) return self._after2(1) unlink(self.tmpname1) unlink(self.tmpname2) ffd = os.open(self.tmpname1, os.O_CREAT | os.O_RDWR | os.O_EXCL, 0666) close_on_exec(ffd, True) self.f = os.fdopen(ffd, 'w+') # this will run in the dofile's directory, so use only basenames here arg1 = basename + ext # target name (including extension) arg2 = basename # target name (without extension) argv = [ 'sh', '-e', dofile, arg1, arg2, # temp output file name state.relpath(os.path.abspath(self.tmpname2), dodir), ] if vars.VERBOSE: argv[1] += 'v' if vars.XTRACE: argv[1] += 'x' if vars.VERBOSE or vars.XTRACE: log_('\n') firstline = open(os.path.join(dodir, dofile)).readline().strip() if firstline.startswith('#!/'): argv[0:2] = firstline[2:].split(' ') log('%s\n' % _nice(t)) self.dodir = dodir self.basename = basename self.ext = ext self.argv = argv sf.is_generated = True sf.save() dof = state.File(name=os.path.join(dodir, dofile)) dof.set_static() dof.save() state.commit() jwack.start_job(t, self._do_subproc, self._after)