def start_job(reason, jobfunc, donefunc): global _mytokens assert(_mytokens <= 1) get_token(reason) assert(_mytokens >= 1) assert(_mytokens == 1) _mytokens -= 1 r,w = _make_pipe(50) pid = os.fork() if pid == 0: # child os.close(r) rv = 201 try: try: rv = jobfunc() or 0 _debug('jobfunc completed (%r, %r)\n' % (jobfunc,rv)) except Exception: import traceback traceback.print_exc() finally: _debug('exit: %d\n' % rv) os._exit(rv) close_on_exec(r, True) os.close(w) pd = Job(reason, pid, donefunc) _waitfds[r] = pd
def start_job(reason, jobfunc, donefunc): assert state.is_flushed() global _mytokens assert _mytokens <= 1 assert _mytokens == 1 # Subprocesses always start with 1 token, so we have to destroy ours # in order for the universe to stay in balance. _destroy_tokens(1) r, w = _make_pipe(50) pid = os.fork() if pid == 0: # child os.close(r) rv = 201 try: try: rv = jobfunc() or 0 _debug('jobfunc completed (%r, %r)\n' % (jobfunc, rv)) except Exception: import traceback traceback.print_exc() finally: _debug('exit: %d\n' % rv) os._exit(rv) close_on_exec(r, True) os.close(w) pd = Job(reason, pid, donefunc) _waitfds[r] = pd
def start_job(name, jobfunc, donefunc): """ Start a job @param name The name of the job @param jobfunc The function representing the job @param donefunc The function to call when the job is done """ global _has_token get_token(name) assert (_has_token) _has_token = False r, w = _make_pipe(50) # Fork a child process to run the job pid = os.fork() if pid == 0: # The child process os.close(r) rv = 201 try: # Run the job rv = jobfunc() or 0 _debug('jobfunc completed (%r, %r)\n' % (jobfunc, rv)) except Exception: import traceback traceback.print_exc() finally: _debug('exit: %d\n' % rv) os._exit(rv) # The main process close_on_exec(r, True) os.close(w) # Add the job completion to _completion_map _completion_map[r] = Completion(name, pid, donefunc)
def start_job(reason, jobfunc, donefunc): assert(state.is_flushed()) global _mytokens assert(_mytokens <= 1) get_token(reason) assert(_mytokens >= 1) assert(_mytokens == 1) _mytokens -= 1 r,w = _make_pipe(50) pid = os.fork() if pid == 0: # child os.close(r) rv = 201 try: try: rv = jobfunc() or 0 _debug('jobfunc completed (%r, %r)\n' % (jobfunc,rv)) except Exception: import traceback traceback.print_exc() finally: _debug('exit: %d\n' % rv) os._exit(rv) close_on_exec(r, True) os.close(w) pd = Job(reason, pid, donefunc) _waitfds[r] = pd
def _open_lock(self): if not self.lockfile: try: os.makedirs(os.path.dirname(self.name)) except: pass self.lockfile = os.open(self.name, os.O_RDWR | os.O_CREAT, 0666) self.close_on_del = True close_on_exec(self.lockfile, True)
def start_job(reason, jobfunc, donefunc): """ Start a job jobfunc: executed in the child process doncfunc: executed in the parent process during a wait or wait_all call """ global _mytokens assert(_mytokens <= 1) get_token(reason) assert(_mytokens >= 1) assert(_mytokens == 1) _mytokens -= 1 r,w = _make_pipe(50) pid = os.fork() if pid == 0: # child os.close(r) os.environ['REDO_JWACK'] = "%s,%d" % (os.environ.get('REDO_JWACK'), w) rv = 201 try: try: rv = jobfunc() or 0 _debug('jobfunc completed (%r, %r)\n' % (jobfunc,rv)) except Exception: import traceback traceback.print_exc() finally: _debug('exit: %d\n' % rv) os._exit(rv) close_on_exec(r, True) os.close(w) pd = Job(reason, pid, donefunc) _waitfds[r] = pd
def build(self): debug3('running build job for %r\n', self.target.name) (dodir, dofile, basedir, basename, ext) = ( self.dodir, self.dofile, self.dobasedir, self.dobasename, self.doext) # this will run in the dofile's directory, so use only basenames here if vars.OLD_ARGS: arg1 = basename # target name (no extension) arg2 = ext # extension (if any), including leading dot else: arg1 = basename + ext # target name (including extension) arg2 = basename # target name (without extension) argv = ['sh', '-e', dofile, arg1, arg2, # temp output file name os.path.relpath(self.tmpname_arg3, dodir), ] if vars.VERBOSE: argv[1] += 'v' if vars.XTRACE: argv[1] += 'x' if vars.VERBOSE or vars.XTRACE: log_e('\n') firstline = open(os.path.join(dodir, dofile)).readline().strip() if firstline.startswith('#!.../'): _, _, interp_argv = firstline.partition("/") interp_argv = interp_argv.split(' ') interpreter = _find_interpreter(self.dodir, interp_argv[0]) if not interpreter: err('%s unable to find interpreter %s.\n', self.dofile, interp_argv[0]) os._exit(208) self.target.add_dep(state.File(interpreter)) argv[0:2] = [interpreter] + interp_argv[1:] elif firstline.startswith('#!/'): argv[0:2] = firstline[2:].split(' ') log('%s\n', self.target.printable_name()) log_cmd("redo", self.target.name + "\n") try: dn = dodir os.environ['REDO_PWD'] = os.path.join(vars.PWD, dn) os.environ['REDO_TARGET'] = basename + ext os.environ['REDO_DEPTH'] = vars.DEPTH + ' ' if dn: os.chdir(dn) l = logger.Logger(self.log_fd, self.tmp_sout_fd) l.fork() os.close(self.tmp_sout_fd) close_on_exec(1, False) if vars.VERBOSE or vars.XTRACE: log_e('* %s\n' % ' '.join(argv)) os.execvp(argv[0], argv) except: import traceback sys.stderr.write(traceback.format_exc()) err('internal exception - see above\n') raise finally: # returns only if there's an exception (exec in other case) os._exit(127)
def __init__(self, fid): self.owned = False self.fid = fid self.lockfile = os.open(os.path.join(vars.BASE, '.redo/lock.%d' % fid), os.O_RDWR | os.O_CREAT, 0666) close_on_exec(self.lockfile, True) assert (_locks.get(fid, 0) == 0) _locks[fid] = 1
def __init__(self, fid): self.owned = False self.fid = fid self.lockfile = os.open(os.path.join(vars.BASE, '.redo/lock.%d' % fid), os.O_RDWR | os.O_CREAT, 0666) close_on_exec(self.lockfile, True) assert(_locks.get(fid,0) == 0) _locks[fid] = 1
def prepare(self): assert self.target.dolock().owned == state.LOCK_EX self.target.build_starting() self.before_t = _try_stat(self.target.name) newstamp = self.target.read_stamp() if newstamp.is_override_or_missing(self.target): if newstamp.is_missing(): # was marked generated, but is now deleted debug3('oldstamp=%r newstamp=%r\n', self.target.stamp, newstamp) self.target.forget() self.target.refresh() elif vars.OVERWRITE: warn('%s: you modified it; overwrite\n', self.target.printable_name()) else: warn('%s: you modified it; skipping\n', self.target.printable_name()) return 0 if self.target.exists_not_dir() and not self.target.is_generated: # an existing source file that was not generated by us. # This step is mentioned by djb in his notes. # For example, a rule called default.c.do could be used to try # to produce hello.c, but we don't want that to happen if # hello.c was created in advance by the end user. if vars.OVERWRITE: warn('%s: exists and not marked as generated; overwrite.\n', self.target.printable_name()) else: warn('%s: exists and not marked as generated; not redoing.\n', self.target.printable_name()) debug2('-- static (%r)\n', self.target.name) return 0 (self.dodir, self.dofile, self.dobasedir, self.dobasename, self.doext) = _find_do_file(self.target) if not self.dofile: if newstamp.is_missing(): err('no rule to make %r\n', self.target.name) return 1 else: self.target.forget() debug2('-- forget (%r)\n', self.target.name) return 0 # no longer a generated target, but exists, so ok self.outdir = self._mkoutdir() # name connected to stdout self.tmpname_sout = self.target.tmpfilename('out.tmp') # name provided as $3 self.tmpname_arg3 = os.path.join(self.outdir, self.target.basename()) # name for the log file unlink(self.tmpname_sout) unlink(self.tmpname_arg3) self.log_fd = logger.open_log(self.target, truncate=True) self.tmp_sout_fd = os.open(self.tmpname_sout, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666) close_on_exec(self.tmp_sout_fd, True) self.tmp_sout_f = os.fdopen(self.tmp_sout_fd, 'w+') return None
def __init__(self, name=None, f=None): self.owned = False self.name = name if not f: self.lockfile = os.open(self.name, os.O_RDWR | os.O_CREAT, 0666) close_on_exec(self.lockfile, True) self.close_on_del = True else: self.lockfile = f self.close_on_del = False self.shared = fcntl.LOCK_SH self.exclusive = fcntl.LOCK_EX
def _do_subproc(self): # careful: REDO_PWD was the PWD relative to the STARTPATH at the time # we *started* building the current target; but that target ran # redo-ifchange, and it might have done it from a different directory # than we started it in. So os.getcwd() might be != REDO_PWD right # now. dn = self.dodir newp = os.path.realpath(dn) os.environ['REDO_PWD'] = state.relpath(newp, vars.STARTDIR) os.environ['REDO_TARGET'] = self.basename + self.ext os.environ['REDO_DEPTH'] = vars.DEPTH + ' ' if dn: os.chdir(dn) os.dup2(self.f.fileno(), 1) os.close(self.f.fileno()) close_on_exec(1, False) if vars.VERBOSE or vars.XTRACE: log_('* %s\n' % ' '.join(self.argv)) os.execvp(self.argv[0], self.argv) assert (0)
def _do_subproc(self): # careful: REDO_PWD was the PWD relative to the STARTPATH at the time # we *started* building the current target; but that target ran # redo-ifchange, and it might have done it from a different directory # than we started it in. So os.getcwd() might be != REDO_PWD right # now. dn = self.dodir newp = os.path.realpath(dn) os.environ['REDO_PWD'] = state.relpath(newp, vars.STARTDIR) os.environ['REDO_TARGET'] = self.basename + self.ext os.environ['REDO_DEPTH'] = vars.DEPTH + ' ' if dn: os.chdir(dn) os.dup2(self.f.fileno(), 1) os.close(self.f.fileno()) close_on_exec(1, False) if vars.VERBOSE or vars.XTRACE: log_('* %s\n' % ' '.join(self.argv)) os.execvp(self.argv[0], self.argv) assert(0)
def cleanup_on_exec(): "Close file descriptors" fds = _find_fds() if fds: a, b = fds close_on_exec(a, True) close_on_exec(b, True) for waitfd in os.environ.get('REDO_JWACK', '').split(','): waitfd = atoi(waitfd, None) if waitfd != None: close_on_exec(waitfd, True) del os.environ['REDO_JWACK'] os.environ['MAKEFLAGS'] = cleanup_makeflags(os.getenv('MAKEFLAGS'))
def fork(self): if not vars.LOG: if vars.OLD_STDOUT or vars.WARN_STDOUT: os.dup2(self.stdoutfd, 1) return pid = os.fork() if pid == 0: os.close(self.fd_std_out) os.close(self.fd_err_out) os.close(self.fd_log_out) os.close(self.fd_err_in) os.close(self.fd_log_in) sysout = sys.stdout if vars.OLD_STDOUT: sysout = None self._main(os.fdopen(self.fd_std_in), "std", sysout, self.stdoutfd) os._exit(0) pid2 = os.fork() if pid2 == 0: os.close(self.fd_std_out) os.close(self.fd_err_out) os.close(self.fd_log_out) os.close(self.fd_std_in) os.close(self.fd_log_in) os.close(self.stdoutfd) self._main(os.fdopen(self.fd_err_in), "err", sys.stderr) os._exit(0) pid3 = os.fork() if pid3 == 0: os.close(self.fd_std_out) os.close(self.fd_err_out) os.close(self.fd_log_out) os.close(self.fd_std_in) os.close(self.fd_err_in) os.close(self.stdoutfd) self._main(os.fdopen(self.fd_log_in), "log", LOGFILE) os._exit(0) os.dup2(self.fd_std_out, 1) os.dup2(self.fd_err_out, 2) os.close(self.fd_std_out) os.close(self.fd_err_out) os.close(self.fd_std_in) os.close(self.fd_err_in) os.close(self.fd_log_in) close_on_exec(self.fd_log_out, False) close_on_exec(1, False) close_on_exec(2, False) if vars.LOGFD: os.close(vars.LOGFD) os.environ["REDO_LOGFD"] = str(self.fd_log_out) vars.reinit() os.close(self.logfd)
def _start_do(self): assert(self.lock.owned) t = self.t sf = self.sf newstamp = sf.read_stamp() if (sf.is_generated and not sf.failed_runid and newstamp != state.STAMP_MISSING and (sf.stamp != newstamp or sf.is_override)): state.warn_override(_nice(t)) sf.set_override() sf.set_checked() sf.save() return self._after2(0) if (os.path.exists(t) and not os.path.exists(t + '/.') and not sf.is_generated): # an existing source file that was not generated by us. # This step is mentioned by djb in his notes. # For example, a rule called default.c.do could be used to try # to produce hello.c, but we don't want that to happen if # hello.c was created by the end user. # FIXME: always refuse to redo any file that was modified outside # of redo? That would make it easy for someone to override a # file temporarily, and could be undone by deleting the file. debug2("-- static (%r)\n" % t) sf.set_static() sf.save() return self._after2(0) sf.zap_deps1() (dodir, dofile, basedir, basename, ext) = _find_do_file(sf) if not dofile: if os.path.exists(t): sf.set_static() sf.save() return self._after2(0) else: err('no rule to make %r\n' % t) return self._after2(1) unlink(self.tmpname1) unlink(self.tmpname2) ffd = os.open(self.tmpname1, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666) close_on_exec(ffd, True) self.f = os.fdopen(ffd, 'w+') # this will run in the dofile's directory, so use only basenames here argv = ['sh', '-e', dofile, basename, # target name (no extension) ext, # extension (if any), including leading dot os.path.join(basedir, os.path.basename(self.tmpname2)) # temp output file name ] if vars.VERBOSE: argv[1] += 'v' if vars.XTRACE: argv[1] += 'x' if vars.VERBOSE or vars.XTRACE: log_('\n') firstline = open(os.path.join(dodir, dofile)).readline().strip() if firstline.startswith('#!/'): argv[0:2] = firstline[2:].split(' ') log('%s\n' % _nice(t)) self.dodir = dodir self.basename = basename self.ext = ext self.argv = argv sf.is_generated = True sf.save() dof = state.File(name=os.path.join(dodir, dofile)) dof.set_static() dof.save() state.commit() jwack.start_job(t, self._do_subproc, self._after)
def cleanup_on_exec(): if LOGFD: close_on_exec(LOGFD, True) close_on_exec(0, True) close_on_exec(1, True) close_on_exec(2, True) STDIO = os.environ.get('REDO_STDIO') if STDIO: try: a, b, c = [int(x) for x in STDIO.split(',')] os.dup2(a, 0) os.dup2(b, 1) os.dup2(c, 2) close_on_exec(a, True) close_on_exec(b, True) close_on_exec(c, True) except: pass for env in ['REDO', 'REDO_STARTDIR', 'REDO_PWD', 'REDO_TARGET', 'REDO_DEPTH', 'REDO_OVERWRITE', 'REDO_DEBUG', 'REDO_DEBUG_PIDS', 'REDO_DEBUG_LOCKS', 'REDO_OLD_ARGS', 'REDO_OLD_STDOUT', 'REDO_WARN_STDOUT', 'REDO_ONLY_LOG', 'REDO_VERBOSE', 'REDO_XTRACE', 'REDO_KEEP_GOING', 'REDO_SHUFFLE', 'REDO_LOGFD', 'REDO_RUNID_FILE', 'REDO_STDIO']: try: del os.environ[env] except: pass
if _db: return _db dbdir = '%s/.redo' % vars.BASE dbfile = '%s/db.sqlite3' % dbdir try: os.mkdir(dbdir) except OSError, e: if e.errno == errno.EEXIST: pass # if it exists, that's okay else: raise _lockfile = os.open(os.path.join(vars.BASE, '.redo/locks'), os.O_RDWR | os.O_CREAT, 0666) close_on_exec(_lockfile, True) must_create = not os.path.exists(dbfile) if not must_create: _db = _connect(dbfile) try: row = _db.cursor().execute("select version from Schema").fetchone() except sqlite3.OperationalError: row = None ver = row and row[0] or None if ver != SCHEMA_VER: # Don't use err() here because this might happen before # redo-log spawns. sys.stderr.write('redo: %s: found v%s (expected v%s)\n' % (dbfile, ver, SCHEMA_VER)) sys.stderr.write(
def _start_do(self): assert (self.lock.owned) t = self.t sf = self.sf newstamp = sf.read_stamp() if (sf.is_generated and newstamp != state.STAMP_MISSING and (sf.stamp != newstamp or sf.is_override)): state.warn_override(_nice(t)) if not sf.is_override: warn('%s - old: %r\n' % (_nice(t), sf.stamp)) warn('%s - new: %r\n' % (_nice(t), newstamp)) sf.set_override() sf.set_checked() sf.save() return self._after2(0) if (os.path.exists(t) and not os.path.isdir(t + '/.') and not sf.is_generated): # an existing source file that was not generated by us. # This step is mentioned by djb in his notes. # For example, a rule called default.c.do could be used to try # to produce hello.c, but we don't want that to happen if # hello.c was created by the end user. debug2("-- static (%r)\n" % t) sf.set_static() sf.save() return self._after2(0) sf.zap_deps1() (dodir, dofile, basedir, basename, ext) = paths.find_do_file(sf) if not dofile: if os.path.exists(t): sf.set_static() sf.save() return self._after2(0) else: err('no rule to make %r\n' % t) return self._after2(1) unlink(self.tmpname1) unlink(self.tmpname2) ffd = os.open(self.tmpname1, os.O_CREAT | os.O_RDWR | os.O_EXCL, 0666) close_on_exec(ffd, True) self.f = os.fdopen(ffd, 'w+') # this will run in the dofile's directory, so use only basenames here arg1 = basename + ext # target name (including extension) arg2 = basename # target name (without extension) argv = [ 'sh', '-e', dofile, arg1, arg2, # temp output file name state.relpath(os.path.abspath(self.tmpname2), dodir), ] if vars.VERBOSE: argv[1] += 'v' if vars.XTRACE: argv[1] += 'x' if vars.VERBOSE or vars.XTRACE: log_('\n') firstline = open(os.path.join(dodir, dofile)).readline().strip() if firstline.startswith('#!/'): argv[0:2] = firstline[2:].split(' ') log('%s\n' % _nice(t)) self.dodir = dodir self.basename = basename self.ext = ext self.argv = argv sf.is_generated = True sf.save() dof = state.File(name=os.path.join(dodir, dofile)) dof.set_static() dof.save() state.commit() jwack.start_job(t, self._do_subproc, self._after)