def _move_extra_results(self, src, dest, rv): assert src assert dest if os.path.isdir(src) and os.path.isdir(dest): for f in os.listdir(src): sp = os.path.join(src, f) dp = os.path.join(dest, f) self._move_extra_results(sp, dp, rv) os.rmdir(src) else: sf = state.File(name=dest) if sf == self.delegate: dest = os.path.join(sf.tmpfilename("out"), sf.basename()) debug("rename %r %r\n", src, dest) os.rename(src, dest) sf.copy_deps_from(self.target) else: sf.dolock().trylock() if sf.dolock().owned == state.LOCK_EX: try: sf.build_starting() debug("rename %r %r\n", src, dest) os.rename(src, dest) sf.copy_deps_from(self.target) sf.build_done(rv) finally: sf.dolock().unlock() else: warn("%s: discarding (parallel build)\n", dest) unlink(src)
def build_starting(self): """Call this when you're about to start building this target.""" if vars.TARGET: with open(self.tmpfilename('parent'), "w") as f: f.write(os.path.relpath(vars.TARGET, self.dir)) depsname = self.tmpfilename('deps2') debug3('build starting: %r\n', depsname) unlink(depsname)
def build_done(self, exitcode): """Call this when you're done building this target.""" depsname = self.tmpfilename('deps2') debug3('build ending: %r\n', depsname) self._add(self.read_stamp(runid=vars.RUNID).stamp) self._add(exitcode) os.utime(depsname, (vars.RUNID, vars.RUNID)) os.rename(depsname, self.tmpfilename('deps')) unlink(self.tmpfilename('parent'))
def prepare(self): assert self.target.dolock().owned == state.LOCK_EX self.target.build_starting() self.before_t = _try_stat(self.target.name) newstamp = self.target.read_stamp() if newstamp.is_override_or_missing(self.target): if newstamp.is_missing(): # was marked generated, but is now deleted debug3('oldstamp=%r newstamp=%r\n', self.target.stamp, newstamp) self.target.forget() self.target.refresh() elif vars.OVERWRITE: warn('%s: you modified it; overwrite\n', self.target.printable_name()) else: warn('%s: you modified it; skipping\n', self.target.printable_name()) return 0 if self.target.exists_not_dir() and not self.target.is_generated: # an existing source file that was not generated by us. # This step is mentioned by djb in his notes. # For example, a rule called default.c.do could be used to try # to produce hello.c, but we don't want that to happen if # hello.c was created in advance by the end user. if vars.OVERWRITE: warn('%s: exists and not marked as generated; overwrite.\n', self.target.printable_name()) else: warn('%s: exists and not marked as generated; not redoing.\n', self.target.printable_name()) debug2('-- static (%r)\n', self.target.name) return 0 (self.dodir, self.dofile, self.dobasedir, self.dobasename, self.doext) = _find_do_file(self.target) if not self.dofile: if newstamp.is_missing(): err('no rule to make %r\n', self.target.name) return 1 else: self.target.forget() debug2('-- forget (%r)\n', self.target.name) return 0 # no longer a generated target, but exists, so ok self.outdir = self._mkoutdir() # name connected to stdout self.tmpname_sout = self.target.tmpfilename('out.tmp') # name provided as $3 self.tmpname_arg3 = os.path.join(self.outdir, self.target.basename()) # name for the log file unlink(self.tmpname_sout) unlink(self.tmpname_arg3) self.log_fd = logger.open_log(self.target, truncate=True) self.tmp_sout_fd = os.open(self.tmpname_sout, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666) close_on_exec(self.tmp_sout_fd, True) self.tmp_sout_f = os.fdopen(self.tmp_sout_fd, 'w+') return None
def open_log(t=None, truncate=False): if not t: t = state.File(vars.TARGET) f = t.tmpfilename('log') flags = os.O_WRONLY|os.O_APPEND|os.O_EXCL if truncate: unlink(f) flags = flags|os.O_CREAT fd = os.open(f, flags, 0666) return f, fd
def build_starting(self): """Call this when you're about to start building this target.""" if vars.TARGET: with open(self.tmpfilename('parent'), "w") as f: f.write(os.path.relpath(vars.TARGET, self.dir)) depsname = self.tmpfilename('deps2') debug3('build starting: %r\n', depsname) unlink(depsname) with open(depsname, 'a') as f: f.write(DEPSFILE_TAG + '\n') st = os.fstat(f.fileno()) f.write('%d %d\n' % (st.st_dev, st.st_ino))
def _after1(self, t, rv): f = self.f before_t = self.before_t after_t = _try_stat(t) st1 = os.fstat(f.fileno()) st2 = _try_stat(self.tmpname2) if (after_t and (not before_t or before_t.st_mtime != after_t.st_mtime) and not stat.S_ISDIR(after_t.st_mode)): err('%s modified %s directly!\n' % (self.argv[2], t)) err('...you should update $3 (a temp file) or stdout, not $1.\n') rv = 206 elif st2 and st1.st_size > 0: err('%s wrote to stdout *and* created $3.\n' % self.argv[2]) err('...you should write status messages to stderr, not stdout.\n') rv = 207 if rv==0: if st2: try: os.rename(self.tmpname2, t) except OSError, e: dnt = os.path.dirname(t) if not os.path.exists(dnt): err('%s: target dir %r does not exist!\n' % (t, dnt)) else: err('%s: rename %s: %s\n' % (t, self.tmpname2, e)) raise os.unlink(self.tmpname1) elif st1.st_size > 0: try: os.rename(self.tmpname1, t) except OSError, e: if e.errno == errno.ENOENT: unlink(t) else: err('%s: can\'t save stdout to %r: %s\n' % (self.argv[2], t, e.strerror)) rv = 1000 if st2: os.unlink(self.tmpname2)
def _after1(self, t, rv): f = self.f before_t = self.before_t after_t = _try_stat(t) st1 = os.fstat(f.fileno()) st2 = _try_stat(self.tmpname2) if (after_t and (not before_t or before_t.st_ctime != after_t.st_ctime) and not stat.S_ISDIR(after_t.st_mode)): err('%s modified %s directly!\n' % (self.argv[2], t)) err('...you should update $3 (a temp file) or stdout, not $1.\n') rv = 206 elif st2 and st1.st_size > 0: err('%s wrote to stdout *and* created $3.\n' % self.argv[2]) err('...you should write status messages to stderr, not stdout.\n') rv = 207 if rv==0: if st2: try: os.rename(self.tmpname2, t) except OSError, e: err('%s: can\'t rename $3 to %r: %s\n' % (self.argv[2], t, e.strerror)) rv = 1000 os.unlink(self.tmpname1) elif st1.st_size > 0: try: os.rename(self.tmpname1, t) except OSError, e: if e.errno == errno.ENOENT: unlink(t) else: err('%s: can\'t save stdout to %r: %s\n' % (self.argv[2], t, e.strerror)) rv = 1000 if st2: os.unlink(self.tmpname2)
def _build(t): if (os.path.exists(t) and not state.is_generated(t) and not os.path.exists('%s.do' % t)): # an existing source file that is not marked as a generated file. # This step is mentioned by djb in his notes. It turns out to be # important to prevent infinite recursion. For example, a rule # called default.c.do could be used to try to produce hello.c, # which is undesirable since hello.c existed already. state.stamp(t) return # success state.start(t) (dofile, basename, ext) = _find_do_file(t) if not dofile: raise BuildError('no rule to make %r' % t) state.stamp(dofile) tmpname = '%s.redo.tmp' % t unlink(tmpname) f = open(tmpname, 'w+') # this will run in the dofile's directory, so use only basenames here argv = ['sh', '-e', os.path.basename(dofile), os.path.basename(basename), # target name (extension removed) ext, # extension (if any), including leading dot os.path.basename(tmpname) # randomized output file name ] if vars.VERBOSE: argv[1] += 'v' log_('\n') log('%s\n' % relpath(t, vars.STARTDIR)) rv = subprocess.call(argv, preexec_fn=lambda: _preexec(t), stdout=f.fileno()) if rv==0: if os.path.exists(tmpname) and os.stat(tmpname).st_size: # there's a race condition here, but if the tmpfile disappears # at *this* point you deserve to get an error, because you're # doing something totally scary. os.rename(tmpname, t) else: unlink(tmpname) state.stamp(t) else: unlink(tmpname) state.unstamp(t) f.close() if rv != 0: raise BuildError('%s: exit code %d' % (t,rv)) if vars.VERBOSE: log('%s (done)\n\n' % relpath(t, vars.STARTDIR))
def done(self, t, rv): assert self.target.dolock().owned == state.LOCK_EX log_cmd("redo_done", self.target.name + "\n") try: after_t = _try_stat(self.target.name) st1 = os.fstat(self.tmp_sout_f.fileno()) st2 = _try_stat(self.tmpname_arg3) if ( after_t and (not self.before_t or self.before_t.st_ctime != after_t.st_ctime) and not stat.S_ISDIR(after_t.st_mode) ): err("%s modified %s directly!\n", self.dofile, self.target.name) err("...you should update $3 (a temp file) or stdout, not $1.\n") rv = 206 elif vars.OLD_STDOUT and st2 and st1.st_size > 0: err("%s wrote to stdout *and* created $3.\n", self.dofile) err("...you should write status messages to stderr, not stdout.\n") rv = 207 elif vars.WARN_STDOUT and st1.st_size > 0: err("%s wrote to stdout, this is not longer supported.\n", self.dofile) err("...you should write status messages to stderr, not stdout.\n") err('...you should write target content to $3 using for example \'exec >"$3"`.\n') if not vars.OLD_STDOUT: rv = 207 if rv == 0: if st2: os.rename(self.tmpname_arg3, self.target.name) os.unlink(self.tmpname_sout) elif vars.OLD_STDOUT and st1.st_size > 0: try: os.rename(self.tmpname_sout, self.target.name) except OSError, e: if e.errno == errno.ENOENT: unlink(self.target.name) else: raise else: # no output generated at all; that's ok unlink(self.tmpname_sout) unlink(self.target.name) if vars.VERBOSE or vars.XTRACE or vars.DEBUG: log("%s (done)\n\n", self.target.printable_name()) else:
def _after1(self, t, rv): f = self.f before_t = self.before_t after_t = _try_stat(t) st1 = os.fstat(f.fileno()) st2 = _try_stat(self.tmpname2) if (after_t and (not before_t or before_t.st_ctime != after_t.st_ctime) and not stat.S_ISDIR(after_t.st_mode)): err('%s modified %s directly!\n' % (self.argv[2], t)) err('...you should update $3 (a temp file) or stdout, not $1.\n') rv = 206 elif st2 and st1.st_size > 0: err('%s wrote to stdout *and* created $3.\n' % self.argv[2]) err('...you should write status messages to stderr, not stdout.\n') rv = 207 if rv==0: if st2: os.rename(self.tmpname2, t) os.unlink(self.tmpname1) elif st1.st_size > 0: try: os.rename(self.tmpname1, t) except OSError, e: if e.errno == errno.ENOENT: unlink(t) else: raise if st2: os.unlink(self.tmpname2) else: # no output generated at all; that's ok unlink(self.tmpname1) unlink(t) sf = self.sf sf.refresh() sf.is_generated = True sf.is_override = False if sf.is_checked() or sf.is_changed(): # it got checked during the run; someone ran redo-stamp. # update_stamp would call set_changed(); we don't want that sf.stamp = sf.read_stamp() else: sf.csum = None sf.update_stamp() sf.set_changed()
def _after1(self, t, rv): f = self.f before_t = self.before_t after_t = _try_stat(t) st1 = os.fstat(f.fileno()) st2 = _try_stat(self.tmpname2) if (after_t and (not before_t or before_t.st_ctime != after_t.st_ctime) and not stat.S_ISDIR(after_t.st_mode)): err('%s modified %s directly!\n' % (self.argv[2], t)) err('...you should update $3 (a temp file) or stdout, not $1.\n') rv = 206 elif st2 and st1.st_size > 0: err('%s wrote to stdout *and* created $3.\n' % self.argv[2]) err('...you should write status messages to stderr, not stdout.\n') rv = 207 if rv == 0: if st2: os.rename(self.tmpname2, t) os.unlink(self.tmpname1) elif st1.st_size > 0: try: os.rename(self.tmpname1, t) except OSError, e: if e.errno == errno.ENOENT: unlink(t) else: raise if st2: os.unlink(self.tmpname2) else: # no output generated at all; that's ok unlink(self.tmpname1) unlink(t) sf = self.sf sf.refresh() sf.is_generated = True sf.is_override = False if sf.is_checked() or sf.is_changed(): # it got checked during the run; someone ran redo-stamp. # update_stamp would call set_changed(); we don't want that sf.stamp = sf.read_stamp() else: sf.csum = None sf.update_stamp() sf.set_changed()
def _start_do(self): assert(self.lock.owned) t = self.t sf = self.sf newstamp = sf.read_stamp() if (sf.is_generated and not sf.failed_runid and newstamp != state.STAMP_MISSING and (sf.stamp != newstamp or sf.is_override)): state.warn_override(_nice(t)) sf.set_override() sf.set_checked() sf.save() return self._after2(0) if (os.path.exists(t) and not os.path.exists(t + '/.') and not sf.is_generated): # an existing source file that was not generated by us. # This step is mentioned by djb in his notes. # For example, a rule called default.c.do could be used to try # to produce hello.c, but we don't want that to happen if # hello.c was created by the end user. # FIXME: always refuse to redo any file that was modified outside # of redo? That would make it easy for someone to override a # file temporarily, and could be undone by deleting the file. debug2("-- static (%r)\n" % t) sf.set_static() sf.save() return self._after2(0) sf.zap_deps1() (dodir, dofile, basedir, basename, ext) = _find_do_file(sf) if not dofile: if os.path.exists(t): sf.set_static() sf.save() return self._after2(0) else: err('no rule to make %r\n' % t) return self._after2(1) unlink(self.tmpname1) unlink(self.tmpname2) ffd = os.open(self.tmpname1, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666) close_on_exec(ffd, True) self.f = os.fdopen(ffd, 'w+') # this will run in the dofile's directory, so use only basenames here argv = ['sh', '-e', dofile, basename, # target name (no extension) ext, # extension (if any), including leading dot os.path.join(basedir, os.path.basename(self.tmpname2)) # temp output file name ] if vars.VERBOSE: argv[1] += 'v' if vars.XTRACE: argv[1] += 'x' if vars.VERBOSE or vars.XTRACE: log_('\n') firstline = open(os.path.join(dodir, dofile)).readline().strip() if firstline.startswith('#!/'): argv[0:2] = firstline[2:].split(' ') log('%s\n' % _nice(t)) self.dodir = dodir self.basename = basename self.ext = ext self.argv = argv sf.is_generated = True sf.save() dof = state.File(name=os.path.join(dodir, dofile)) dof.set_static() dof.save() state.commit() jwack.start_job(t, self._do_subproc, self._after)
unlink(self.tmpname1) unlink(t) sf = self.sf sf.refresh() sf.is_generated = True sf.is_override = False if sf.is_checked() or sf.is_changed(): # it got checked during the run; someone ran redo-stamp. # update_stamp would call set_changed(); we don't want that sf.stamp = sf.read_stamp() else: sf.csum = None sf.update_stamp() sf.set_changed() else: unlink(self.tmpname1) unlink(self.tmpname2) sf = self.sf sf.set_failed() sf.zap_deps2() sf.save() f.close() if rv != 0: err('%s: exit code %d\n' % (_nice(t),rv)) else: if vars.VERBOSE or vars.XTRACE or vars.DEBUG: log('%s (done)\n\n' % _nice(t)) return rv def _after2(self, rv): try:
def refresh(self): if self.name == ALWAYS: self.stamp_mtime = str(vars.RUNID) self.exitcode = 0 self.deps = [] self.is_generated = True self.stamp = Stamp(str(vars.RUNID)) return assert(not self.name.startswith('/')) try: # read the state file f = open(self.tmpfilename('deps')) except IOError: try: # okay, check for the file itself st = os.stat(self.name) except OSError: # it doesn't exist at all yet self.stamp_mtime = 0 # no stamp file self.exitcode = 0 self.deps = [] self.stamp = Stamp(STAMP_MISSING) self.runid = None self.is_generated = True else: # it's a source file (without a .deps file) self.stamp_mtime = 0 # no stamp file self.exitcode = 0 self.deps = [] self.is_generated = False self.stamp = self.read_stamp(st=st) self.runid = self.stamp.runid() else: # it's a target (with a .deps file) st = os.fstat(f.fileno()) lines = f.read().strip().split('\n') version = None device = None inode = None try: version = lines.pop(0) device, inode = [int(i) for i in lines.pop(0).split(" ")] except: pass if version != DEPSFILE_TAG or device != st.st_dev or inode != st.st_ino: # It is an old .deps file, consider it missing self.stamp_mtime = 0 # no stamp file self.exitcode = 0 self.deps = [] self.stamp = Stamp(STAMP_OLD) self.runid = None self.is_generated = True else: # Read .deps file self.stamp_mtime = int(st.st_mtime) self.exitcode = int(lines.pop(-1)) self.is_generated = True self.stamp = Stamp(lines.pop(-1)) if not self.exists() and not self.stamp.is_missing(): #debug3('deleted %s (mark as missing)\n', self.name) #self.stamp.stamp = STAMP_MISSING debug3('deleted %s (delete .deps and rescan)\n', self.name) unlink(self.tmpfilename('deps')) return self.refresh() self.runid = self.stamp.runid() self.deps = [line.split(' ', 1) for line in lines] # if the next line fails, it means that the .dep file is not # correctly formatted while self.deps and self.deps[-1][1] == '.': # a line added by redo-stamp self.stamp.csum = self.deps.pop(-1)[0] for i in range(len(self.deps)): self.deps[i][0] = Stamp(auto_detect=self.deps[i][0])
def unstamp(t, fromdir=None): unlink(_stampname(t, fromdir))
def forget(self): """Turn a 'target' file back into a 'source' file.""" debug3('forget(%s)\n', self.name) unlink(self.tmpfilename('deps'))
must_create = not os.path.exists(dbfile) if not must_create: _db = _connect(dbfile) try: row = _db.cursor().execute("select version from Schema").fetchone() except sqlite3.OperationalError: row = None ver = row and row[0] or None if ver != SCHEMA_VER: err("state database: discarding v%s (wanted v%s)\n" % (ver, SCHEMA_VER)) must_create = True _db = None if must_create: unlink(dbfile) _db = _connect(dbfile) _db.execute("create table Schema " " (version int)") _db.execute("create table Runid " " (id integer primary key autoincrement)") _db.execute("create table Files " " (name not null primary key, " " is_generated int, " " is_override int, " " checked_runid int, " " changed_runid int, " " failed_runid int, " " stamp, " " csum)") _db.execute("create table Deps "
must_create = not os.path.exists(dbfile) if not must_create: _db = _connect(dbfile) try: row = _db.cursor().execute("select version from Schema").fetchone() except sqlite3.OperationalError: row = None ver = row and row[0] or None if ver != SCHEMA_VER: err("state database: discarding v%s (wanted v%s)\n" % (ver, SCHEMA_VER)) must_create = True _db = None if must_create: unlink(dbfile) _db = _connect(dbfile) _db.execute("create table Schema " " (version int)") _db.execute("create table Runid " " (id integer primary key autoincrement)") _db.execute("create table Files " " (name not null primary key, " " is_generated int, " " is_override int, " " checked_runid int, " " changed_runid int, " " failed_runid int, " " stamp, " " csum)") _db.execute("create table Deps " " (target int, "
os.unlink(self.tmpname_sout) elif vars.OLD_STDOUT and st1.st_size > 0: try: os.rename(self.tmpname_sout, self.target.name) except OSError, e: if e.errno == errno.ENOENT: unlink(self.target.name) else: raise else: # no output generated at all; that's ok unlink(self.tmpname_sout) unlink(self.target.name) if vars.VERBOSE or vars.XTRACE or vars.DEBUG: log('%s (done)\n\n', self.target.printable_name()) else: unlink(self.tmpname_sout) unlink(self.tmpname_arg3) if rv != 0: if vars.ONLY_LOG: logger.print_log(self.target) err('%s: exit code %d\n', self.target.printable_name(), rv) self.target.build_done(exitcode=rv) self.target.refresh() self._move_extra_results(self.outdir, self.target.dirname() or ".", rv) self.result[0] += rv self.result[1] += 1 if self.parent: self.parent.add_dep(self.target)
def _start_do(self): assert (self.lock.owned) t = self.t sf = self.sf newstamp = sf.read_stamp() if (sf.is_generated and newstamp != state.STAMP_MISSING and (sf.stamp != newstamp or sf.is_override)): state.warn_override(_nice(t)) if not sf.is_override: warn('%s - old: %r\n' % (_nice(t), sf.stamp)) warn('%s - new: %r\n' % (_nice(t), newstamp)) sf.set_override() sf.set_checked() sf.save() return self._after2(0) if (os.path.exists(t) and not os.path.isdir(t + '/.') and not sf.is_generated): # an existing source file that was not generated by us. # This step is mentioned by djb in his notes. # For example, a rule called default.c.do could be used to try # to produce hello.c, but we don't want that to happen if # hello.c was created by the end user. debug2("-- static (%r)\n" % t) sf.set_static() sf.save() return self._after2(0) sf.zap_deps1() (dodir, dofile, basedir, basename, ext) = paths.find_do_file(sf) if not dofile: if os.path.exists(t): sf.set_static() sf.save() return self._after2(0) else: err('no rule to make %r\n' % t) return self._after2(1) unlink(self.tmpname1) unlink(self.tmpname2) ffd = os.open(self.tmpname1, os.O_CREAT | os.O_RDWR | os.O_EXCL, 0666) close_on_exec(ffd, True) self.f = os.fdopen(ffd, 'w+') # this will run in the dofile's directory, so use only basenames here arg1 = basename + ext # target name (including extension) arg2 = basename # target name (without extension) argv = [ 'sh', '-e', dofile, arg1, arg2, # temp output file name state.relpath(os.path.abspath(self.tmpname2), dodir), ] if vars.VERBOSE: argv[1] += 'v' if vars.XTRACE: argv[1] += 'x' if vars.VERBOSE or vars.XTRACE: log_('\n') firstline = open(os.path.join(dodir, dofile)).readline().strip() if firstline.startswith('#!/'): argv[0:2] = firstline[2:].split(' ') log('%s\n' % _nice(t)) self.dodir = dodir self.basename = basename self.ext = ext self.argv = argv sf.is_generated = True sf.save() dof = state.File(name=os.path.join(dodir, dofile)) dof.set_static() dof.save() state.commit() jwack.start_job(t, self._do_subproc, self._after)
raise os.unlink(self.tmpname1) elif st1.st_size > 0: try: os.rename(self.tmpname1, t) except OSError, e: if e.errno == errno.ENOENT: unlink(t) else: err('%s: can\'t save stdout to %r: %s\n' % (self.argv[2], t, e.strerror)) rv = 1000 if st2: os.unlink(self.tmpname2) else: # no output generated at all; that's ok unlink(self.tmpname1) unlink(t) sf = self.sf sf.refresh() sf.is_generated = True sf.is_override = False if sf.is_checked() or sf.is_changed(): # it got checked during the run; someone ran redo-stamp. # update_stamp would call set_changed(); we don't want that sf.stamp = sf.read_stamp() else: sf.csum = None sf.update_stamp() sf.set_changed() else: unlink(self.tmpname1)