def build(f, any_errors, should_build, add_dep_to=None, delegate=None, re_do=True): if f.dolock(): if f.check_deadlocks(): err("%s: recursive dependency, breaking deadlock\n", f.printable_name()) any_errors[0] += 1 any_errors[1] += 1 else: jwack.get_token(f) f.dolock().waitlock() if any_errors[0] and not vars.KEEP_GOING: return False f.refresh() debug3('think about building %r\n', f.name) dirty = should_build(f) while dirty and dirty != deps.DIRTY: # FIXME: bring back the old (targetname) notation in the output # when we need to do this. And add comments. for t2 in dirty: if not build(t2, any_errors, should_build, delegate, re_do): return False jwack.wait_all() dirty = should_build(f) if dirty: job = BuildJob(f, any_errors, add_dep_to, delegate, re_do) add_dep_to = None job.schedule_job() else: f.dolock().unlock() if add_dep_to: f.refresh() add_dep_to.add_dep(f) return True
def add_dep(self, mode, dep): src = File(name=dep) debug3('add-dep: "%s" < %s "%s"\n' % (self.name, mode, src.name)) assert(self.id != src.id) _write("insert or replace into Deps " " (target, mode, source, delete_me) values (?,?,?,?)", [self.id, mode, src.id, False])
def build(self): debug3('running build job for %r\n', self.target.name) (dodir, dofile, basedir, basename, ext) = ( self.dodir, self.dofile, self.dobasedir, self.dobasename, self.doext) # this will run in the dofile's directory, so use only basenames here if vars.OLD_ARGS: arg1 = basename # target name (no extension) arg2 = ext # extension (if any), including leading dot else: arg1 = basename + ext # target name (including extension) arg2 = basename # target name (without extension) argv = ['sh', '-e', dofile, arg1, arg2, # temp output file name os.path.relpath(self.tmpname_arg3, dodir), ] if vars.VERBOSE: argv[1] += 'v' if vars.XTRACE: argv[1] += 'x' if vars.VERBOSE or vars.XTRACE: log_e('\n') firstline = open(os.path.join(dodir, dofile)).readline().strip() if firstline.startswith('#!.../'): _, _, interp_argv = firstline.partition("/") interp_argv = interp_argv.split(' ') interpreter = _find_interpreter(self.dodir, interp_argv[0]) if not interpreter: err('%s unable to find interpreter %s.\n', self.dofile, interp_argv[0]) os._exit(208) self.target.add_dep(state.File(interpreter)) argv[0:2] = [interpreter] + interp_argv[1:] elif firstline.startswith('#!/'): argv[0:2] = firstline[2:].split(' ') log('%s\n', self.target.printable_name()) log_cmd("redo", self.target.name + "\n") try: dn = dodir os.environ['REDO_PWD'] = os.path.join(vars.PWD, dn) os.environ['REDO_TARGET'] = basename + ext os.environ['REDO_DEPTH'] = vars.DEPTH + ' ' if dn: os.chdir(dn) l = logger.Logger(self.log_fd, self.tmp_sout_fd) l.fork() os.close(self.tmp_sout_fd) close_on_exec(1, False) if vars.VERBOSE or vars.XTRACE: log_e('* %s\n' % ' '.join(argv)) os.execvp(argv[0], argv) except: import traceback sys.stderr.write(traceback.format_exc()) err('internal exception - see above\n') raise finally: # returns only if there's an exception (exec in other case) os._exit(127)
def _add(self, line): depsname = self.tmpfilename('deps2') debug3('_add(%s) to %r\n', line, depsname) #assert os.path.exists(depsname) line = str(line) assert('\n' not in line) with open(depsname, 'a') as f: f.write(line + '\n')
def add_dep(self, mode, dep): src = File(name=dep) debug3('add-dep: "%s" < %s "%s"\n' % (self.name, mode, src.name)) assert (self.id != src.id) _write( "insert or replace into Deps " " (target, mode, source, delete_me) values (?,?,?,?)", [self.id, mode, src.id, False])
def build_starting(self): """Call this when you're about to start building this target.""" if vars.TARGET: with open(self.tmpfilename('parent'), "w") as f: f.write(os.path.relpath(vars.TARGET, self.dir)) depsname = self.tmpfilename('deps2') debug3('build starting: %r\n', depsname) unlink(depsname)
def build_done(self, exitcode): """Call this when you're done building this target.""" depsname = self.tmpfilename('deps2') debug3('build ending: %r\n', depsname) self._add(self.read_stamp(runid=vars.RUNID).stamp) self._add(exitcode) os.utime(depsname, (vars.RUNID, vars.RUNID)) os.rename(depsname, self.tmpfilename('deps')) unlink(self.tmpfilename('parent'))
def prepare(self): assert self.target.dolock().owned == state.LOCK_EX self.target.build_starting() self.before_t = _try_stat(self.target.name) newstamp = self.target.read_stamp() if newstamp.is_override_or_missing(self.target): if newstamp.is_missing(): # was marked generated, but is now deleted debug3('oldstamp=%r newstamp=%r\n', self.target.stamp, newstamp) self.target.forget() self.target.refresh() elif vars.OVERWRITE: warn('%s: you modified it; overwrite\n', self.target.printable_name()) else: warn('%s: you modified it; skipping\n', self.target.printable_name()) return 0 if self.target.exists_not_dir() and not self.target.is_generated: # an existing source file that was not generated by us. # This step is mentioned by djb in his notes. # For example, a rule called default.c.do could be used to try # to produce hello.c, but we don't want that to happen if # hello.c was created in advance by the end user. if vars.OVERWRITE: warn('%s: exists and not marked as generated; overwrite.\n', self.target.printable_name()) else: warn('%s: exists and not marked as generated; not redoing.\n', self.target.printable_name()) debug2('-- static (%r)\n', self.target.name) return 0 (self.dodir, self.dofile, self.dobasedir, self.dobasename, self.doext) = _find_do_file(self.target) if not self.dofile: if newstamp.is_missing(): err('no rule to make %r\n', self.target.name) return 1 else: self.target.forget() debug2('-- forget (%r)\n', self.target.name) return 0 # no longer a generated target, but exists, so ok self.outdir = self._mkoutdir() # name connected to stdout self.tmpname_sout = self.target.tmpfilename('out.tmp') # name provided as $3 self.tmpname_arg3 = os.path.join(self.outdir, self.target.basename()) # name for the log file unlink(self.tmpname_sout) unlink(self.tmpname_arg3) self.log_fd = logger.open_log(self.target, truncate=True) self.tmp_sout_fd = os.open(self.tmpname_sout, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666) close_on_exec(self.tmp_sout_fd, True) self.tmp_sout_f = os.fdopen(self.tmp_sout_fd, 'w+') return None
def build_starting(self): """Call this when you're about to start building this target.""" if vars.TARGET: with open(self.tmpfilename('parent'), "w") as f: f.write(os.path.relpath(vars.TARGET, self.dir)) depsname = self.tmpfilename('deps2') debug3('build starting: %r\n', depsname) unlink(depsname) with open(depsname, 'a') as f: f.write(DEPSFILE_TAG + '\n') st = os.fstat(f.fileno()) f.write('%d %d\n' % (st.st_dev, st.st_ino))
def add_dep(self, file): """Mark the given File() object as a dependency of this target. The filesystem file it refers to may or may not exist. If it doesn't exist, creating the file is considered a "modified" event and will result in this target being rebuilt. """ if file.name == ALWAYS: relname = file.name else: relname = os.path.relpath(file.name, self.dir) debug3('add-dep: %r < %r %r\n', self.name, file.stamp, relname) assert('\n' not in file.name) assert(isinstance(file.stamp, Stamp)) self._add('%s %s' % (file.stamp.csum_or_stamp(), relname))
def _find_uncovered_dn(self, generator, break_early=False): is_covered = self.nsec3_chain.covers while True: for ptlabel,dn_hash in self._prehash_iter: if not is_covered(dn_hash): dn = name.DomainName(name.Label(ptlabel), *self.zone.labels) owner_b32 = util.base32_ext_hex_encode( dn_hash).lower() hashed_dn = name.DomainName( name.Label(owner_b32), *self.zone.labels) log.debug3('found uncovered dn: ', str(dn), '; hashed: ', str(hashed_dn)) return dn,dn_hash self.stats['tested_hashes'] += len(self._prehash_list) hashes, label_counter_state = self._hash_queues.next().recv() if self._label_counter_state < label_counter_state: self._label_counter_state = label_counter_state self._prehash_list = hashes self._prehash_iter = iter(hashes) log.update() if break_early: return None,None
def forget(self): """Turn a 'target' file back into a 'source' file.""" debug3('forget(%s)\n', self.name) unlink(self.tmpfilename('deps'))
def isdirty(f, depth, expect_stamp): assert(isinstance(expect_stamp, state.Stamp)) debug('%s?%s\n', depth, f.name) debug3('%sexpect: %r\n', depth, expect_stamp) debug3('%sold: %r\n', depth, f.stamp) if not f.is_generated and expect_stamp.is_none() and f.exists(): debug('%s-- CLEAN (static)\n', depth) return CLEAN if f.exitcode: debug('%s-- DIRTY (failed last time)\n', depth) return DIRTY if not expect_stamp.is_missing() and f.stamp.is_missing() and not f.stamp.runid(): debug('%s-- DIRTY (never built)\n', depth) return DIRTY if f.stamp.is_old(): debug('%s-- DIRTY (from old redo)\n', depth) return DIRTY if not f.stamp or f.stamp.is_none(): debug('%s-- DIRTY (no stamp)\n', depth) return DIRTY newstamp = f.read_stamp() debug3('%snew: %r\n', depth, newstamp) if newstamp.is_override_or_missing(f) and not newstamp.is_missing(): if vars.OVERWRITE: debug('%s-- DIRTY (override)\n', depth) return DIRTY else: debug('%s-- CLEAN (override)\n', depth) return CLEAN if newstamp.is_stamp_dirty(f): if newstamp.is_missing(): debug('%s-- DIRTY (missing)\n', depth) else: debug('%s-- DIRTY (mtime)\n', depth) return [f] if f.stamp.is_csum() else DIRTY must_build = [] for stamp2, f2 in f.deps: dirty = CLEAN if f2 == state.ALWAYS: if f.stamp_mtime >= vars.RUNID: # has already been checked during this session debug('%s-- CLEAN (always, checked)\n', depth) else: debug('%s-- DIRTY (always)\n', depth) dirty = DIRTY else: f2 = state.File(f2, f.dir) sub = isdirty(f2, depth = depth + ' ', expect_stamp = stamp2) if sub: debug('%s-- DIRTY (sub)\n', depth) dirty = sub if not f.stamp.is_csum(): # f is a "normal" target: dirty f2 means f is instantly dirty if dirty: # if dirty==DIRTY, this means f is definitely dirty. # if dirty==[...], it's a list of the uncertain children. return dirty else: # f is "checksummable": dirty f2 means f needs to redo, # but f might turn out to be clean after that (ie. our parent # might not be dirty). if dirty == DIRTY: # f2 is definitely dirty, so f definitely needs to # redo. However, after that, f might turn out to be # unchanged. return [f] elif isinstance(dirty, list): # our child f2 might be dirty, but it's not sure yet. It's # given us a list of targets we have to redo in order to # be sure. must_build += dirty if must_build: # f is *maybe* dirty because at least one of its children is maybe # dirty. must_build has accumulated a list of "topmost" uncertain # objects in the tree. If we build all those, we can then # redo-ifchange f and it won't have any uncertainty next time. return must_build if expect_stamp.is_dirty(f): # This must be after we checked the children. Before, we didn't knew # if the current target was dirty or not debug('%s-- DIRTY (parent)\n', depth) return DIRTY # if we get here, it's because the target is clean debug2('%s-- CLEAN (dropped off)\n', depth) return CLEAN
def refresh(self): if self.name == ALWAYS: self.stamp_mtime = str(vars.RUNID) self.exitcode = 0 self.deps = [] self.is_generated = True self.stamp = Stamp(str(vars.RUNID)) return assert(not self.name.startswith('/')) try: # read the state file f = open(self.tmpfilename('deps')) except IOError: try: # okay, check for the file itself st = os.stat(self.name) except OSError: # it doesn't exist at all yet self.stamp_mtime = 0 # no stamp file self.exitcode = 0 self.deps = [] self.stamp = Stamp(STAMP_MISSING) self.runid = None self.is_generated = True else: # it's a source file (without a .deps file) self.stamp_mtime = 0 # no stamp file self.exitcode = 0 self.deps = [] self.is_generated = False self.stamp = self.read_stamp(st=st) self.runid = self.stamp.runid() else: # it's a target (with a .deps file) st = os.fstat(f.fileno()) lines = f.read().strip().split('\n') version = None device = None inode = None try: version = lines.pop(0) device, inode = [int(i) for i in lines.pop(0).split(" ")] except: pass if version != DEPSFILE_TAG or device != st.st_dev or inode != st.st_ino: # It is an old .deps file, consider it missing self.stamp_mtime = 0 # no stamp file self.exitcode = 0 self.deps = [] self.stamp = Stamp(STAMP_OLD) self.runid = None self.is_generated = True else: # Read .deps file self.stamp_mtime = int(st.st_mtime) self.exitcode = int(lines.pop(-1)) self.is_generated = True self.stamp = Stamp(lines.pop(-1)) if not self.exists() and not self.stamp.is_missing(): #debug3('deleted %s (mark as missing)\n', self.name) #self.stamp.stamp = STAMP_MISSING debug3('deleted %s (delete .deps and rescan)\n', self.name) unlink(self.tmpfilename('deps')) return self.refresh() self.runid = self.stamp.runid() self.deps = [line.split(' ', 1) for line in lines] # if the next line fails, it means that the .dep file is not # correctly formatted while self.deps and self.deps[-1][1] == '.': # a line added by redo-stamp self.stamp.csum = self.deps.pop(-1)[0] for i in range(len(self.deps)): self.deps[i][0] = Stamp(auto_detect=self.deps[i][0])