def build(f, any_errors, should_build, add_dep_to=None, delegate=None, re_do=True):
if f.dolock():
if f.check_deadlocks():
err("%s: recursive dependency, breaking deadlock\n", f.printable_name())
any_errors[0] += 1
any_errors[1] += 1
else:
jwack.get_token(f)
f.dolock().waitlock()
if any_errors[0] and not vars.KEEP_GOING:
return False
f.refresh()
debug3('think about building %r\n', f.name)
dirty = should_build(f)
while dirty and dirty != deps.DIRTY:
# FIXME: bring back the old (targetname) notation in the output
# when we need to do this. And add comments.
for t2 in dirty:
if not build(t2, any_errors, should_build, delegate, re_do):
return False
jwack.wait_all()
dirty = should_build(f)
if dirty:
job = BuildJob(f, any_errors, add_dep_to, delegate, re_do)
add_dep_to = None
job.schedule_job()
else:
f.dolock().unlock()
if add_dep_to:
f.refresh()
add_dep_to.add_dep(f)
return True
def add_dep(self, mode, dep):
src = File(name=dep)
debug3('add-dep: "%s" < %s "%s"\n' % (self.name, mode, src.name))
assert(self.id != src.id)
_write("insert or replace into Deps "
" (target, mode, source, delete_me) values (?,?,?,?)",
[self.id, mode, src.id, False])
def build(self):
debug3('running build job for %r\n', self.target.name)
(dodir, dofile, basedir, basename, ext) = (
self.dodir, self.dofile, self.dobasedir, self.dobasename, self.doext)
# this will run in the dofile's directory, so use only basenames here
if vars.OLD_ARGS:
arg1 = basename # target name (no extension)
arg2 = ext # extension (if any), including leading dot
else:
arg1 = basename + ext # target name (including extension)
arg2 = basename # target name (without extension)
argv = ['sh', '-e',
dofile,
arg1,
arg2,
# temp output file name
os.path.relpath(self.tmpname_arg3, dodir),
]
if vars.VERBOSE: argv[1] += 'v'
if vars.XTRACE: argv[1] += 'x'
if vars.VERBOSE or vars.XTRACE: log_e('\n')
firstline = open(os.path.join(dodir, dofile)).readline().strip()
if firstline.startswith('#!.../'):
_, _, interp_argv = firstline.partition("/")
interp_argv = interp_argv.split(' ')
interpreter = _find_interpreter(self.dodir, interp_argv[0])
if not interpreter:
err('%s unable to find interpreter %s.\n', self.dofile, interp_argv[0])
os._exit(208)
self.target.add_dep(state.File(interpreter))
argv[0:2] = [interpreter] + interp_argv[1:]
elif firstline.startswith('#!/'):
argv[0:2] = firstline[2:].split(' ')
log('%s\n', self.target.printable_name())
log_cmd("redo", self.target.name + "\n")
try:
dn = dodir
os.environ['REDO_PWD'] = os.path.join(vars.PWD, dn)
os.environ['REDO_TARGET'] = basename + ext
os.environ['REDO_DEPTH'] = vars.DEPTH + ' '
if dn:
os.chdir(dn)
l = logger.Logger(self.log_fd, self.tmp_sout_fd)
l.fork()
os.close(self.tmp_sout_fd)
close_on_exec(1, False)
if vars.VERBOSE or vars.XTRACE: log_e('* %s\n' % ' '.join(argv))
os.execvp(argv[0], argv)
except:
import traceback
sys.stderr.write(traceback.format_exc())
err('internal exception - see above\n')
raise
finally:
# returns only if there's an exception (exec in other case)
os._exit(127)
def _add(self, line):
depsname = self.tmpfilename('deps2')
debug3('_add(%s) to %r\n', line, depsname)
#assert os.path.exists(depsname)
line = str(line)
assert('\n' not in line)
with open(depsname, 'a') as f:
f.write(line + '\n')
def add_dep(self, mode, dep):
src = File(name=dep)
debug3('add-dep: "%s" < %s "%s"\n' % (self.name, mode, src.name))
assert (self.id != src.id)
_write(
"insert or replace into Deps "
" (target, mode, source, delete_me) values (?,?,?,?)",
[self.id, mode, src.id, False])
def build_starting(self):
"""Call this when you're about to start building this target."""
if vars.TARGET:
with open(self.tmpfilename('parent'), "w") as f:
f.write(os.path.relpath(vars.TARGET, self.dir))
depsname = self.tmpfilename('deps2')
debug3('build starting: %r\n', depsname)
unlink(depsname)
def build_done(self, exitcode):
"""Call this when you're done building this target."""
depsname = self.tmpfilename('deps2')
debug3('build ending: %r\n', depsname)
self._add(self.read_stamp(runid=vars.RUNID).stamp)
self._add(exitcode)
os.utime(depsname, (vars.RUNID, vars.RUNID))
os.rename(depsname, self.tmpfilename('deps'))
unlink(self.tmpfilename('parent'))
def prepare(self):
assert self.target.dolock().owned == state.LOCK_EX
self.target.build_starting()
self.before_t = _try_stat(self.target.name)
newstamp = self.target.read_stamp()
if newstamp.is_override_or_missing(self.target):
if newstamp.is_missing():
# was marked generated, but is now deleted
debug3('oldstamp=%r newstamp=%r\n', self.target.stamp, newstamp)
self.target.forget()
self.target.refresh()
elif vars.OVERWRITE:
warn('%s: you modified it; overwrite\n', self.target.printable_name())
else:
warn('%s: you modified it; skipping\n', self.target.printable_name())
return 0
if self.target.exists_not_dir() and not self.target.is_generated:
# an existing source file that was not generated by us.
# This step is mentioned by djb in his notes.
# For example, a rule called default.c.do could be used to try
# to produce hello.c, but we don't want that to happen if
# hello.c was created in advance by the end user.
if vars.OVERWRITE:
warn('%s: exists and not marked as generated; overwrite.\n',
self.target.printable_name())
else:
warn('%s: exists and not marked as generated; not redoing.\n',
self.target.printable_name())
debug2('-- static (%r)\n', self.target.name)
return 0
(self.dodir, self.dofile, self.dobasedir, self.dobasename, self.doext) = _find_do_file(self.target)
if not self.dofile:
if newstamp.is_missing():
err('no rule to make %r\n', self.target.name)
return 1
else:
self.target.forget()
debug2('-- forget (%r)\n', self.target.name)
return 0 # no longer a generated target, but exists, so ok
self.outdir = self._mkoutdir()
# name connected to stdout
self.tmpname_sout = self.target.tmpfilename('out.tmp')
# name provided as $3
self.tmpname_arg3 = os.path.join(self.outdir, self.target.basename())
# name for the log file
unlink(self.tmpname_sout)
unlink(self.tmpname_arg3)
self.log_fd = logger.open_log(self.target, truncate=True)
self.tmp_sout_fd = os.open(self.tmpname_sout, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666)
close_on_exec(self.tmp_sout_fd, True)
self.tmp_sout_f = os.fdopen(self.tmp_sout_fd, 'w+')
return None
def build_starting(self):
"""Call this when you're about to start building this target."""
if vars.TARGET:
with open(self.tmpfilename('parent'), "w") as f:
f.write(os.path.relpath(vars.TARGET, self.dir))
depsname = self.tmpfilename('deps2')
debug3('build starting: %r\n', depsname)
unlink(depsname)
with open(depsname, 'a') as f:
f.write(DEPSFILE_TAG + '\n')
st = os.fstat(f.fileno())
f.write('%d %d\n' % (st.st_dev, st.st_ino))
def add_dep(self, file):
"""Mark the given File() object as a dependency of this target.
The filesystem file it refers to may or may not exist. If it doesn't
exist, creating the file is considered a "modified" event and will
result in this target being rebuilt.
"""
if file.name == ALWAYS:
relname = file.name
else:
relname = os.path.relpath(file.name, self.dir)
debug3('add-dep: %r < %r %r\n', self.name, file.stamp, relname)
assert('\n' not in file.name)
assert(isinstance(file.stamp, Stamp))
self._add('%s %s' % (file.stamp.csum_or_stamp(), relname))
def _find_uncovered_dn(self, generator, break_early=False):
is_covered = self.nsec3_chain.covers
while True:
for ptlabel,dn_hash in self._prehash_iter:
if not is_covered(dn_hash):
dn = name.DomainName(name.Label(ptlabel), *self.zone.labels)
owner_b32 = util.base32_ext_hex_encode( dn_hash).lower()
hashed_dn = name.DomainName( name.Label(owner_b32), *self.zone.labels)
log.debug3('found uncovered dn: ', str(dn), '; hashed: ', str(hashed_dn))
return dn,dn_hash
self.stats['tested_hashes'] += len(self._prehash_list)
hashes, label_counter_state = self._hash_queues.next().recv()
if self._label_counter_state < label_counter_state:
self._label_counter_state = label_counter_state
self._prehash_list = hashes
self._prehash_iter = iter(hashes)
log.update()
if break_early:
return None,None
def forget(self):
"""Turn a 'target' file back into a 'source' file."""
debug3('forget(%s)\n', self.name)
unlink(self.tmpfilename('deps'))
def isdirty(f, depth, expect_stamp):
assert(isinstance(expect_stamp, state.Stamp))
debug('%s?%s\n', depth, f.name)
debug3('%sexpect: %r\n', depth, expect_stamp)
debug3('%sold: %r\n', depth, f.stamp)
if not f.is_generated and expect_stamp.is_none() and f.exists():
debug('%s-- CLEAN (static)\n', depth)
return CLEAN
if f.exitcode:
debug('%s-- DIRTY (failed last time)\n', depth)
return DIRTY
if not expect_stamp.is_missing() and f.stamp.is_missing() and not f.stamp.runid():
debug('%s-- DIRTY (never built)\n', depth)
return DIRTY
if f.stamp.is_old():
debug('%s-- DIRTY (from old redo)\n', depth)
return DIRTY
if not f.stamp or f.stamp.is_none():
debug('%s-- DIRTY (no stamp)\n', depth)
return DIRTY
newstamp = f.read_stamp()
debug3('%snew: %r\n', depth, newstamp)
if newstamp.is_override_or_missing(f) and not newstamp.is_missing():
if vars.OVERWRITE:
debug('%s-- DIRTY (override)\n', depth)
return DIRTY
else:
debug('%s-- CLEAN (override)\n', depth)
return CLEAN
if newstamp.is_stamp_dirty(f):
if newstamp.is_missing():
debug('%s-- DIRTY (missing)\n', depth)
else:
debug('%s-- DIRTY (mtime)\n', depth)
return [f] if f.stamp.is_csum() else DIRTY
must_build = []
for stamp2, f2 in f.deps:
dirty = CLEAN
if f2 == state.ALWAYS:
if f.stamp_mtime >= vars.RUNID:
# has already been checked during this session
debug('%s-- CLEAN (always, checked)\n', depth)
else:
debug('%s-- DIRTY (always)\n', depth)
dirty = DIRTY
else:
f2 = state.File(f2, f.dir)
sub = isdirty(f2, depth = depth + ' ',
expect_stamp = stamp2)
if sub:
debug('%s-- DIRTY (sub)\n', depth)
dirty = sub
if not f.stamp.is_csum():
# f is a "normal" target: dirty f2 means f is instantly dirty
if dirty:
# if dirty==DIRTY, this means f is definitely dirty.
# if dirty==[...], it's a list of the uncertain children.
return dirty
else:
# f is "checksummable": dirty f2 means f needs to redo,
# but f might turn out to be clean after that (ie. our parent
# might not be dirty).
if dirty == DIRTY:
# f2 is definitely dirty, so f definitely needs to
# redo. However, after that, f might turn out to be
# unchanged.
return [f]
elif isinstance(dirty, list):
# our child f2 might be dirty, but it's not sure yet. It's
# given us a list of targets we have to redo in order to
# be sure.
must_build += dirty
if must_build:
# f is *maybe* dirty because at least one of its children is maybe
# dirty. must_build has accumulated a list of "topmost" uncertain
# objects in the tree. If we build all those, we can then
# redo-ifchange f and it won't have any uncertainty next time.
return must_build
if expect_stamp.is_dirty(f):
# This must be after we checked the children. Before, we didn't knew
# if the current target was dirty or not
debug('%s-- DIRTY (parent)\n', depth)
return DIRTY
# if we get here, it's because the target is clean
debug2('%s-- CLEAN (dropped off)\n', depth)
return CLEAN
def refresh(self):
if self.name == ALWAYS:
self.stamp_mtime = str(vars.RUNID)
self.exitcode = 0
self.deps = []
self.is_generated = True
self.stamp = Stamp(str(vars.RUNID))
return
assert(not self.name.startswith('/'))
try:
# read the state file
f = open(self.tmpfilename('deps'))
except IOError:
try:
# okay, check for the file itself
st = os.stat(self.name)
except OSError:
# it doesn't exist at all yet
self.stamp_mtime = 0 # no stamp file
self.exitcode = 0
self.deps = []
self.stamp = Stamp(STAMP_MISSING)
self.runid = None
self.is_generated = True
else:
# it's a source file (without a .deps file)
self.stamp_mtime = 0 # no stamp file
self.exitcode = 0
self.deps = []
self.is_generated = False
self.stamp = self.read_stamp(st=st)
self.runid = self.stamp.runid()
else:
# it's a target (with a .deps file)
st = os.fstat(f.fileno())
lines = f.read().strip().split('\n')
version = None
device = None
inode = None
try:
version = lines.pop(0)
device, inode = [int(i) for i in lines.pop(0).split(" ")]
except: pass
if version != DEPSFILE_TAG or device != st.st_dev or inode != st.st_ino:
# It is an old .deps file, consider it missing
self.stamp_mtime = 0 # no stamp file
self.exitcode = 0
self.deps = []
self.stamp = Stamp(STAMP_OLD)
self.runid = None
self.is_generated = True
else:
# Read .deps file
self.stamp_mtime = int(st.st_mtime)
self.exitcode = int(lines.pop(-1))
self.is_generated = True
self.stamp = Stamp(lines.pop(-1))
if not self.exists() and not self.stamp.is_missing():
#debug3('deleted %s (mark as missing)\n', self.name)
#self.stamp.stamp = STAMP_MISSING
debug3('deleted %s (delete .deps and rescan)\n', self.name)
unlink(self.tmpfilename('deps'))
return self.refresh()
self.runid = self.stamp.runid()
self.deps = [line.split(' ', 1) for line in lines]
# if the next line fails, it means that the .dep file is not
# correctly formatted
while self.deps and self.deps[-1][1] == '.':
# a line added by redo-stamp
self.stamp.csum = self.deps.pop(-1)[0]
for i in range(len(self.deps)):
self.deps[i][0] = Stamp(auto_detect=self.deps[i][0])