def stack_container(container, config):
announce("Stacking " + container + " container")
if config.has_key('run') and not config['run']:
note("container has 'run' key set to " + str(config['run']) +
", skipping")
return
command = ['docker', 'run', '-d', '--name', get_container(container)]
if config.has_key('options'):
for option in config['options']:
command = command + [parse_template(option)]
if config.has_key('hostname'):
command = command + ['--hostname', get_value(config, 'hostname')]
# else:
# command = command + ['--hostname', container]
if config.has_key('dns'):
command = command + ['--dns', get_value(config, 'dns')]
if config.has_key('volumes'):
for volumespec in config['volumes']:
volumespec_parsed = parse_template(volumespec)
if volumespec_parsed.startswith("/"):
command = command + ['-v', volumespec_parsed]
else:
command = command + [
'-v', mem.VolumePath + "/" + mem.Project + "/" +
container + "/" + volumespec_parsed
]
if config.has_key('ports'):
for portspec in config['ports']:
command = command + ['-p', parse_template(portspec)]
if config.has_key('environment'):
for envvar in config['environment']:
command = command + [
'-e',
parse_template(envvar) + '=' +
parse_template(config['environment'][envvar])
]
if config.has_key('links'):
for link in config['links']:
command = command + [
"--link=" + get_container(parse_template(link))
]
command.append(get_image(config))
if config.has_key('command'):
command = command + shlex.split(get_value(config, 'command'))
note(" ".join(command))
quietcall(command)
if config.has_key('upwhen'):
wait_for_up(container, config)
def action_ls(name, args):
"""list available build names or builds"""
parser = OptionParser("""\
usage: %%prog %s [build-name]
With no arguments, list the available build names on 'llvmlab'. With a build
name, list the available builds for that builder.\
""" % name)
(opts, args) = parser.parse_args(args)
if not len(args):
available_buildnames = llvmlab.fetch_builders()
available_buildnames.sort()
for item in available_buildnames:
print item
return available_buildnames
for name in args:
if len(args) > 1:
if name is not args[0]:
print
print '%s:' % name
available_builds = list(llvmlab.fetch_builds(name))
available_builds.sort()
available_builds.reverse()
for build in available_builds:
print build.tobasename(include_suffix=False)
min_rev = min([x.revision for x in available_builds])
max_rev = max([x.revision for x in available_builds])
note("Summary: found {} builds: r{}-r{}".format(
len(available_builds), min_rev, max_rev))
return available_builds
def execute(self, verbose=False):
if verbose:
note('executing: %s' % ' '.join("'%s'" % arg
for arg in self.command))
start_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
start_time = time.time()
p = subprocess.Popen(self.command,
stdout=open(self.stdout_path, 'w'),
stderr=open(self.stderr_path, 'w'),
env=self.env)
self.result = p.wait() == 0
end_time = time.time()
end_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
self.metrics["user_time"] = end_rusage.ru_utime - start_rusage.ru_utime
self.metrics["sys_time"] = end_rusage.ru_stime - start_rusage.ru_stime
self.metrics["wall_time"] = end_time - start_time
if verbose:
note("command executed in -- "
"user: %.4fs, wall: %.4fs, sys: %.4fs" %
(self.metrics["user_time"], self.metrics["wall_time"],
self.metrics["sys_time"]))
def action_ls(name, args):
"""list available build names or builds"""
parser = OptionParser("""\
usage: %%prog %s [build-name]
With no arguments, list the available build names on 'llvmlab'. With a build
name, list the available builds for that builder.\
""" % name)
(opts, args) = parser.parse_args(args)
if not len(args):
available_buildnames = llvmlab.fetch_builders()
available_buildnames.sort()
for item in available_buildnames:
print item
return available_buildnames
for name in args:
if len(args) > 1:
if name is not args[0]:
print
print '%s:' % name
available_builds = list(llvmlab.fetch_builds(name))
available_builds.sort()
available_builds.reverse()
for build in available_builds:
print build.tobasename(include_suffix=False)
min_rev = min([x.revision for x in available_builds])
max_rev = max([x.revision for x in available_builds])
note("Summary: found {} builds: r{}-r{}".format(len(available_builds),
min_rev, max_rev))
return available_builds
def execute(self, verbose=False):
if verbose:
note('executing: %s' % ' '.join("'%s'" % arg
for arg in self.command))
start_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
start_time = time.time()
p = subprocess.Popen(self.command,
stdout=open(self.stdout_path, 'w'),
stderr=open(self.stderr_path, 'w'),
env=self.env)
self.result = p.wait() == 0
end_time = time.time()
end_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
self.metrics["user_time"] = end_rusage.ru_utime - start_rusage.ru_utime
self.metrics["sys_time"] = end_rusage.ru_stime - start_rusage.ru_stime
self.metrics["wall_time"] = end_time - start_time
if verbose:
note("command executed in -- "
"user: %.4fs, wall: %.4fs, sys: %.4fs" % (
self.metrics["user_time"], self.metrics["wall_time"],
self.metrics["sys_time"]))
def unstack_container(container):
announce("Unstacking " + container + " container")
command = ['docker', 'stop', get_container(container)]
note(" ".join(command))
quietcall(command)
command = ['docker', 'rm', '--force', get_container(container)]
note(" ".join(command))
quietcall(command)
def unstack_container(container):
announce("Unstacking " + container + " container")
command = ['docker', 'stop', get_container(container)]
note(" ".join(command))
quietcall(command)
command = ['docker', 'rm', '--force', get_container(container)]
note(" ".join(command))
quietcall(command)
def enter_container(commands):
which = commands.popleft()
run = ["/bin/bash"]
if len(commands) > 0:
run = list(commands)
announce("Entering '" + which + "' container and running: " + str(run))
command = ['docker', 'exec', '-ti', get_container(which)] + run
note(" ".join(command))
call(command)
def enter_container(commands):
which = commands.popleft()
run = ["/bin/bash"]
if len(commands) > 0:
run = list(commands)
announce("Entering '" + which + "' container and running: " + str(run))
command = ['docker', 'exec', '-ti', get_container(which)] + run
note(" ".join(command))
call(command)
def stack_container(container, config):
announce("Stacking " + container + " container")
if config.has_key('run') and not config['run']:
note("container has 'run' key set to " + str(config['run']) + ", skipping")
return
command = ['docker', 'run', '-d', '--name', get_container(container)]
if config.has_key('options'):
for option in config['options']:
command = command + [parse_template(option)]
if config.has_key('hostname'):
command = command + ['--hostname', get_value(config, 'hostname')]
# else:
# command = command + ['--hostname', container]
if config.has_key('dns'):
command = command + ['--dns', get_value(config, 'dns')]
if config.has_key('volumes'):
for volumespec in config['volumes']:
volumespec_parsed = parse_template(volumespec)
if volumespec_parsed.startswith("/"):
command = command + ['-v', volumespec_parsed]
elif volumespec_parsed.startswith('./'):
# docker complains if path starts with "." so expand CWD here
cwd = os.path.dirname(os.path.realpath(volumespec_parsed.split(':')[0]))
realpath = cwd + '/' + volumespec_parsed.strip('./')
command = command + ['-v', realpath]
else:
command = command + ['-v', mem.VolumePath + '/' + mem.Project +
'/' + container + '/' + volumespec_parsed]
if config.has_key('ports'):
for portspec in config['ports']:
command = command + ['-p', parse_template(portspec)]
if config.has_key('environment'):
for envvar in config['environment']:
command = command + ['-e', parse_template(envvar) + '=' + parse_template(config['environment'][envvar])]
if config.has_key('links'):
for link in config['links']:
command = command + ["--link=" + get_container(parse_template(link))]
command.append(get_image(config))
if config.has_key('command'):
command = command + shlex.split(get_value(config, 'command'))
note(" ".join(command))
quietcall(command)
if config.has_key('upwhen'):
wait_for_up(container, config)
def proc(args):
langs = [l for l in args.langs.split(",") if l != ""]
c = Classifier(folder=args.model_dir, langs=langs)
if args.u is not None:
c.DROP_RATIO = args.u
if args.verbose:
util.note("Drop ratio: {}".format(c.DROP_RATIO))
if args.s:
for line in sys.stdin:
print(c.classify(line.decode('utf-8'), verbose=args.verbose))
else:
print(c.classify(sys.stdin.read().decode('utf-8'),
verbose=args.verbose))
def get_langs(self, langs=[]):
if langs == []:
return self.langs
else:
langs = set(langs)
active_langs = self.langs & langs
if len(langs) != len(active_langs):
missing = langs - active_langs - self.langs_warned
if missing:
# only warn once per lang
self.langs_warned.update(missing)
util.note("WARNING: No language model for {}".format(
"/".join(missing)))
return active_langs
def freq_of_text_file(self, fil):
freq = {}
for nl, strline in enumerate(fil.readlines()):
try:
line = strline.decode('utf-8')
except UnicodeDecodeError as e:
if self.unicode_warned == 0:
util.note(
"WARNING: Line {} gave {}, skipping ... "
"(not warning again)".format(nl, e))
self.unicode_warned += 1
continue
freq = self.freq_of_text(line, freq)
if self.unicode_warned != 0:
util.note("Saw {} UnicodeDecodeErrors".format(self.unicode_warned))
return freq
def __init__(self, folder=None, langs=[], verbose=False):
if folder is None:
folder = os.path.join(here, 'lm')
self.cmodels = {}
self.wmodels = {}
ext = '.lm'
fnames = []
folder_glob = os.path.join(folder, '*' + ext)
found_fnames = glob.glob(os.path.normcase(folder_glob))
if len(found_fnames) == 0:
raise ValueError("No language files found in %s" % (folder,))
if len(langs) == 0:
fnames = found_fnames
else:
fnames = [os.path.join(folder, lang + ext) for lang in langs]
not_found = set(fnames) - set(found_fnames)
if len(not_found) != 0:
raise ValueError(
"Unknown language(s): " + ", ".join(not_found))
for fname in fnames:
lang = util.basename_noext(fname, ext)
self.cmodels[lang] = CharModel(lang).of_model_file(
open(fname, 'r'), fname)
if verbose:
util.note("Loaded %s" % (fname,))
fname_wm = os.path.join(folder, lang + '.wm')
# fname_wmgz = os.path.join(folder, lang+'.wm.gz')
if os.path.exists(fname_wm):
self.wmodels[lang] = WordModel(lang).of_model_file(
open(fname_wm, 'r'), fname_wm)
if verbose:
util.note("Loaded %s" % (fname_wm,))
else:
self.wmodels[lang] = WordModel(lang).of_freq({})
if len(self.cmodels) == 0:
raise ValueError("No character models created!")
else:
self.langs = set(self.cmodels.keys())
self.langs_warned = set()
def __init__(self, folder, exts=['.txt', '.txt.gz'], Model=CharModel,
verbose=False):
self.models = {}
for ext in exts:
files = glob.glob(os.path.normcase(os.path.join(folder, '*' + ext)))
for fname in files:
if verbose:
msg = "Processing %s" % (fname,)
if os.path.getsize(fname) > 5000000:
msg += " (this may take a while)"
util.note(msg)
sys.stderr.flush()
lang = util.basename_noext(fname, ext)
self.models[lang] = Model(lang).of_text_file(
self.open_corpus(fname))
if len(self.models) == 0:
raise Exception(
"No suitable files found matching {}/*.{}{}{}!".format(
folder, "{", ",".join(exts), "}"))
def fill_container(container, config):
announce("Filling " + container + " container")
if config.has_key('build'):
build = get_value(config, 'build')
tag = mem.Project + "/" + build
if config.has_key('tag'):
tag = get_value(config, 'tag')
target = build
if build[:4] == 'git:':
if not config.has_key('tag'):
raise Exception(
"If you are using a git repo for 'build' you must also specify 'tag'"
)
tag = get_value(config, 'tag')
url = build[4:]
print("Cloning " + tag + " from " + url, end="")
dir = "_grua_" + tag.replace("/", "_")
if os.path.isdir(dir):
shutil.rmtree(dir)
command = ['git', 'clone', url, dir]
note(" ".join(command))
call(command)
target = dir
mention("building " + container + " ( " + target + " ) " +
" with tag '" + tag + "'")
command = ['docker', 'build', '-t', tag, target]
note(" ".join(command))
quietcall(command)
else:
mention(container + " uses an image. Pulling " +
get_value(config, 'image'))
command = ['docker', 'pull', get_value(config, 'image')]
note(" ".join(command))
quietcall(command)
def fill_container(container, config):
announce("Filling " + container + " container")
if config.has_key('build'):
build = get_value(config, 'build')
tag = mem.Project + "/" + build
if config.has_key('tag'):
tag = get_value(config, 'tag')
target = build
if build[:4] == 'git:':
if not config.has_key('tag'):
raise Exception("If you are using a git repo for 'build' you must also specify 'tag'")
tag = get_value(config, 'tag')
url = build[4:]
print ( "Cloning " + tag + " from " + url, end="" )
dir = "_grua_" + tag.replace("/", "_")
if os.path.isdir(dir):
shutil.rmtree(dir)
command = ['git', 'clone', url, dir]
note(" ".join(command))
call(command)
target = dir
mention("building " + container + " ( " + target + " ) " + " with tag '" + tag + "'")
command = ['docker', 'build', '-t', tag, target]
note(" ".join(command))
quietcall(command)
else:
mention(container + " uses an image. Pulling " + get_value(config, 'image'))
command = ['docker', 'pull', get_value(config, 'image')]
note(" ".join(command))
quietcall(command)
def classify_full(self, intext, langs=[], verbose=False):
active_langs = self.get_langs(langs)
text = ensure_unicode(intext)
ingram = CharModel().of_text(text)
cscored = {l: model.compare(ingram)
for l, model in self.cmodels.iteritems()
if l in active_langs}
cranked = util.sort_by_value(cscored)
cbest = cranked[0]
cfiltered = {l: d for l, d in cranked
if d <= cbest[1] * self.DROP_RATIO}
if len(cfiltered) <= 1:
if verbose:
util.note("lm gave: {} as only result for input: {}".format(
cfiltered, text))
return list(cfiltered.iteritems())
else:
# Along with compare_tc, implements text_cat.pl line
# 442 and on:
wscored = {l: model.compare_tc(text, cscored[l])
for l, model in self.wmodels.iteritems()
if l in cfiltered}
cwcombined = {l: (cscored[l] - wscore)
for l, wscore in wscored.iteritems()}
cwranked = util.sort_by_value(cwcombined)
if verbose:
if cranked[:len(cwranked)] == cwranked:
util.note(
"lm gave: {}\t\twm gave no change\t\tfor"
"input: {}".format(
pretty_tbl(cranked), text))
else:
util.note(
"lm gave: {}\t\twm-weighted to: "
"{}\t\tfor input: {}".format(
pretty_tbl(cranked), pretty_tbl(cwranked), text))
return cwranked
def action_fetch(name, args):
"""fetch a build from the server"""
parser = OptionParser("""\
usage: %%prog %(name)s [options] builder [build-name]
Fetch the build from the named builder which matchs build-name. If no match is
found, get the first build before the given name. If no build name is given,
the most recent build is fetched.
The available builders can be listed using:
%%prog ls
The available builds can be listed using:
%%prog ls builder""" % locals())
parser.add_option("-f",
"--force",
dest="force",
help=("always download and extract, overwriting any"
"existing files"),
action="store_true",
default=False)
parser.add_option("",
"--update-link",
dest="update_link",
metavar="PATH",
help=("update a symbolic link at PATH to point to the "
"fetched build (on success)"),
action="store",
default=None)
parser.add_option("-d",
"--dry-run",
dest='dry_run',
help=("Perform all operations except the actual "
"downloading and extracting of any files"),
action="store_true",
default=False)
(opts, args) = parser.parse_args(args)
if len(args) == 0:
parser.error("please specify a builder name")
elif len(args) == 1:
builder, = args
build_name = None
elif len(args) == 2:
builder, build_name = args
else:
parser.error("invalid number of arguments")
builds = list(llvmlab.fetch_builds(builder))
if not builds:
parser.error("no builds for builder: %r" % builder)
build = get_best_match(builds, build_name)
if not build:
parser.error("no match for build %r" % build_name)
path = build.tobasename()
if build_name is not None and not path.startswith(build_name):
note('no exact match, fetching %r' % path)
# Get the paths to extract to.
root_path = path
builddir_path = build.tobasename(include_suffix=False)
if not opts.dry_run:
# Check that the download and extract paths are clean.
for p in (root_path, builddir_path):
if os.path.exists(p):
# If we are using --force, then clean the path.
if opts.force:
shutil.rmtree(p, ignore_errors=True)
continue
fatal('current directory is not clean, %r exists' % p)
llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path)
print 'downloaded root: %s' % root_path
print 'extracted path : %s' % builddir_path
# Update the symbolic link, if requested.
if not opts.dry_run and opts.update_link:
# Remove the existing path.
try:
os.unlink(opts.update_link)
except OSError as e:
if e.errno != errno.ENOENT:
fatal('unable to update symbolic link at %r, cannot unlink' %
(opts.update_link))
# Create the symbolic link.
os.symlink(os.path.abspath(builddir_path), opts.update_link)
print 'updated link at: %s' % opts.update_link
return os.path.abspath(builddir_path)
def action_bisect(name, args):
"""find first failing build using binary search"""
parser = OptionParser("""\
usage: %%prog %(name)s [options] ... test command args ...
Look for the first published build where a test failed, using the builds on
llvmlab. The command arguments are executed once per build tested, but each
argument is first subject to string interpolation. The syntax is
"%%(VARIABLE)FORMAT" where FORMAT is a standard printf format, and VARIABLE is
one of:
'sandbox' - the path to the sandbox directory.
'path' - the path to the build under test.
'revision' - the revision number of the build.
'build' - the build number of the build under test.
'clang' - the path to the clang binary of the build if it exists.
'clang++' - the path to the clang++ binary of the build if it exists.
'libltodir' - the path to the directory containing libLTO.dylib, if it
exists.
Each test is run in a sandbox directory. By default, sandbox directories are
temporary directories which are created and destroyed for each test (see
--sandbox).
For use in auxiliary test scripts, each test is also run with each variable
available in the environment as TEST_<variable name> (variables are converted
to uppercase). For example, a test script could use "TEST_PATH" to find the
path to the build under test.
The stdout and stderr of the command are logged to files inside the sandbox
directory. Use an explicit sandbox directory if you would like to look at
them.
It is possible to run multiple distinct commands for each test by separating
them in the command line arguments by '----'. The failure of any command causes
the entire test to fail.\
""" % locals())
parser.add_option("-b", "--build", dest="build_name", metavar="STR",
help="name of build to fetch",
action="store", default=DEFAULT_BUILDER)
parser.add_option("-s", "--sandbox", dest="sandbox",
help="directory to use as a sandbox",
action="store", default=None)
parser.add_option("-v", "--verbose", dest="verbose",
help="output more test notermation",
action="store_true", default=False)
parser.add_option("-V", "--very-verbose", dest="very_verbose",
help="output even more test notermation",
action="store_true", default=False)
parser.add_option("", "--show-output", dest="show_command_output",
help="display command output",
action="store_true", default=False)
parser.add_option("", "--single-step", dest="single_step",
help="single step instead of binary stepping",
action="store_true", default=False)
parser.add_option("", "--min-rev", dest="min_rev",
help="minimum revision to test",
type="int", action="store", default=None)
parser.add_option("", "--max-rev", dest="max_rev",
help="maximum revision to test",
type="int", action="store", default=None)
parser.disable_interspersed_args()
(opts, args) = parser.parse_args(args)
if opts.build_name is None:
parser.error("no build name given (see --build)")
# Very verbose implies verbose.
opts.verbose |= opts.very_verbose
start_time = time.time()
available_builds = list(llvmlab.fetch_builds(opts.build_name))
available_builds.sort()
available_builds.reverse()
if opts.very_verbose:
note("fetched builds in %.2fs" % (time.time() - start_time,))
if opts.min_rev is not None:
available_builds = [b for b in available_builds
if b.revision >= opts.min_rev]
if opts.max_rev is not None:
available_builds = [b for b in available_builds
if b.revision <= opts.max_rev]
def predicate(item):
# Run the sandboxed test.
test_result, _ = execute_sandboxed_test(
opts.sandbox, opts.build_name, item, args, verbose=opts.verbose,
very_verbose=opts.very_verbose,
show_command_output=opts.show_command_output or opts.very_verbose)
# Print status.
print '%s: %s' % (('FAIL', 'PASS')[test_result],
item.tobasename(include_suffix=False))
return test_result
if opts.single_step:
for item in available_builds:
if predicate(item):
break
else:
item = None
else:
if opts.min_rev is None or opts.max_rev is None:
# Gallop to find initial search range, under the assumption that we
# are most likely looking for something at the head of this list.
search_space = algorithm.gallop(predicate, available_builds)
else:
# If both min and max revisions are specified,
# don't gallop - bisect the given range.
search_space = available_builds
item = algorithm.bisect(predicate, search_space)
if item is None:
fatal('unable to find any passing build!')
print '%s: first working build' % item.tobasename(include_suffix=False)
index = available_builds.index(item)
if index == 0:
print 'no failing builds!?'
else:
print '%s: next failing build' % available_builds[index-1].tobasename(
include_suffix=False)
def save(self, folder, ext='.lm', verbose=False):
for lang, model in self.models.iteritems():
fname = os.path.join(folder, lang + ext)
model.to_model_file(open(fname, 'w'))
if verbose and len(self.models) != 0:
util.note("Wrote {%s}%s" % (",".join(self.models.keys()), ext))
def action_fetch(name, args):
"""fetch a build from the server"""
parser = OptionParser("""\
usage: %%prog %(name)s [options] builder [build-name]
Fetch the build from the named builder which matchs build-name. If no match is
found, get the first build before the given name. If no build name is given,
the most recent build is fetched.
The available builders can be listed using:
%%prog ls
The available builds can be listed using:
%%prog ls builder""" % locals())
parser.add_option("-f", "--force", dest="force",
help=("always download and extract, overwriting any"
"existing files"),
action="store_true", default=False)
parser.add_option("", "--update-link", dest="update_link", metavar="PATH",
help=("update a symbolic link at PATH to point to the "
"fetched build (on success)"),
action="store", default=None)
parser.add_option("-d", "--dry-run", dest='dry_run',
help=("Perform all operations except the actual "
"downloading and extracting of any files"),
action="store_true", default=False)
(opts, args) = parser.parse_args(args)
if len(args) == 0:
parser.error("please specify a builder name")
elif len(args) == 1:
builder, = args
build_name = None
elif len(args) == 2:
builder, build_name = args
else:
parser.error("invalid number of arguments")
builds = list(llvmlab.fetch_builds(builder))
if not builds:
parser.error("no builds for builder: %r" % builder)
build = get_best_match(builds, build_name)
if not build:
parser.error("no match for build %r" % build_name)
path = build.tobasename()
if build_name is not None and not path.startswith(build_name):
note('no exact match, fetching %r' % path)
# Get the paths to extract to.
root_path = path
builddir_path = build.tobasename(include_suffix=False)
if not opts.dry_run:
# Check that the download and extract paths are clean.
for p in (root_path, builddir_path):
if os.path.exists(p):
# If we are using --force, then clean the path.
if opts.force:
shutil.rmtree(p, ignore_errors=True)
continue
fatal('current directory is not clean, %r exists' % p)
llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path)
print 'downloaded root: %s' % root_path
print 'extracted path : %s' % builddir_path
# Update the symbolic link, if requested.
if not opts.dry_run and opts.update_link:
# Remove the existing path.
try:
os.unlink(opts.update_link)
except OSError as e:
if e.errno != errno.ENOENT:
fatal('unable to update symbolic link at %r, cannot unlink' % (
opts.update_link))
# Create the symbolic link.
os.symlink(os.path.abspath(builddir_path), opts.update_link)
print 'updated link at: %s' % opts.update_link
return os.path.abspath(builddir_path)
def execute_sandboxed_test(sandbox, builder, build, args,
verbose=False, very_verbose=False,
add_path_variables=True,
show_command_output=False,
reuse_sandbox=False):
def split_command_filters(command):
for i, arg in enumerate(command):
if arg[:2] != "%%" or arg[-2:] != "%%":
break
else:
fatal("invalid command: %s, only contains filter "
"specifications" % ("".join('"%s"' % a for a in command)))
return ([a[2:-2] for a in command[:i]],
command[i:])
path = build.tobasename(include_suffix=False)
fullpath = build.tobasename()
if verbose:
note('testing %r' % path)
# Create the sandbox directory, if it doesn't exist.
is_temp = False
if sandbox is None:
sandbox = tempfile.mkdtemp()
is_temp = True
else:
# Make absolute.
sandbox = os.path.abspath(sandbox)
if not os.path.exists(sandbox):
os.mkdir(sandbox)
# Compute paths and make sure sandbox is clean.
root_path = os.path.join(sandbox, fullpath)
builddir_path = os.path.join(sandbox, path)
need_build = True
if reuse_sandbox and (os.path.exists(root_path) and
os.path.exists(builddir_path)):
need_build = False
else:
for p in (root_path, builddir_path):
if os.path.exists(p):
fatal('sandbox is not clean, %r exists' % p)
# Fetch and extract the build.
if need_build:
start_time = time.time()
llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path)
if very_verbose:
note("extracted build in %.2fs" % (time.time() - start_time,))
# Attempt to find clang/clang++ in the downloaded build.
def find_binary(name):
x = subprocess.check_output(['find', builddir_path, '-name', name])\
.strip().split("\n")[0]
if x == '':
x = None
return x
clang_path = find_binary('clang')
clangpp_path = find_binary('clang++')
liblto_path = find_binary('libLTO.dylib')
if liblto_path is not None:
liblto_dir = os.path.dirname(liblto_path)
else:
liblto_dir = None
# Construct the interpolation variables.
options = {'sandbox': sandbox,
'path': builddir_path,
'revision': build.revision,
'build': build.build,
'clang': clang_path,
'clang++': clangpp_path,
'libltodir': liblto_dir}
# Inject environment variables.
env = os.environ.copy()
for key, value in options.items():
env['TEST_%s' % key.upper()] = str(value)
# Extend the environment to include the path to the extracted build.
#
# FIXME: Ideally, we would be able to read some kind of configuration
# notermation about a builder so that we could just set this up, it doesn't
# necessarily here as hard-coded notermation.
if add_path_variables:
path_extensions = []
dyld_library_path_extensions = []
toolchains_dir = os.path.join(builddir_path,
('Applications/Xcode.app/Contents/'
'Developer/Toolchains'))
toolchain_paths = []
if os.path.exists(toolchains_dir):
toolchain_paths = [os.path.join(toolchains_dir, name, 'usr')
for name in os.listdir(toolchains_dir)]
for package_root in ['', 'Developer/usr/'] + toolchain_paths:
p = os.path.join(builddir_path, package_root, 'bin')
if os.path.exists(p):
path_extensions.append(p)
p = os.path.join(builddir_path, package_root, 'lib')
if os.path.exists(p):
dyld_library_path_extensions.append(p)
if path_extensions:
env['PATH'] = os.pathsep.join(
path_extensions + [os.environ.get('PATH', '')])
if dyld_library_path_extensions:
env['DYLD_LIBRARY_PATH'] = os.pathsep.join(
dyld_library_path_extensions + [
os.environ.get('DYLD_LIBRARY_PATH', '')])
# Split the arguments into distinct commands.
#
# Extended command syntax allows running multiple commands by separating
# them with '----'.
test_commands = util.list_split(args, "----")
# Split command specs into filters and commands.
test_commands = [split_command_filters(spec) for spec in test_commands]
# Execute the test.
command_objects = []
interpolated_variables = False
for i, (filters, command) in enumerate(test_commands):
# Interpolate arguments.
old_command = command
command = [a % options for a in command]
if old_command != command:
interpolated_variables = True
# Create the command object...
stdout_log_path = os.path.join(sandbox, '%s.%d.stdout' % (path, i))
stderr_log_path = os.path.join(sandbox, '%s.%d.stderr' % (path, i))
cmd_object = Command(command, stdout_log_path, stderr_log_path, env)
command_objects.append(cmd_object)
# Execute the command.
try:
cmd_object.execute(verbose=verbose)
except OSError, e:
# Python's exceptions are horribly to read, and this one is
# incredibly common when people don't use the right syntax (or
# misspell something) when writing a predicate. Detect this and
# notify the user.
if e.errno == errno.ENOENT:
fatal("invalid command, executable doesn't exist: %r" % (
cmd_object.command[0],))
elif e.errno == errno.ENOEXEC:
fatal("invalid command, executable has a bad format. Did you "
"forget to put a #! at the top of a script?: %r"
% (cmd_object.command[0],))
else:
# Otherwise raise the error again.
raise e
# Evaluate the filters.
for filter in filters:
cmd_object.evaluate_filter_spec(filter)
if show_command_output:
for p, type in ((stdout_log_path, "stdout"),
(stderr_log_path, "stderr")):
if not os.path.exists(p):
continue
f = open(p)
data = f.read()
f.close()
if data:
print ("-- command %s (note: suppressed by default, "
"see sandbox dir for log files) --" % (type))
print "--\n%s--\n" % data
test_result = cmd_object.result
if not test_result:
break
f.close()
if data:
print ("-- command %s (note: suppressed by default, "
"see sandbox dir for log files) --" % (type))
print "--\n%s--\n" % data
test_result = cmd_object.result
if not test_result:
break
if not interpolated_variables:
warning('no substitutions found. Fetched root ignored?')
# Remove the temporary directory.
if is_temp:
if shell.execute(['rm', '-rf', sandbox]) != 0:
note('unable to remove sandbox dir %r' % path)
return test_result, command_objects
def get_best_match(builds, name, key=lambda x: x):
builds = list(builds)
builds.sort(key=key)
if name is None and builds:
return builds[-1]
to_find = llvmlab.Build.frombasename(name, None)
best = None
for item in builds:
def empty_container(container, config):
announce("Emptying image " + container)
command = ['docker', 'rmi', get_image(config)]
note(" ".join(command))
quietcall(command)
def action_bisect(name, args):
"""find first failing build using binary search"""
parser = OptionParser("""\
usage: %%prog %(name)s [options] ... test command args ...
Look for the first published build where a test failed, using the builds on
llvmlab. The command arguments are executed once per build tested, but each
argument is first subject to string interpolation. The syntax is
"%%(VARIABLE)FORMAT" where FORMAT is a standard printf format, and VARIABLE is
one of:
'sandbox' - the path to the sandbox directory.
'path' - the path to the build under test.
'revision' - the revision number of the build.
'build' - the build number of the build under test.
'clang' - the path to the clang binary of the build if it exists.
'clang++' - the path to the clang++ binary of the build if it exists.
'libltodir' - the path to the directory containing libLTO.dylib, if it
exists.
Each test is run in a sandbox directory. By default, sandbox directories are
temporary directories which are created and destroyed for each test (see
--sandbox).
For use in auxiliary test scripts, each test is also run with each variable
available in the environment as TEST_<variable name> (variables are converted
to uppercase). For example, a test script could use "TEST_PATH" to find the
path to the build under test.
The stdout and stderr of the command are logged to files inside the sandbox
directory. Use an explicit sandbox directory if you would like to look at
them.
It is possible to run multiple distinct commands for each test by separating
them in the command line arguments by '----'. The failure of any command causes
the entire test to fail.\
""" % locals())
parser.add_option("-b",
"--build",
dest="build_name",
metavar="STR",
help="name of build to fetch",
action="store",
default=DEFAULT_BUILDER)
parser.add_option("-s",
"--sandbox",
dest="sandbox",
help="directory to use as a sandbox",
action="store",
default=None)
parser.add_option("-v",
"--verbose",
dest="verbose",
help="output more test notermation",
action="store_true",
default=False)
parser.add_option("-V",
"--very-verbose",
dest="very_verbose",
help="output even more test notermation",
action="store_true",
default=False)
parser.add_option("",
"--show-output",
dest="show_command_output",
help="display command output",
action="store_true",
default=False)
parser.add_option("",
"--single-step",
dest="single_step",
help="single step instead of binary stepping",
action="store_true",
default=False)
parser.add_option("",
"--min-rev",
dest="min_rev",
help="minimum revision to test",
type="int",
action="store",
default=None)
parser.add_option("",
"--max-rev",
dest="max_rev",
help="maximum revision to test",
type="int",
action="store",
default=None)
parser.disable_interspersed_args()
(opts, args) = parser.parse_args(args)
if opts.build_name is None:
parser.error("no build name given (see --build)")
# Very verbose implies verbose.
opts.verbose |= opts.very_verbose
start_time = time.time()
available_builds = list(llvmlab.fetch_builds(opts.build_name))
available_builds.sort()
available_builds.reverse()
if opts.very_verbose:
note("fetched builds in %.2fs" % (time.time() - start_time, ))
if opts.min_rev is not None:
available_builds = [
b for b in available_builds if b.revision >= opts.min_rev
]
if opts.max_rev is not None:
available_builds = [
b for b in available_builds if b.revision <= opts.max_rev
]
def predicate(item):
# Run the sandboxed test.
test_result, _ = execute_sandboxed_test(
opts.sandbox,
opts.build_name,
item,
args,
verbose=opts.verbose,
very_verbose=opts.very_verbose,
show_command_output=opts.show_command_output or opts.very_verbose)
# Print status.
print '%s: %s' % (('FAIL', 'PASS')[test_result],
item.tobasename(include_suffix=False))
return test_result
if opts.single_step:
for item in available_builds:
if predicate(item):
break
else:
item = None
else:
if opts.min_rev is None or opts.max_rev is None:
# Gallop to find initial search range, under the assumption that we
# are most likely looking for something at the head of this list.
search_space = algorithm.gallop(predicate, available_builds)
else:
# If both min and max revisions are specified,
# don't gallop - bisect the given range.
search_space = available_builds
item = algorithm.bisect(predicate, search_space)
if item is None:
fatal('unable to find any passing build!')
print '%s: first working build' % item.tobasename(include_suffix=False)
index = available_builds.index(item)
if index == 0:
print 'no failing builds!?'
else:
print '%s: next failing build' % available_builds[
index - 1].tobasename(include_suffix=False)
def empty_container(container, config):
announce("Emptying image " + container)
command = ['docker', 'rmi', get_image(config)]
note(" ".join(command))
quietcall(command)
def execute_sandboxed_test(sandbox,
builder,
build,
args,
verbose=False,
very_verbose=False,
add_path_variables=True,
show_command_output=False,
reuse_sandbox=False):
def split_command_filters(command):
for i, arg in enumerate(command):
if arg[:2] != "%%" or arg[-2:] != "%%":
break
else:
fatal("invalid command: %s, only contains filter "
"specifications" % ("".join('"%s"' % a for a in command)))
return ([a[2:-2] for a in command[:i]], command[i:])
path = build.tobasename(include_suffix=False)
fullpath = build.tobasename()
if verbose:
note('testing %r' % path)
# Create the sandbox directory, if it doesn't exist.
is_temp = False
if sandbox is None:
sandbox = tempfile.mkdtemp()
is_temp = True
else:
# Make absolute.
sandbox = os.path.abspath(sandbox)
if not os.path.exists(sandbox):
os.mkdir(sandbox)
# Compute paths and make sure sandbox is clean.
root_path = os.path.join(sandbox, fullpath)
builddir_path = os.path.join(sandbox, path)
need_build = True
if reuse_sandbox and (os.path.exists(root_path)
and os.path.exists(builddir_path)):
need_build = False
else:
for p in (root_path, builddir_path):
if os.path.exists(p):
fatal('sandbox is not clean, %r exists' % p)
# Fetch and extract the build.
if need_build:
start_time = time.time()
llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path)
if very_verbose:
note("extracted build in %.2fs" % (time.time() - start_time, ))
# Attempt to find clang/clang++ in the downloaded build.
def find_binary(name):
x = subprocess.check_output(['find', builddir_path, '-name', name])\
.strip().split("\n")[0]
if x == '':
x = None
return x
clang_path = find_binary('clang')
clangpp_path = find_binary('clang++')
liblto_path = find_binary('libLTO.dylib')
if liblto_path is not None:
liblto_dir = os.path.dirname(liblto_path)
else:
liblto_dir = None
# Construct the interpolation variables.
options = {
'sandbox': sandbox,
'path': builddir_path,
'revision': build.revision,
'build': build.build,
'clang': clang_path,
'clang++': clangpp_path,
'libltodir': liblto_dir
}
# Inject environment variables.
env = os.environ.copy()
for key, value in options.items():
env['TEST_%s' % key.upper()] = str(value)
# Extend the environment to include the path to the extracted build.
#
# FIXME: Ideally, we would be able to read some kind of configuration
# notermation about a builder so that we could just set this up, it doesn't
# necessarily here as hard-coded notermation.
if add_path_variables:
path_extensions = []
dyld_library_path_extensions = []
toolchains_dir = os.path.join(builddir_path,
('Applications/Xcode.app/Contents/'
'Developer/Toolchains'))
toolchain_paths = []
if os.path.exists(toolchains_dir):
toolchain_paths = [
os.path.join(toolchains_dir, name, 'usr')
for name in os.listdir(toolchains_dir)
]
for package_root in ['', 'Developer/usr/'] + toolchain_paths:
p = os.path.join(builddir_path, package_root, 'bin')
if os.path.exists(p):
path_extensions.append(p)
p = os.path.join(builddir_path, package_root, 'lib')
if os.path.exists(p):
dyld_library_path_extensions.append(p)
if path_extensions:
env['PATH'] = os.pathsep.join(path_extensions +
[os.environ.get('PATH', '')])
if dyld_library_path_extensions:
env['DYLD_LIBRARY_PATH'] = os.pathsep.join(
dyld_library_path_extensions +
[os.environ.get('DYLD_LIBRARY_PATH', '')])
# Split the arguments into distinct commands.
#
# Extended command syntax allows running multiple commands by separating
# them with '----'.
test_commands = util.list_split(args, "----")
# Split command specs into filters and commands.
test_commands = [split_command_filters(spec) for spec in test_commands]
# Execute the test.
command_objects = []
interpolated_variables = False
for i, (filters, command) in enumerate(test_commands):
# Interpolate arguments.
old_command = command
command = [a % options for a in command]
if old_command != command:
interpolated_variables = True
# Create the command object...
stdout_log_path = os.path.join(sandbox, '%s.%d.stdout' % (path, i))
stderr_log_path = os.path.join(sandbox, '%s.%d.stderr' % (path, i))
cmd_object = Command(command, stdout_log_path, stderr_log_path, env)
command_objects.append(cmd_object)
# Execute the command.
try:
cmd_object.execute(verbose=verbose)
except OSError, e:
# Python's exceptions are horribly to read, and this one is
# incredibly common when people don't use the right syntax (or
# misspell something) when writing a predicate. Detect this and
# notify the user.
if e.errno == errno.ENOENT:
fatal("invalid command, executable doesn't exist: %r" %
(cmd_object.command[0], ))
elif e.errno == errno.ENOEXEC:
fatal("invalid command, executable has a bad format. Did you "
"forget to put a #! at the top of a script?: %r" %
(cmd_object.command[0], ))
else:
# Otherwise raise the error again.
raise e
# Evaluate the filters.
for filter in filters:
cmd_object.evaluate_filter_spec(filter)
if show_command_output:
for p, type in ((stdout_log_path, "stdout"), (stderr_log_path,
"stderr")):
if not os.path.exists(p):
continue
f = open(p)
data = f.read()
f.close()
if data:
print(
"-- command %s (note: suppressed by default, "
"see sandbox dir for log files) --" % (type))
print "--\n%s--\n" % data
test_result = cmd_object.result
if not test_result:
break
if data:
print(
"-- command %s (note: suppressed by default, "
"see sandbox dir for log files) --" % (type))
print "--\n%s--\n" % data
test_result = cmd_object.result
if not test_result:
break
if not interpolated_variables:
warning('no substitutions found. Fetched root ignored?')
# Remove the temporary directory.
if is_temp:
if shell.execute(['rm', '-rf', sandbox]) != 0:
note('unable to remove sandbox dir %r' % path)
return test_result, command_objects
def get_best_match(builds, name, key=lambda x: x):
builds = list(builds)
builds.sort(key=key)
if name is None and builds:
return builds[-1]
to_find = llvmlab.Build.frombasename(name, None)
best = None
for item in builds:
