def stack_container(container, config): announce("Stacking " + container + " container") if config.has_key('run') and not config['run']: note("container has 'run' key set to " + str(config['run']) + ", skipping") return command = ['docker', 'run', '-d', '--name', get_container(container)] if config.has_key('options'): for option in config['options']: command = command + [parse_template(option)] if config.has_key('hostname'): command = command + ['--hostname', get_value(config, 'hostname')] # else: # command = command + ['--hostname', container] if config.has_key('dns'): command = command + ['--dns', get_value(config, 'dns')] if config.has_key('volumes'): for volumespec in config['volumes']: volumespec_parsed = parse_template(volumespec) if volumespec_parsed.startswith("/"): command = command + ['-v', volumespec_parsed] else: command = command + [ '-v', mem.VolumePath + "/" + mem.Project + "/" + container + "/" + volumespec_parsed ] if config.has_key('ports'): for portspec in config['ports']: command = command + ['-p', parse_template(portspec)] if config.has_key('environment'): for envvar in config['environment']: command = command + [ '-e', parse_template(envvar) + '=' + parse_template(config['environment'][envvar]) ] if config.has_key('links'): for link in config['links']: command = command + [ "--link=" + get_container(parse_template(link)) ] command.append(get_image(config)) if config.has_key('command'): command = command + shlex.split(get_value(config, 'command')) note(" ".join(command)) quietcall(command) if config.has_key('upwhen'): wait_for_up(container, config)
def action_ls(name, args): """list available build names or builds""" parser = OptionParser("""\ usage: %%prog %s [build-name] With no arguments, list the available build names on 'llvmlab'. With a build name, list the available builds for that builder.\ """ % name) (opts, args) = parser.parse_args(args) if not len(args): available_buildnames = llvmlab.fetch_builders() available_buildnames.sort() for item in available_buildnames: print item return available_buildnames for name in args: if len(args) > 1: if name is not args[0]: print print '%s:' % name available_builds = list(llvmlab.fetch_builds(name)) available_builds.sort() available_builds.reverse() for build in available_builds: print build.tobasename(include_suffix=False) min_rev = min([x.revision for x in available_builds]) max_rev = max([x.revision for x in available_builds]) note("Summary: found {} builds: r{}-r{}".format( len(available_builds), min_rev, max_rev)) return available_builds
def execute(self, verbose=False): if verbose: note('executing: %s' % ' '.join("'%s'" % arg for arg in self.command)) start_rusage = resource.getrusage(resource.RUSAGE_CHILDREN) start_time = time.time() p = subprocess.Popen(self.command, stdout=open(self.stdout_path, 'w'), stderr=open(self.stderr_path, 'w'), env=self.env) self.result = p.wait() == 0 end_time = time.time() end_rusage = resource.getrusage(resource.RUSAGE_CHILDREN) self.metrics["user_time"] = end_rusage.ru_utime - start_rusage.ru_utime self.metrics["sys_time"] = end_rusage.ru_stime - start_rusage.ru_stime self.metrics["wall_time"] = end_time - start_time if verbose: note("command executed in -- " "user: %.4fs, wall: %.4fs, sys: %.4fs" % (self.metrics["user_time"], self.metrics["wall_time"], self.metrics["sys_time"]))
def action_ls(name, args): """list available build names or builds""" parser = OptionParser("""\ usage: %%prog %s [build-name] With no arguments, list the available build names on 'llvmlab'. With a build name, list the available builds for that builder.\ """ % name) (opts, args) = parser.parse_args(args) if not len(args): available_buildnames = llvmlab.fetch_builders() available_buildnames.sort() for item in available_buildnames: print item return available_buildnames for name in args: if len(args) > 1: if name is not args[0]: print print '%s:' % name available_builds = list(llvmlab.fetch_builds(name)) available_builds.sort() available_builds.reverse() for build in available_builds: print build.tobasename(include_suffix=False) min_rev = min([x.revision for x in available_builds]) max_rev = max([x.revision for x in available_builds]) note("Summary: found {} builds: r{}-r{}".format(len(available_builds), min_rev, max_rev)) return available_builds
def execute(self, verbose=False): if verbose: note('executing: %s' % ' '.join("'%s'" % arg for arg in self.command)) start_rusage = resource.getrusage(resource.RUSAGE_CHILDREN) start_time = time.time() p = subprocess.Popen(self.command, stdout=open(self.stdout_path, 'w'), stderr=open(self.stderr_path, 'w'), env=self.env) self.result = p.wait() == 0 end_time = time.time() end_rusage = resource.getrusage(resource.RUSAGE_CHILDREN) self.metrics["user_time"] = end_rusage.ru_utime - start_rusage.ru_utime self.metrics["sys_time"] = end_rusage.ru_stime - start_rusage.ru_stime self.metrics["wall_time"] = end_time - start_time if verbose: note("command executed in -- " "user: %.4fs, wall: %.4fs, sys: %.4fs" % ( self.metrics["user_time"], self.metrics["wall_time"], self.metrics["sys_time"]))
def unstack_container(container): announce("Unstacking " + container + " container") command = ['docker', 'stop', get_container(container)] note(" ".join(command)) quietcall(command) command = ['docker', 'rm', '--force', get_container(container)] note(" ".join(command)) quietcall(command)
def enter_container(commands): which = commands.popleft() run = ["/bin/bash"] if len(commands) > 0: run = list(commands) announce("Entering '" + which + "' container and running: " + str(run)) command = ['docker', 'exec', '-ti', get_container(which)] + run note(" ".join(command)) call(command)
def stack_container(container, config): announce("Stacking " + container + " container") if config.has_key('run') and not config['run']: note("container has 'run' key set to " + str(config['run']) + ", skipping") return command = ['docker', 'run', '-d', '--name', get_container(container)] if config.has_key('options'): for option in config['options']: command = command + [parse_template(option)] if config.has_key('hostname'): command = command + ['--hostname', get_value(config, 'hostname')] # else: # command = command + ['--hostname', container] if config.has_key('dns'): command = command + ['--dns', get_value(config, 'dns')] if config.has_key('volumes'): for volumespec in config['volumes']: volumespec_parsed = parse_template(volumespec) if volumespec_parsed.startswith("/"): command = command + ['-v', volumespec_parsed] elif volumespec_parsed.startswith('./'): # docker complains if path starts with "." so expand CWD here cwd = os.path.dirname(os.path.realpath(volumespec_parsed.split(':')[0])) realpath = cwd + '/' + volumespec_parsed.strip('./') command = command + ['-v', realpath] else: command = command + ['-v', mem.VolumePath + '/' + mem.Project + '/' + container + '/' + volumespec_parsed] if config.has_key('ports'): for portspec in config['ports']: command = command + ['-p', parse_template(portspec)] if config.has_key('environment'): for envvar in config['environment']: command = command + ['-e', parse_template(envvar) + '=' + parse_template(config['environment'][envvar])] if config.has_key('links'): for link in config['links']: command = command + ["--link=" + get_container(parse_template(link))] command.append(get_image(config)) if config.has_key('command'): command = command + shlex.split(get_value(config, 'command')) note(" ".join(command)) quietcall(command) if config.has_key('upwhen'): wait_for_up(container, config)
def proc(args): langs = [l for l in args.langs.split(",") if l != ""] c = Classifier(folder=args.model_dir, langs=langs) if args.u is not None: c.DROP_RATIO = args.u if args.verbose: util.note("Drop ratio: {}".format(c.DROP_RATIO)) if args.s: for line in sys.stdin: print(c.classify(line.decode('utf-8'), verbose=args.verbose)) else: print(c.classify(sys.stdin.read().decode('utf-8'), verbose=args.verbose))
def get_langs(self, langs=[]): if langs == []: return self.langs else: langs = set(langs) active_langs = self.langs & langs if len(langs) != len(active_langs): missing = langs - active_langs - self.langs_warned if missing: # only warn once per lang self.langs_warned.update(missing) util.note("WARNING: No language model for {}".format( "/".join(missing))) return active_langs
def freq_of_text_file(self, fil): freq = {} for nl, strline in enumerate(fil.readlines()): try: line = strline.decode('utf-8') except UnicodeDecodeError as e: if self.unicode_warned == 0: util.note( "WARNING: Line {} gave {}, skipping ... " "(not warning again)".format(nl, e)) self.unicode_warned += 1 continue freq = self.freq_of_text(line, freq) if self.unicode_warned != 0: util.note("Saw {} UnicodeDecodeErrors".format(self.unicode_warned)) return freq
def __init__(self, folder=None, langs=[], verbose=False): if folder is None: folder = os.path.join(here, 'lm') self.cmodels = {} self.wmodels = {} ext = '.lm' fnames = [] folder_glob = os.path.join(folder, '*' + ext) found_fnames = glob.glob(os.path.normcase(folder_glob)) if len(found_fnames) == 0: raise ValueError("No language files found in %s" % (folder,)) if len(langs) == 0: fnames = found_fnames else: fnames = [os.path.join(folder, lang + ext) for lang in langs] not_found = set(fnames) - set(found_fnames) if len(not_found) != 0: raise ValueError( "Unknown language(s): " + ", ".join(not_found)) for fname in fnames: lang = util.basename_noext(fname, ext) self.cmodels[lang] = CharModel(lang).of_model_file( open(fname, 'r'), fname) if verbose: util.note("Loaded %s" % (fname,)) fname_wm = os.path.join(folder, lang + '.wm') # fname_wmgz = os.path.join(folder, lang+'.wm.gz') if os.path.exists(fname_wm): self.wmodels[lang] = WordModel(lang).of_model_file( open(fname_wm, 'r'), fname_wm) if verbose: util.note("Loaded %s" % (fname_wm,)) else: self.wmodels[lang] = WordModel(lang).of_freq({}) if len(self.cmodels) == 0: raise ValueError("No character models created!") else: self.langs = set(self.cmodels.keys()) self.langs_warned = set()
def __init__(self, folder, exts=['.txt', '.txt.gz'], Model=CharModel, verbose=False): self.models = {} for ext in exts: files = glob.glob(os.path.normcase(os.path.join(folder, '*' + ext))) for fname in files: if verbose: msg = "Processing %s" % (fname,) if os.path.getsize(fname) > 5000000: msg += " (this may take a while)" util.note(msg) sys.stderr.flush() lang = util.basename_noext(fname, ext) self.models[lang] = Model(lang).of_text_file( self.open_corpus(fname)) if len(self.models) == 0: raise Exception( "No suitable files found matching {}/*.{}{}{}!".format( folder, "{", ",".join(exts), "}"))
def fill_container(container, config): announce("Filling " + container + " container") if config.has_key('build'): build = get_value(config, 'build') tag = mem.Project + "/" + build if config.has_key('tag'): tag = get_value(config, 'tag') target = build if build[:4] == 'git:': if not config.has_key('tag'): raise Exception( "If you are using a git repo for 'build' you must also specify 'tag'" ) tag = get_value(config, 'tag') url = build[4:] print("Cloning " + tag + " from " + url, end="") dir = "_grua_" + tag.replace("/", "_") if os.path.isdir(dir): shutil.rmtree(dir) command = ['git', 'clone', url, dir] note(" ".join(command)) call(command) target = dir mention("building " + container + " ( " + target + " ) " + " with tag '" + tag + "'") command = ['docker', 'build', '-t', tag, target] note(" ".join(command)) quietcall(command) else: mention(container + " uses an image. Pulling " + get_value(config, 'image')) command = ['docker', 'pull', get_value(config, 'image')] note(" ".join(command)) quietcall(command)
def fill_container(container, config): announce("Filling " + container + " container") if config.has_key('build'): build = get_value(config, 'build') tag = mem.Project + "/" + build if config.has_key('tag'): tag = get_value(config, 'tag') target = build if build[:4] == 'git:': if not config.has_key('tag'): raise Exception("If you are using a git repo for 'build' you must also specify 'tag'") tag = get_value(config, 'tag') url = build[4:] print ( "Cloning " + tag + " from " + url, end="" ) dir = "_grua_" + tag.replace("/", "_") if os.path.isdir(dir): shutil.rmtree(dir) command = ['git', 'clone', url, dir] note(" ".join(command)) call(command) target = dir mention("building " + container + " ( " + target + " ) " + " with tag '" + tag + "'") command = ['docker', 'build', '-t', tag, target] note(" ".join(command)) quietcall(command) else: mention(container + " uses an image. Pulling " + get_value(config, 'image')) command = ['docker', 'pull', get_value(config, 'image')] note(" ".join(command)) quietcall(command)
def classify_full(self, intext, langs=[], verbose=False): active_langs = self.get_langs(langs) text = ensure_unicode(intext) ingram = CharModel().of_text(text) cscored = {l: model.compare(ingram) for l, model in self.cmodels.iteritems() if l in active_langs} cranked = util.sort_by_value(cscored) cbest = cranked[0] cfiltered = {l: d for l, d in cranked if d <= cbest[1] * self.DROP_RATIO} if len(cfiltered) <= 1: if verbose: util.note("lm gave: {} as only result for input: {}".format( cfiltered, text)) return list(cfiltered.iteritems()) else: # Along with compare_tc, implements text_cat.pl line # 442 and on: wscored = {l: model.compare_tc(text, cscored[l]) for l, model in self.wmodels.iteritems() if l in cfiltered} cwcombined = {l: (cscored[l] - wscore) for l, wscore in wscored.iteritems()} cwranked = util.sort_by_value(cwcombined) if verbose: if cranked[:len(cwranked)] == cwranked: util.note( "lm gave: {}\t\twm gave no change\t\tfor" "input: {}".format( pretty_tbl(cranked), text)) else: util.note( "lm gave: {}\t\twm-weighted to: " "{}\t\tfor input: {}".format( pretty_tbl(cranked), pretty_tbl(cwranked), text)) return cwranked
def action_fetch(name, args): """fetch a build from the server""" parser = OptionParser("""\ usage: %%prog %(name)s [options] builder [build-name] Fetch the build from the named builder which matchs build-name. If no match is found, get the first build before the given name. If no build name is given, the most recent build is fetched. The available builders can be listed using: %%prog ls The available builds can be listed using: %%prog ls builder""" % locals()) parser.add_option("-f", "--force", dest="force", help=("always download and extract, overwriting any" "existing files"), action="store_true", default=False) parser.add_option("", "--update-link", dest="update_link", metavar="PATH", help=("update a symbolic link at PATH to point to the " "fetched build (on success)"), action="store", default=None) parser.add_option("-d", "--dry-run", dest='dry_run', help=("Perform all operations except the actual " "downloading and extracting of any files"), action="store_true", default=False) (opts, args) = parser.parse_args(args) if len(args) == 0: parser.error("please specify a builder name") elif len(args) == 1: builder, = args build_name = None elif len(args) == 2: builder, build_name = args else: parser.error("invalid number of arguments") builds = list(llvmlab.fetch_builds(builder)) if not builds: parser.error("no builds for builder: %r" % builder) build = get_best_match(builds, build_name) if not build: parser.error("no match for build %r" % build_name) path = build.tobasename() if build_name is not None and not path.startswith(build_name): note('no exact match, fetching %r' % path) # Get the paths to extract to. root_path = path builddir_path = build.tobasename(include_suffix=False) if not opts.dry_run: # Check that the download and extract paths are clean. for p in (root_path, builddir_path): if os.path.exists(p): # If we are using --force, then clean the path. if opts.force: shutil.rmtree(p, ignore_errors=True) continue fatal('current directory is not clean, %r exists' % p) llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path) print 'downloaded root: %s' % root_path print 'extracted path : %s' % builddir_path # Update the symbolic link, if requested. if not opts.dry_run and opts.update_link: # Remove the existing path. try: os.unlink(opts.update_link) except OSError as e: if e.errno != errno.ENOENT: fatal('unable to update symbolic link at %r, cannot unlink' % (opts.update_link)) # Create the symbolic link. os.symlink(os.path.abspath(builddir_path), opts.update_link) print 'updated link at: %s' % opts.update_link return os.path.abspath(builddir_path)
def action_bisect(name, args): """find first failing build using binary search""" parser = OptionParser("""\ usage: %%prog %(name)s [options] ... test command args ... Look for the first published build where a test failed, using the builds on llvmlab. The command arguments are executed once per build tested, but each argument is first subject to string interpolation. The syntax is "%%(VARIABLE)FORMAT" where FORMAT is a standard printf format, and VARIABLE is one of: 'sandbox' - the path to the sandbox directory. 'path' - the path to the build under test. 'revision' - the revision number of the build. 'build' - the build number of the build under test. 'clang' - the path to the clang binary of the build if it exists. 'clang++' - the path to the clang++ binary of the build if it exists. 'libltodir' - the path to the directory containing libLTO.dylib, if it exists. Each test is run in a sandbox directory. By default, sandbox directories are temporary directories which are created and destroyed for each test (see --sandbox). For use in auxiliary test scripts, each test is also run with each variable available in the environment as TEST_<variable name> (variables are converted to uppercase). For example, a test script could use "TEST_PATH" to find the path to the build under test. The stdout and stderr of the command are logged to files inside the sandbox directory. Use an explicit sandbox directory if you would like to look at them. It is possible to run multiple distinct commands for each test by separating them in the command line arguments by '----'. The failure of any command causes the entire test to fail.\ """ % locals()) parser.add_option("-b", "--build", dest="build_name", metavar="STR", help="name of build to fetch", action="store", default=DEFAULT_BUILDER) parser.add_option("-s", "--sandbox", dest="sandbox", help="directory to use as a sandbox", action="store", default=None) parser.add_option("-v", "--verbose", dest="verbose", help="output more test notermation", action="store_true", default=False) parser.add_option("-V", "--very-verbose", dest="very_verbose", help="output even more test notermation", action="store_true", default=False) parser.add_option("", "--show-output", dest="show_command_output", help="display command output", action="store_true", default=False) parser.add_option("", "--single-step", dest="single_step", help="single step instead of binary stepping", action="store_true", default=False) parser.add_option("", "--min-rev", dest="min_rev", help="minimum revision to test", type="int", action="store", default=None) parser.add_option("", "--max-rev", dest="max_rev", help="maximum revision to test", type="int", action="store", default=None) parser.disable_interspersed_args() (opts, args) = parser.parse_args(args) if opts.build_name is None: parser.error("no build name given (see --build)") # Very verbose implies verbose. opts.verbose |= opts.very_verbose start_time = time.time() available_builds = list(llvmlab.fetch_builds(opts.build_name)) available_builds.sort() available_builds.reverse() if opts.very_verbose: note("fetched builds in %.2fs" % (time.time() - start_time,)) if opts.min_rev is not None: available_builds = [b for b in available_builds if b.revision >= opts.min_rev] if opts.max_rev is not None: available_builds = [b for b in available_builds if b.revision <= opts.max_rev] def predicate(item): # Run the sandboxed test. test_result, _ = execute_sandboxed_test( opts.sandbox, opts.build_name, item, args, verbose=opts.verbose, very_verbose=opts.very_verbose, show_command_output=opts.show_command_output or opts.very_verbose) # Print status. print '%s: %s' % (('FAIL', 'PASS')[test_result], item.tobasename(include_suffix=False)) return test_result if opts.single_step: for item in available_builds: if predicate(item): break else: item = None else: if opts.min_rev is None or opts.max_rev is None: # Gallop to find initial search range, under the assumption that we # are most likely looking for something at the head of this list. search_space = algorithm.gallop(predicate, available_builds) else: # If both min and max revisions are specified, # don't gallop - bisect the given range. search_space = available_builds item = algorithm.bisect(predicate, search_space) if item is None: fatal('unable to find any passing build!') print '%s: first working build' % item.tobasename(include_suffix=False) index = available_builds.index(item) if index == 0: print 'no failing builds!?' else: print '%s: next failing build' % available_builds[index-1].tobasename( include_suffix=False)
def save(self, folder, ext='.lm', verbose=False): for lang, model in self.models.iteritems(): fname = os.path.join(folder, lang + ext) model.to_model_file(open(fname, 'w')) if verbose and len(self.models) != 0: util.note("Wrote {%s}%s" % (",".join(self.models.keys()), ext))
def action_fetch(name, args): """fetch a build from the server""" parser = OptionParser("""\ usage: %%prog %(name)s [options] builder [build-name] Fetch the build from the named builder which matchs build-name. If no match is found, get the first build before the given name. If no build name is given, the most recent build is fetched. The available builders can be listed using: %%prog ls The available builds can be listed using: %%prog ls builder""" % locals()) parser.add_option("-f", "--force", dest="force", help=("always download and extract, overwriting any" "existing files"), action="store_true", default=False) parser.add_option("", "--update-link", dest="update_link", metavar="PATH", help=("update a symbolic link at PATH to point to the " "fetched build (on success)"), action="store", default=None) parser.add_option("-d", "--dry-run", dest='dry_run', help=("Perform all operations except the actual " "downloading and extracting of any files"), action="store_true", default=False) (opts, args) = parser.parse_args(args) if len(args) == 0: parser.error("please specify a builder name") elif len(args) == 1: builder, = args build_name = None elif len(args) == 2: builder, build_name = args else: parser.error("invalid number of arguments") builds = list(llvmlab.fetch_builds(builder)) if not builds: parser.error("no builds for builder: %r" % builder) build = get_best_match(builds, build_name) if not build: parser.error("no match for build %r" % build_name) path = build.tobasename() if build_name is not None and not path.startswith(build_name): note('no exact match, fetching %r' % path) # Get the paths to extract to. root_path = path builddir_path = build.tobasename(include_suffix=False) if not opts.dry_run: # Check that the download and extract paths are clean. for p in (root_path, builddir_path): if os.path.exists(p): # If we are using --force, then clean the path. if opts.force: shutil.rmtree(p, ignore_errors=True) continue fatal('current directory is not clean, %r exists' % p) llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path) print 'downloaded root: %s' % root_path print 'extracted path : %s' % builddir_path # Update the symbolic link, if requested. if not opts.dry_run and opts.update_link: # Remove the existing path. try: os.unlink(opts.update_link) except OSError as e: if e.errno != errno.ENOENT: fatal('unable to update symbolic link at %r, cannot unlink' % ( opts.update_link)) # Create the symbolic link. os.symlink(os.path.abspath(builddir_path), opts.update_link) print 'updated link at: %s' % opts.update_link return os.path.abspath(builddir_path)
def execute_sandboxed_test(sandbox, builder, build, args, verbose=False, very_verbose=False, add_path_variables=True, show_command_output=False, reuse_sandbox=False): def split_command_filters(command): for i, arg in enumerate(command): if arg[:2] != "%%" or arg[-2:] != "%%": break else: fatal("invalid command: %s, only contains filter " "specifications" % ("".join('"%s"' % a for a in command))) return ([a[2:-2] for a in command[:i]], command[i:]) path = build.tobasename(include_suffix=False) fullpath = build.tobasename() if verbose: note('testing %r' % path) # Create the sandbox directory, if it doesn't exist. is_temp = False if sandbox is None: sandbox = tempfile.mkdtemp() is_temp = True else: # Make absolute. sandbox = os.path.abspath(sandbox) if not os.path.exists(sandbox): os.mkdir(sandbox) # Compute paths and make sure sandbox is clean. root_path = os.path.join(sandbox, fullpath) builddir_path = os.path.join(sandbox, path) need_build = True if reuse_sandbox and (os.path.exists(root_path) and os.path.exists(builddir_path)): need_build = False else: for p in (root_path, builddir_path): if os.path.exists(p): fatal('sandbox is not clean, %r exists' % p) # Fetch and extract the build. if need_build: start_time = time.time() llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path) if very_verbose: note("extracted build in %.2fs" % (time.time() - start_time,)) # Attempt to find clang/clang++ in the downloaded build. def find_binary(name): x = subprocess.check_output(['find', builddir_path, '-name', name])\ .strip().split("\n")[0] if x == '': x = None return x clang_path = find_binary('clang') clangpp_path = find_binary('clang++') liblto_path = find_binary('libLTO.dylib') if liblto_path is not None: liblto_dir = os.path.dirname(liblto_path) else: liblto_dir = None # Construct the interpolation variables. options = {'sandbox': sandbox, 'path': builddir_path, 'revision': build.revision, 'build': build.build, 'clang': clang_path, 'clang++': clangpp_path, 'libltodir': liblto_dir} # Inject environment variables. env = os.environ.copy() for key, value in options.items(): env['TEST_%s' % key.upper()] = str(value) # Extend the environment to include the path to the extracted build. # # FIXME: Ideally, we would be able to read some kind of configuration # notermation about a builder so that we could just set this up, it doesn't # necessarily here as hard-coded notermation. if add_path_variables: path_extensions = [] dyld_library_path_extensions = [] toolchains_dir = os.path.join(builddir_path, ('Applications/Xcode.app/Contents/' 'Developer/Toolchains')) toolchain_paths = [] if os.path.exists(toolchains_dir): toolchain_paths = [os.path.join(toolchains_dir, name, 'usr') for name in os.listdir(toolchains_dir)] for package_root in ['', 'Developer/usr/'] + toolchain_paths: p = os.path.join(builddir_path, package_root, 'bin') if os.path.exists(p): path_extensions.append(p) p = os.path.join(builddir_path, package_root, 'lib') if os.path.exists(p): dyld_library_path_extensions.append(p) if path_extensions: env['PATH'] = os.pathsep.join( path_extensions + [os.environ.get('PATH', '')]) if dyld_library_path_extensions: env['DYLD_LIBRARY_PATH'] = os.pathsep.join( dyld_library_path_extensions + [ os.environ.get('DYLD_LIBRARY_PATH', '')]) # Split the arguments into distinct commands. # # Extended command syntax allows running multiple commands by separating # them with '----'. test_commands = util.list_split(args, "----") # Split command specs into filters and commands. test_commands = [split_command_filters(spec) for spec in test_commands] # Execute the test. command_objects = [] interpolated_variables = False for i, (filters, command) in enumerate(test_commands): # Interpolate arguments. old_command = command command = [a % options for a in command] if old_command != command: interpolated_variables = True # Create the command object... stdout_log_path = os.path.join(sandbox, '%s.%d.stdout' % (path, i)) stderr_log_path = os.path.join(sandbox, '%s.%d.stderr' % (path, i)) cmd_object = Command(command, stdout_log_path, stderr_log_path, env) command_objects.append(cmd_object) # Execute the command. try: cmd_object.execute(verbose=verbose) except OSError, e: # Python's exceptions are horribly to read, and this one is # incredibly common when people don't use the right syntax (or # misspell something) when writing a predicate. Detect this and # notify the user. if e.errno == errno.ENOENT: fatal("invalid command, executable doesn't exist: %r" % ( cmd_object.command[0],)) elif e.errno == errno.ENOEXEC: fatal("invalid command, executable has a bad format. Did you " "forget to put a #! at the top of a script?: %r" % (cmd_object.command[0],)) else: # Otherwise raise the error again. raise e # Evaluate the filters. for filter in filters: cmd_object.evaluate_filter_spec(filter) if show_command_output: for p, type in ((stdout_log_path, "stdout"), (stderr_log_path, "stderr")): if not os.path.exists(p): continue f = open(p) data = f.read() f.close() if data: print ("-- command %s (note: suppressed by default, " "see sandbox dir for log files) --" % (type)) print "--\n%s--\n" % data test_result = cmd_object.result if not test_result: break
f.close() if data: print ("-- command %s (note: suppressed by default, " "see sandbox dir for log files) --" % (type)) print "--\n%s--\n" % data test_result = cmd_object.result if not test_result: break if not interpolated_variables: warning('no substitutions found. Fetched root ignored?') # Remove the temporary directory. if is_temp: if shell.execute(['rm', '-rf', sandbox]) != 0: note('unable to remove sandbox dir %r' % path) return test_result, command_objects def get_best_match(builds, name, key=lambda x: x): builds = list(builds) builds.sort(key=key) if name is None and builds: return builds[-1] to_find = llvmlab.Build.frombasename(name, None) best = None for item in builds:
def empty_container(container, config): announce("Emptying image " + container) command = ['docker', 'rmi', get_image(config)] note(" ".join(command)) quietcall(command)
def action_bisect(name, args): """find first failing build using binary search""" parser = OptionParser("""\ usage: %%prog %(name)s [options] ... test command args ... Look for the first published build where a test failed, using the builds on llvmlab. The command arguments are executed once per build tested, but each argument is first subject to string interpolation. The syntax is "%%(VARIABLE)FORMAT" where FORMAT is a standard printf format, and VARIABLE is one of: 'sandbox' - the path to the sandbox directory. 'path' - the path to the build under test. 'revision' - the revision number of the build. 'build' - the build number of the build under test. 'clang' - the path to the clang binary of the build if it exists. 'clang++' - the path to the clang++ binary of the build if it exists. 'libltodir' - the path to the directory containing libLTO.dylib, if it exists. Each test is run in a sandbox directory. By default, sandbox directories are temporary directories which are created and destroyed for each test (see --sandbox). For use in auxiliary test scripts, each test is also run with each variable available in the environment as TEST_<variable name> (variables are converted to uppercase). For example, a test script could use "TEST_PATH" to find the path to the build under test. The stdout and stderr of the command are logged to files inside the sandbox directory. Use an explicit sandbox directory if you would like to look at them. It is possible to run multiple distinct commands for each test by separating them in the command line arguments by '----'. The failure of any command causes the entire test to fail.\ """ % locals()) parser.add_option("-b", "--build", dest="build_name", metavar="STR", help="name of build to fetch", action="store", default=DEFAULT_BUILDER) parser.add_option("-s", "--sandbox", dest="sandbox", help="directory to use as a sandbox", action="store", default=None) parser.add_option("-v", "--verbose", dest="verbose", help="output more test notermation", action="store_true", default=False) parser.add_option("-V", "--very-verbose", dest="very_verbose", help="output even more test notermation", action="store_true", default=False) parser.add_option("", "--show-output", dest="show_command_output", help="display command output", action="store_true", default=False) parser.add_option("", "--single-step", dest="single_step", help="single step instead of binary stepping", action="store_true", default=False) parser.add_option("", "--min-rev", dest="min_rev", help="minimum revision to test", type="int", action="store", default=None) parser.add_option("", "--max-rev", dest="max_rev", help="maximum revision to test", type="int", action="store", default=None) parser.disable_interspersed_args() (opts, args) = parser.parse_args(args) if opts.build_name is None: parser.error("no build name given (see --build)") # Very verbose implies verbose. opts.verbose |= opts.very_verbose start_time = time.time() available_builds = list(llvmlab.fetch_builds(opts.build_name)) available_builds.sort() available_builds.reverse() if opts.very_verbose: note("fetched builds in %.2fs" % (time.time() - start_time, )) if opts.min_rev is not None: available_builds = [ b for b in available_builds if b.revision >= opts.min_rev ] if opts.max_rev is not None: available_builds = [ b for b in available_builds if b.revision <= opts.max_rev ] def predicate(item): # Run the sandboxed test. test_result, _ = execute_sandboxed_test( opts.sandbox, opts.build_name, item, args, verbose=opts.verbose, very_verbose=opts.very_verbose, show_command_output=opts.show_command_output or opts.very_verbose) # Print status. print '%s: %s' % (('FAIL', 'PASS')[test_result], item.tobasename(include_suffix=False)) return test_result if opts.single_step: for item in available_builds: if predicate(item): break else: item = None else: if opts.min_rev is None or opts.max_rev is None: # Gallop to find initial search range, under the assumption that we # are most likely looking for something at the head of this list. search_space = algorithm.gallop(predicate, available_builds) else: # If both min and max revisions are specified, # don't gallop - bisect the given range. search_space = available_builds item = algorithm.bisect(predicate, search_space) if item is None: fatal('unable to find any passing build!') print '%s: first working build' % item.tobasename(include_suffix=False) index = available_builds.index(item) if index == 0: print 'no failing builds!?' else: print '%s: next failing build' % available_builds[ index - 1].tobasename(include_suffix=False)
def execute_sandboxed_test(sandbox, builder, build, args, verbose=False, very_verbose=False, add_path_variables=True, show_command_output=False, reuse_sandbox=False): def split_command_filters(command): for i, arg in enumerate(command): if arg[:2] != "%%" or arg[-2:] != "%%": break else: fatal("invalid command: %s, only contains filter " "specifications" % ("".join('"%s"' % a for a in command))) return ([a[2:-2] for a in command[:i]], command[i:]) path = build.tobasename(include_suffix=False) fullpath = build.tobasename() if verbose: note('testing %r' % path) # Create the sandbox directory, if it doesn't exist. is_temp = False if sandbox is None: sandbox = tempfile.mkdtemp() is_temp = True else: # Make absolute. sandbox = os.path.abspath(sandbox) if not os.path.exists(sandbox): os.mkdir(sandbox) # Compute paths and make sure sandbox is clean. root_path = os.path.join(sandbox, fullpath) builddir_path = os.path.join(sandbox, path) need_build = True if reuse_sandbox and (os.path.exists(root_path) and os.path.exists(builddir_path)): need_build = False else: for p in (root_path, builddir_path): if os.path.exists(p): fatal('sandbox is not clean, %r exists' % p) # Fetch and extract the build. if need_build: start_time = time.time() llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path) if very_verbose: note("extracted build in %.2fs" % (time.time() - start_time, )) # Attempt to find clang/clang++ in the downloaded build. def find_binary(name): x = subprocess.check_output(['find', builddir_path, '-name', name])\ .strip().split("\n")[0] if x == '': x = None return x clang_path = find_binary('clang') clangpp_path = find_binary('clang++') liblto_path = find_binary('libLTO.dylib') if liblto_path is not None: liblto_dir = os.path.dirname(liblto_path) else: liblto_dir = None # Construct the interpolation variables. options = { 'sandbox': sandbox, 'path': builddir_path, 'revision': build.revision, 'build': build.build, 'clang': clang_path, 'clang++': clangpp_path, 'libltodir': liblto_dir } # Inject environment variables. env = os.environ.copy() for key, value in options.items(): env['TEST_%s' % key.upper()] = str(value) # Extend the environment to include the path to the extracted build. # # FIXME: Ideally, we would be able to read some kind of configuration # notermation about a builder so that we could just set this up, it doesn't # necessarily here as hard-coded notermation. if add_path_variables: path_extensions = [] dyld_library_path_extensions = [] toolchains_dir = os.path.join(builddir_path, ('Applications/Xcode.app/Contents/' 'Developer/Toolchains')) toolchain_paths = [] if os.path.exists(toolchains_dir): toolchain_paths = [ os.path.join(toolchains_dir, name, 'usr') for name in os.listdir(toolchains_dir) ] for package_root in ['', 'Developer/usr/'] + toolchain_paths: p = os.path.join(builddir_path, package_root, 'bin') if os.path.exists(p): path_extensions.append(p) p = os.path.join(builddir_path, package_root, 'lib') if os.path.exists(p): dyld_library_path_extensions.append(p) if path_extensions: env['PATH'] = os.pathsep.join(path_extensions + [os.environ.get('PATH', '')]) if dyld_library_path_extensions: env['DYLD_LIBRARY_PATH'] = os.pathsep.join( dyld_library_path_extensions + [os.environ.get('DYLD_LIBRARY_PATH', '')]) # Split the arguments into distinct commands. # # Extended command syntax allows running multiple commands by separating # them with '----'. test_commands = util.list_split(args, "----") # Split command specs into filters and commands. test_commands = [split_command_filters(spec) for spec in test_commands] # Execute the test. command_objects = [] interpolated_variables = False for i, (filters, command) in enumerate(test_commands): # Interpolate arguments. old_command = command command = [a % options for a in command] if old_command != command: interpolated_variables = True # Create the command object... stdout_log_path = os.path.join(sandbox, '%s.%d.stdout' % (path, i)) stderr_log_path = os.path.join(sandbox, '%s.%d.stderr' % (path, i)) cmd_object = Command(command, stdout_log_path, stderr_log_path, env) command_objects.append(cmd_object) # Execute the command. try: cmd_object.execute(verbose=verbose) except OSError, e: # Python's exceptions are horribly to read, and this one is # incredibly common when people don't use the right syntax (or # misspell something) when writing a predicate. Detect this and # notify the user. if e.errno == errno.ENOENT: fatal("invalid command, executable doesn't exist: %r" % (cmd_object.command[0], )) elif e.errno == errno.ENOEXEC: fatal("invalid command, executable has a bad format. Did you " "forget to put a #! at the top of a script?: %r" % (cmd_object.command[0], )) else: # Otherwise raise the error again. raise e # Evaluate the filters. for filter in filters: cmd_object.evaluate_filter_spec(filter) if show_command_output: for p, type in ((stdout_log_path, "stdout"), (stderr_log_path, "stderr")): if not os.path.exists(p): continue f = open(p) data = f.read() f.close() if data: print( "-- command %s (note: suppressed by default, " "see sandbox dir for log files) --" % (type)) print "--\n%s--\n" % data test_result = cmd_object.result if not test_result: break
if data: print( "-- command %s (note: suppressed by default, " "see sandbox dir for log files) --" % (type)) print "--\n%s--\n" % data test_result = cmd_object.result if not test_result: break if not interpolated_variables: warning('no substitutions found. Fetched root ignored?') # Remove the temporary directory. if is_temp: if shell.execute(['rm', '-rf', sandbox]) != 0: note('unable to remove sandbox dir %r' % path) return test_result, command_objects def get_best_match(builds, name, key=lambda x: x): builds = list(builds) builds.sort(key=key) if name is None and builds: return builds[-1] to_find = llvmlab.Build.frombasename(name, None) best = None for item in builds: