def link_configuration(cfg: Path, host_cfg: Path) -> None:
"""Ensure that ``cfg`` is a symlink to ``host_cfg``.
:param cfg: path to ``configuration.nix``
:param host_cfg: path to ``hosts/$(hostname)-configuration.nix``
"""
if cfg.is_symlink():
cfg_dest = cfg.parent / os.readlink(cfg)
if cfg_dest.samefile(host_cfg):
info(f"{p(cfg)} is already a symlink pointing to {p(host_cfg)}")
else:
warn(
f"{p(cfg)} is a symlink pointing to {p(cfg_dest)}, not {p(host_cfg)}; updating it"
)
if not DRY_RUN:
cfg.unlink()
cfg.symlink_to(host_cfg)
elif cfg.exists():
info(
f"{p(cfg)} already exists and is a regular file; moving it to {p(host_cfg)}"
)
if host_cfg.exists():
fatal(f"{p(host_cfg)} already exists!")
elif not DRY_RUN:
cfg.rename(host_cfg)
else:
info(f"{p(cfg)} doesn't exist; creating it as a link to {p(host_cfg)}")
if not DRY_RUN:
cfg.symlink_to(host_cfg)
def _check_tracing():
import defines
if defines.TRACING_ON:
return
fatal("Tracing is not enabled. Compile with TRACING_ON")
def __init__(self, value):
try:
self.value = float(value)
except:
fatal("invalid argument: %r" % time)
warning("'max_time' filter is deprecated, use "
"'user_time < %.4f' filter expression" % self.value)
def __init__(self, value):
try:
self.value = float(value)
except:
fatal("invalid argument: %r" % time)
warning("'max_time' filter is deprecated, use "
"'user_time < %.4f' filter expression" % self.value)
def pull(args: Args) -> None:
"""Update ``args.repo`` with ``git pull``."""
if args.pull:
cmd("git pull --no-edit")
proc = subprocess.run(args.sudo_prefix + ["git", "pull", "--no-edit"],
check=False,
cwd=args.repo)
if proc.returncode != 0:
# git pull failed, maybe reset?
if args.reset:
run_or_fatal(
args.sudo_prefix +
["git", "reset", "--hard", args.remote_branch],
log=True,
cwd=args.repo,
)
else:
fatal(
"`git pull` failed. Pass `--reset` to reset the repository."
)
info(f"{args.repo} is now at commit:")
subprocess.run(["git", "log", "HEAD^1..HEAD", "--oneline"],
check=False,
cwd=args.repo)
def postList(emailList):
emailFile = {"file": open(emailList, "rb")}
r = requests.post(API_URL, files=emailFile)
resp = r.json()
if resp["status"] != "ok":
util.fatal("%s" % resp["message"])
else:
results = {}
report = '''
File: {filename}
Count: {count}
Purchased/harvested: {p_flag}
Deliverability: {d_flag}
Forced signups: {f_flag}
Spamtrap/blacklist: {s_flag}
Overall: {o_flag}
'''
for k, v in resp.iteritems():
if isinstance(v, dict):
results[k] = ",".join([
"{key}: {message}".format(key=x, message=y)
for x, y in v.iteritems()
])
else:
results[k] = v
print report.format(**results)
def main():
parser = OptionParser(usage="%prog")
parser.add_option('-b', '--benchmarks', action='store_true', default=False, dest='benchmarks', help='analyze benchmark applications')
parser.add_option('-w', '--websites', action='store_true', default=False, dest='websites', help='end-to-end website analysis')
parser.add_option('-y', '--targetpages', action='store_true', default=False, dest='targetpages', help='prototype target website analysis')
parser.add_option('-m', '--micro', action='store_true', default=False, dest='micro', help='analyze microbenchmark applications')
parser.add_option('-x', '--exploit', action='store_true', default=False, dest='exploit', help='analyze exploit applications')
#parser.add_option('-i', '--interpreter', action='store_true', default=False, dest='interpreter', help='test semantics as an interpreter (currently unsupported)')
parser.add_option('-e', '--overwrite', action='store_true', default=False, dest='overwrite', help='overwrite expected output')
#parser.add_option('-s', '--semantics', action='store_true', default=False, dest='semantics', help='test semantics')
parser.add_option('-g', '--debug', action='store_true', default=False, dest='debug', help='generate debug output')
parser.add_option('-u', '--url', action='store', default=None, dest='url', help='analyze HTML/JS at given URL')
parser.add_option('-Y', '--policy', action='store', default=None, dest='policy', help='policy file to apply to URL')
parser.add_option('-p', '--refine', action='store', default=None, dest='refine', help='maximum of number of predicates to learn')
parser.add_option('-z', '--syntax-only', action='store_true', default=False, dest='syntaxonly', help='use syntax-only (not semantic) analysis')
parser.add_option('-d', '--service', action='store_true', default=False, dest='service', help='run JAM as a service')
parser.add_option('-a', '--apps', action='append', default=None, dest='apps', help='limit to the given app(s)')
opts, args = parser.parse_args()
if len(args) != 0:
parser.error("Invalid number of arguments")
allmods = True
if opts.benchmarks or opts.micro or opts.exploit or opts.websites or opts.targetpages or opts.url is not None:
allmods = False
ref = opts.refine
if ref is not None:
try:
ref = int(ref)
if ref < -1:
raise
except:
fatal('Invalid refinement limit: %s (should be positive integer, or -1 for unlimited)' % (opts.refine))
if opts.service:
start_jam_service(debug=opts.debug)
#if opts.interpreter:
# err('Interpreter tests are currently out-of-order')
# run_interpreter_tests(opts.debug)
if opts.url is not None:
pol = load_policy(opts.policy)
run_website(opts.url, pol, debug=opts.debug, overwrite=opts.overwrite, refine=ref, synonly=opts.syntaxonly, service=opts.service)
if allmods or opts.micro:
run_microbenchmarks(opts.debug, opts.overwrite, refine=ref, synonly=opts.syntaxonly, service=opts.service, apps=opts.apps)
if allmods or opts.exploit:
run_exploits(opts.debug, opts.overwrite, refine=ref, synonly=opts.syntaxonly, service=opts.service, apps=opts.apps)
if allmods or opts.benchmarks:
run_benchmarks(opts.debug, opts.overwrite, refine=ref, synonly=opts.syntaxonly, service=opts.service, apps=opts.apps)
if allmods or opts.websites:
run_websites(opts.debug, opts.overwrite, refine=ref, synonly=opts.syntaxonly, service=opts.service, apps=opts.apps)
if allmods or opts.targetpages:
run_targetpages(opts.debug, opts.overwrite, refine=ref, synonly=opts.syntaxonly, service=opts.service, apps=opts.apps)
if opts.service:
close_jam_service()
def get_nsc(ns3_dir):
print """
#
# Get NSC
#
"""
# Skip downloading NSC on OS X due to HFS+ case insensitive issues
# Skip downloading NSC on Cygwin because of fundamental incompatibilities.
if sys.platform in ['darwin', 'cygwin']:
print "Architecture (%s) does not support NSC... skipping" % (
sys.platform, )
raise RuntimeError
# (peek into the ns-3 wscript and extract the required nsc version)
internet_stack_wscript = open(
os.path.join(ns3_dir, "src", "internet", "wscript"), "rt")
required_nsc_version = None
for line in internet_stack_wscript:
if 'NSC_RELEASE_NAME' in line:
required_nsc_version = eval(line.split('=')[1].strip())
break
internet_stack_wscript.close()
if required_nsc_version is None:
fatal("Unable to detect NSC required version")
print "Required NSC version: ", required_nsc_version
def nsc_clone():
print "Retrieving nsc from " + constants.NSC_REPO
run_command(
['hg', 'clone', constants.NSC_REPO, constants.LOCAL_NSC_PATH])
def nsc_update():
print "Pulling nsc updates from " + constants.NSC_REPO
run_command([
'hg', '--cwd', constants.LOCAL_NSC_PATH, 'pull', '-u',
constants.NSC_REPO
])
def nsc_download():
local_file = required_nsc_version + ".tar.bz2"
remote_file = constants.NSC_RELEASE_URL + "/" + local_file
print "Retrieving nsc from " + remote_file
urllib.urlretrieve(remote_file, local_file)
print "Uncompressing " + local_file
run_command(["tar", "-xjf", local_file])
print "Rename %s as %s" % (required_nsc_version,
constants.LOCAL_NSC_PATH)
os.rename(required_nsc_version, constants.LOCAL_NSC_PATH)
if not os.path.exists(os.path.join(ns3_dir, '.hg')):
nsc_download()
elif not os.path.exists(constants.LOCAL_NSC_PATH):
nsc_clone()
else:
nsc_update()
return (constants.LOCAL_NSC_PATH, required_nsc_version)
def split_command_filters(command):
for i, arg in enumerate(command):
if arg[:2] != "%%" or arg[-2:] != "%%":
break
else:
fatal("invalid command: %s, only contains filter "
"specifications" % ("".join('"%s"' % a for a in command)))
return ([a[2:-2] for a in command[:i]], command[i:])
def get_netanim(ns3_dir):
print """
#
# Get NetAnim
#
"""
if sys.platform in ['cygwin']:
print "Architecture (%s) does not support NetAnim... skipping" % (sys.platform)
raise RuntimeError
# (peek into the ns-3 wscript and extract the required netanim version)
try:
# For the recent versions
netanim_wscript = open(os.path.join(ns3_dir, "src", "netanim", "wscript"), "rt")
except:
print "Unable to detect NetAnim required version.Skipping download"
pass
return
required_netanim_version = None
for line in netanim_wscript:
if 'NETANIM_RELEASE_NAME' in line:
required_netanim_version = eval(line.split('=')[1].strip())
break
netanim_wscript.close()
if required_netanim_version is None:
fatal("Unable to detect NetAnim required version")
print "Required NetAnim version: ", required_netanim_version
def netanim_clone():
print "Retrieving NetAnim from " + constants.NETANIM_REPO
run_command(['hg', 'clone', constants.NETANIM_REPO, constants.LOCAL_NETANIM_PATH])
def netanim_update():
print "Pulling NetAnim updates from " + constants.NETANIM_REPO
run_command(['hg', '--cwd', constants.LOCAL_NETANIM_PATH, 'pull', '-u', constants.NETANIM_REPO])
def netanim_download():
local_file = required_netanim_version + ".tar.bz2"
remote_file = constants.NETANIM_RELEASE_URL + "/" + local_file
print "Retrieving NetAnim from " + remote_file
urllib.urlretrieve(remote_file, local_file)
print "Uncompressing " + local_file
run_command(["tar", "-xjf", local_file])
print "Rename %s as %s" % (required_netanim_version, constants.LOCAL_NETANIM_PATH)
os.rename(required_netanim_version, constants.LOCAL_NETANIM_PATH)
if not os.path.exists(os.path.join(ns3_dir, '.hg')):
netanim_download()
elif not os.path.exists(constants.LOCAL_NETANIM_PATH):
netanim_clone()
else:
netanim_update()
return (constants.LOCAL_NETANIM_PATH, required_netanim_version)
def backup_object(self, obj: str) -> None:
"""Backup an object. Must be a directory or ordinary file."""
logging.info(f'backup: {obj}')
if os.path.isdir(obj):
self.backup_dir(obj)
elif os.path.isfile(obj):
self.backup_file(obj)
else:
util.fatal(f'ABORT: {obj} is not a file or directory')
def get_nsc(ns3_dir):
print """
#
# Get NSC
#
"""
# Skip downloading NSC on OS X due to HFS+ case insensitive issues
# Skip downloading NSC on Cygwin because of fundamental incompatibilities.
if sys.platform in ['darwin', 'cygwin']:
print "Architecture (%s) does not support NSC... skipping" % (sys.platform,)
raise RuntimeError
# (peek into the ns-3 wscript and extract the required nsc version)
try:
# For the recent versions
internet_stack_wscript = open(os.path.join(ns3_dir, "src", "internet", "wscript"), "rt")
except IOError:
# For the old versions (ns-3.10 and before)
internet_stack_wscript = open(os.path.join(ns3_dir, "src", "internet-stack", "wscript"), "rt")
required_nsc_version = None
for line in internet_stack_wscript:
if 'NSC_RELEASE_NAME' in line:
required_nsc_version = eval(line.split('=')[1].strip())
break
internet_stack_wscript.close()
if required_nsc_version is None:
fatal("Unable to detect NSC required version")
print "Required NSC version: ", required_nsc_version
def nsc_clone():
print "Retrieving nsc from " + constants.NSC_REPO
run_command(['hg', 'clone', constants.NSC_REPO, constants.LOCAL_NSC_PATH])
def nsc_update():
print "Pulling nsc updates from " + constants.NSC_REPO
run_command(['hg', '--cwd', constants.LOCAL_NSC_PATH, 'pull', '-u', constants.NSC_REPO])
def nsc_download():
local_file = required_nsc_version + ".tar.bz2"
remote_file = constants.NSC_RELEASE_URL + "/" + local_file
print "Retrieving nsc from " + remote_file
urllib.urlretrieve(remote_file, local_file)
print "Uncompressing " + local_file
run_command(["tar", "-xjf", local_file])
print "Rename %s as %s" % (required_nsc_version, constants.LOCAL_NSC_PATH)
os.rename(required_nsc_version, constants.LOCAL_NSC_PATH)
if not os.path.exists(os.path.join(ns3_dir, '.hg')):
nsc_download()
elif not os.path.exists(constants.LOCAL_NSC_PATH):
nsc_clone()
else:
nsc_update()
return (constants.LOCAL_NSC_PATH, required_nsc_version)
def split_command_filters(command):
for i, arg in enumerate(command):
if arg[:2] != "%%" or arg[-2:] != "%%":
break
else:
fatal("invalid command: %s, only contains filter "
"specifications" % ("".join('"%s"' % a for a in command)))
return ([a[2:-2] for a in command[:i]],
command[i:])
def __init__(self, list_name):
self.numlist = []
self.list_name = list_name
try:
with open(list_name, "r") as f:
self.numlist = [int(line) for line in f]
except IOError:
pass #no numlist.txt, ignore
except ValueError:
util.fatal("Malformed numlist.txt")
def interp(self):
p = Parser(self.fname, self.text)
try:
node = p.parse()
except ParserError, e:
ctx = "Traceback: \n"
for f in callstack:
ctx = ctx + str(f) + '\n'
output = ctx + str(e)
fatal(output)
def __init__(self, repo: str):
self.repo = repo
self.lock_name = repo + LOCK_FILE
self.objects = repo + OBJECTS
self.backups = repo + BACKUPS
self.all_backups = self.get_all_backups()
self.sig_file = repo + SIG
if not self.quick_verify():
util.fatal(f'{repo} is not a repository')
self.bytes_written = 0
def interp(self):
p = Parser(self.fname, self.text)
try:
node = p.parse()
except ParserError, e:
ctx = "Traceback: \n"
for f in callstack:
ctx = ctx + str(f) + '\n'
output = ctx + str(e)
fatal(output)
def type_check(self):
p = Parser(self.fname)
try:
node = p.parse()
except ParserError, e:
ctx = "Traceback: \n"
for f in callstack:
ctx = ctx + str(f) + '\n'
output = ctx + str(e)
fatal(output)
def instantiate(ckpt_dir=None):
from m5 import options
root = objects.Root.getInstance()
if not root:
fatal("Need to instantiate Root() before calling instantiate()")
# we need to fix the global frequency
ticks.fixGlobalFrequency()
# Make sure SimObject-valued params are in the configuration
# hierarchy so we catch them with future descendants() walks
for obj in root.descendants(): obj.adoptOrphanParams()
# Unproxy in sorted order for determinism
for obj in root.descendants(): obj.unproxyParams()
if options.dump_config:
ini_file = file(os.path.join(options.outdir, options.dump_config), 'w')
# Print ini sections in sorted order for easier diffing
for obj in sorted(root.descendants(), key=lambda o: o.path()):
obj.print_ini(ini_file)
ini_file.close()
# Initialize the global statistics
stats.initSimStats()
# Create the C++ sim objects and connect ports
for obj in root.descendants(): obj.createCCObject()
for obj in root.descendants(): obj.connectPorts()
# Do a second pass to finish initializing the sim objects
for obj in root.descendants(): obj.init()
# Do a third pass to initialize statistics
for obj in root.descendants(): obj.regStats()
for obj in root.descendants(): obj.regFormulas()
# We're done registering statistics. Enable the stats package now.
stats.enable()
# Restore checkpoint (if any)
if ckpt_dir:
ckpt = internal.core.getCheckpoint(ckpt_dir)
internal.core.unserializeGlobals(ckpt);
for obj in root.descendants(): obj.loadState(ckpt)
need_resume.append(root)
else:
for obj in root.descendants(): obj.initState()
# Reset to put the stats in a consistent state.
stats.reset()
def create(repo: str) -> None:
"""Create a new repository in 'repo'.
Make sure 'repo' is an empty directory.
Call create_contents() to initialize it.
"""
logging.info(f'creating repo: {repo}')
if os.path.isdir(repo):
if os.listdir(repo) == []:
_create_repo_contents(repo)
logging.info('repo creation complete')
else:
util.fatal(f'directory {repo} is not empty')
else:
util.fatal(f'{repo} is not a directory.')
def pt_newtool(args):
'''newtool - Create a new tool-style program
usage: pytool newtool <program-name> <prefix>
Creates executable file <program-name>.py with skeletal
contents. The structure of the program is such that it is easy
to add and describe new subfunctions.
'''
p = optparse.OptionParser()
(o, a) = p.parse_args(args)
if a == [] or len(a) != 2:
U.fatal('usage: pytool newtool <program-name> <prefix>')
else:
lname = a[0]
pname = lname + '.py'
prefix = a[1]
are_we_overwriting([lname, pname])
f = open(pname, 'w')
f.writelines(['#!/usr/bin/env python\n',
'"""\n',
'%s - program description\n' % lname,
'"""\n',
'\n',
'from bscr import util as U\n',
'import optparse\n',
'import os\n',
'import re\n',
'import sys\n',
'\n',
'# ----------------------------------------------------'
+ '-----------------------\n',
'def %s_example(argv):\n' % prefix,
' print("this is an example")\n',
"\n",
'# ----------------------------------------------------'
+ '-----------------------\n',
"if __name__ == '__main__':\n",
" U.dispatch('__main__', '%s', sys.argv)\n" % prefix,
])
f.close()
os.chmod(pname, 0755)
os.symlink(os.path.abspath(pname), lname)
def save(self, id):
id = str(id)
"write the template id to disk at self.location"
fname = os.path.join(self.location, id)
if not os.path.exists(fname):
template = self.cache[id]
if template is None:
return
xml = BallotTemplate.Template_to_XML(template)
util.writeto(fname, xml)
if template.image is not None:
try:
im = _fixup(template.image, template.rot, template.xoff,
template.yoff)
im.save(fname + ".jpg")
except IOError:
util.fatal("could not save image of template")
self.log.info("new template %s saved", fname)
def main() -> None:
args = docopt.docopt(__doc__, version=VERSION)
logging.basicConfig(filename=LOG_FILE, filemode='a',
format='%(levelname)s - %(message)s', level=LOG_LEVEL)
now = time.asctime()
logging.info(f'log start: {now}')
logging.info(f'args: {args}')
repo_file = args['REPO']
if args['create']:
Repo.create(repo_file)
elif args['verify']:
repos = Repo(repo_file)
logging.info(f'Verifying repo {repo_file}')
n = 0
if args['--hashes']:
repos.verify_all_hashes(delete=args['-d'])
n += 1
if args['--backups']:
repos.verify_backups()
n += 1
if args['--orphans']:
repos.find_orphans(delete=args['-d'])
n += 1
if n == 0:
util.msg('You must use --hashes, --backups, or --orphans')
elif args['unlock']:
# XXX Could this be: Repo(repo_file).unlock() ?
repos = Repo(repo_file)
repos.unlock()
elif args['backup']:
arg_prefix = args['--prefix']
prefix = arg_prefix if arg_prefix else 'backup'
if not alphanum.match(prefix):
util.fatal('prefix must be alphanumeric')
want_verify = args['--verify']
backup.run_backup(args['--repo'], prefix, args['FILE'],
verify=want_verify,
verbose=args['-v'])
elif args['restore']:
restore.restore(repo_file)
now = time.asctime()
logging.info(f'log end: {now}')
def fetch_builds(name):
"""
fetch_builds(name) -> [(path, revision, build), ...]
Get a list of available builds for the named builder.
"""
# Handle only_use_cache setting.
prefs = util.get_prefs()
if prefs.getboolean("ci", "only_use_cache"):
cache_path = os.path.join(prefs.path, "ci", "build_cache")
cache_build_path = os.path.join(cache_path, name)
items = os.listdir(cache_build_path)
assert False, "Unimplemented?" + str(items)
# Otherwise, load the builder map.
buildermap = load_builder_map()
# If the builder isn't in the builder map, do a forced load of the builder
# map.
if name not in buildermap.builders:
buildermap = load_builder_map(reload=True)
# If the builder doesn't exist, report an error.
builder_artifacts = buildermap.builders.get(name)
if builder_artifacts is None:
fatal("unknown builder name: %r" % (name,))
# Otherwise, load the builder list.
server_builds = gcs.fetch_builds(builder_artifacts)
builds = []
for path in server_builds['items']:
build = Build.frombasename(path['name'], path['mediaLink'])
# Ignore any links which don't at least have a revision component.
if build.revision is not None:
builds.append(build)
# If there were no builds, report an error.
if not builds:
fatal("builder %r may be misconfigured (no items)" % (name,))
# Sort the builds, to make sure we return them ordered properly.
builds.sort()
return builds
def fl_edit(args):
"""edit - edit files in place
usage: fl edit [-i <suffix>] -e <edit-cmd> file1 file2 file3 ...
"""
p = optparse.OptionParser()
p.add_option('-d', '--debug',
default=False, action='store_true', dest='debug',
help='run the debugger')
p.add_option('-e', '--edit',
default='', action='store', dest='edit_cmd',
help='edit command')
p.add_option('-i', '--init',
default='', action='store', dest='suffix',
help='suffix for original files')
(o, a) = p.parse_args(args)
if o.debug:
pdb.set_trace()
if o.suffix == '':
suffix = 'original'
else:
suffix = o.suffix
if o.edit_cmd == '':
util.fatal("usage: fl edit [-i <suffix>] -e <cmd> f1 f2 ...")
ec = o.edit_cmd.split(o.edit_cmd[1])
if all([ec[0] != 's', ec[0] != 'y']):
raise bscr.Error("Only 's' and 'y' supported for -e right now")
if 4 != len(ec):
raise bscr.Error("usage: ... -e '[sy]/before/after/' ...")
(op, prev, post) = ec[0:3]
if 0 == len(a):
util.fatal("no files on command line to edit")
else:
for filename in a:
editfile(filename, op, prev, post, suffix)
def fetch_build_to_path(builder, build, root_path, builddir_path):
path = build.tobasename()
# Check whether we are using a build cache and get the cached build path if
# so.
prefs = util.get_prefs()
cache_build_path = None
if prefs.getboolean("ci", "cache_builds"):
cache_path = os.path.join(prefs.path, "ci", "build_cache")
cache_build_path = os.path.join(cache_path, builder, path)
# Copy the build from the cache or download it.
if cache_build_path and os.path.exists(cache_build_path):
shutil.copy(cache_build_path, root_path)
else:
# Load the builder map.
buildermap = load_builder_map()
# If the builder isn't in the builder map, do a forced reload of the
# builder map.
if builder not in buildermap.builders:
buildermap = load_builder_map(reload=True)
# If the builder doesn't exist, report an error.
builder_artifacts = buildermap.builders.get(builder)
if builder_artifacts is None:
fatal("unknown builder name: %r" % (builder,))
# Otherwise create the build url.
gcs.get_compiler(build.url, root_path)
# Copy the build into the cache, if enabled.
if cache_build_path is not None:
shell.mkdir_p(os.path.dirname(cache_build_path))
shutil.copy(root_path, cache_build_path)
# Create the directory for the build.
os.mkdir(builddir_path)
# Extract the build.
if shell.execute(['tar', '-xf', root_path, '-C', builddir_path]):
fatal('unable to extract %r to %r' % (root_path, builddir_path))
def run_backup(repo_name: str, prefix: str, paths: List[str],
verify=True,
verbose=False) -> None:
"""Perform a backup to the repository of all dirs/files in the list.
Make sure we have an unlocked DDU repository. Process all items in
'paths'. If an item is '-', read path names from stdin.
"""
logging.info(f"backup list: {paths}")
logging.info(f"backup repo: {repo_name}")
print(f"Backing up to {repo_name} with prefix '{prefix}'.")
repos = Repo(repo_name)
if repos.lock():
bkup = Backup(repo_name, prefix, verbose=verbose)
for g in paths:
if g == '-':
for f in sys.stdin:
if not f:
continue
f = f.rstrip()
if os.path.isfile(f) or os.path.isdir(f):
bkup.backup_object(f)
else:
util.warning(f'{f} is not a file or directory')
else:
if util.os_is_windows:
# You have to use globbing on Windows. Shell is too stupid.
path_list = glob.glob(g)
if path_list == []:
util.warning(f'File/dir {g} not found.')
else:
for f in glob.glob(g):
bkup.backup_object(f)
else:
bkup.backup_object(g)
bkup.finish()
repos.unlock()
if verify:
repos.verify_backups(only=bkup.bk_name)
else:
util.fatal(f'repo {repo_name} is LOCKED')
def create_test_from_file(fl, name, group, policy):
txt = fl.read()
fl.close()
appdir = os.path.join(TESTS_DIR, group, name)
if os.path.exists(appdir):
if OVERWRITE:
if not os.path.isdir(appdir):
fatal("Unable to overwrite file: %s" % appdir)
warn("Creating in existing directory: %s" % appdir)
else:
fatal("Not overwriting existing directory: %s" % appdir)
prepare_dir(appdir)
inputdir = os.path.join(appdir, 'source-input')
if os.path.exists(inputdir):
assert OVERWRITE
if not os.path.isdir(inputdir):
fatal("Unable to overwrite non-directory: %s" % inputdir)
else:
os.makedirs(inputdir)
tgtfile = "%s.js" % name
tgtpath = os.path.join(inputdir, tgtfile)
tgtfl = open(tgtpath, 'w')
tgtfl.write(txt)
tgtfl.close()
def run_server():
cfg_file = tevsgui_get_args.get_args()
config.get(cfg_file)
logfile = os.path.join(const.root, "logs/processing_log.txt")
bv = BallotClass.LoadBallotFactoryForVendorStyle(const.layout_brand)
bsv = BallotClass.LoadBallotSideFactoryForVendorStyle(const.layout_brand)
logging.basicConfig(
filename=logfile,
format='%(asctime)s %(levelname)s %(module)s %(message)s',
level=logging.INFO)
logging.info("Processing server starting.")
prepopulate_cache()
# establish a connection to the database
# connect to db and open cursor
dbc = None
if const.use_db:
try:
dbc = db.PostgresDB(database=const.dbname, user=const.dbuser)
except db.DatabaseError:
util.fatal("Could not connect to database!")
else:
dbc = db.NullDB()
# Instantiate and bind to localhost:8000
try:
server = AsyncXMLRPCServer(('', 8000), SimpleXMLRPCRequestHandler)
except socket.error as e:
print "A service is already running on address 8000."
sys.exit()
# Register example object instance
server.register_instance(ProcessBallots(const.root, bv, bsv, dbc))
print "Ballot processing / scraping service now ready on address 8000."
print "Using %s as root directory." % (const.root, )
# run!
server.serve_forever()
def main():
parser = OptionParser(usage="%prog")
parser.add_option('-n', '--name', action='store', default=None, dest='name', help='name of the new test')
parser.add_option('-g', '--group', action='store', default=None, dest='group', help='test group to add to')
parser.add_option('-f', '--file', action='store', default=None, dest='file', help='input source file')
parser.add_option('-u', '--url', action='store', default=None, dest='url', help='URL for website test case')
parser.add_option('-F', '--overwrite', action='store_true', default=False, dest='overwrite', help='overwrite existing files')
parser.add_option('-v', '--verbose', action='store_true', default=False, dest='verbose', help='generate verbose output')
parser.add_option('-Y', '--policy', action='store', default=None, dest='policy', help='policy file to use for the test case')
opts, args = parser.parse_args()
if len(args) != 0:
parser.error("Invalid number of arguments")
if opts.group not in TEST_GROUPS:
fatal("Invalid test group: %s" % opts.group)
global VERBOSE, OVERWRITE
VERBOSE = opts.verbose
OVERWRITE = opts.overwrite
if opts.url is not None:
group = opts.group
if group is None:
group = 'sites'
if group != 'sites':
fatal("Invalid group for URL test: %s" % group)
create_test_for_url(opts.url, opts.name, opts.policy)
else:
if opts.file is None:
if opts.name is None:
fatal("Name (-n) must be provided for stdin source")
appname = opts.name
fl = sys.stdin
else:
if not os.path.isfile(opts.file):
fatal("Unable to access source file: %s" % opts.file)
filename = os.path.basename(opts.file)
appname = os.path.splitext(filename)[0]
fl = open(opts.file, 'r')
group = opts.group
if group is None: group = 'bench'
create_test_from_file(fl, appname, group, opts.policy)
class TypeChecker(object):
def __init__(self, fname):
self.fname = fname
def type_check(self):
p = Parser(self.fname)
try:
node = p.parse()
except ParserError, e:
ctx = "Traceback: \n"
for f in callstack:
ctx = ctx + str(f) + '\n'
output = ctx + str(e)
fatal(output)
tbl = SymTable.init_type_table()
try:
return node.interp(tbl)
except TypeCheckError, e:
fatal(str(e))
class Interpreter(object):
def __init__(self, fname, text=None):
self.fname = fname
self.text = text
def interp(self):
p = Parser(self.fname, self.text)
try:
node = p.parse()
except ParserError, e:
ctx = "Traceback: \n"
for f in callstack:
ctx = ctx + str(f) + '\n'
output = ctx + str(e)
fatal(output)
tbl = SymTable.init_value_table()
try:
return node.interp(tbl)
except InterpError, e:
fatal(str(e))
def action_exec(name, args):
"""execute a command against a published root"""
parser = OptionParser("""\
usage: %%prog %(name)s [options] ... test command args ...
Executes the given command against the latest published build. The syntax for
commands (and exit code) is exactly the same as for the 'bisect' tool, so this
command is useful for testing bisect test commands.
See 'bisect' for more notermation on the exact test syntax.\
""" % locals())
parser.add_option("-b", "--build", dest="build_name", metavar="STR",
help="name of build to fetch",
action="store", default=DEFAULT_BUILDER)
parser.add_option("-s", "--sandbox", dest="sandbox",
help="directory to use as a sandbox",
action="store", default=None)
parser.add_option("", "--min-rev", dest="min_rev",
help="minimum revision to test",
type="int", action="store", default=None)
parser.add_option("", "--max-rev", dest="max_rev",
help="maximum revision to test",
type="int", action="store", default=None)
parser.add_option("", "--near", dest="near_build",
help="use a build near NAME",
type="str", action="store", metavar="NAME", default=None)
parser.disable_interspersed_args()
(opts, args) = parser.parse_args(args)
if opts.build_name is None:
parser.error("no build name given (see --build)")
available_builds = list(llvmlab.fetch_builds(opts.build_name))
available_builds.sort()
available_builds.reverse()
if opts.min_rev is not None:
available_builds = [b for b in available_builds
if b.revision >= opts.min_rev]
if opts.max_rev is not None:
available_builds = [b for b in available_builds
if b.revision <= opts.max_rev]
if len(available_builds) == 0:
fatal("No builds available for builder name: %s" % opts.build_name)
# Find the best match, if requested.
if opts.near_build:
build = get_best_match(available_builds, opts.near_build)
if not build:
parser.error("no match for build %r" % opts.near_build)
else:
# Otherwise, take the latest build.
build = available_builds[0]
test_result, _ = execute_sandboxed_test(
opts.sandbox, opts.build_name, build, args, verbose=True,
show_command_output=True)
print '%s: %s' % (('FAIL', 'PASS')[test_result],
build.tobasename(include_suffix=False))
raise SystemExit(test_result != True)
def instantiate(ckpt_dir=None):
from m5 import options
root = objects.Root.getInstance()
if not root:
fatal("Need to instantiate Root() before calling instantiate()")
# we need to fix the global frequency
ticks.fixGlobalFrequency()
# Make sure SimObject-valued params are in the configuration
# hierarchy so we catch them with future descendants() walks
for obj in root.descendants(): obj.adoptOrphanParams()
# Unproxy in sorted order for determinism
for obj in root.descendants(): obj.unproxyParams()
if options.dump_config:
ini_file = file(os.path.join(options.outdir, options.dump_config), 'w')
# Print ini sections in sorted order for easier diffing
for obj in sorted(root.descendants(), key=lambda o: o.path()):
obj.print_ini(ini_file)
ini_file.close()
if options.json_config:
try:
import json
json_file = file(os.path.join(options.outdir, options.json_config), 'w')
d = root.get_config_as_dict()
json.dump(d, json_file, indent=4)
json_file.close()
except ImportError:
pass
do_dot(root, options.outdir, options.dot_config)
# Initialize the global statistics
stats.initSimStats()
# Create the C++ sim objects and connect ports
for obj in root.descendants(): obj.createCCObject()
for obj in root.descendants(): obj.connectPorts()
# Do a second pass to finish initializing the sim objects
for obj in root.descendants(): obj.init()
# Do a third pass to initialize statistics
for obj in root.descendants(): obj.regStats()
# Do a fourth pass to initialize probe points
for obj in root.descendants(): obj.regProbePoints()
# Do a fifth pass to connect probe listeners
for obj in root.descendants(): obj.regProbeListeners()
# We're done registering statistics. Enable the stats package now.
stats.enable()
# Restore checkpoint (if any)
if ckpt_dir:
_drain_manager.preCheckpointRestore()
ckpt = internal.core.getCheckpoint(ckpt_dir)
internal.core.unserializeGlobals(ckpt);
for obj in root.descendants(): obj.loadState(ckpt)
else:
for obj in root.descendants(): obj.initState()
# Check to see if any of the stat events are in the past after resuming from
# a checkpoint, If so, this call will shift them to be at a valid time.
updateStatEvents()
def get_pybindgen(ns3_dir):
print """
#
# Get PyBindGen
#
"""
if sys.platform in ['cygwin']:
print "Architecture (%s) does not support PyBindGen ... skipping" % (sys.platform,)
raise RuntimeError
# (peek into the ns-3 wscript and extract the required pybindgen version)
ns3_python_wscript = open(os.path.join(ns3_dir, "bindings", "python", "wscript"), "rt")
required_pybindgen_version = None
for line in ns3_python_wscript:
if 'REQUIRED_PYBINDGEN_VERSION' in line:
required_pybindgen_version = eval(line.split('=')[1].strip())
ns3_python_wscript.close()
break
if required_pybindgen_version is None:
fatal("Unable to detect pybindgen required version")
print "Required pybindgen version: ", required_pybindgen_version
# work around http_proxy handling bug in bzr
# if 'http_proxy' in os.environ and 'https_proxy' not in os.environ:
# os.environ['https_proxy'] = os.environ['http_proxy']
if 'post' in required_pybindgen_version:
# given a version like '0.17.0.post41+ngd10fa60', the last 7 characters
# are part of a git hash, which should be enough to identify a revision
rev = required_pybindgen_version[-7:]
else:
rev = required_pybindgen_version
if os.path.exists(constants.LOCAL_PYBINDGEN_PATH):
print "Trying to update pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip."
try:
run_command(["git", "fetch", constants.PYBINDGEN_BRANCH],
cwd=constants.LOCAL_PYBINDGEN_PATH)
except KeyboardInterrupt:
print "Interrupted; Python bindings will be disabled."
else:
print "Update was successful."
else:
print "Trying to fetch pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip."
try:
run_command(["git", "clone", constants.PYBINDGEN_BRANCH,
constants.LOCAL_PYBINDGEN_PATH])
except KeyboardInterrupt:
print "Interrupted; Python bindings will be disabled."
shutil.rmtree(constants.LOCAL_PYBINDGEN_PATH, True)
return False
print "Fetch was successful."
run_command(["git", "checkout", rev, "-q"],
cwd=constants.LOCAL_PYBINDGEN_PATH)
## This causes the version to be generated
run_command(["python", "setup.py", "clean"],
cwd=constants.LOCAL_PYBINDGEN_PATH)
return (constants.LOCAL_PYBINDGEN_PATH, required_pybindgen_version)
def action_bisect(name, args):
"""find first failing build using binary search"""
parser = OptionParser("""\
usage: %%prog %(name)s [options] ... test command args ...
Look for the first published build where a test failed, using the builds on
llvmlab. The command arguments are executed once per build tested, but each
argument is first subject to string interpolation. The syntax is
"%%(VARIABLE)FORMAT" where FORMAT is a standard printf format, and VARIABLE is
one of:
'sandbox' - the path to the sandbox directory.
'path' - the path to the build under test.
'revision' - the revision number of the build.
'build' - the build number of the build under test.
'clang' - the path to the clang binary of the build if it exists.
'clang++' - the path to the clang++ binary of the build if it exists.
'libltodir' - the path to the directory containing libLTO.dylib, if it
exists.
Each test is run in a sandbox directory. By default, sandbox directories are
temporary directories which are created and destroyed for each test (see
--sandbox).
For use in auxiliary test scripts, each test is also run with each variable
available in the environment as TEST_<variable name> (variables are converted
to uppercase). For example, a test script could use "TEST_PATH" to find the
path to the build under test.
The stdout and stderr of the command are logged to files inside the sandbox
directory. Use an explicit sandbox directory if you would like to look at
them.
It is possible to run multiple distinct commands for each test by separating
them in the command line arguments by '----'. The failure of any command causes
the entire test to fail.\
""" % locals())
parser.add_option("-b", "--build", dest="build_name", metavar="STR",
help="name of build to fetch",
action="store", default=DEFAULT_BUILDER)
parser.add_option("-s", "--sandbox", dest="sandbox",
help="directory to use as a sandbox",
action="store", default=None)
parser.add_option("-v", "--verbose", dest="verbose",
help="output more test notermation",
action="store_true", default=False)
parser.add_option("-V", "--very-verbose", dest="very_verbose",
help="output even more test notermation",
action="store_true", default=False)
parser.add_option("", "--show-output", dest="show_command_output",
help="display command output",
action="store_true", default=False)
parser.add_option("", "--single-step", dest="single_step",
help="single step instead of binary stepping",
action="store_true", default=False)
parser.add_option("", "--min-rev", dest="min_rev",
help="minimum revision to test",
type="int", action="store", default=None)
parser.add_option("", "--max-rev", dest="max_rev",
help="maximum revision to test",
type="int", action="store", default=None)
parser.disable_interspersed_args()
(opts, args) = parser.parse_args(args)
if opts.build_name is None:
parser.error("no build name given (see --build)")
# Very verbose implies verbose.
opts.verbose |= opts.very_verbose
start_time = time.time()
available_builds = list(llvmlab.fetch_builds(opts.build_name))
available_builds.sort()
available_builds.reverse()
if opts.very_verbose:
note("fetched builds in %.2fs" % (time.time() - start_time,))
if opts.min_rev is not None:
available_builds = [b for b in available_builds
if b.revision >= opts.min_rev]
if opts.max_rev is not None:
available_builds = [b for b in available_builds
if b.revision <= opts.max_rev]
def predicate(item):
# Run the sandboxed test.
test_result, _ = execute_sandboxed_test(
opts.sandbox, opts.build_name, item, args, verbose=opts.verbose,
very_verbose=opts.very_verbose,
show_command_output=opts.show_command_output or opts.very_verbose)
# Print status.
print '%s: %s' % (('FAIL', 'PASS')[test_result],
item.tobasename(include_suffix=False))
return test_result
if opts.single_step:
for item in available_builds:
if predicate(item):
break
else:
item = None
else:
if opts.min_rev is None or opts.max_rev is None:
# Gallop to find initial search range, under the assumption that we
# are most likely looking for something at the head of this list.
search_space = algorithm.gallop(predicate, available_builds)
else:
# If both min and max revisions are specified,
# don't gallop - bisect the given range.
search_space = available_builds
item = algorithm.bisect(predicate, search_space)
if item is None:
fatal('unable to find any passing build!')
print '%s: first working build' % item.tobasename(include_suffix=False)
index = available_builds.index(item)
if index == 0:
print 'no failing builds!?'
else:
print '%s: next failing build' % available_builds[index-1].tobasename(
include_suffix=False)
def create_test_for_url(url, name, policy):
fatal("URL test case creation not yet implemented")
def action_fetch(name, args):
"""fetch a build from the server"""
parser = OptionParser("""\
usage: %%prog %(name)s [options] builder [build-name]
Fetch the build from the named builder which matchs build-name. If no match is
found, get the first build before the given name. If no build name is given,
the most recent build is fetched.
The available builders can be listed using:
%%prog ls
The available builds can be listed using:
%%prog ls builder""" % locals())
parser.add_option("-f", "--force", dest="force",
help=("always download and extract, overwriting any"
"existing files"),
action="store_true", default=False)
parser.add_option("", "--update-link", dest="update_link", metavar="PATH",
help=("update a symbolic link at PATH to point to the "
"fetched build (on success)"),
action="store", default=None)
parser.add_option("-d", "--dry-run", dest='dry_run',
help=("Perform all operations except the actual "
"downloading and extracting of any files"),
action="store_true", default=False)
(opts, args) = parser.parse_args(args)
if len(args) == 0:
parser.error("please specify a builder name")
elif len(args) == 1:
builder, = args
build_name = None
elif len(args) == 2:
builder, build_name = args
else:
parser.error("invalid number of arguments")
builds = list(llvmlab.fetch_builds(builder))
if not builds:
parser.error("no builds for builder: %r" % builder)
build = get_best_match(builds, build_name)
if not build:
parser.error("no match for build %r" % build_name)
path = build.tobasename()
if build_name is not None and not path.startswith(build_name):
note('no exact match, fetching %r' % path)
# Get the paths to extract to.
root_path = path
builddir_path = build.tobasename(include_suffix=False)
if not opts.dry_run:
# Check that the download and extract paths are clean.
for p in (root_path, builddir_path):
if os.path.exists(p):
# If we are using --force, then clean the path.
if opts.force:
shutil.rmtree(p, ignore_errors=True)
continue
fatal('current directory is not clean, %r exists' % p)
llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path)
print 'downloaded root: %s' % root_path
print 'extracted path : %s' % builddir_path
# Update the symbolic link, if requested.
if not opts.dry_run and opts.update_link:
# Remove the existing path.
try:
os.unlink(opts.update_link)
except OSError as e:
if e.errno != errno.ENOENT:
fatal('unable to update symbolic link at %r, cannot unlink' % (
opts.update_link))
# Create the symbolic link.
os.symlink(os.path.abspath(builddir_path), opts.update_link)
print 'updated link at: %s' % opts.update_link
return os.path.abspath(builddir_path)
import config
if __name__ == "__main__":
config.get()
bv = LoadBallotFactoryForVendorStyle(const.layout_brand)
bsv = LoadBallotSideFactoryForVendorStyle(const.layout_brand)
# establish a connection to the database
# connect to db and open cursor
dbc = None
if const.use_db:
try:
dbc = db.PostgresDB(database=const.dbname, user=const.dbuser)
except db.DatabaseError:
util.fatal("Could not connect to database!")
else:
dbc = db.NullDB()
prepopulate_cache()
# Create a ballot from one image file.
# This should create an instance for the style named as "brand" in tevs.cfg
#b = bv("harttestfront.jpg",None)
# WARNING -- The display program(s)
# will expect a numeric filename
# in the correct location specified by tevs.cfg
# hart
filename1 = "/home/mitch/data/hart/unproc/000/000001.jpg"
filename2 = "/home/mitch/data/hart/unproc/000/000002.jpg"
filename3 = "/home/mitch/data/hart/unproc/000/000003.jpg"
def execute_sandboxed_test(sandbox, builder, build, args,
verbose=False, very_verbose=False,
add_path_variables=True,
show_command_output=False,
reuse_sandbox=False):
def split_command_filters(command):
for i, arg in enumerate(command):
if arg[:2] != "%%" or arg[-2:] != "%%":
break
else:
fatal("invalid command: %s, only contains filter "
"specifications" % ("".join('"%s"' % a for a in command)))
return ([a[2:-2] for a in command[:i]],
command[i:])
path = build.tobasename(include_suffix=False)
fullpath = build.tobasename()
if verbose:
note('testing %r' % path)
# Create the sandbox directory, if it doesn't exist.
is_temp = False
if sandbox is None:
sandbox = tempfile.mkdtemp()
is_temp = True
else:
# Make absolute.
sandbox = os.path.abspath(sandbox)
if not os.path.exists(sandbox):
os.mkdir(sandbox)
# Compute paths and make sure sandbox is clean.
root_path = os.path.join(sandbox, fullpath)
builddir_path = os.path.join(sandbox, path)
need_build = True
if reuse_sandbox and (os.path.exists(root_path) and
os.path.exists(builddir_path)):
need_build = False
else:
for p in (root_path, builddir_path):
if os.path.exists(p):
fatal('sandbox is not clean, %r exists' % p)
# Fetch and extract the build.
if need_build:
start_time = time.time()
llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path)
if very_verbose:
note("extracted build in %.2fs" % (time.time() - start_time,))
# Attempt to find clang/clang++ in the downloaded build.
def find_binary(name):
x = subprocess.check_output(['find', builddir_path, '-name', name])\
.strip().split("\n")[0]
if x == '':
x = None
return x
clang_path = find_binary('clang')
clangpp_path = find_binary('clang++')
liblto_path = find_binary('libLTO.dylib')
if liblto_path is not None:
liblto_dir = os.path.dirname(liblto_path)
else:
liblto_dir = None
# Construct the interpolation variables.
options = {'sandbox': sandbox,
'path': builddir_path,
'revision': build.revision,
'build': build.build,
'clang': clang_path,
'clang++': clangpp_path,
'libltodir': liblto_dir}
# Inject environment variables.
env = os.environ.copy()
for key, value in options.items():
env['TEST_%s' % key.upper()] = str(value)
# Extend the environment to include the path to the extracted build.
#
# FIXME: Ideally, we would be able to read some kind of configuration
# notermation about a builder so that we could just set this up, it doesn't
# necessarily here as hard-coded notermation.
if add_path_variables:
path_extensions = []
dyld_library_path_extensions = []
toolchains_dir = os.path.join(builddir_path,
('Applications/Xcode.app/Contents/'
'Developer/Toolchains'))
toolchain_paths = []
if os.path.exists(toolchains_dir):
toolchain_paths = [os.path.join(toolchains_dir, name, 'usr')
for name in os.listdir(toolchains_dir)]
for package_root in ['', 'Developer/usr/'] + toolchain_paths:
p = os.path.join(builddir_path, package_root, 'bin')
if os.path.exists(p):
path_extensions.append(p)
p = os.path.join(builddir_path, package_root, 'lib')
if os.path.exists(p):
dyld_library_path_extensions.append(p)
if path_extensions:
env['PATH'] = os.pathsep.join(
path_extensions + [os.environ.get('PATH', '')])
if dyld_library_path_extensions:
env['DYLD_LIBRARY_PATH'] = os.pathsep.join(
dyld_library_path_extensions + [
os.environ.get('DYLD_LIBRARY_PATH', '')])
# Split the arguments into distinct commands.
#
# Extended command syntax allows running multiple commands by separating
# them with '----'.
test_commands = util.list_split(args, "----")
# Split command specs into filters and commands.
test_commands = [split_command_filters(spec) for spec in test_commands]
# Execute the test.
command_objects = []
interpolated_variables = False
for i, (filters, command) in enumerate(test_commands):
# Interpolate arguments.
old_command = command
command = [a % options for a in command]
if old_command != command:
interpolated_variables = True
# Create the command object...
stdout_log_path = os.path.join(sandbox, '%s.%d.stdout' % (path, i))
stderr_log_path = os.path.join(sandbox, '%s.%d.stderr' % (path, i))
cmd_object = Command(command, stdout_log_path, stderr_log_path, env)
command_objects.append(cmd_object)
# Execute the command.
try:
cmd_object.execute(verbose=verbose)
except OSError, e:
# Python's exceptions are horribly to read, and this one is
# incredibly common when people don't use the right syntax (or
# misspell something) when writing a predicate. Detect this and
# notify the user.
if e.errno == errno.ENOENT:
fatal("invalid command, executable doesn't exist: %r" % (
cmd_object.command[0],))
elif e.errno == errno.ENOEXEC:
fatal("invalid command, executable has a bad format. Did you "
"forget to put a #! at the top of a script?: %r"
% (cmd_object.command[0],))
else:
# Otherwise raise the error again.
raise e
# Evaluate the filters.
for filter in filters:
cmd_object.evaluate_filter_spec(filter)
if show_command_output:
for p, type in ((stdout_log_path, "stdout"),
(stderr_log_path, "stderr")):
if not os.path.exists(p):
continue
f = open(p)
data = f.read()
f.close()
if data:
print ("-- command %s (note: suppressed by default, "
"see sandbox dir for log files) --" % (type))
print "--\n%s--\n" % data
test_result = cmd_object.result
if not test_result:
break
def get_pybindgen(ns3_dir):
print """
#
# Get PyBindGen
#
"""
if sys.platform in ['cygwin']:
print "Architecture (%s) does not support PyBindGen ... skipping" % (sys.platform,)
raise RuntimeError
# (peek into the ns-3 wscript and extract the required pybindgen version)
ns3_python_wscript = open(os.path.join(ns3_dir, "bindings", "python", "wscript"), "rt")
required_pybindgen_version = None
for line in ns3_python_wscript:
if 'REQUIRED_PYBINDGEN_VERSION' in line:
required_pybindgen_version = eval(line.split('=')[1].strip())
ns3_python_wscript.close()
break
if required_pybindgen_version is None:
fatal("Unable to detect pybindgen required version")
print "Required pybindgen version: ", '.'.join([str(x) for x in required_pybindgen_version])
# work around http_proxy handling bug in bzr
if 'http_proxy' in os.environ and 'https_proxy' not in os.environ:
os.environ['https_proxy'] = os.environ['http_proxy']
if len(required_pybindgen_version) == 4:
rev = "-rrevno:%i" % required_pybindgen_version[3]
else:
rev = "-rtag:%s" % '.'.join([str(x) for x in required_pybindgen_version])
if os.path.exists(constants.LOCAL_PYBINDGEN_PATH):
print "Trying to update pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip."
try:
run_command(["bzr", "pull", rev, "-d", constants.LOCAL_PYBINDGEN_PATH, constants.PYBINDGEN_BRANCH])
except KeyboardInterrupt:
print "Interrupted; Python bindings will be disabled."
else:
print "Update was successful."
else:
print "Trying to fetch pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip."
try:
run_command(["bzr", "checkout", rev, constants.PYBINDGEN_BRANCH, constants.LOCAL_PYBINDGEN_PATH])
except KeyboardInterrupt:
print "Interrupted; Python bindings will be disabled."
shutil.rmtree(constants.LOCAL_PYBINDGEN_PATH, True)
return False
print "Fetch was successful."
## generate a fake version.py file in pybindgen it's safer this
## way, since the normal version generation process requires
## bazaar python bindings, which may not be available.
vfile = open(os.path.join(constants.LOCAL_PYBINDGEN_PATH, "pybindgen", "version.py"), "wt")
vfile.write("""
# (fake version generated by ns-3)
__version__ = %r
""" % list(required_pybindgen_version))
vfile.close()
return (constants.LOCAL_PYBINDGEN_PATH, '.'.join([str(x) for x in required_pybindgen_version]))
def execute_sandboxed_test(sandbox,
builder,
build,
args,
verbose=False,
very_verbose=False,
add_path_variables=True,
show_command_output=False,
reuse_sandbox=False):
def split_command_filters(command):
for i, arg in enumerate(command):
if arg[:2] != "%%" or arg[-2:] != "%%":
break
else:
fatal("invalid command: %s, only contains filter "
"specifications" % ("".join('"%s"' % a for a in command)))
return ([a[2:-2] for a in command[:i]], command[i:])
path = build.tobasename(include_suffix=False)
fullpath = build.tobasename()
if verbose:
note('testing %r' % path)
# Create the sandbox directory, if it doesn't exist.
is_temp = False
if sandbox is None:
sandbox = tempfile.mkdtemp()
is_temp = True
else:
# Make absolute.
sandbox = os.path.abspath(sandbox)
if not os.path.exists(sandbox):
os.mkdir(sandbox)
# Compute paths and make sure sandbox is clean.
root_path = os.path.join(sandbox, fullpath)
builddir_path = os.path.join(sandbox, path)
need_build = True
if reuse_sandbox and (os.path.exists(root_path)
and os.path.exists(builddir_path)):
need_build = False
else:
for p in (root_path, builddir_path):
if os.path.exists(p):
fatal('sandbox is not clean, %r exists' % p)
# Fetch and extract the build.
if need_build:
start_time = time.time()
llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path)
if very_verbose:
note("extracted build in %.2fs" % (time.time() - start_time, ))
# Attempt to find clang/clang++ in the downloaded build.
def find_binary(name):
x = subprocess.check_output(['find', builddir_path, '-name', name])\
.strip().split("\n")[0]
if x == '':
x = None
return x
clang_path = find_binary('clang')
clangpp_path = find_binary('clang++')
liblto_path = find_binary('libLTO.dylib')
if liblto_path is not None:
liblto_dir = os.path.dirname(liblto_path)
else:
liblto_dir = None
# Construct the interpolation variables.
options = {
'sandbox': sandbox,
'path': builddir_path,
'revision': build.revision,
'build': build.build,
'clang': clang_path,
'clang++': clangpp_path,
'libltodir': liblto_dir
}
# Inject environment variables.
env = os.environ.copy()
for key, value in options.items():
env['TEST_%s' % key.upper()] = str(value)
# Extend the environment to include the path to the extracted build.
#
# FIXME: Ideally, we would be able to read some kind of configuration
# notermation about a builder so that we could just set this up, it doesn't
# necessarily here as hard-coded notermation.
if add_path_variables:
path_extensions = []
dyld_library_path_extensions = []
toolchains_dir = os.path.join(builddir_path,
('Applications/Xcode.app/Contents/'
'Developer/Toolchains'))
toolchain_paths = []
if os.path.exists(toolchains_dir):
toolchain_paths = [
os.path.join(toolchains_dir, name, 'usr')
for name in os.listdir(toolchains_dir)
]
for package_root in ['', 'Developer/usr/'] + toolchain_paths:
p = os.path.join(builddir_path, package_root, 'bin')
if os.path.exists(p):
path_extensions.append(p)
p = os.path.join(builddir_path, package_root, 'lib')
if os.path.exists(p):
dyld_library_path_extensions.append(p)
if path_extensions:
env['PATH'] = os.pathsep.join(path_extensions +
[os.environ.get('PATH', '')])
if dyld_library_path_extensions:
env['DYLD_LIBRARY_PATH'] = os.pathsep.join(
dyld_library_path_extensions +
[os.environ.get('DYLD_LIBRARY_PATH', '')])
# Split the arguments into distinct commands.
#
# Extended command syntax allows running multiple commands by separating
# them with '----'.
test_commands = util.list_split(args, "----")
# Split command specs into filters and commands.
test_commands = [split_command_filters(spec) for spec in test_commands]
# Execute the test.
command_objects = []
interpolated_variables = False
for i, (filters, command) in enumerate(test_commands):
# Interpolate arguments.
old_command = command
command = [a % options for a in command]
if old_command != command:
interpolated_variables = True
# Create the command object...
stdout_log_path = os.path.join(sandbox, '%s.%d.stdout' % (path, i))
stderr_log_path = os.path.join(sandbox, '%s.%d.stderr' % (path, i))
cmd_object = Command(command, stdout_log_path, stderr_log_path, env)
command_objects.append(cmd_object)
# Execute the command.
try:
cmd_object.execute(verbose=verbose)
except OSError, e:
# Python's exceptions are horribly to read, and this one is
# incredibly common when people don't use the right syntax (or
# misspell something) when writing a predicate. Detect this and
# notify the user.
if e.errno == errno.ENOENT:
fatal("invalid command, executable doesn't exist: %r" %
(cmd_object.command[0], ))
elif e.errno == errno.ENOEXEC:
fatal("invalid command, executable has a bad format. Did you "
"forget to put a #! at the top of a script?: %r" %
(cmd_object.command[0], ))
else:
# Otherwise raise the error again.
raise e
# Evaluate the filters.
for filter in filters:
cmd_object.evaluate_filter_spec(filter)
if show_command_output:
for p, type in ((stdout_log_path, "stdout"), (stderr_log_path,
"stderr")):
if not os.path.exists(p):
continue
f = open(p)
data = f.read()
f.close()
if data:
print(
"-- command %s (note: suppressed by default, "
"see sandbox dir for log files) --" % (type))
print "--\n%s--\n" % data
test_result = cmd_object.result
if not test_result:
break
def get_pybindgen(ns3_dir):
print("""
#
# Get PyBindGen
#
""")
if sys.platform in ['cygwin']:
print("Architecture (%s) does not support PyBindGen ... skipping" % (sys.platform,))
raise RuntimeError
# (peek into the ns-3 wscript and extract the required pybindgen version)
ns3_python_wscript = open(os.path.join(ns3_dir, "bindings", "python", "wscript"), "rt")
required_pybindgen_version = None
for line in ns3_python_wscript:
if line.startswith('REQUIRED_PYBINDGEN_VERSION'):
required_pybindgen_version = eval(line.split('=')[1].strip())
ns3_python_wscript.close()
break
if required_pybindgen_version is None:
fatal("Unable to detect pybindgen required version")
print("Required pybindgen version: ", required_pybindgen_version)
# work around http_proxy handling bug in bzr
# if 'http_proxy' in os.environ and 'https_proxy' not in os.environ:
# os.environ['https_proxy'] = os.environ['http_proxy']
if 'post' in required_pybindgen_version:
# given a version like '0.17.0.post41+ngd10fa60', the last 7 characters
# are part of a git hash, which should be enough to identify a revision
rev = required_pybindgen_version[-7:]
else:
rev = required_pybindgen_version
if os.path.exists(constants.LOCAL_PYBINDGEN_PATH):
print("Trying to update pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip.")
try:
run_command(["git", "fetch", constants.PYBINDGEN_BRANCH],
cwd=constants.LOCAL_PYBINDGEN_PATH)
except KeyboardInterrupt:
print("Interrupted; Python bindings will be disabled.")
else:
print("Update was successful.")
else:
print("Trying to fetch pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip.")
try:
run_command(["git", "clone", constants.PYBINDGEN_BRANCH,
constants.LOCAL_PYBINDGEN_PATH])
except KeyboardInterrupt:
print("Interrupted; Python bindings will be disabled.")
shutil.rmtree(constants.LOCAL_PYBINDGEN_PATH, True)
return False
print("Fetch was successful.")
run_command(["git", "checkout", rev, "-q"],
cwd=constants.LOCAL_PYBINDGEN_PATH)
## This causes the version to be generated
try:
import setuptools
run_command(["python3", "setup.py", "clean"],
cwd=constants.LOCAL_PYBINDGEN_PATH)
except ImportError:
print("Warning: Pybindgen setup not successful, are setuptools installed?")
raise RuntimeError
return (constants.LOCAL_PYBINDGEN_PATH, required_pybindgen_version)
def action_exec(name, args):
"""execute a command against a published root"""
parser = OptionParser("""\
usage: %%prog %(name)s [options] ... test command args ...
Executes the given command against the latest published build. The syntax for
commands (and exit code) is exactly the same as for the 'bisect' tool, so this
command is useful for testing bisect test commands.
See 'bisect' for more notermation on the exact test syntax.\
""" % locals())
parser.add_option("-b",
"--build",
dest="build_name",
metavar="STR",
help="name of build to fetch",
action="store",
default=DEFAULT_BUILDER)
parser.add_option("-s",
"--sandbox",
dest="sandbox",
help="directory to use as a sandbox",
action="store",
default=None)
parser.add_option("",
"--min-rev",
dest="min_rev",
help="minimum revision to test",
type="int",
action="store",
default=None)
parser.add_option("",
"--max-rev",
dest="max_rev",
help="maximum revision to test",
type="int",
action="store",
default=None)
parser.add_option("",
"--near",
dest="near_build",
help="use a build near NAME",
type="str",
action="store",
metavar="NAME",
default=None)
parser.disable_interspersed_args()
(opts, args) = parser.parse_args(args)
if opts.build_name is None:
parser.error("no build name given (see --build)")
available_builds = list(llvmlab.fetch_builds(opts.build_name))
available_builds.sort()
available_builds.reverse()
if opts.min_rev is not None:
available_builds = [
b for b in available_builds if b.revision >= opts.min_rev
]
if opts.max_rev is not None:
available_builds = [
b for b in available_builds if b.revision <= opts.max_rev
]
if len(available_builds) == 0:
fatal("No builds available for builder name: %s" % opts.build_name)
# Find the best match, if requested.
if opts.near_build:
build = get_best_match(available_builds, opts.near_build)
if not build:
parser.error("no match for build %r" % opts.near_build)
else:
# Otherwise, take the latest build.
build = available_builds[0]
test_result, _ = execute_sandboxed_test(opts.sandbox,
opts.build_name,
build,
args,
verbose=True,
show_command_output=True)
print '%s: %s' % (
('FAIL', 'PASS')[test_result], build.tobasename(include_suffix=False))
raise SystemExit(test_result != True)
def main():
miss_counter = 0
# get command line arguments
cfg_file = get_args()
# read configuration from tevs.cfg and set constants for this run
config.get(cfg_file)
util.mkdirp(const.root)
log = config.logger(const.logfilename)
#create initial top level dirs, if they do not exist
for p in (
"%s" % ("templates"),
"%s%d" % ("template_images", os.getpid()),
"%s%d" % ("composite_images", os.getpid()),
"results",
"proc",
"errors"):
util.mkdirp(util.root(p))
next_ballot = next.File(util.root("nexttoprocess.txt"), const.num_pages)
try:
ballotfrom = Ballot.LoadBallotType(const.layout_brand)
except KeyError as e:
util.fatal("No such ballot type: " + const.layout_brand + ": check " + cfg_file)
# allow all instances to share a common template location,
# though need per-pid locs for template_images and composite_images
cache = Ballot.TemplateCache(util.root("templates"))
extensions = Ballot.Extensions(template_cache=cache)
# connect to db and open cursor
if const.use_db:
try:
dbc = db.PostgresDB(const.dbname, const.dbuser)
except db.DatabaseError:
util.fatal("Could not connect to database")
else:
dbc = db.NullDB()
total_proc, total_unproc = 0, 0
base = os.path.basename
# While ballot images exist in the directory specified in tevs.cfg,
# create ballot from images, get landmarks, get layout code, get votes.
# Write votes to database and results directory. Repeat.
#from guppy import hpy;hp=hpy();hp.setref();import gc;gc.disable();gc.collect();hp.setref()
try:
for n in next_ballot:
gc.collect()
unprocs = [incomingn(n + m) for m in range(const.num_pages)]
if not os.path.exists(unprocs[0]):
miss_counter += 1
log.info(base(unprocs[0]) + " does not exist. No more records to process")
if miss_counter > 10:
break
continue
#for i, f in enumerate(unprocs[1:]):
# if not os.path.exists(f):
# log.info(base(f) + " does not exist. Cannot proceed.")
# for j in range(i):
# log.info(base(unprocs[j]) + " will NOT be processed")
# total_unproc += mark_error(None, *unprocs[:i-1])
#Processing
log.info("Processing %s:\n %s" %
(n, "\n".join("\t%s" % base(u) for u in unprocs))
)
try:
ballot = ballotfrom(unprocs, extensions)
results = ballot.ProcessPages()
except BallotException as e:
total_unproc += mark_error(e, *unprocs)
log.exception("Could not process ballot")
continue
csv = Ballot.results_to_CSV(results)
#moz = Ballot.results_to_mosaic(results)
#Write all data
#make dirs:
proc1d = dirn("proc", n)
resultsd = dirn("results", n)
resultsfilename = filen(resultsd, n)
for p in (proc1d, resultsd):
util.mkdirp(p)
try:
results_to_vop_files(results,resultsfilename)
except Exception as e:
print e
#write csv and mosaic
util.genwriteto(resultsfilename + ".txt", csv)
#write to the database
try:
dbc.insert(ballot)
except db.DatabaseError:
#dbc does not commit if there is an error, just need to remove
#partial files
remove_partial(resultsfilename + ".txt")
remove_partial(resultsfilename + const.filename_extension)
util.fatal("Could not commit vote information to database")
#Post-processing
# move the images from unproc to proc
procs = [filen(proc1d, n + m) + const.filename_extension
for m in range(const.num_pages)]
for a, b in zip(unprocs, procs):
try:
os.rename(a, b)
except OSError as e:
util.fatal("Could not rename %s", a)
total_proc += const.num_pages
log.info("%d images processed", const.num_pages)
#hp.heap().dump('prof.hpy');hp.setref();gc.collect();hp.setref();hp.heap().dump('prof.hpy')
finally:
cache.save_all()
dbc.close()
next_ballot.save()
log.info("%d images processed", total_proc)
if total_unproc > 0:
log.warning("%d images NOT processed.", total_unproc)
def instantiate(ckpt_dir=None):
from m5 import options
root = objects.Root.getInstance()
if not root:
fatal("Need to instantiate Root() before calling instantiate()")
# we need to fix the global frequency
ticks.fixGlobalFrequency()
# Make sure SimObject-valued params are in the configuration
# hierarchy so we catch them with future descendants() walks
for obj in root.descendants():
obj.adoptOrphanParams()
# Unproxy in sorted order for determinism
for obj in root.descendants():
obj.unproxyParams()
if options.dump_config:
ini_file = file(os.path.join(options.outdir, options.dump_config), 'w')
# Print ini sections in sorted order for easier diffing
for obj in sorted(root.descendants(), key=lambda o: o.path()):
obj.print_ini(ini_file)
ini_file.close()
if options.json_config:
try:
import json
json_file = file(os.path.join(options.outdir, options.json_config),
'w')
d = root.get_config_as_dict()
json.dump(d, json_file, indent=4)
json_file.close()
except ImportError:
pass
do_dot(root, options.outdir, options.dot_config)
# Initialize the global statistics
stats.initSimStats()
# Create the C++ sim objects and connect ports
for obj in root.descendants():
obj.createCCObject()
for obj in root.descendants():
obj.connectPorts()
# Do a second pass to finish initializing the sim objects
for obj in root.descendants():
obj.init()
# Do a third pass to initialize statistics
for obj in root.descendants():
obj.regStats()
# We're done registering statistics. Enable the stats package now.
stats.enable()
# Restore checkpoint (if any)
if ckpt_dir:
ckpt = internal.core.getCheckpoint(ckpt_dir)
internal.core.unserializeGlobals(ckpt)
for obj in root.descendants():
obj.loadState(ckpt)
need_resume.append(root)
else:
for obj in root.descendants():
obj.initState()
# Check to see if any of the stat events are in the past after resuming from
# a checkpoint, If so, this call will shift them to be at a valid time.
updateStatEvents()
def action_fetch(name, args):
"""fetch a build from the server"""
parser = OptionParser("""\
usage: %%prog %(name)s [options] builder [build-name]
Fetch the build from the named builder which matchs build-name. If no match is
found, get the first build before the given name. If no build name is given,
the most recent build is fetched.
The available builders can be listed using:
%%prog ls
The available builds can be listed using:
%%prog ls builder""" % locals())
parser.add_option("-f",
"--force",
dest="force",
help=("always download and extract, overwriting any"
"existing files"),
action="store_true",
default=False)
parser.add_option("",
"--update-link",
dest="update_link",
metavar="PATH",
help=("update a symbolic link at PATH to point to the "
"fetched build (on success)"),
action="store",
default=None)
parser.add_option("-d",
"--dry-run",
dest='dry_run',
help=("Perform all operations except the actual "
"downloading and extracting of any files"),
action="store_true",
default=False)
(opts, args) = parser.parse_args(args)
if len(args) == 0:
parser.error("please specify a builder name")
elif len(args) == 1:
builder, = args
build_name = None
elif len(args) == 2:
builder, build_name = args
else:
parser.error("invalid number of arguments")
builds = list(llvmlab.fetch_builds(builder))
if not builds:
parser.error("no builds for builder: %r" % builder)
build = get_best_match(builds, build_name)
if not build:
parser.error("no match for build %r" % build_name)
path = build.tobasename()
if build_name is not None and not path.startswith(build_name):
note('no exact match, fetching %r' % path)
# Get the paths to extract to.
root_path = path
builddir_path = build.tobasename(include_suffix=False)
if not opts.dry_run:
# Check that the download and extract paths are clean.
for p in (root_path, builddir_path):
if os.path.exists(p):
# If we are using --force, then clean the path.
if opts.force:
shutil.rmtree(p, ignore_errors=True)
continue
fatal('current directory is not clean, %r exists' % p)
llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path)
print 'downloaded root: %s' % root_path
print 'extracted path : %s' % builddir_path
# Update the symbolic link, if requested.
if not opts.dry_run and opts.update_link:
# Remove the existing path.
try:
os.unlink(opts.update_link)
except OSError as e:
if e.errno != errno.ENOENT:
fatal('unable to update symbolic link at %r, cannot unlink' %
(opts.update_link))
# Create the symbolic link.
os.symlink(os.path.abspath(builddir_path), opts.update_link)
print 'updated link at: %s' % opts.update_link
return os.path.abspath(builddir_path)
def check_tracing():
if defines.TRACING_ON:
return
fatal("Tracing is not enabled. Compile with TRACING_ON")
def action_bisect(name, args):
"""find first failing build using binary search"""
parser = OptionParser("""\
usage: %%prog %(name)s [options] ... test command args ...
Look for the first published build where a test failed, using the builds on
llvmlab. The command arguments are executed once per build tested, but each
argument is first subject to string interpolation. The syntax is
"%%(VARIABLE)FORMAT" where FORMAT is a standard printf format, and VARIABLE is
one of:
'sandbox' - the path to the sandbox directory.
'path' - the path to the build under test.
'revision' - the revision number of the build.
'build' - the build number of the build under test.
'clang' - the path to the clang binary of the build if it exists.
'clang++' - the path to the clang++ binary of the build if it exists.
'libltodir' - the path to the directory containing libLTO.dylib, if it
exists.
Each test is run in a sandbox directory. By default, sandbox directories are
temporary directories which are created and destroyed for each test (see
--sandbox).
For use in auxiliary test scripts, each test is also run with each variable
available in the environment as TEST_<variable name> (variables are converted
to uppercase). For example, a test script could use "TEST_PATH" to find the
path to the build under test.
The stdout and stderr of the command are logged to files inside the sandbox
directory. Use an explicit sandbox directory if you would like to look at
them.
It is possible to run multiple distinct commands for each test by separating
them in the command line arguments by '----'. The failure of any command causes
the entire test to fail.\
""" % locals())
parser.add_option("-b",
"--build",
dest="build_name",
metavar="STR",
help="name of build to fetch",
action="store",
default=DEFAULT_BUILDER)
parser.add_option("-s",
"--sandbox",
dest="sandbox",
help="directory to use as a sandbox",
action="store",
default=None)
parser.add_option("-v",
"--verbose",
dest="verbose",
help="output more test notermation",
action="store_true",
default=False)
parser.add_option("-V",
"--very-verbose",
dest="very_verbose",
help="output even more test notermation",
action="store_true",
default=False)
parser.add_option("",
"--show-output",
dest="show_command_output",
help="display command output",
action="store_true",
default=False)
parser.add_option("",
"--single-step",
dest="single_step",
help="single step instead of binary stepping",
action="store_true",
default=False)
parser.add_option("",
"--min-rev",
dest="min_rev",
help="minimum revision to test",
type="int",
action="store",
default=None)
parser.add_option("",
"--max-rev",
dest="max_rev",
help="maximum revision to test",
type="int",
action="store",
default=None)
parser.disable_interspersed_args()
(opts, args) = parser.parse_args(args)
if opts.build_name is None:
parser.error("no build name given (see --build)")
# Very verbose implies verbose.
opts.verbose |= opts.very_verbose
start_time = time.time()
available_builds = list(llvmlab.fetch_builds(opts.build_name))
available_builds.sort()
available_builds.reverse()
if opts.very_verbose:
note("fetched builds in %.2fs" % (time.time() - start_time, ))
if opts.min_rev is not None:
available_builds = [
b for b in available_builds if b.revision >= opts.min_rev
]
if opts.max_rev is not None:
available_builds = [
b for b in available_builds if b.revision <= opts.max_rev
]
def predicate(item):
# Run the sandboxed test.
test_result, _ = execute_sandboxed_test(
opts.sandbox,
opts.build_name,
item,
args,
verbose=opts.verbose,
very_verbose=opts.very_verbose,
show_command_output=opts.show_command_output or opts.very_verbose)
# Print status.
print '%s: %s' % (('FAIL', 'PASS')[test_result],
item.tobasename(include_suffix=False))
return test_result
if opts.single_step:
for item in available_builds:
if predicate(item):
break
else:
item = None
else:
if opts.min_rev is None or opts.max_rev is None:
# Gallop to find initial search range, under the assumption that we
# are most likely looking for something at the head of this list.
search_space = algorithm.gallop(predicate, available_builds)
else:
# If both min and max revisions are specified,
# don't gallop - bisect the given range.
search_space = available_builds
item = algorithm.bisect(predicate, search_space)
if item is None:
fatal('unable to find any passing build!')
print '%s: first working build' % item.tobasename(include_suffix=False)
index = available_builds.index(item)
if index == 0:
print 'no failing builds!?'
else:
print '%s: next failing build' % available_builds[
index - 1].tobasename(include_suffix=False)
