def link_configuration(cfg: Path, host_cfg: Path) -> None: """Ensure that ``cfg`` is a symlink to ``host_cfg``. :param cfg: path to ``configuration.nix`` :param host_cfg: path to ``hosts/$(hostname)-configuration.nix`` """ if cfg.is_symlink(): cfg_dest = cfg.parent / os.readlink(cfg) if cfg_dest.samefile(host_cfg): info(f"{p(cfg)} is already a symlink pointing to {p(host_cfg)}") else: warn( f"{p(cfg)} is a symlink pointing to {p(cfg_dest)}, not {p(host_cfg)}; updating it" ) if not DRY_RUN: cfg.unlink() cfg.symlink_to(host_cfg) elif cfg.exists(): info( f"{p(cfg)} already exists and is a regular file; moving it to {p(host_cfg)}" ) if host_cfg.exists(): fatal(f"{p(host_cfg)} already exists!") elif not DRY_RUN: cfg.rename(host_cfg) else: info(f"{p(cfg)} doesn't exist; creating it as a link to {p(host_cfg)}") if not DRY_RUN: cfg.symlink_to(host_cfg)
def _check_tracing(): import defines if defines.TRACING_ON: return fatal("Tracing is not enabled. Compile with TRACING_ON")
def __init__(self, value): try: self.value = float(value) except: fatal("invalid argument: %r" % time) warning("'max_time' filter is deprecated, use " "'user_time < %.4f' filter expression" % self.value)
def pull(args: Args) -> None: """Update ``args.repo`` with ``git pull``.""" if args.pull: cmd("git pull --no-edit") proc = subprocess.run(args.sudo_prefix + ["git", "pull", "--no-edit"], check=False, cwd=args.repo) if proc.returncode != 0: # git pull failed, maybe reset? if args.reset: run_or_fatal( args.sudo_prefix + ["git", "reset", "--hard", args.remote_branch], log=True, cwd=args.repo, ) else: fatal( "`git pull` failed. Pass `--reset` to reset the repository." ) info(f"{args.repo} is now at commit:") subprocess.run(["git", "log", "HEAD^1..HEAD", "--oneline"], check=False, cwd=args.repo)
def postList(emailList): emailFile = {"file": open(emailList, "rb")} r = requests.post(API_URL, files=emailFile) resp = r.json() if resp["status"] != "ok": util.fatal("%s" % resp["message"]) else: results = {} report = ''' File: {filename} Count: {count} Purchased/harvested: {p_flag} Deliverability: {d_flag} Forced signups: {f_flag} Spamtrap/blacklist: {s_flag} Overall: {o_flag} ''' for k, v in resp.iteritems(): if isinstance(v, dict): results[k] = ",".join([ "{key}: {message}".format(key=x, message=y) for x, y in v.iteritems() ]) else: results[k] = v print report.format(**results)
def main(): parser = OptionParser(usage="%prog") parser.add_option('-b', '--benchmarks', action='store_true', default=False, dest='benchmarks', help='analyze benchmark applications') parser.add_option('-w', '--websites', action='store_true', default=False, dest='websites', help='end-to-end website analysis') parser.add_option('-y', '--targetpages', action='store_true', default=False, dest='targetpages', help='prototype target website analysis') parser.add_option('-m', '--micro', action='store_true', default=False, dest='micro', help='analyze microbenchmark applications') parser.add_option('-x', '--exploit', action='store_true', default=False, dest='exploit', help='analyze exploit applications') #parser.add_option('-i', '--interpreter', action='store_true', default=False, dest='interpreter', help='test semantics as an interpreter (currently unsupported)') parser.add_option('-e', '--overwrite', action='store_true', default=False, dest='overwrite', help='overwrite expected output') #parser.add_option('-s', '--semantics', action='store_true', default=False, dest='semantics', help='test semantics') parser.add_option('-g', '--debug', action='store_true', default=False, dest='debug', help='generate debug output') parser.add_option('-u', '--url', action='store', default=None, dest='url', help='analyze HTML/JS at given URL') parser.add_option('-Y', '--policy', action='store', default=None, dest='policy', help='policy file to apply to URL') parser.add_option('-p', '--refine', action='store', default=None, dest='refine', help='maximum of number of predicates to learn') parser.add_option('-z', '--syntax-only', action='store_true', default=False, dest='syntaxonly', help='use syntax-only (not semantic) analysis') parser.add_option('-d', '--service', action='store_true', default=False, dest='service', help='run JAM as a service') parser.add_option('-a', '--apps', action='append', default=None, dest='apps', help='limit to the given app(s)') opts, args = parser.parse_args() if len(args) != 0: parser.error("Invalid number of arguments") allmods = True if opts.benchmarks or opts.micro or opts.exploit or opts.websites or opts.targetpages or opts.url is not None: allmods = False ref = opts.refine if ref is not None: try: ref = int(ref) if ref < -1: raise except: fatal('Invalid refinement limit: %s (should be positive integer, or -1 for unlimited)' % (opts.refine)) if opts.service: start_jam_service(debug=opts.debug) #if opts.interpreter: # err('Interpreter tests are currently out-of-order') # run_interpreter_tests(opts.debug) if opts.url is not None: pol = load_policy(opts.policy) run_website(opts.url, pol, debug=opts.debug, overwrite=opts.overwrite, refine=ref, synonly=opts.syntaxonly, service=opts.service) if allmods or opts.micro: run_microbenchmarks(opts.debug, opts.overwrite, refine=ref, synonly=opts.syntaxonly, service=opts.service, apps=opts.apps) if allmods or opts.exploit: run_exploits(opts.debug, opts.overwrite, refine=ref, synonly=opts.syntaxonly, service=opts.service, apps=opts.apps) if allmods or opts.benchmarks: run_benchmarks(opts.debug, opts.overwrite, refine=ref, synonly=opts.syntaxonly, service=opts.service, apps=opts.apps) if allmods or opts.websites: run_websites(opts.debug, opts.overwrite, refine=ref, synonly=opts.syntaxonly, service=opts.service, apps=opts.apps) if allmods or opts.targetpages: run_targetpages(opts.debug, opts.overwrite, refine=ref, synonly=opts.syntaxonly, service=opts.service, apps=opts.apps) if opts.service: close_jam_service()
def get_nsc(ns3_dir): print """ # # Get NSC # """ # Skip downloading NSC on OS X due to HFS+ case insensitive issues # Skip downloading NSC on Cygwin because of fundamental incompatibilities. if sys.platform in ['darwin', 'cygwin']: print "Architecture (%s) does not support NSC... skipping" % ( sys.platform, ) raise RuntimeError # (peek into the ns-3 wscript and extract the required nsc version) internet_stack_wscript = open( os.path.join(ns3_dir, "src", "internet", "wscript"), "rt") required_nsc_version = None for line in internet_stack_wscript: if 'NSC_RELEASE_NAME' in line: required_nsc_version = eval(line.split('=')[1].strip()) break internet_stack_wscript.close() if required_nsc_version is None: fatal("Unable to detect NSC required version") print "Required NSC version: ", required_nsc_version def nsc_clone(): print "Retrieving nsc from " + constants.NSC_REPO run_command( ['hg', 'clone', constants.NSC_REPO, constants.LOCAL_NSC_PATH]) def nsc_update(): print "Pulling nsc updates from " + constants.NSC_REPO run_command([ 'hg', '--cwd', constants.LOCAL_NSC_PATH, 'pull', '-u', constants.NSC_REPO ]) def nsc_download(): local_file = required_nsc_version + ".tar.bz2" remote_file = constants.NSC_RELEASE_URL + "/" + local_file print "Retrieving nsc from " + remote_file urllib.urlretrieve(remote_file, local_file) print "Uncompressing " + local_file run_command(["tar", "-xjf", local_file]) print "Rename %s as %s" % (required_nsc_version, constants.LOCAL_NSC_PATH) os.rename(required_nsc_version, constants.LOCAL_NSC_PATH) if not os.path.exists(os.path.join(ns3_dir, '.hg')): nsc_download() elif not os.path.exists(constants.LOCAL_NSC_PATH): nsc_clone() else: nsc_update() return (constants.LOCAL_NSC_PATH, required_nsc_version)
def split_command_filters(command): for i, arg in enumerate(command): if arg[:2] != "%%" or arg[-2:] != "%%": break else: fatal("invalid command: %s, only contains filter " "specifications" % ("".join('"%s"' % a for a in command))) return ([a[2:-2] for a in command[:i]], command[i:])
def get_netanim(ns3_dir): print """ # # Get NetAnim # """ if sys.platform in ['cygwin']: print "Architecture (%s) does not support NetAnim... skipping" % (sys.platform) raise RuntimeError # (peek into the ns-3 wscript and extract the required netanim version) try: # For the recent versions netanim_wscript = open(os.path.join(ns3_dir, "src", "netanim", "wscript"), "rt") except: print "Unable to detect NetAnim required version.Skipping download" pass return required_netanim_version = None for line in netanim_wscript: if 'NETANIM_RELEASE_NAME' in line: required_netanim_version = eval(line.split('=')[1].strip()) break netanim_wscript.close() if required_netanim_version is None: fatal("Unable to detect NetAnim required version") print "Required NetAnim version: ", required_netanim_version def netanim_clone(): print "Retrieving NetAnim from " + constants.NETANIM_REPO run_command(['hg', 'clone', constants.NETANIM_REPO, constants.LOCAL_NETANIM_PATH]) def netanim_update(): print "Pulling NetAnim updates from " + constants.NETANIM_REPO run_command(['hg', '--cwd', constants.LOCAL_NETANIM_PATH, 'pull', '-u', constants.NETANIM_REPO]) def netanim_download(): local_file = required_netanim_version + ".tar.bz2" remote_file = constants.NETANIM_RELEASE_URL + "/" + local_file print "Retrieving NetAnim from " + remote_file urllib.urlretrieve(remote_file, local_file) print "Uncompressing " + local_file run_command(["tar", "-xjf", local_file]) print "Rename %s as %s" % (required_netanim_version, constants.LOCAL_NETANIM_PATH) os.rename(required_netanim_version, constants.LOCAL_NETANIM_PATH) if not os.path.exists(os.path.join(ns3_dir, '.hg')): netanim_download() elif not os.path.exists(constants.LOCAL_NETANIM_PATH): netanim_clone() else: netanim_update() return (constants.LOCAL_NETANIM_PATH, required_netanim_version)
def backup_object(self, obj: str) -> None: """Backup an object. Must be a directory or ordinary file.""" logging.info(f'backup: {obj}') if os.path.isdir(obj): self.backup_dir(obj) elif os.path.isfile(obj): self.backup_file(obj) else: util.fatal(f'ABORT: {obj} is not a file or directory')
def get_nsc(ns3_dir): print """ # # Get NSC # """ # Skip downloading NSC on OS X due to HFS+ case insensitive issues # Skip downloading NSC on Cygwin because of fundamental incompatibilities. if sys.platform in ['darwin', 'cygwin']: print "Architecture (%s) does not support NSC... skipping" % (sys.platform,) raise RuntimeError # (peek into the ns-3 wscript and extract the required nsc version) try: # For the recent versions internet_stack_wscript = open(os.path.join(ns3_dir, "src", "internet", "wscript"), "rt") except IOError: # For the old versions (ns-3.10 and before) internet_stack_wscript = open(os.path.join(ns3_dir, "src", "internet-stack", "wscript"), "rt") required_nsc_version = None for line in internet_stack_wscript: if 'NSC_RELEASE_NAME' in line: required_nsc_version = eval(line.split('=')[1].strip()) break internet_stack_wscript.close() if required_nsc_version is None: fatal("Unable to detect NSC required version") print "Required NSC version: ", required_nsc_version def nsc_clone(): print "Retrieving nsc from " + constants.NSC_REPO run_command(['hg', 'clone', constants.NSC_REPO, constants.LOCAL_NSC_PATH]) def nsc_update(): print "Pulling nsc updates from " + constants.NSC_REPO run_command(['hg', '--cwd', constants.LOCAL_NSC_PATH, 'pull', '-u', constants.NSC_REPO]) def nsc_download(): local_file = required_nsc_version + ".tar.bz2" remote_file = constants.NSC_RELEASE_URL + "/" + local_file print "Retrieving nsc from " + remote_file urllib.urlretrieve(remote_file, local_file) print "Uncompressing " + local_file run_command(["tar", "-xjf", local_file]) print "Rename %s as %s" % (required_nsc_version, constants.LOCAL_NSC_PATH) os.rename(required_nsc_version, constants.LOCAL_NSC_PATH) if not os.path.exists(os.path.join(ns3_dir, '.hg')): nsc_download() elif not os.path.exists(constants.LOCAL_NSC_PATH): nsc_clone() else: nsc_update() return (constants.LOCAL_NSC_PATH, required_nsc_version)
def __init__(self, list_name): self.numlist = [] self.list_name = list_name try: with open(list_name, "r") as f: self.numlist = [int(line) for line in f] except IOError: pass #no numlist.txt, ignore except ValueError: util.fatal("Malformed numlist.txt")
def interp(self): p = Parser(self.fname, self.text) try: node = p.parse() except ParserError, e: ctx = "Traceback: \n" for f in callstack: ctx = ctx + str(f) + '\n' output = ctx + str(e) fatal(output)
def __init__(self, repo: str): self.repo = repo self.lock_name = repo + LOCK_FILE self.objects = repo + OBJECTS self.backups = repo + BACKUPS self.all_backups = self.get_all_backups() self.sig_file = repo + SIG if not self.quick_verify(): util.fatal(f'{repo} is not a repository') self.bytes_written = 0
def type_check(self): p = Parser(self.fname) try: node = p.parse() except ParserError, e: ctx = "Traceback: \n" for f in callstack: ctx = ctx + str(f) + '\n' output = ctx + str(e) fatal(output)
def instantiate(ckpt_dir=None): from m5 import options root = objects.Root.getInstance() if not root: fatal("Need to instantiate Root() before calling instantiate()") # we need to fix the global frequency ticks.fixGlobalFrequency() # Make sure SimObject-valued params are in the configuration # hierarchy so we catch them with future descendants() walks for obj in root.descendants(): obj.adoptOrphanParams() # Unproxy in sorted order for determinism for obj in root.descendants(): obj.unproxyParams() if options.dump_config: ini_file = file(os.path.join(options.outdir, options.dump_config), 'w') # Print ini sections in sorted order for easier diffing for obj in sorted(root.descendants(), key=lambda o: o.path()): obj.print_ini(ini_file) ini_file.close() # Initialize the global statistics stats.initSimStats() # Create the C++ sim objects and connect ports for obj in root.descendants(): obj.createCCObject() for obj in root.descendants(): obj.connectPorts() # Do a second pass to finish initializing the sim objects for obj in root.descendants(): obj.init() # Do a third pass to initialize statistics for obj in root.descendants(): obj.regStats() for obj in root.descendants(): obj.regFormulas() # We're done registering statistics. Enable the stats package now. stats.enable() # Restore checkpoint (if any) if ckpt_dir: ckpt = internal.core.getCheckpoint(ckpt_dir) internal.core.unserializeGlobals(ckpt); for obj in root.descendants(): obj.loadState(ckpt) need_resume.append(root) else: for obj in root.descendants(): obj.initState() # Reset to put the stats in a consistent state. stats.reset()
def create(repo: str) -> None: """Create a new repository in 'repo'. Make sure 'repo' is an empty directory. Call create_contents() to initialize it. """ logging.info(f'creating repo: {repo}') if os.path.isdir(repo): if os.listdir(repo) == []: _create_repo_contents(repo) logging.info('repo creation complete') else: util.fatal(f'directory {repo} is not empty') else: util.fatal(f'{repo} is not a directory.')
def pt_newtool(args): '''newtool - Create a new tool-style program usage: pytool newtool <program-name> <prefix> Creates executable file <program-name>.py with skeletal contents. The structure of the program is such that it is easy to add and describe new subfunctions. ''' p = optparse.OptionParser() (o, a) = p.parse_args(args) if a == [] or len(a) != 2: U.fatal('usage: pytool newtool <program-name> <prefix>') else: lname = a[0] pname = lname + '.py' prefix = a[1] are_we_overwriting([lname, pname]) f = open(pname, 'w') f.writelines(['#!/usr/bin/env python\n', '"""\n', '%s - program description\n' % lname, '"""\n', '\n', 'from bscr import util as U\n', 'import optparse\n', 'import os\n', 'import re\n', 'import sys\n', '\n', '# ----------------------------------------------------' + '-----------------------\n', 'def %s_example(argv):\n' % prefix, ' print("this is an example")\n', "\n", '# ----------------------------------------------------' + '-----------------------\n', "if __name__ == '__main__':\n", " U.dispatch('__main__', '%s', sys.argv)\n" % prefix, ]) f.close() os.chmod(pname, 0755) os.symlink(os.path.abspath(pname), lname)
def save(self, id): id = str(id) "write the template id to disk at self.location" fname = os.path.join(self.location, id) if not os.path.exists(fname): template = self.cache[id] if template is None: return xml = BallotTemplate.Template_to_XML(template) util.writeto(fname, xml) if template.image is not None: try: im = _fixup(template.image, template.rot, template.xoff, template.yoff) im.save(fname + ".jpg") except IOError: util.fatal("could not save image of template") self.log.info("new template %s saved", fname)
def main() -> None: args = docopt.docopt(__doc__, version=VERSION) logging.basicConfig(filename=LOG_FILE, filemode='a', format='%(levelname)s - %(message)s', level=LOG_LEVEL) now = time.asctime() logging.info(f'log start: {now}') logging.info(f'args: {args}') repo_file = args['REPO'] if args['create']: Repo.create(repo_file) elif args['verify']: repos = Repo(repo_file) logging.info(f'Verifying repo {repo_file}') n = 0 if args['--hashes']: repos.verify_all_hashes(delete=args['-d']) n += 1 if args['--backups']: repos.verify_backups() n += 1 if args['--orphans']: repos.find_orphans(delete=args['-d']) n += 1 if n == 0: util.msg('You must use --hashes, --backups, or --orphans') elif args['unlock']: # XXX Could this be: Repo(repo_file).unlock() ? repos = Repo(repo_file) repos.unlock() elif args['backup']: arg_prefix = args['--prefix'] prefix = arg_prefix if arg_prefix else 'backup' if not alphanum.match(prefix): util.fatal('prefix must be alphanumeric') want_verify = args['--verify'] backup.run_backup(args['--repo'], prefix, args['FILE'], verify=want_verify, verbose=args['-v']) elif args['restore']: restore.restore(repo_file) now = time.asctime() logging.info(f'log end: {now}')
def fetch_builds(name): """ fetch_builds(name) -> [(path, revision, build), ...] Get a list of available builds for the named builder. """ # Handle only_use_cache setting. prefs = util.get_prefs() if prefs.getboolean("ci", "only_use_cache"): cache_path = os.path.join(prefs.path, "ci", "build_cache") cache_build_path = os.path.join(cache_path, name) items = os.listdir(cache_build_path) assert False, "Unimplemented?" + str(items) # Otherwise, load the builder map. buildermap = load_builder_map() # If the builder isn't in the builder map, do a forced load of the builder # map. if name not in buildermap.builders: buildermap = load_builder_map(reload=True) # If the builder doesn't exist, report an error. builder_artifacts = buildermap.builders.get(name) if builder_artifacts is None: fatal("unknown builder name: %r" % (name,)) # Otherwise, load the builder list. server_builds = gcs.fetch_builds(builder_artifacts) builds = [] for path in server_builds['items']: build = Build.frombasename(path['name'], path['mediaLink']) # Ignore any links which don't at least have a revision component. if build.revision is not None: builds.append(build) # If there were no builds, report an error. if not builds: fatal("builder %r may be misconfigured (no items)" % (name,)) # Sort the builds, to make sure we return them ordered properly. builds.sort() return builds
def fl_edit(args): """edit - edit files in place usage: fl edit [-i <suffix>] -e <edit-cmd> file1 file2 file3 ... """ p = optparse.OptionParser() p.add_option('-d', '--debug', default=False, action='store_true', dest='debug', help='run the debugger') p.add_option('-e', '--edit', default='', action='store', dest='edit_cmd', help='edit command') p.add_option('-i', '--init', default='', action='store', dest='suffix', help='suffix for original files') (o, a) = p.parse_args(args) if o.debug: pdb.set_trace() if o.suffix == '': suffix = 'original' else: suffix = o.suffix if o.edit_cmd == '': util.fatal("usage: fl edit [-i <suffix>] -e <cmd> f1 f2 ...") ec = o.edit_cmd.split(o.edit_cmd[1]) if all([ec[0] != 's', ec[0] != 'y']): raise bscr.Error("Only 's' and 'y' supported for -e right now") if 4 != len(ec): raise bscr.Error("usage: ... -e '[sy]/before/after/' ...") (op, prev, post) = ec[0:3] if 0 == len(a): util.fatal("no files on command line to edit") else: for filename in a: editfile(filename, op, prev, post, suffix)
def fetch_build_to_path(builder, build, root_path, builddir_path): path = build.tobasename() # Check whether we are using a build cache and get the cached build path if # so. prefs = util.get_prefs() cache_build_path = None if prefs.getboolean("ci", "cache_builds"): cache_path = os.path.join(prefs.path, "ci", "build_cache") cache_build_path = os.path.join(cache_path, builder, path) # Copy the build from the cache or download it. if cache_build_path and os.path.exists(cache_build_path): shutil.copy(cache_build_path, root_path) else: # Load the builder map. buildermap = load_builder_map() # If the builder isn't in the builder map, do a forced reload of the # builder map. if builder not in buildermap.builders: buildermap = load_builder_map(reload=True) # If the builder doesn't exist, report an error. builder_artifacts = buildermap.builders.get(builder) if builder_artifacts is None: fatal("unknown builder name: %r" % (builder,)) # Otherwise create the build url. gcs.get_compiler(build.url, root_path) # Copy the build into the cache, if enabled. if cache_build_path is not None: shell.mkdir_p(os.path.dirname(cache_build_path)) shutil.copy(root_path, cache_build_path) # Create the directory for the build. os.mkdir(builddir_path) # Extract the build. if shell.execute(['tar', '-xf', root_path, '-C', builddir_path]): fatal('unable to extract %r to %r' % (root_path, builddir_path))
def run_backup(repo_name: str, prefix: str, paths: List[str], verify=True, verbose=False) -> None: """Perform a backup to the repository of all dirs/files in the list. Make sure we have an unlocked DDU repository. Process all items in 'paths'. If an item is '-', read path names from stdin. """ logging.info(f"backup list: {paths}") logging.info(f"backup repo: {repo_name}") print(f"Backing up to {repo_name} with prefix '{prefix}'.") repos = Repo(repo_name) if repos.lock(): bkup = Backup(repo_name, prefix, verbose=verbose) for g in paths: if g == '-': for f in sys.stdin: if not f: continue f = f.rstrip() if os.path.isfile(f) or os.path.isdir(f): bkup.backup_object(f) else: util.warning(f'{f} is not a file or directory') else: if util.os_is_windows: # You have to use globbing on Windows. Shell is too stupid. path_list = glob.glob(g) if path_list == []: util.warning(f'File/dir {g} not found.') else: for f in glob.glob(g): bkup.backup_object(f) else: bkup.backup_object(g) bkup.finish() repos.unlock() if verify: repos.verify_backups(only=bkup.bk_name) else: util.fatal(f'repo {repo_name} is LOCKED')
def create_test_from_file(fl, name, group, policy): txt = fl.read() fl.close() appdir = os.path.join(TESTS_DIR, group, name) if os.path.exists(appdir): if OVERWRITE: if not os.path.isdir(appdir): fatal("Unable to overwrite file: %s" % appdir) warn("Creating in existing directory: %s" % appdir) else: fatal("Not overwriting existing directory: %s" % appdir) prepare_dir(appdir) inputdir = os.path.join(appdir, 'source-input') if os.path.exists(inputdir): assert OVERWRITE if not os.path.isdir(inputdir): fatal("Unable to overwrite non-directory: %s" % inputdir) else: os.makedirs(inputdir) tgtfile = "%s.js" % name tgtpath = os.path.join(inputdir, tgtfile) tgtfl = open(tgtpath, 'w') tgtfl.write(txt) tgtfl.close()
def run_server(): cfg_file = tevsgui_get_args.get_args() config.get(cfg_file) logfile = os.path.join(const.root, "logs/processing_log.txt") bv = BallotClass.LoadBallotFactoryForVendorStyle(const.layout_brand) bsv = BallotClass.LoadBallotSideFactoryForVendorStyle(const.layout_brand) logging.basicConfig( filename=logfile, format='%(asctime)s %(levelname)s %(module)s %(message)s', level=logging.INFO) logging.info("Processing server starting.") prepopulate_cache() # establish a connection to the database # connect to db and open cursor dbc = None if const.use_db: try: dbc = db.PostgresDB(database=const.dbname, user=const.dbuser) except db.DatabaseError: util.fatal("Could not connect to database!") else: dbc = db.NullDB() # Instantiate and bind to localhost:8000 try: server = AsyncXMLRPCServer(('', 8000), SimpleXMLRPCRequestHandler) except socket.error as e: print "A service is already running on address 8000." sys.exit() # Register example object instance server.register_instance(ProcessBallots(const.root, bv, bsv, dbc)) print "Ballot processing / scraping service now ready on address 8000." print "Using %s as root directory." % (const.root, ) # run! server.serve_forever()
def main(): parser = OptionParser(usage="%prog") parser.add_option('-n', '--name', action='store', default=None, dest='name', help='name of the new test') parser.add_option('-g', '--group', action='store', default=None, dest='group', help='test group to add to') parser.add_option('-f', '--file', action='store', default=None, dest='file', help='input source file') parser.add_option('-u', '--url', action='store', default=None, dest='url', help='URL for website test case') parser.add_option('-F', '--overwrite', action='store_true', default=False, dest='overwrite', help='overwrite existing files') parser.add_option('-v', '--verbose', action='store_true', default=False, dest='verbose', help='generate verbose output') parser.add_option('-Y', '--policy', action='store', default=None, dest='policy', help='policy file to use for the test case') opts, args = parser.parse_args() if len(args) != 0: parser.error("Invalid number of arguments") if opts.group not in TEST_GROUPS: fatal("Invalid test group: %s" % opts.group) global VERBOSE, OVERWRITE VERBOSE = opts.verbose OVERWRITE = opts.overwrite if opts.url is not None: group = opts.group if group is None: group = 'sites' if group != 'sites': fatal("Invalid group for URL test: %s" % group) create_test_for_url(opts.url, opts.name, opts.policy) else: if opts.file is None: if opts.name is None: fatal("Name (-n) must be provided for stdin source") appname = opts.name fl = sys.stdin else: if not os.path.isfile(opts.file): fatal("Unable to access source file: %s" % opts.file) filename = os.path.basename(opts.file) appname = os.path.splitext(filename)[0] fl = open(opts.file, 'r') group = opts.group if group is None: group = 'bench' create_test_from_file(fl, appname, group, opts.policy)
class TypeChecker(object): def __init__(self, fname): self.fname = fname def type_check(self): p = Parser(self.fname) try: node = p.parse() except ParserError, e: ctx = "Traceback: \n" for f in callstack: ctx = ctx + str(f) + '\n' output = ctx + str(e) fatal(output) tbl = SymTable.init_type_table() try: return node.interp(tbl) except TypeCheckError, e: fatal(str(e))
class Interpreter(object): def __init__(self, fname, text=None): self.fname = fname self.text = text def interp(self): p = Parser(self.fname, self.text) try: node = p.parse() except ParserError, e: ctx = "Traceback: \n" for f in callstack: ctx = ctx + str(f) + '\n' output = ctx + str(e) fatal(output) tbl = SymTable.init_value_table() try: return node.interp(tbl) except InterpError, e: fatal(str(e))
def action_exec(name, args): """execute a command against a published root""" parser = OptionParser("""\ usage: %%prog %(name)s [options] ... test command args ... Executes the given command against the latest published build. The syntax for commands (and exit code) is exactly the same as for the 'bisect' tool, so this command is useful for testing bisect test commands. See 'bisect' for more notermation on the exact test syntax.\ """ % locals()) parser.add_option("-b", "--build", dest="build_name", metavar="STR", help="name of build to fetch", action="store", default=DEFAULT_BUILDER) parser.add_option("-s", "--sandbox", dest="sandbox", help="directory to use as a sandbox", action="store", default=None) parser.add_option("", "--min-rev", dest="min_rev", help="minimum revision to test", type="int", action="store", default=None) parser.add_option("", "--max-rev", dest="max_rev", help="maximum revision to test", type="int", action="store", default=None) parser.add_option("", "--near", dest="near_build", help="use a build near NAME", type="str", action="store", metavar="NAME", default=None) parser.disable_interspersed_args() (opts, args) = parser.parse_args(args) if opts.build_name is None: parser.error("no build name given (see --build)") available_builds = list(llvmlab.fetch_builds(opts.build_name)) available_builds.sort() available_builds.reverse() if opts.min_rev is not None: available_builds = [b for b in available_builds if b.revision >= opts.min_rev] if opts.max_rev is not None: available_builds = [b for b in available_builds if b.revision <= opts.max_rev] if len(available_builds) == 0: fatal("No builds available for builder name: %s" % opts.build_name) # Find the best match, if requested. if opts.near_build: build = get_best_match(available_builds, opts.near_build) if not build: parser.error("no match for build %r" % opts.near_build) else: # Otherwise, take the latest build. build = available_builds[0] test_result, _ = execute_sandboxed_test( opts.sandbox, opts.build_name, build, args, verbose=True, show_command_output=True) print '%s: %s' % (('FAIL', 'PASS')[test_result], build.tobasename(include_suffix=False)) raise SystemExit(test_result != True)
def instantiate(ckpt_dir=None): from m5 import options root = objects.Root.getInstance() if not root: fatal("Need to instantiate Root() before calling instantiate()") # we need to fix the global frequency ticks.fixGlobalFrequency() # Make sure SimObject-valued params are in the configuration # hierarchy so we catch them with future descendants() walks for obj in root.descendants(): obj.adoptOrphanParams() # Unproxy in sorted order for determinism for obj in root.descendants(): obj.unproxyParams() if options.dump_config: ini_file = file(os.path.join(options.outdir, options.dump_config), 'w') # Print ini sections in sorted order for easier diffing for obj in sorted(root.descendants(), key=lambda o: o.path()): obj.print_ini(ini_file) ini_file.close() if options.json_config: try: import json json_file = file(os.path.join(options.outdir, options.json_config), 'w') d = root.get_config_as_dict() json.dump(d, json_file, indent=4) json_file.close() except ImportError: pass do_dot(root, options.outdir, options.dot_config) # Initialize the global statistics stats.initSimStats() # Create the C++ sim objects and connect ports for obj in root.descendants(): obj.createCCObject() for obj in root.descendants(): obj.connectPorts() # Do a second pass to finish initializing the sim objects for obj in root.descendants(): obj.init() # Do a third pass to initialize statistics for obj in root.descendants(): obj.regStats() # Do a fourth pass to initialize probe points for obj in root.descendants(): obj.regProbePoints() # Do a fifth pass to connect probe listeners for obj in root.descendants(): obj.regProbeListeners() # We're done registering statistics. Enable the stats package now. stats.enable() # Restore checkpoint (if any) if ckpt_dir: _drain_manager.preCheckpointRestore() ckpt = internal.core.getCheckpoint(ckpt_dir) internal.core.unserializeGlobals(ckpt); for obj in root.descendants(): obj.loadState(ckpt) else: for obj in root.descendants(): obj.initState() # Check to see if any of the stat events are in the past after resuming from # a checkpoint, If so, this call will shift them to be at a valid time. updateStatEvents()
def get_pybindgen(ns3_dir): print """ # # Get PyBindGen # """ if sys.platform in ['cygwin']: print "Architecture (%s) does not support PyBindGen ... skipping" % (sys.platform,) raise RuntimeError # (peek into the ns-3 wscript and extract the required pybindgen version) ns3_python_wscript = open(os.path.join(ns3_dir, "bindings", "python", "wscript"), "rt") required_pybindgen_version = None for line in ns3_python_wscript: if 'REQUIRED_PYBINDGEN_VERSION' in line: required_pybindgen_version = eval(line.split('=')[1].strip()) ns3_python_wscript.close() break if required_pybindgen_version is None: fatal("Unable to detect pybindgen required version") print "Required pybindgen version: ", required_pybindgen_version # work around http_proxy handling bug in bzr # if 'http_proxy' in os.environ and 'https_proxy' not in os.environ: # os.environ['https_proxy'] = os.environ['http_proxy'] if 'post' in required_pybindgen_version: # given a version like '0.17.0.post41+ngd10fa60', the last 7 characters # are part of a git hash, which should be enough to identify a revision rev = required_pybindgen_version[-7:] else: rev = required_pybindgen_version if os.path.exists(constants.LOCAL_PYBINDGEN_PATH): print "Trying to update pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip." try: run_command(["git", "fetch", constants.PYBINDGEN_BRANCH], cwd=constants.LOCAL_PYBINDGEN_PATH) except KeyboardInterrupt: print "Interrupted; Python bindings will be disabled." else: print "Update was successful." else: print "Trying to fetch pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip." try: run_command(["git", "clone", constants.PYBINDGEN_BRANCH, constants.LOCAL_PYBINDGEN_PATH]) except KeyboardInterrupt: print "Interrupted; Python bindings will be disabled." shutil.rmtree(constants.LOCAL_PYBINDGEN_PATH, True) return False print "Fetch was successful." run_command(["git", "checkout", rev, "-q"], cwd=constants.LOCAL_PYBINDGEN_PATH) ## This causes the version to be generated run_command(["python", "setup.py", "clean"], cwd=constants.LOCAL_PYBINDGEN_PATH) return (constants.LOCAL_PYBINDGEN_PATH, required_pybindgen_version)
def action_bisect(name, args): """find first failing build using binary search""" parser = OptionParser("""\ usage: %%prog %(name)s [options] ... test command args ... Look for the first published build where a test failed, using the builds on llvmlab. The command arguments are executed once per build tested, but each argument is first subject to string interpolation. The syntax is "%%(VARIABLE)FORMAT" where FORMAT is a standard printf format, and VARIABLE is one of: 'sandbox' - the path to the sandbox directory. 'path' - the path to the build under test. 'revision' - the revision number of the build. 'build' - the build number of the build under test. 'clang' - the path to the clang binary of the build if it exists. 'clang++' - the path to the clang++ binary of the build if it exists. 'libltodir' - the path to the directory containing libLTO.dylib, if it exists. Each test is run in a sandbox directory. By default, sandbox directories are temporary directories which are created and destroyed for each test (see --sandbox). For use in auxiliary test scripts, each test is also run with each variable available in the environment as TEST_<variable name> (variables are converted to uppercase). For example, a test script could use "TEST_PATH" to find the path to the build under test. The stdout and stderr of the command are logged to files inside the sandbox directory. Use an explicit sandbox directory if you would like to look at them. It is possible to run multiple distinct commands for each test by separating them in the command line arguments by '----'. The failure of any command causes the entire test to fail.\ """ % locals()) parser.add_option("-b", "--build", dest="build_name", metavar="STR", help="name of build to fetch", action="store", default=DEFAULT_BUILDER) parser.add_option("-s", "--sandbox", dest="sandbox", help="directory to use as a sandbox", action="store", default=None) parser.add_option("-v", "--verbose", dest="verbose", help="output more test notermation", action="store_true", default=False) parser.add_option("-V", "--very-verbose", dest="very_verbose", help="output even more test notermation", action="store_true", default=False) parser.add_option("", "--show-output", dest="show_command_output", help="display command output", action="store_true", default=False) parser.add_option("", "--single-step", dest="single_step", help="single step instead of binary stepping", action="store_true", default=False) parser.add_option("", "--min-rev", dest="min_rev", help="minimum revision to test", type="int", action="store", default=None) parser.add_option("", "--max-rev", dest="max_rev", help="maximum revision to test", type="int", action="store", default=None) parser.disable_interspersed_args() (opts, args) = parser.parse_args(args) if opts.build_name is None: parser.error("no build name given (see --build)") # Very verbose implies verbose. opts.verbose |= opts.very_verbose start_time = time.time() available_builds = list(llvmlab.fetch_builds(opts.build_name)) available_builds.sort() available_builds.reverse() if opts.very_verbose: note("fetched builds in %.2fs" % (time.time() - start_time,)) if opts.min_rev is not None: available_builds = [b for b in available_builds if b.revision >= opts.min_rev] if opts.max_rev is not None: available_builds = [b for b in available_builds if b.revision <= opts.max_rev] def predicate(item): # Run the sandboxed test. test_result, _ = execute_sandboxed_test( opts.sandbox, opts.build_name, item, args, verbose=opts.verbose, very_verbose=opts.very_verbose, show_command_output=opts.show_command_output or opts.very_verbose) # Print status. print '%s: %s' % (('FAIL', 'PASS')[test_result], item.tobasename(include_suffix=False)) return test_result if opts.single_step: for item in available_builds: if predicate(item): break else: item = None else: if opts.min_rev is None or opts.max_rev is None: # Gallop to find initial search range, under the assumption that we # are most likely looking for something at the head of this list. search_space = algorithm.gallop(predicate, available_builds) else: # If both min and max revisions are specified, # don't gallop - bisect the given range. search_space = available_builds item = algorithm.bisect(predicate, search_space) if item is None: fatal('unable to find any passing build!') print '%s: first working build' % item.tobasename(include_suffix=False) index = available_builds.index(item) if index == 0: print 'no failing builds!?' else: print '%s: next failing build' % available_builds[index-1].tobasename( include_suffix=False)
def create_test_for_url(url, name, policy): fatal("URL test case creation not yet implemented")
def action_fetch(name, args): """fetch a build from the server""" parser = OptionParser("""\ usage: %%prog %(name)s [options] builder [build-name] Fetch the build from the named builder which matchs build-name. If no match is found, get the first build before the given name. If no build name is given, the most recent build is fetched. The available builders can be listed using: %%prog ls The available builds can be listed using: %%prog ls builder""" % locals()) parser.add_option("-f", "--force", dest="force", help=("always download and extract, overwriting any" "existing files"), action="store_true", default=False) parser.add_option("", "--update-link", dest="update_link", metavar="PATH", help=("update a symbolic link at PATH to point to the " "fetched build (on success)"), action="store", default=None) parser.add_option("-d", "--dry-run", dest='dry_run', help=("Perform all operations except the actual " "downloading and extracting of any files"), action="store_true", default=False) (opts, args) = parser.parse_args(args) if len(args) == 0: parser.error("please specify a builder name") elif len(args) == 1: builder, = args build_name = None elif len(args) == 2: builder, build_name = args else: parser.error("invalid number of arguments") builds = list(llvmlab.fetch_builds(builder)) if not builds: parser.error("no builds for builder: %r" % builder) build = get_best_match(builds, build_name) if not build: parser.error("no match for build %r" % build_name) path = build.tobasename() if build_name is not None and not path.startswith(build_name): note('no exact match, fetching %r' % path) # Get the paths to extract to. root_path = path builddir_path = build.tobasename(include_suffix=False) if not opts.dry_run: # Check that the download and extract paths are clean. for p in (root_path, builddir_path): if os.path.exists(p): # If we are using --force, then clean the path. if opts.force: shutil.rmtree(p, ignore_errors=True) continue fatal('current directory is not clean, %r exists' % p) llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path) print 'downloaded root: %s' % root_path print 'extracted path : %s' % builddir_path # Update the symbolic link, if requested. if not opts.dry_run and opts.update_link: # Remove the existing path. try: os.unlink(opts.update_link) except OSError as e: if e.errno != errno.ENOENT: fatal('unable to update symbolic link at %r, cannot unlink' % ( opts.update_link)) # Create the symbolic link. os.symlink(os.path.abspath(builddir_path), opts.update_link) print 'updated link at: %s' % opts.update_link return os.path.abspath(builddir_path)
import config if __name__ == "__main__": config.get() bv = LoadBallotFactoryForVendorStyle(const.layout_brand) bsv = LoadBallotSideFactoryForVendorStyle(const.layout_brand) # establish a connection to the database # connect to db and open cursor dbc = None if const.use_db: try: dbc = db.PostgresDB(database=const.dbname, user=const.dbuser) except db.DatabaseError: util.fatal("Could not connect to database!") else: dbc = db.NullDB() prepopulate_cache() # Create a ballot from one image file. # This should create an instance for the style named as "brand" in tevs.cfg #b = bv("harttestfront.jpg",None) # WARNING -- The display program(s) # will expect a numeric filename # in the correct location specified by tevs.cfg # hart filename1 = "/home/mitch/data/hart/unproc/000/000001.jpg" filename2 = "/home/mitch/data/hart/unproc/000/000002.jpg" filename3 = "/home/mitch/data/hart/unproc/000/000003.jpg"
def execute_sandboxed_test(sandbox, builder, build, args, verbose=False, very_verbose=False, add_path_variables=True, show_command_output=False, reuse_sandbox=False): def split_command_filters(command): for i, arg in enumerate(command): if arg[:2] != "%%" or arg[-2:] != "%%": break else: fatal("invalid command: %s, only contains filter " "specifications" % ("".join('"%s"' % a for a in command))) return ([a[2:-2] for a in command[:i]], command[i:]) path = build.tobasename(include_suffix=False) fullpath = build.tobasename() if verbose: note('testing %r' % path) # Create the sandbox directory, if it doesn't exist. is_temp = False if sandbox is None: sandbox = tempfile.mkdtemp() is_temp = True else: # Make absolute. sandbox = os.path.abspath(sandbox) if not os.path.exists(sandbox): os.mkdir(sandbox) # Compute paths and make sure sandbox is clean. root_path = os.path.join(sandbox, fullpath) builddir_path = os.path.join(sandbox, path) need_build = True if reuse_sandbox and (os.path.exists(root_path) and os.path.exists(builddir_path)): need_build = False else: for p in (root_path, builddir_path): if os.path.exists(p): fatal('sandbox is not clean, %r exists' % p) # Fetch and extract the build. if need_build: start_time = time.time() llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path) if very_verbose: note("extracted build in %.2fs" % (time.time() - start_time,)) # Attempt to find clang/clang++ in the downloaded build. def find_binary(name): x = subprocess.check_output(['find', builddir_path, '-name', name])\ .strip().split("\n")[0] if x == '': x = None return x clang_path = find_binary('clang') clangpp_path = find_binary('clang++') liblto_path = find_binary('libLTO.dylib') if liblto_path is not None: liblto_dir = os.path.dirname(liblto_path) else: liblto_dir = None # Construct the interpolation variables. options = {'sandbox': sandbox, 'path': builddir_path, 'revision': build.revision, 'build': build.build, 'clang': clang_path, 'clang++': clangpp_path, 'libltodir': liblto_dir} # Inject environment variables. env = os.environ.copy() for key, value in options.items(): env['TEST_%s' % key.upper()] = str(value) # Extend the environment to include the path to the extracted build. # # FIXME: Ideally, we would be able to read some kind of configuration # notermation about a builder so that we could just set this up, it doesn't # necessarily here as hard-coded notermation. if add_path_variables: path_extensions = [] dyld_library_path_extensions = [] toolchains_dir = os.path.join(builddir_path, ('Applications/Xcode.app/Contents/' 'Developer/Toolchains')) toolchain_paths = [] if os.path.exists(toolchains_dir): toolchain_paths = [os.path.join(toolchains_dir, name, 'usr') for name in os.listdir(toolchains_dir)] for package_root in ['', 'Developer/usr/'] + toolchain_paths: p = os.path.join(builddir_path, package_root, 'bin') if os.path.exists(p): path_extensions.append(p) p = os.path.join(builddir_path, package_root, 'lib') if os.path.exists(p): dyld_library_path_extensions.append(p) if path_extensions: env['PATH'] = os.pathsep.join( path_extensions + [os.environ.get('PATH', '')]) if dyld_library_path_extensions: env['DYLD_LIBRARY_PATH'] = os.pathsep.join( dyld_library_path_extensions + [ os.environ.get('DYLD_LIBRARY_PATH', '')]) # Split the arguments into distinct commands. # # Extended command syntax allows running multiple commands by separating # them with '----'. test_commands = util.list_split(args, "----") # Split command specs into filters and commands. test_commands = [split_command_filters(spec) for spec in test_commands] # Execute the test. command_objects = [] interpolated_variables = False for i, (filters, command) in enumerate(test_commands): # Interpolate arguments. old_command = command command = [a % options for a in command] if old_command != command: interpolated_variables = True # Create the command object... stdout_log_path = os.path.join(sandbox, '%s.%d.stdout' % (path, i)) stderr_log_path = os.path.join(sandbox, '%s.%d.stderr' % (path, i)) cmd_object = Command(command, stdout_log_path, stderr_log_path, env) command_objects.append(cmd_object) # Execute the command. try: cmd_object.execute(verbose=verbose) except OSError, e: # Python's exceptions are horribly to read, and this one is # incredibly common when people don't use the right syntax (or # misspell something) when writing a predicate. Detect this and # notify the user. if e.errno == errno.ENOENT: fatal("invalid command, executable doesn't exist: %r" % ( cmd_object.command[0],)) elif e.errno == errno.ENOEXEC: fatal("invalid command, executable has a bad format. Did you " "forget to put a #! at the top of a script?: %r" % (cmd_object.command[0],)) else: # Otherwise raise the error again. raise e # Evaluate the filters. for filter in filters: cmd_object.evaluate_filter_spec(filter) if show_command_output: for p, type in ((stdout_log_path, "stdout"), (stderr_log_path, "stderr")): if not os.path.exists(p): continue f = open(p) data = f.read() f.close() if data: print ("-- command %s (note: suppressed by default, " "see sandbox dir for log files) --" % (type)) print "--\n%s--\n" % data test_result = cmd_object.result if not test_result: break
def get_pybindgen(ns3_dir): print """ # # Get PyBindGen # """ if sys.platform in ['cygwin']: print "Architecture (%s) does not support PyBindGen ... skipping" % (sys.platform,) raise RuntimeError # (peek into the ns-3 wscript and extract the required pybindgen version) ns3_python_wscript = open(os.path.join(ns3_dir, "bindings", "python", "wscript"), "rt") required_pybindgen_version = None for line in ns3_python_wscript: if 'REQUIRED_PYBINDGEN_VERSION' in line: required_pybindgen_version = eval(line.split('=')[1].strip()) ns3_python_wscript.close() break if required_pybindgen_version is None: fatal("Unable to detect pybindgen required version") print "Required pybindgen version: ", '.'.join([str(x) for x in required_pybindgen_version]) # work around http_proxy handling bug in bzr if 'http_proxy' in os.environ and 'https_proxy' not in os.environ: os.environ['https_proxy'] = os.environ['http_proxy'] if len(required_pybindgen_version) == 4: rev = "-rrevno:%i" % required_pybindgen_version[3] else: rev = "-rtag:%s" % '.'.join([str(x) for x in required_pybindgen_version]) if os.path.exists(constants.LOCAL_PYBINDGEN_PATH): print "Trying to update pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip." try: run_command(["bzr", "pull", rev, "-d", constants.LOCAL_PYBINDGEN_PATH, constants.PYBINDGEN_BRANCH]) except KeyboardInterrupt: print "Interrupted; Python bindings will be disabled." else: print "Update was successful." else: print "Trying to fetch pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip." try: run_command(["bzr", "checkout", rev, constants.PYBINDGEN_BRANCH, constants.LOCAL_PYBINDGEN_PATH]) except KeyboardInterrupt: print "Interrupted; Python bindings will be disabled." shutil.rmtree(constants.LOCAL_PYBINDGEN_PATH, True) return False print "Fetch was successful." ## generate a fake version.py file in pybindgen it's safer this ## way, since the normal version generation process requires ## bazaar python bindings, which may not be available. vfile = open(os.path.join(constants.LOCAL_PYBINDGEN_PATH, "pybindgen", "version.py"), "wt") vfile.write(""" # (fake version generated by ns-3) __version__ = %r """ % list(required_pybindgen_version)) vfile.close() return (constants.LOCAL_PYBINDGEN_PATH, '.'.join([str(x) for x in required_pybindgen_version]))
def execute_sandboxed_test(sandbox, builder, build, args, verbose=False, very_verbose=False, add_path_variables=True, show_command_output=False, reuse_sandbox=False): def split_command_filters(command): for i, arg in enumerate(command): if arg[:2] != "%%" or arg[-2:] != "%%": break else: fatal("invalid command: %s, only contains filter " "specifications" % ("".join('"%s"' % a for a in command))) return ([a[2:-2] for a in command[:i]], command[i:]) path = build.tobasename(include_suffix=False) fullpath = build.tobasename() if verbose: note('testing %r' % path) # Create the sandbox directory, if it doesn't exist. is_temp = False if sandbox is None: sandbox = tempfile.mkdtemp() is_temp = True else: # Make absolute. sandbox = os.path.abspath(sandbox) if not os.path.exists(sandbox): os.mkdir(sandbox) # Compute paths and make sure sandbox is clean. root_path = os.path.join(sandbox, fullpath) builddir_path = os.path.join(sandbox, path) need_build = True if reuse_sandbox and (os.path.exists(root_path) and os.path.exists(builddir_path)): need_build = False else: for p in (root_path, builddir_path): if os.path.exists(p): fatal('sandbox is not clean, %r exists' % p) # Fetch and extract the build. if need_build: start_time = time.time() llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path) if very_verbose: note("extracted build in %.2fs" % (time.time() - start_time, )) # Attempt to find clang/clang++ in the downloaded build. def find_binary(name): x = subprocess.check_output(['find', builddir_path, '-name', name])\ .strip().split("\n")[0] if x == '': x = None return x clang_path = find_binary('clang') clangpp_path = find_binary('clang++') liblto_path = find_binary('libLTO.dylib') if liblto_path is not None: liblto_dir = os.path.dirname(liblto_path) else: liblto_dir = None # Construct the interpolation variables. options = { 'sandbox': sandbox, 'path': builddir_path, 'revision': build.revision, 'build': build.build, 'clang': clang_path, 'clang++': clangpp_path, 'libltodir': liblto_dir } # Inject environment variables. env = os.environ.copy() for key, value in options.items(): env['TEST_%s' % key.upper()] = str(value) # Extend the environment to include the path to the extracted build. # # FIXME: Ideally, we would be able to read some kind of configuration # notermation about a builder so that we could just set this up, it doesn't # necessarily here as hard-coded notermation. if add_path_variables: path_extensions = [] dyld_library_path_extensions = [] toolchains_dir = os.path.join(builddir_path, ('Applications/Xcode.app/Contents/' 'Developer/Toolchains')) toolchain_paths = [] if os.path.exists(toolchains_dir): toolchain_paths = [ os.path.join(toolchains_dir, name, 'usr') for name in os.listdir(toolchains_dir) ] for package_root in ['', 'Developer/usr/'] + toolchain_paths: p = os.path.join(builddir_path, package_root, 'bin') if os.path.exists(p): path_extensions.append(p) p = os.path.join(builddir_path, package_root, 'lib') if os.path.exists(p): dyld_library_path_extensions.append(p) if path_extensions: env['PATH'] = os.pathsep.join(path_extensions + [os.environ.get('PATH', '')]) if dyld_library_path_extensions: env['DYLD_LIBRARY_PATH'] = os.pathsep.join( dyld_library_path_extensions + [os.environ.get('DYLD_LIBRARY_PATH', '')]) # Split the arguments into distinct commands. # # Extended command syntax allows running multiple commands by separating # them with '----'. test_commands = util.list_split(args, "----") # Split command specs into filters and commands. test_commands = [split_command_filters(spec) for spec in test_commands] # Execute the test. command_objects = [] interpolated_variables = False for i, (filters, command) in enumerate(test_commands): # Interpolate arguments. old_command = command command = [a % options for a in command] if old_command != command: interpolated_variables = True # Create the command object... stdout_log_path = os.path.join(sandbox, '%s.%d.stdout' % (path, i)) stderr_log_path = os.path.join(sandbox, '%s.%d.stderr' % (path, i)) cmd_object = Command(command, stdout_log_path, stderr_log_path, env) command_objects.append(cmd_object) # Execute the command. try: cmd_object.execute(verbose=verbose) except OSError, e: # Python's exceptions are horribly to read, and this one is # incredibly common when people don't use the right syntax (or # misspell something) when writing a predicate. Detect this and # notify the user. if e.errno == errno.ENOENT: fatal("invalid command, executable doesn't exist: %r" % (cmd_object.command[0], )) elif e.errno == errno.ENOEXEC: fatal("invalid command, executable has a bad format. Did you " "forget to put a #! at the top of a script?: %r" % (cmd_object.command[0], )) else: # Otherwise raise the error again. raise e # Evaluate the filters. for filter in filters: cmd_object.evaluate_filter_spec(filter) if show_command_output: for p, type in ((stdout_log_path, "stdout"), (stderr_log_path, "stderr")): if not os.path.exists(p): continue f = open(p) data = f.read() f.close() if data: print( "-- command %s (note: suppressed by default, " "see sandbox dir for log files) --" % (type)) print "--\n%s--\n" % data test_result = cmd_object.result if not test_result: break
def get_pybindgen(ns3_dir): print(""" # # Get PyBindGen # """) if sys.platform in ['cygwin']: print("Architecture (%s) does not support PyBindGen ... skipping" % (sys.platform,)) raise RuntimeError # (peek into the ns-3 wscript and extract the required pybindgen version) ns3_python_wscript = open(os.path.join(ns3_dir, "bindings", "python", "wscript"), "rt") required_pybindgen_version = None for line in ns3_python_wscript: if line.startswith('REQUIRED_PYBINDGEN_VERSION'): required_pybindgen_version = eval(line.split('=')[1].strip()) ns3_python_wscript.close() break if required_pybindgen_version is None: fatal("Unable to detect pybindgen required version") print("Required pybindgen version: ", required_pybindgen_version) # work around http_proxy handling bug in bzr # if 'http_proxy' in os.environ and 'https_proxy' not in os.environ: # os.environ['https_proxy'] = os.environ['http_proxy'] if 'post' in required_pybindgen_version: # given a version like '0.17.0.post41+ngd10fa60', the last 7 characters # are part of a git hash, which should be enough to identify a revision rev = required_pybindgen_version[-7:] else: rev = required_pybindgen_version if os.path.exists(constants.LOCAL_PYBINDGEN_PATH): print("Trying to update pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip.") try: run_command(["git", "fetch", constants.PYBINDGEN_BRANCH], cwd=constants.LOCAL_PYBINDGEN_PATH) except KeyboardInterrupt: print("Interrupted; Python bindings will be disabled.") else: print("Update was successful.") else: print("Trying to fetch pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip.") try: run_command(["git", "clone", constants.PYBINDGEN_BRANCH, constants.LOCAL_PYBINDGEN_PATH]) except KeyboardInterrupt: print("Interrupted; Python bindings will be disabled.") shutil.rmtree(constants.LOCAL_PYBINDGEN_PATH, True) return False print("Fetch was successful.") run_command(["git", "checkout", rev, "-q"], cwd=constants.LOCAL_PYBINDGEN_PATH) ## This causes the version to be generated try: import setuptools run_command(["python3", "setup.py", "clean"], cwd=constants.LOCAL_PYBINDGEN_PATH) except ImportError: print("Warning: Pybindgen setup not successful, are setuptools installed?") raise RuntimeError return (constants.LOCAL_PYBINDGEN_PATH, required_pybindgen_version)
def action_exec(name, args): """execute a command against a published root""" parser = OptionParser("""\ usage: %%prog %(name)s [options] ... test command args ... Executes the given command against the latest published build. The syntax for commands (and exit code) is exactly the same as for the 'bisect' tool, so this command is useful for testing bisect test commands. See 'bisect' for more notermation on the exact test syntax.\ """ % locals()) parser.add_option("-b", "--build", dest="build_name", metavar="STR", help="name of build to fetch", action="store", default=DEFAULT_BUILDER) parser.add_option("-s", "--sandbox", dest="sandbox", help="directory to use as a sandbox", action="store", default=None) parser.add_option("", "--min-rev", dest="min_rev", help="minimum revision to test", type="int", action="store", default=None) parser.add_option("", "--max-rev", dest="max_rev", help="maximum revision to test", type="int", action="store", default=None) parser.add_option("", "--near", dest="near_build", help="use a build near NAME", type="str", action="store", metavar="NAME", default=None) parser.disable_interspersed_args() (opts, args) = parser.parse_args(args) if opts.build_name is None: parser.error("no build name given (see --build)") available_builds = list(llvmlab.fetch_builds(opts.build_name)) available_builds.sort() available_builds.reverse() if opts.min_rev is not None: available_builds = [ b for b in available_builds if b.revision >= opts.min_rev ] if opts.max_rev is not None: available_builds = [ b for b in available_builds if b.revision <= opts.max_rev ] if len(available_builds) == 0: fatal("No builds available for builder name: %s" % opts.build_name) # Find the best match, if requested. if opts.near_build: build = get_best_match(available_builds, opts.near_build) if not build: parser.error("no match for build %r" % opts.near_build) else: # Otherwise, take the latest build. build = available_builds[0] test_result, _ = execute_sandboxed_test(opts.sandbox, opts.build_name, build, args, verbose=True, show_command_output=True) print '%s: %s' % ( ('FAIL', 'PASS')[test_result], build.tobasename(include_suffix=False)) raise SystemExit(test_result != True)
def main(): miss_counter = 0 # get command line arguments cfg_file = get_args() # read configuration from tevs.cfg and set constants for this run config.get(cfg_file) util.mkdirp(const.root) log = config.logger(const.logfilename) #create initial top level dirs, if they do not exist for p in ( "%s" % ("templates"), "%s%d" % ("template_images", os.getpid()), "%s%d" % ("composite_images", os.getpid()), "results", "proc", "errors"): util.mkdirp(util.root(p)) next_ballot = next.File(util.root("nexttoprocess.txt"), const.num_pages) try: ballotfrom = Ballot.LoadBallotType(const.layout_brand) except KeyError as e: util.fatal("No such ballot type: " + const.layout_brand + ": check " + cfg_file) # allow all instances to share a common template location, # though need per-pid locs for template_images and composite_images cache = Ballot.TemplateCache(util.root("templates")) extensions = Ballot.Extensions(template_cache=cache) # connect to db and open cursor if const.use_db: try: dbc = db.PostgresDB(const.dbname, const.dbuser) except db.DatabaseError: util.fatal("Could not connect to database") else: dbc = db.NullDB() total_proc, total_unproc = 0, 0 base = os.path.basename # While ballot images exist in the directory specified in tevs.cfg, # create ballot from images, get landmarks, get layout code, get votes. # Write votes to database and results directory. Repeat. #from guppy import hpy;hp=hpy();hp.setref();import gc;gc.disable();gc.collect();hp.setref() try: for n in next_ballot: gc.collect() unprocs = [incomingn(n + m) for m in range(const.num_pages)] if not os.path.exists(unprocs[0]): miss_counter += 1 log.info(base(unprocs[0]) + " does not exist. No more records to process") if miss_counter > 10: break continue #for i, f in enumerate(unprocs[1:]): # if not os.path.exists(f): # log.info(base(f) + " does not exist. Cannot proceed.") # for j in range(i): # log.info(base(unprocs[j]) + " will NOT be processed") # total_unproc += mark_error(None, *unprocs[:i-1]) #Processing log.info("Processing %s:\n %s" % (n, "\n".join("\t%s" % base(u) for u in unprocs)) ) try: ballot = ballotfrom(unprocs, extensions) results = ballot.ProcessPages() except BallotException as e: total_unproc += mark_error(e, *unprocs) log.exception("Could not process ballot") continue csv = Ballot.results_to_CSV(results) #moz = Ballot.results_to_mosaic(results) #Write all data #make dirs: proc1d = dirn("proc", n) resultsd = dirn("results", n) resultsfilename = filen(resultsd, n) for p in (proc1d, resultsd): util.mkdirp(p) try: results_to_vop_files(results,resultsfilename) except Exception as e: print e #write csv and mosaic util.genwriteto(resultsfilename + ".txt", csv) #write to the database try: dbc.insert(ballot) except db.DatabaseError: #dbc does not commit if there is an error, just need to remove #partial files remove_partial(resultsfilename + ".txt") remove_partial(resultsfilename + const.filename_extension) util.fatal("Could not commit vote information to database") #Post-processing # move the images from unproc to proc procs = [filen(proc1d, n + m) + const.filename_extension for m in range(const.num_pages)] for a, b in zip(unprocs, procs): try: os.rename(a, b) except OSError as e: util.fatal("Could not rename %s", a) total_proc += const.num_pages log.info("%d images processed", const.num_pages) #hp.heap().dump('prof.hpy');hp.setref();gc.collect();hp.setref();hp.heap().dump('prof.hpy') finally: cache.save_all() dbc.close() next_ballot.save() log.info("%d images processed", total_proc) if total_unproc > 0: log.warning("%d images NOT processed.", total_unproc)
def instantiate(ckpt_dir=None): from m5 import options root = objects.Root.getInstance() if not root: fatal("Need to instantiate Root() before calling instantiate()") # we need to fix the global frequency ticks.fixGlobalFrequency() # Make sure SimObject-valued params are in the configuration # hierarchy so we catch them with future descendants() walks for obj in root.descendants(): obj.adoptOrphanParams() # Unproxy in sorted order for determinism for obj in root.descendants(): obj.unproxyParams() if options.dump_config: ini_file = file(os.path.join(options.outdir, options.dump_config), 'w') # Print ini sections in sorted order for easier diffing for obj in sorted(root.descendants(), key=lambda o: o.path()): obj.print_ini(ini_file) ini_file.close() if options.json_config: try: import json json_file = file(os.path.join(options.outdir, options.json_config), 'w') d = root.get_config_as_dict() json.dump(d, json_file, indent=4) json_file.close() except ImportError: pass do_dot(root, options.outdir, options.dot_config) # Initialize the global statistics stats.initSimStats() # Create the C++ sim objects and connect ports for obj in root.descendants(): obj.createCCObject() for obj in root.descendants(): obj.connectPorts() # Do a second pass to finish initializing the sim objects for obj in root.descendants(): obj.init() # Do a third pass to initialize statistics for obj in root.descendants(): obj.regStats() # We're done registering statistics. Enable the stats package now. stats.enable() # Restore checkpoint (if any) if ckpt_dir: ckpt = internal.core.getCheckpoint(ckpt_dir) internal.core.unserializeGlobals(ckpt) for obj in root.descendants(): obj.loadState(ckpt) need_resume.append(root) else: for obj in root.descendants(): obj.initState() # Check to see if any of the stat events are in the past after resuming from # a checkpoint, If so, this call will shift them to be at a valid time. updateStatEvents()
def action_fetch(name, args): """fetch a build from the server""" parser = OptionParser("""\ usage: %%prog %(name)s [options] builder [build-name] Fetch the build from the named builder which matchs build-name. If no match is found, get the first build before the given name. If no build name is given, the most recent build is fetched. The available builders can be listed using: %%prog ls The available builds can be listed using: %%prog ls builder""" % locals()) parser.add_option("-f", "--force", dest="force", help=("always download and extract, overwriting any" "existing files"), action="store_true", default=False) parser.add_option("", "--update-link", dest="update_link", metavar="PATH", help=("update a symbolic link at PATH to point to the " "fetched build (on success)"), action="store", default=None) parser.add_option("-d", "--dry-run", dest='dry_run', help=("Perform all operations except the actual " "downloading and extracting of any files"), action="store_true", default=False) (opts, args) = parser.parse_args(args) if len(args) == 0: parser.error("please specify a builder name") elif len(args) == 1: builder, = args build_name = None elif len(args) == 2: builder, build_name = args else: parser.error("invalid number of arguments") builds = list(llvmlab.fetch_builds(builder)) if not builds: parser.error("no builds for builder: %r" % builder) build = get_best_match(builds, build_name) if not build: parser.error("no match for build %r" % build_name) path = build.tobasename() if build_name is not None and not path.startswith(build_name): note('no exact match, fetching %r' % path) # Get the paths to extract to. root_path = path builddir_path = build.tobasename(include_suffix=False) if not opts.dry_run: # Check that the download and extract paths are clean. for p in (root_path, builddir_path): if os.path.exists(p): # If we are using --force, then clean the path. if opts.force: shutil.rmtree(p, ignore_errors=True) continue fatal('current directory is not clean, %r exists' % p) llvmlab.fetch_build_to_path(builder, build, root_path, builddir_path) print 'downloaded root: %s' % root_path print 'extracted path : %s' % builddir_path # Update the symbolic link, if requested. if not opts.dry_run and opts.update_link: # Remove the existing path. try: os.unlink(opts.update_link) except OSError as e: if e.errno != errno.ENOENT: fatal('unable to update symbolic link at %r, cannot unlink' % (opts.update_link)) # Create the symbolic link. os.symlink(os.path.abspath(builddir_path), opts.update_link) print 'updated link at: %s' % opts.update_link return os.path.abspath(builddir_path)
def check_tracing(): if defines.TRACING_ON: return fatal("Tracing is not enabled. Compile with TRACING_ON")
def action_bisect(name, args): """find first failing build using binary search""" parser = OptionParser("""\ usage: %%prog %(name)s [options] ... test command args ... Look for the first published build where a test failed, using the builds on llvmlab. The command arguments are executed once per build tested, but each argument is first subject to string interpolation. The syntax is "%%(VARIABLE)FORMAT" where FORMAT is a standard printf format, and VARIABLE is one of: 'sandbox' - the path to the sandbox directory. 'path' - the path to the build under test. 'revision' - the revision number of the build. 'build' - the build number of the build under test. 'clang' - the path to the clang binary of the build if it exists. 'clang++' - the path to the clang++ binary of the build if it exists. 'libltodir' - the path to the directory containing libLTO.dylib, if it exists. Each test is run in a sandbox directory. By default, sandbox directories are temporary directories which are created and destroyed for each test (see --sandbox). For use in auxiliary test scripts, each test is also run with each variable available in the environment as TEST_<variable name> (variables are converted to uppercase). For example, a test script could use "TEST_PATH" to find the path to the build under test. The stdout and stderr of the command are logged to files inside the sandbox directory. Use an explicit sandbox directory if you would like to look at them. It is possible to run multiple distinct commands for each test by separating them in the command line arguments by '----'. The failure of any command causes the entire test to fail.\ """ % locals()) parser.add_option("-b", "--build", dest="build_name", metavar="STR", help="name of build to fetch", action="store", default=DEFAULT_BUILDER) parser.add_option("-s", "--sandbox", dest="sandbox", help="directory to use as a sandbox", action="store", default=None) parser.add_option("-v", "--verbose", dest="verbose", help="output more test notermation", action="store_true", default=False) parser.add_option("-V", "--very-verbose", dest="very_verbose", help="output even more test notermation", action="store_true", default=False) parser.add_option("", "--show-output", dest="show_command_output", help="display command output", action="store_true", default=False) parser.add_option("", "--single-step", dest="single_step", help="single step instead of binary stepping", action="store_true", default=False) parser.add_option("", "--min-rev", dest="min_rev", help="minimum revision to test", type="int", action="store", default=None) parser.add_option("", "--max-rev", dest="max_rev", help="maximum revision to test", type="int", action="store", default=None) parser.disable_interspersed_args() (opts, args) = parser.parse_args(args) if opts.build_name is None: parser.error("no build name given (see --build)") # Very verbose implies verbose. opts.verbose |= opts.very_verbose start_time = time.time() available_builds = list(llvmlab.fetch_builds(opts.build_name)) available_builds.sort() available_builds.reverse() if opts.very_verbose: note("fetched builds in %.2fs" % (time.time() - start_time, )) if opts.min_rev is not None: available_builds = [ b for b in available_builds if b.revision >= opts.min_rev ] if opts.max_rev is not None: available_builds = [ b for b in available_builds if b.revision <= opts.max_rev ] def predicate(item): # Run the sandboxed test. test_result, _ = execute_sandboxed_test( opts.sandbox, opts.build_name, item, args, verbose=opts.verbose, very_verbose=opts.very_verbose, show_command_output=opts.show_command_output or opts.very_verbose) # Print status. print '%s: %s' % (('FAIL', 'PASS')[test_result], item.tobasename(include_suffix=False)) return test_result if opts.single_step: for item in available_builds: if predicate(item): break else: item = None else: if opts.min_rev is None or opts.max_rev is None: # Gallop to find initial search range, under the assumption that we # are most likely looking for something at the head of this list. search_space = algorithm.gallop(predicate, available_builds) else: # If both min and max revisions are specified, # don't gallop - bisect the given range. search_space = available_builds item = algorithm.bisect(predicate, search_space) if item is None: fatal('unable to find any passing build!') print '%s: first working build' % item.tobasename(include_suffix=False) index = available_builds.index(item) if index == 0: print 'no failing builds!?' else: print '%s: next failing build' % available_builds[ index - 1].tobasename(include_suffix=False)