def importAndBisect(self, good, bad, testcondition=None, args_for_condition=[]): #Convenience function for API use only conditionscript = ximport.importRelativeOrAbsolute(testcondition) self.bisect(good, bad, testcondition=conditionscript, args_for_condition=args_for_condition)
def interesting(cliArgs, tempPrefix): (options, rangeMin, rangeMax, arguments) = parseOptions(cliArgs) conditionScript = ximport.importRelativeOrAbsolute(arguments[0]) conditionArgs = arguments[1:] if hasattr(conditionScript, "init"): conditionScript.init(conditionArgs) assert (rangeMax - rangeMin) >= 0 for i in xrange(rangeMin, rangeMax + 1): # This doesn't do anything if RANGENUM is not found. replacedConditionArgs = [s.replace('RANGENUM', str(i)) for s in conditionArgs] print 'Range number ' + str(i) + ':', if conditionScript.interesting(replacedConditionArgs, tempPrefix): return True return False
def interesting(cliArgs, tempPrefix): (rangeMin, rangeMax, arguments) = parseOptions(cliArgs) conditionScript = ximport.importRelativeOrAbsolute(arguments[0]) conditionArgs = arguments[1:] if hasattr(conditionScript, "init"): conditionScript.init(conditionArgs) assert (rangeMax - rangeMin) >= 0 for i in xrange(rangeMin, rangeMax + 1): # This doesn't do anything if RANGENUM is not found. replacedConditionArgs = [ s.replace('RANGENUM', str(i)) for s in conditionArgs ] print 'Range number ' + str(i) + ':', if conditionScript.interesting(replacedConditionArgs, tempPrefix): return True return False
def externalTestAndLabel(options, interestingness): '''Make use of interestingness scripts to decide whether the changeset is good or bad.''' conditionScript = ximport.importRelativeOrAbsolute(interestingness[0]) conditionArgPrefix = interestingness[1:] def inner(shellFilename, hgHash): conditionArgs = conditionArgPrefix + [shellFilename] + options.paramList tempDir = tempfile.mkdtemp(prefix="abExtTestAndLabel-" + hgHash) tempPrefix = os.path.join(tempDir, 't') if hasattr(conditionScript, "init"): # Since we're changing the js shell name, call init() again! conditionScript.init(conditionArgs) if conditionScript.interesting(conditionArgs, tempPrefix): innerResult = ('bad', 'interesting') else: innerResult = ('good', 'not interesting') if os.path.isdir(tempDir): sps.rmTreeIncludingReadOnly(tempDir) return innerResult return inner
def cli(): #Command line interface usage = """usage: %prog --good=[changeset] --bad=[changeset] [options] \n %prog --single=[changeset] -e """ parser = OptionParser(usage=usage, version="%prog " + progVersion) parser.disable_interspersed_args() group1 = OptionGroup(parser, "Global Options", "") group1.add_option( "-j", "--cores", dest="cores", default=cpuCount(), help= "Max simultaneous make jobs (default: %default, the number of cores on this system)", metavar="[numjobs]") group1.add_option("-m", "--mozconfig", dest="mozconf", help="external mozconfig if so desired", metavar="[mozconf path]", default=False) group1.add_option("-f", "--freshtrunk", action="store_true", dest="makeClean", default=False, help="Delete old trunk and use a fresh one") group1.add_option( "-d", "--deletetrunk", action="store_true", dest="deleteTrunk", default=False, help= "Cleanup Flag: deletes the temp directory / trunk generated. Do not use with other flags, this is just for convenient cleanup." ) group2 = OptionGroup(parser, "Bisector Options", "These are options for bisecting on changesets. Dates are retrieved from pushlog " \ "and are not as reliable as exact changeset numbers.") group2.add_option("-g", "--good", dest="good", help="Last known good revision", metavar="[changeset or date]") group2.add_option("-b", "--bad", dest="bad", help="Broken commit revision", metavar="[changeset or date]") group3 = OptionGroup(parser, "Single Changeset Options", "These are options for building a single changeset") group3.add_option("-s", "--single", dest="single", help="Build a single changeset", metavar="[changeset]") group3.add_option("-e", "--run", action="store_true", dest="run", default=False, help="run a single changeset") group4 = OptionGroup( parser, "Binary building options", "These are options for building binaries from a single changeset") group4.add_option("-x", "--binary", action="store_true", dest="binary", default=False, help="build binary and return path to it") group4.add_option( "-q", "--revision", dest="revision", default=None, metavar="[changeset]", help="revision number for single changeset to build binary from") group5 = OptionGroup(parser, "Automatic Testing Options", "Options for using an automated test instead of interactive prompting for bisection. " \ "Please read documentation on how to write testing functions for this script.") group5.add_option("-c", "--condition", dest="condition", default=None, metavar="[condtest.py -opt1 -opt2]", help="External condition for bisecting. " \ "Note: THIS MUST BE THE LAST OPTION CALLED.") group6 = OptionGroup( parser, "Remote Options", "If you don't have a build environment you can push to tryserver to build. Warning: not recommended -- very very slow. Uses a trypusher server (see http://github.com/samliu/moztrypusher). Another option here is to use mozilla's remote build cache to avoid a lot of building. Warning: breaks support for the automated test." ) group6.add_option("-t", "--try", action="store_true", dest="trypusher", help="Build remotely with trypusher", default=False) group6.add_option("-n", "--host", dest="tryhost", metavar="[trypusher server hostname]", help="Trypusher host", default="localhost") group6.add_option("-p", "--port", dest="tryport", metavar="[trypusher server port]", help="Trypusher Port", default=8080) group2.add_option( "-r", "--remote", action="store_true", dest="remote", help="Use remote build cache to avoid extra builds (NOT YET WORKING)", default=False) group7 = OptionGroup( parser, "Broken and Unstable Options", "Caution: use these options at your own risk. " "They aren't recommended.") group7.add_option( "-R", "--repo", dest="repoURL", help="alternative mercurial repo to bisect NOTE: NEVER BEEN TESTED", metavar="valid repository url") parser.add_option_group(group1) parser.add_option_group(group2) parser.add_option_group(group3) parser.add_option_group(group4) parser.add_option_group(group5) parser.add_option_group(group6) parser.add_option_group(group7) (options, args_for_condition) = parser.parse_args() # If a user only wants to make clean or has supplied no options: if (not options.good or not options.bad ) and not options.single and not options.deleteTrunk: if options.makeClean: #Make a clean trunk and quit. commitBuilder = Builder(clean=options.makeClean) else: # Not enough relevant parameters were given # Print a help message and quit. print """Use -h flag for available options.""" print """To bisect, you must specify both a good and a bad date. (-g and -b flags are not optional).""" print """You can also use the --single=[chset] flag to build and --run to run a single changeset.""" quit() # Set up a trunk for either bisection or build. mozConfiguration = options.mozconf commitBuilder = Builder(clean=options.makeClean, mozconf=mozConfiguration, tryhost=options.tryhost, tryport=options.tryport, remote=options.remote, tryPusher=options.trypusher, deleteTrunk=options.deleteTrunk) if options.cores: commitBuilder.cores = options.cores commitBuilder.mozconfigure() # For building single commits: if options.single: if options.run: commitBuilder.buildAndRun(changeset=options.single) print "Firefox successfully built and ran!" elif options.binary: commitBuilder.getBinary(revision=options.single) else: commitBuilder.build(changeset=options.single) print "Local trunk built. Not running." # For bisections: elif options.good and options.bad: print "Begin interactive commit bisect!\n" if options.repoURL: commitBuilder.repoURL = options.repoURL print "Using alternative repository " + options.repoURL conditionscript = None if options.condition: conditionscript = ximport.importRelativeOrAbsolute( options.condition) if options.remote: #Build remotely! pass commitBuilder.bisect(options.good, options.bad, testcondition=conditionscript, args_for_condition=args_for_condition) # Should not get here. else: print "Invalid input. Please try again."
allPositionalArgs = args if len(args) == 0: # No arguments; not even a condition was specified usage() return if len(args) > 1: testcaseFilename = args[-1] # can be overridden by --testcase in processOptions processOptions(opts) if testcaseFilename == None: usageError("No testcase specified (use --testcase or last condition arg)") conditionScript = ximport.importRelativeOrAbsolute(args[0]) conditionArgs = args[1:] e = testcaseFilename.rsplit(".", 1) if len(e) > 1: testcaseExtension = "." + e[1] readTestcase() if hasattr(conditionScript, "init"): conditionScript.init(conditionArgs) try: if tempDir == None: createTempDir()
def cli(): #Command line interface usage = """usage: %prog --good=[changeset] --bad=[changeset] [options] \n %prog --single=[changeset] -e """ parser = OptionParser(usage=usage,version="%prog "+progVersion) parser.disable_interspersed_args() group1 = OptionGroup(parser, "Global Options", "") group1.add_option("-j", "--cores", dest="cores", default=cpuCount(), help="Max simultaneous make jobs (default: %default, the number of cores on this system)", metavar="[numjobs]") group1.add_option("-m", "--mozconfig", dest="mozconf", help="external mozconfig if so desired", metavar="[mozconf path]", default=False) group1.add_option("-f", "--freshtrunk", action = "store_true", dest="makeClean", default=False, help="Delete old trunk and use a fresh one") group2 = OptionGroup(parser, "Bisector Options", "These are options for bisecting on changesets. Dates are retrieved from pushlog " \ "and are not as reliable as exact changeset numbers.") group2.add_option("-g", "--good", dest="good", help="Last known good revision", metavar="[changeset or date]") group2.add_option("-b", "--bad", dest="bad", help="Broken commit revision", metavar="[changeset or date]") group3 = OptionGroup(parser, "Single Changeset Options", "These are options for building a single changeset") group3.add_option("-s", "--single", dest="single", help="Build a single changeset", metavar="[changeset]") group3.add_option("-e", "--run", action="store_true", dest="run", default=False, help="run a single changeset") group4 = OptionGroup(parser, "Binary building options", "These are options for building binaries from a single changeset") group4.add_option("-x", "--binary", action="store_true", dest="binary", default=False, help="build binary and return path to it") group4.add_option("-q", "--revision", dest="revision", default=None, metavar="[changeset]", help="revision number for single changeset to build binary from") group5 = OptionGroup(parser, "Automatic Testing Options", "Options for using an automated test instead of interactive prompting for bisection. " \ "Please read documentation on how to write testing functions for this script.") group5.add_option("-c", "--condition", dest="condition", default=None, metavar="[condtest.py -opt1 -opt2]", help="External condition for bisecting. " \ "Note: THIS MUST BE THE LAST OPTION CALLED.") group6 = OptionGroup(parser, "Remote Options", "If you don't have a build environment you can push to tryserver to build. Warning: not recommended -- very very slow. Uses a trypusher server (see http://github.com/samliu/moztrypusher). Another option here is to use mozilla's remote build cache to avoid a lot of building. Warning: breaks support for the automated test.") group6.add_option("-t", "--try", action="store_true", dest="trypusher", help="Build remotely with trypusher", default=False) group6.add_option("-n", "--host",dest="tryhost", metavar="[trypusher server hostname]", help="Trypusher host", default="localhost") group6.add_option("-p", "--port", dest="tryport", metavar="[trypusher server port]", help="Trypusher Port", default=8080) group2.add_option("-r", "--remote", action="store_true", dest="remote", help="Use remote build cache to avoid extra builds (NOT YET WORKING)", default=False) group7 = OptionGroup(parser, "Broken and Unstable Options", "Caution: use these options at your own risk. " "They aren't recommended.") group7.add_option("-R", "--repo", dest="repoURL", help="alternative mercurial repo to bisect NOTE: NEVER BEEN TESTED", metavar="valid repository url") parser.add_option_group(group1) parser.add_option_group(group2) parser.add_option_group(group3) parser.add_option_group(group4) parser.add_option_group(group5) parser.add_option_group(group6) parser.add_option_group(group7) (options, args_for_condition) = parser.parse_args() # If a user only wants to make clean or has supplied no options: if (not options.good or not options.bad) and not options.single: if options.makeClean: #Make a clean trunk and quit. commitBuilder = Builder(clean=options.makeClean) else: # Not enough relevant parameters were given # Print a help message and quit. print """Use -h flag for available options.""" print """To bisect, you must specify both a good and a bad date. (-g and -b flags are not optional).""" print """You can also use the --single=[chset] flag to build and --run to run a single changeset.""" quit() # Set up a trunk for either bisection or build. mozConfiguration = options.mozconf commitBuilder = Builder(clean=options.makeClean, mozconf=mozConfiguration, tryhost=options.tryhost, tryport=options.tryport, remote=options.remote, tryPusher=options.trypusher) if options.cores: commitBuilder.cores = options.cores commitBuilder.mozconfigure() # For building single commits: if options.single: if options.run: commitBuilder.buildAndRun(changeset=options.single) print "Firefox successfully built and ran!" elif options.binary: commitBuilder.getBinary(revision=options.single) else: commitBuilder.build(changeset=options.single) print "Local trunk built. Not running." # For bisections: elif options.good and options.bad: print "Begin interactive commit bisect!\n" if options.repoURL: commitBuilder.repoURL = options.repoURL print "Using alternative repository "+options.repoURL conditionscript = None if options.condition: conditionscript = ximport.importRelativeOrAbsolute(options.condition) if options.remote: #Build remotely! pass commitBuilder.bisect(options.good,options.bad, testcondition=conditionscript, args_for_condition=args_for_condition) # Should not get here. else: print "Invalid input. Please try again."
def importAndBisect(self, good, bad, testcondition=None, args_for_condition=[]): #Convenience function for API use only conditionscript = ximport.importRelativeOrAbsolute(testcondition) self.bisect(good,bad, testcondition=conditionscript, args_for_condition=args_for_condition)
def processOptions(): global atom, conditionArgs, conditionScript, strategy, testcaseExtension, testcaseFilename, tempDir global minimizeRepeat, minimizeMin, minimizeMax, minimizeChunkStart, minimizeRepeatFirstRound, stopAfterTime parser = argparse.ArgumentParser( description="Lithium, an automated testcase reduction tool by Jesse Ruderman.", epilog="See doc/using.html for more information.", usage="./lithium.py [options] condition [condition options] file-to-reduce\n\n" "example: " "./lithium.py crashes 120 ~/tracemonkey/js/src/debug/js -j a.js\n" " Lithium will reduce a.js subject to the condition that the following\n" " crashes in 120 seconds:\n" " ~/tracemonkey/js/src/debug/js -j a.js") grp_opt = parser.add_argument_group(description="Lithium options") grp_opt.add_argument( "--testcase", help="testcase file. default: last argument is used.") grp_opt.add_argument( "--tempdir", help="specify the directory to use as temporary directory.") grp_opt.add_argument( "-c", "--char", action="store_true", help="Don't treat lines as atomic units; treat the file as a sequence of characters rather than a sequence of lines.") grp_opt.add_argument( "--strategy", default="minimize", choices=["minimize", "check-only"], help="reduction strategy to use. default: minimize") grp_add = parser.add_argument_group(description="Additional options for the default strategy") grp_add.add_argument( "--min", type=int, default=1, help="must be a power of two. default: 1") grp_add.add_argument( "--max", type=int, default=pow(2, 30), help="must be a power of two. default: about half of the file") grp_add.add_argument( "--repeat", default="last", choices=["always", "last", "never"], help="Whether to repeat a chunk size if chunks are removed. default: last") grp_add.add_argument( "--chunksize", type=int, default=None, help="Shortcut for repeat=never, min=n, max=n. chunk size must be a power of two.") grp_add.add_argument( "--chunkstart", type=int, default=0, help="For the first round only, start n chars/lines into the file. Best for max to divide n. [Mostly intended for internal use]") grp_add.add_argument( "--repeatfirstround", default=False, action="store_true", help="Treat the first round as if it removed chunks; possibly repeat it. [Mostly intended for internal use]") grp_add.add_argument( "--maxruntime", type=int, default=None, help="If reduction takes more than n seconds, stop (and print instructions for continuing).") grp_ext = parser.add_argument_group(description="Condition, condition options and file-to-reduce") grp_ext.add_argument( "extra_args", action="append", nargs=argparse.REMAINDER, help="condition [condition options] file-to-reduce") args = parser.parse_args() tempDir = args.tempdir atom = "char" if args.char else "line" strategy = args.strategy if args.chunksize: minimizeMin = args.chunksize minimizeMax = args.chunksize minimizeRepeat = "never" else: minimizeMin = args.min minimizeMax = args.max minimizeRepeat = args.repeat minimizeChunkStart = args.chunkstart minimizeRepeatFirstRound = args.repeatfirstround if args.maxruntime: stopAfterTime = time.time() + args.maxruntime extra_args = args.extra_args[0] if args.testcase: testcaseFilename = args.testcase elif len(extra_args) > 0: testcaseFilename = extra_args[-1] # can be overridden by --testcase in processOptions else: print "No testcase specified (use --testcase or last condition arg)" return False testcaseExtension = os.path.splitext(testcaseFilename)[1] if not isPowerOfTwo(minimizeMin) or not isPowerOfTwo(minimizeMax): print "Min/Max must be powers of two." return False conditionScript = ximport.importRelativeOrAbsolute(extra_args[0]) conditionArgs = extra_args[1:] return True
def bisectRecurse(self, testfile=None, testpath=None, testcondition=None, args_for_condition=[]): #Recursively build, run, and prompt verdict = "" try: self.build() except Exception: verdict = "skip" if verdict == "skip": pass elif testfile==None and testpath==None and testcondition==None: #Not using a test, interactive bisect begin! self.run() elif testcondition != None: #Using Jesse's idea: import any testing script and run it as the truth condition conditionscript = ximport.importRelativeOrAbsolute(testcondition) args_to_pass = [self.objdir] + args_for_condition conditionscript.init(args_to_pass) #TODO: refactor to use directories with revision numbers tmpdir = tempfile.mkdtemp() #If conditionscript is true, that means there was a bug. #If conditionscript is false, that means the bug is not present. verdict = conditionscript.interesting(args_to_pass,tmpdir) verdict = "bad" if verdict else "good" else: if testfile == None: #Using External testfile #1. Clear self.mochitest_tmp try: shutil.rmtree(self.mochitest_tmp) except: pass #2. Move file from testpath to self.mochitest_tmp try: os.mkdir(self.mochitest_tmp) dst = os.path.join(self.mochitest_tmp,"test_bug999999.html") subprocess.call(['touch',dst]) print "copy " + str(testpath) + " to " +dst shutil.copy(str(testpath),dst) except: print "Unable to generate test path, quitting. Is your inputted test a valid mochitest?" quit() #3. Make mochitest use the commitbuilder directory. testfile="commitbuilder" #DEBUG #testfile = "content/base/test/test_CrossSiteXHR.html" print "Trying testfile "+str(testfile) #TODO: # 1. Copy relevant test to a directory of my choice #sts = os.system("TEST_PATH="+testpath+" EXTRA_TEST_ARGS='--close-when-done' make -C " +self.objdir+ " mochitest-plain") sts = subprocess.call(['make','-C',self.objdir,'mochitest-plain'],stdout=open('/dev/null','w'),env={'TEST_PATH': testfile, 'EXTRA_TEST_ARGS':"--close-when-done"}) shutil.rmtree(self.mochitest_tmp) #cleanup if sts != 0: verdict = "bad" print "============================" print "Verdict: FAILED test, bad changeset detected!" print "============================" else: verdict = "good" print "============================" print "Verdict: PASSED test, good changeset detected!" print "============================" while verdict not in ["good", "bad", "skip"]: verdict = raw_input("Was this commit good or bad? (type 'good', 'bad', or 'skip'): ") # do hg bisect --good, --bad, or --skip verdictCommand = self.hgPrefix+["bisect","--"+verdict] print " ".join(verdictCommand) retval = captureStdout(verdictCommand) print str(retval) # HACK if retval[1] == 'h': #if retval starts with "the" then we can quit quit() elif retval[1] == 'e': #if retval starts with "testing" then it needs to keep going print "\n" else: print "Something went wrong! :(" quit() self.bisectRecurse(testfile=testfile, testpath=testpath, testcondition=testcondition, args_for_condition=args_for_condition)
def processOptions(): global atom, conditionArgs, conditionScript, strategy, testcaseExtension, testcaseFilename, tempDir global minimizeRepeat, minimizeMin, minimizeMax, minimizeChunkStart, minimizeRepeatFirstRound, stopAfterTime parser = argparse.ArgumentParser( description= "Lithium, an automated testcase reduction tool by Jesse Ruderman.", epilog="See doc/using.html for more information.", usage= "./lithium.py [options] condition [condition options] file-to-reduce\n\n" "example: " "./lithium.py crashes 120 ~/tracemonkey/js/src/debug/js -j a.js\n" " Lithium will reduce a.js subject to the condition that the following\n" " crashes in 120 seconds:\n" " ~/tracemonkey/js/src/debug/js -j a.js") grp_opt = parser.add_argument_group(description="Lithium options") grp_opt.add_argument("--testcase", help="testcase file. default: last argument is used.") grp_opt.add_argument( "--tempdir", help="specify the directory to use as temporary directory.") grp_opt.add_argument( "-c", "--char", action="store_true", help= "Don't treat lines as atomic units; treat the file as a sequence of characters rather than a sequence of lines." ) grp_opt.add_argument("--strategy", default="minimize", choices=["minimize", "check-only"], help="reduction strategy to use. default: minimize") grp_add = parser.add_argument_group( description="Additional options for the default strategy") grp_add.add_argument("--min", type=int, default=1, help="must be a power of two. default: 1") grp_add.add_argument( "--max", type=int, default=pow(2, 30), help="must be a power of two. default: about half of the file") grp_add.add_argument( "--repeat", default="last", choices=["always", "last", "never"], help= "Whether to repeat a chunk size if chunks are removed. default: last") grp_add.add_argument( "--chunksize", type=int, default=None, help= "Shortcut for repeat=never, min=n, max=n. chunk size must be a power of two." ) grp_add.add_argument( "--chunkstart", type=int, default=0, help= "For the first round only, start n chars/lines into the file. Best for max to divide n. [Mostly intended for internal use]" ) grp_add.add_argument( "--repeatfirstround", default=False, action="store_true", help= "Treat the first round as if it removed chunks; possibly repeat it. [Mostly intended for internal use]" ) grp_add.add_argument( "--maxruntime", type=int, default=None, help= "If reduction takes more than n seconds, stop (and print instructions for continuing)." ) grp_ext = parser.add_argument_group( description="Condition, condition options and file-to-reduce") grp_ext.add_argument("extra_args", action="append", nargs=argparse.REMAINDER, help="condition [condition options] file-to-reduce") args = parser.parse_args() tempDir = args.tempdir atom = "char" if args.char else "line" strategy = args.strategy if args.chunksize: minimizeMin = args.chunksize minimizeMax = args.chunksize minimizeRepeat = "never" else: minimizeMin = args.min minimizeMax = args.max minimizeRepeat = args.repeat minimizeChunkStart = args.chunkstart minimizeRepeatFirstRound = args.repeatfirstround if args.maxruntime: stopAfterTime = time.time() + args.maxruntime extra_args = args.extra_args[0] if args.testcase: testcaseFilename = args.testcase elif len(extra_args) > 0: testcaseFilename = extra_args[ -1] # can be overridden by --testcase in processOptions else: print "No testcase specified (use --testcase or last condition arg)" return False testcaseExtension = os.path.splitext(testcaseFilename)[1] if not isPowerOfTwo(minimizeMin) or not isPowerOfTwo(minimizeMax): print "Min/Max must be powers of two." return False conditionScript = ximport.importRelativeOrAbsolute(extra_args[0]) conditionArgs = extra_args[1:] return True