def main(): # We can't use sys.path[0] to determine the script directory # because it doesn't work under a debugger test_directory = os.path.dirname(os.path.realpath(__file__)) parser = OptionParser(usage="""\ Run lldb test suite using a separate process for each test file. Each test will run with a time limit of 10 minutes by default. Override the default time limit of 10 minutes by setting the environment variable LLDB_TEST_TIMEOUT. E.g., export LLDB_TEST_TIMEOUT=10m Override the time limit for individual tests by setting the environment variable LLDB_[TEST NAME]_TIMEOUT. E.g., export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=2m Set to "0" to run without time limit. E.g., export LLDB_TEST_TIMEOUT=0 or export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0 """) parser.add_option( '-o', '--options', type='string', action='store', dest='dotest_options', help="""The options passed to 'dotest.py' if specified.""") parser.add_option( '-s', '--output-on-success', action='store_true', dest='output_on_success', default=False, help="""Print full output of 'dotest.py' even when it succeeds.""") parser.add_option( '-t', '--threads', type='int', dest='num_threads', help="""The number of threads to use when running tests separately.""") opts, args = parser.parse_args() dotest_option_string = opts.dotest_options is_posix = (os.name == "posix") dotest_argv = (shlex.split(dotest_option_string, posix=is_posix) if dotest_option_string else []) parser = dotest_args.create_parser() global dotest_options global output_on_success output_on_success = opts.output_on_success dotest_options = dotest_args.parse_args(parser, dotest_argv) if not dotest_options.s: # no session log directory, we need to add this to prevent # every dotest invocation from creating its own directory import datetime # The windows platforms don't like ':' in the pathname. timestamp_started = datetime.datetime.now().strftime("%F-%H_%M_%S") dotest_argv.append('-s') dotest_argv.append(timestamp_started) dotest_options.s = timestamp_started session_dir = os.path.join(os.getcwd(), dotest_options.s) # The root directory was specified on the command line if len(args) == 0: test_subdir = test_directory else: test_subdir = os.path.join(test_directory, args[0]) # clean core files in test tree from previous runs (Linux) cores = find('core.*', test_subdir) for core in cores: os.unlink(core) if opts.num_threads: num_threads = opts.num_threads else: num_threads_str = os.environ.get("LLDB_TEST_THREADS") if num_threads_str: num_threads = int(num_threads_str) else: num_threads = multiprocessing.cpu_count() if num_threads < 1: num_threads = 1 system_info = " ".join(platform.uname()) (timed_out, passed, failed, unexpected_successes, pass_count, fail_count) = walk_and_invoke(test_directory, test_subdir, dotest_argv, num_threads) timed_out = set(timed_out) num_test_files = len(passed) + len(failed) num_test_cases = pass_count + fail_count # move core files into session dir cores = find('core.*', test_subdir) for core in cores: dst = core.replace(test_directory, "")[1:] dst = dst.replace(os.path.sep, "-") os.rename(core, os.path.join(session_dir, dst)) # remove expected timeouts from failures expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name) for xtime in expected_timeout: if xtime in timed_out: timed_out.remove(xtime) failed.remove(xtime) result = "ExpectedTimeout" elif xtime in passed: result = "UnexpectedCompletion" else: result = None # failed if result: test_name = os.path.splitext(xtime)[0] touch(os.path.join(session_dir, "{}-{}".format(result, test_name))) print sys.stdout.write("Ran %d test suites" % num_test_files) if num_test_files > 0: sys.stdout.write(" (%d failed) (%f%%)" % (len(failed), 100.0 * len(failed) / num_test_files)) print sys.stdout.write("Ran %d test cases" % num_test_cases) if num_test_cases > 0: sys.stdout.write(" (%d failed) (%f%%)" % (fail_count, 100.0 * fail_count / num_test_cases)) print exit_code = 0 if len(failed) > 0: failed.sort() print "Failing Tests (%d)" % len(failed) for f in failed: print "%s: LLDB (suite) :: %s (%s)" % ("TIMEOUT" if f in timed_out else "FAIL", f, system_info) exit_code = 1 if len(unexpected_successes) > 0: unexpected_successes.sort() print "\nUnexpected Successes (%d)" % len(unexpected_successes) for u in unexpected_successes: print "UNEXPECTED SUCCESS: LLDB (suite) :: %s (%s)" % (u, system_info) sys.exit(exit_code)
def main(print_details_on_success, num_threads, test_subdir): """Run dotest.py in inferior mode in parallel. @param print_details_on_success the parsed value of the output-on-success command line argument. When True, details of a successful dotest inferior are printed even when everything succeeds. The normal behavior is to not print any details when all the inferior tests pass. @param num_threads the parsed value of the num-threads command line argument. @param test_subdir optionally specifies a subdir to limit testing within. May be None if the entire test tree is to be used. This subdir is assumed to be relative to the lldb/test root of the test hierarchy. """ dotest_argv = sys.argv[1:] global output_on_success output_on_success = print_details_on_success # We can't use sys.path[0] to determine the script directory # because it doesn't work under a debugger test_directory = os.path.dirname(os.path.realpath(__file__)) parser = OptionParser(usage="""\ Run lldb test suite using a separate process for each test file. Each test will run with a time limit of 10 minutes by default. Override the default time limit of 10 minutes by setting the environment variable LLDB_TEST_TIMEOUT. E.g., export LLDB_TEST_TIMEOUT=10m Override the time limit for individual tests by setting the environment variable LLDB_[TEST NAME]_TIMEOUT. E.g., export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=2m Set to "0" to run without time limit. E.g., export LLDB_TEST_TIMEOUT=0 or export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0 """) parser = dotest_args.create_parser() global dotest_options dotest_options = dotest_args.parse_args(parser, dotest_argv) if not dotest_options.s: # no session log directory, we need to add this to prevent # every dotest invocation from creating its own directory import datetime # The windows platforms don't like ':' in the pathname. timestamp_started = datetime.datetime.now().strftime("%F-%H_%M_%S") dotest_argv.append('-s') dotest_argv.append(timestamp_started) dotest_options.s = timestamp_started session_dir = os.path.join(os.getcwd(), dotest_options.s) # The root directory was specified on the command line if test_subdir and len(test_subdir) > 0: test_subdir = os.path.join(test_directory, test_subdir) else: test_subdir = test_directory # clean core files in test tree from previous runs (Linux) cores = find('core.*', test_subdir) for core in cores: os.unlink(core) if not num_threads: num_threads_str = os.environ.get("LLDB_TEST_THREADS") if num_threads_str: num_threads = int(num_threads_str) else: num_threads = multiprocessing.cpu_count() if num_threads < 1: num_threads = 1 system_info = " ".join(platform.uname()) (timed_out, passed, failed, unexpected_successes, pass_count, fail_count) = walk_and_invoke( test_directory, test_subdir, dotest_argv, num_threads) timed_out = set(timed_out) num_test_files = len(passed) + len(failed) num_test_cases = pass_count + fail_count # move core files into session dir cores = find('core.*', test_subdir) for core in cores: dst = core.replace(test_directory, "")[1:] dst = dst.replace(os.path.sep, "-") os.rename(core, os.path.join(session_dir, dst)) # remove expected timeouts from failures expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name) for xtime in expected_timeout: if xtime in timed_out: timed_out.remove(xtime) failed.remove(xtime) result = "ExpectedTimeout" elif xtime in passed: result = "UnexpectedCompletion" else: result = None # failed if result: test_name = os.path.splitext(xtime)[0] touch(os.path.join(session_dir, "{}-{}".format(result, test_name))) print sys.stdout.write("Ran %d test suites" % num_test_files) if num_test_files > 0: sys.stdout.write(" (%d failed) (%f%%)" % ( len(failed), 100.0 * len(failed) / num_test_files)) print sys.stdout.write("Ran %d test cases" % num_test_cases) if num_test_cases > 0: sys.stdout.write(" (%d failed) (%f%%)" % ( fail_count, 100.0 * fail_count / num_test_cases)) print exit_code = 0 if len(failed) > 0: failed.sort() print "Failing Tests (%d)" % len(failed) for f in failed: print "%s: LLDB (suite) :: %s (%s)" % ( "TIMEOUT" if f in timed_out else "FAIL", f, system_info ) exit_code = 1 if len(unexpected_successes) > 0: unexpected_successes.sort() print "\nUnexpected Successes (%d)" % len(unexpected_successes) for u in unexpected_successes: print "UNEXPECTED SUCCESS: LLDB (suite) :: %s (%s)" % (u, system_info) sys.exit(exit_code)
def main(print_details_on_success, num_threads, test_subdir, test_runner_name, results_formatter): """Run dotest.py in inferior mode in parallel. @param print_details_on_success the parsed value of the output-on-success command line argument. When True, details of a successful dotest inferior are printed even when everything succeeds. The normal behavior is to not print any details when all the inferior tests pass. @param num_threads the parsed value of the num-threads command line argument. @param test_subdir optionally specifies a subdir to limit testing within. May be None if the entire test tree is to be used. This subdir is assumed to be relative to the lldb/test root of the test hierarchy. @param test_runner_name if specified, contains the test runner name which selects the strategy used to run the isolated and optionally concurrent test runner. Specify None to allow the system to choose the most appropriate test runner given desired thread count and OS type. @param results_formatter if specified, provides the TestResultsFormatter instance that will format and output test result data from the side-channel test results. When specified, inferior dotest calls will send test results side-channel data over a socket to the parallel test runner, which will forward them on to results_formatter. """ # Do not shut down on sighup. if hasattr(signal, 'SIGHUP'): signal.signal(signal.SIGHUP, signal.SIG_IGN) dotest_argv = sys.argv[1:] global output_on_success, RESULTS_FORMATTER output_on_success = print_details_on_success RESULTS_FORMATTER = results_formatter # We can't use sys.path[0] to determine the script directory # because it doesn't work under a debugger parser = dotest_args.create_parser() global dotest_options dotest_options = dotest_args.parse_args(parser, dotest_argv) adjust_inferior_options(dotest_argv) session_dir = os.path.join(os.getcwd(), dotest_options.s) # The root directory was specified on the command line test_directory = os.path.dirname(os.path.realpath(__file__)) if test_subdir and len(test_subdir) > 0: test_subdir = os.path.join(test_directory, test_subdir) else: test_subdir = test_directory # clean core files in test tree from previous runs (Linux) cores = find('core.*', test_subdir) for core in cores: os.unlink(core) system_info = " ".join(platform.uname()) # Figure out which testrunner strategy we'll use. runner_strategies_by_name = get_test_runner_strategies(num_threads) # If the user didn't specify a test runner strategy, determine # the default now based on number of threads and OS type. if not test_runner_name: test_runner_name = default_test_runner_name(num_threads) if test_runner_name not in runner_strategies_by_name: raise Exception( "specified testrunner name '{}' unknown. Valid choices: {}".format( test_runner_name, runner_strategies_by_name.keys())) test_runner_func = runner_strategies_by_name[test_runner_name] summary_results = walk_and_invoke( test_directory, test_subdir, dotest_argv, num_threads, test_runner_func) (timed_out, passed, failed, unexpected_successes, pass_count, fail_count) = summary_results # The results formatter - if present - is done now. Tell it to # terminate. if results_formatter is not None: results_formatter.send_terminate_as_needed() timed_out = set(timed_out) num_test_files = len(passed) + len(failed) num_test_cases = pass_count + fail_count # move core files into session dir cores = find('core.*', test_subdir) for core in cores: dst = core.replace(test_directory, "")[1:] dst = dst.replace(os.path.sep, "-") os.rename(core, os.path.join(session_dir, dst)) # remove expected timeouts from failures expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name) for xtime in expected_timeout: if xtime in timed_out: timed_out.remove(xtime) failed.remove(xtime) result = "ExpectedTimeout" elif xtime in passed: result = "UnexpectedCompletion" else: result = None # failed if result: test_name = os.path.splitext(xtime)[0] touch(os.path.join(session_dir, "{}-{}".format(result, test_name))) print sys.stdout.write("Ran %d test suites" % num_test_files) if num_test_files > 0: sys.stdout.write(" (%d failed) (%f%%)" % ( len(failed), 100.0 * len(failed) / num_test_files)) print sys.stdout.write("Ran %d test cases" % num_test_cases) if num_test_cases > 0: sys.stdout.write(" (%d failed) (%f%%)" % ( fail_count, 100.0 * fail_count / num_test_cases)) print exit_code = 0 if len(failed) > 0: failed.sort() print "Failing Tests (%d)" % len(failed) for f in failed: print "%s: LLDB (suite) :: %s (%s)" % ( "TIMEOUT" if f in timed_out else "FAIL", f, system_info ) exit_code = 1 if len(unexpected_successes) > 0: unexpected_successes.sort() print "\nUnexpected Successes (%d)" % len(unexpected_successes) for u in unexpected_successes: print "UNEXPECTED SUCCESS: LLDB (suite) :: %s (%s)" % (u, system_info) sys.exit(exit_code)
def main(): # We can't use sys.path[0] to determine the script directory # because it doesn't work under a debugger test_directory = os.path.dirname(os.path.realpath(__file__)) parser = OptionParser(usage="""\ Run lldb test suite using a separate process for each test file. Each test will run with a time limit of 10 minutes by default. Override the default time limit of 10 minutes by setting the environment variable LLDB_TEST_TIMEOUT. E.g., export LLDB_TEST_TIMEOUT=10m Override the time limit for individual tests by setting the environment variable LLDB_[TEST NAME]_TIMEOUT. E.g., export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=2m Set to "0" to run without time limit. E.g., export LLDB_TEST_TIMEOUT=0 or export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0 """) parser.add_option('-o', '--options', type='string', action='store', dest='dotest_options', help="""The options passed to 'dotest.py' if specified.""") parser.add_option('-t', '--threads', type='int', dest='num_threads', help="""The number of threads to use when running tests separately.""") opts, args = parser.parse_args() dotest_option_string = opts.dotest_options is_posix = (os.name == "posix") dotest_argv = shlex.split(dotest_option_string, posix=is_posix) if dotest_option_string else [] parser = dotest_args.create_parser() dotest_options = dotest_args.parse_args(parser, dotest_argv) if not dotest_options.s: # no session log directory, we need to add this to prevent # every dotest invocation from creating its own directory import datetime # The windows platforms don't like ':' in the pathname. timestamp_started = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S") dotest_argv.append('-s') dotest_argv.append(timestamp_started) dotest_options.s = timestamp_started session_dir = os.path.join(os.getcwd(), dotest_options.s) # The root directory was specified on the command line if len(args) == 0: test_subdir = test_directory else: test_subdir = os.path.join(test_directory, args[0]) # clean core files in test tree from previous runs (Linux) cores = find('core.*', test_subdir) for core in cores: os.unlink(core) if opts.num_threads: num_threads = opts.num_threads else: num_threads_str = os.environ.get("LLDB_TEST_THREADS") if num_threads_str: num_threads = int(num_threads_str) else: num_threads = multiprocessing.cpu_count() if num_threads < 1: num_threads = 1 global default_timeout default_timeout = getDefaultTimeout(dotest_options.lldb_platform_name) system_info = " ".join(platform.uname()) (timed_out, failed, passed, all_fails, all_passes) = walk_and_invoke(test_directory, test_subdir, dotest_argv, num_threads) timed_out = set(timed_out) num_test_files = len(failed) + len(passed) num_tests = all_fails + all_passes # move core files into session dir cores = find('core.*', test_subdir) for core in cores: dst = core.replace(test_directory, "")[1:] dst = dst.replace(os.path.sep, "-") os.rename(core, os.path.join(session_dir, dst)) # remove expected timeouts from failures expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name) for xtime in expected_timeout: if xtime in timed_out: timed_out.remove(xtime) failed.remove(xtime) result = "ExpectedTimeout" elif xtime in passed: result = "UnexpectedCompletion" else: result = None # failed if result: test_name = os.path.splitext(xtime)[0] touch(os.path.join(session_dir, "{}-{}".format(result, test_name))) print print "Ran %d test suites (%d failed) (%f%%)" % (num_test_files, len(failed), (100.0 * len(failed) / num_test_files) if num_test_files > 0 else float('NaN')) print "Ran %d test cases (%d failed) (%f%%)" % (num_tests, all_fails, (100.0 * all_fails / num_tests) if num_tests > 0 else float('NaN')) if len(failed) > 0: failed.sort() print "Failing Tests (%d)" % len(failed) for f in failed: print "%s: LLDB (suite) :: %s (%s)" % ( "TIMEOUT" if f in timed_out else "FAIL", f, system_info ) sys.exit(1) sys.exit(0)