def test_shows_last_run_first_run(self): ui, cmd = self.get_test_ui_and_cmd() cmd.repository_factory = memory.RepositoryFactory() repo = cmd.repository_factory.initialise(ui.here) inserter = repo.get_inserter() inserter.startTestRun() inserter.status(test_id='failing', test_status='fail') inserter.status(test_id='ok', test_status='success') inserter.stopTestRun() id = inserter.get_id() self.assertEqual(1, cmd.execute()) # We should have seen test outputs (of the failure) and summary data. self.assertEqual([('results', Wildcard), ('summary', False, 2, None, Wildcard, Wildcard, [ ('id', id, None), ('failures', 1, None) ])], ui.outputs) suite = ui.outputs[0][1] result = testtools.StreamSummary() result.startTestRun() try: suite.run(result) finally: result.stopTestRun() self.assertEqual(1, len(result.errors)) self.assertEqual(2, result.testsRun)
def test_load_new_shows_test_failure_details(self): if v2_avail: buffer = BytesIO() stream = subunit.StreamResultToBytes(buffer) stream.status(test_id='foo', test_status='inprogress') stream.status(test_id='foo', test_status='fail', file_name="traceback", mime_type='text/plain;charset=utf8', file_bytes=b'arg\n') subunit_bytes = buffer.getvalue() else: subunit_bytes = b'test: foo\nfailure: foo [\narg\n]\n' ui = UI([('subunit', subunit_bytes)]) cmd = load.load(ui) ui.set_command(cmd) cmd.repository_factory = memory.RepositoryFactory() cmd.repository_factory.initialise(ui.here) self.assertEqual(1, cmd.execute()) suite = ui.outputs[0][1] self.assertEqual([('results', Wildcard), ('summary', False, 1, None, Wildcard, None, [ ('id', 0, None), ('failures', 1, None) ])], ui.outputs) result = testtools.StreamSummary() result.startTestRun() try: suite.run(result) finally: result.stopTestRun() self.assertEqual(1, result.testsRun) self.assertEqual(1, len(result.errors))
def _run_tests(cmd, failing, analyze_isolation, isolated, until_failure, subunit_out=False, combine_id=None, repo_type='file', repo_url=None, pretty_out=True, color=False, stdout=sys.stdout, abbreviate=False, suppress_attachments=False): """Run the tests cmd was parameterised with.""" cmd.setUp() try: def run_tests(): run_procs = [('subunit', output.ReturnCodeToSubunit(proc)) for proc in cmd.run_tests()] if not run_procs: stdout.write("The specified regex doesn't match with anything") return 1 return load.load((None, None), in_streams=run_procs, subunit_out=subunit_out, repo_type=repo_type, repo_url=repo_url, run_id=combine_id, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments) if not until_failure: return run_tests() else: while True: result = run_tests() # If we're using subunit output we want to make sure to check # the result from the repository because load() returns 0 # always on subunit output if subunit_out: repo = util.get_repo_open(repo_type, repo_url) summary = testtools.StreamSummary() last_run = repo.get_latest_run().get_subunit_stream() stream = subunit.ByteStreamToStreamResult(last_run) summary.startTestRun() try: stream.run(summary) finally: summary.stopTestRun() if not summary.wasSuccessful(): result = 1 if result: return result finally: cmd.cleanUp()
def _make_result(self, repo): testcommand = self.command_factory(self.ui, repo) if self.ui.options.list: list_result = testtools.StreamSummary() return list_result, list_result else: return self.ui.make_result(repo.latest_id, testcommand)
def __init__(self, stream_file, attachments=False, attr_regex=None, targets=None, use_wall_time=False, non_subunit_name=None): if targets is None: targets = [] else: targets = targets[:] self.use_wall_time = use_wall_time self.stream_file = stream_file self.stream = subunit.ByteStreamToStreamResult( self.stream_file, non_subunit_name=non_subunit_name) starts = testtools.StreamResult() summary = testtools.StreamSummary() outcomes = testtools.StreamToDict(functools.partial( self.parse_outcome)) targets.extend([starts, outcomes, summary]) self.result = testtools.CopyStreamResult(targets) self.results = {} self.attachments = attachments if attr_regex: self.attr_regex = re.compile(attr_regex) # NOTE(mtreinish): Default to the previous implicit regex if None is # specified for backwards compat else: self.attr_regex = re.compile('\[(.*)\]')
def _get_run_details(stream_file, stdout): stream = subunit.ByteStreamToStreamResult(stream_file, non_subunit_name='stdout') global start_times global stop_times start_times = [] stop_times = [] def collect_data(stream, test): global start_times global stop_times start_times.append(test['timestamps'][0]) stop_times.append(test['timestamps'][1]) outcomes = testtools.StreamToDict(functools.partial(collect_data, stdout)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result = testtools.StreamResultRouter(result) cat = subunit.test_results.CatFiles(stdout) result.add_rule(cat, 'test_id', test_id=None) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() successful = results.wasSuccessful(summary) if start_times and stop_times: start_time = min(start_times) stop_time = max(stop_times) run_time = subunit_trace.get_duration([start_time, stop_time]) else: run_time = '---' successful = '---' start_time = '---' return {'passed': successful, 'runtime': run_time, 'start': start_time}
def main(): args = parse_args() stream = subunit.ByteStreamToStreamResult(sys.stdin, non_subunit_name='stdout') outcomes = testtools.StreamToDict( functools.partial(show_outcome, sys.stdout, print_failures=args.print_failures, failonly=args.failonly)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result = testtools.StreamResultRouter(result) cat = subunit.test_results.CatFiles(sys.stdout) result.add_rule(cat, 'test_id', test_id=None) start_time = datetime.datetime.utcnow() result.startTestRun() try: stream.run(result) finally: result.stopTestRun() stop_time = datetime.datetime.utcnow() elapsed_time = stop_time - start_time if count_tests('status', '.*') == 0: print("The test run didn't actually run any tests") exit(1) if args.post_fails: print_fails(sys.stdout) print_summary(sys.stdout, elapsed_time) exit(0 if summary.wasSuccessful() else 1)
def make_result(get_id, output=sys.stdout): serializer = subunit.StreamResultToBytes(output) # By pass user transforms - just forward it all, result = serializer # and interpret everything as success. summary = testtools.StreamSummary() summary.startTestRun() summary.stopTestRun() return result, summary
def trace(stdin, stdout, print_failures=False, failonly=False, enable_diff=False, abbreviate=False, color=False, post_fails=False, no_summary=False, suppress_attachments=False, all_attachments=False, show_binary_attachments=False): stream = subunit.ByteStreamToStreamResult(stdin, non_subunit_name='stdout') outcomes = testtools.StreamToDict( functools.partial(show_outcome, stdout, print_failures=print_failures, failonly=failonly, enable_diff=enable_diff, abbreviate=abbreviate, enable_color=color, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result = testtools.StreamResultRouter(result) cat = subunit.test_results.CatFiles(stdout) result.add_rule(cat, 'test_id', test_id=None) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() start_times = [] stop_times = [] for worker in RESULTS: start_times += [x['timestamps'][0] for x in RESULTS[worker]] stop_times += [x['timestamps'][1] for x in RESULTS[worker]] start_time = min(start_times) stop_time = max(stop_times) elapsed_time = stop_time - start_time if count_tests('status', '.*') == 0: print("The test run didn't actually run any tests", file=sys.stderr) return 1 if post_fails: print_fails(stdout) if not no_summary: print_summary(stdout, elapsed_time) # NOTE(mtreinish): Ideally this should live in testtools streamSummary # this is just in place until the behavior lands there (if it ever does) if count_tests('status', '^success$') == 0: print("\nNo tests were successful during the run", file=sys.stderr) return 1 return 0 if results.wasSuccessful(summary) else 1
def get_failing(self, repo): """Analyze a failing stream from repo and return it.""" run = repo.get_failing() analyzer = testtools.StreamSummary() analyzer.startTestRun() try: run.get_test().run(analyzer) finally: analyzer.stopTestRun() return analyzer
def get_last_run(self, repo): """Return the results from a stream.""" run = repo.get_test_run(repo.latest_id()) analyzer = testtools.StreamSummary() analyzer.startTestRun() try: run.get_test().run(analyzer) finally: analyzer.stopTestRun() return analyzer
def _make_result(repo, list_tests=False): if list_tests: list_result = testtools.StreamSummary() return list_result, list_result else: def _get_id(): return repo.get_latest_run().get_id() output_result = results.CLITestResult(_get_id, sys.stdout, None) summary_result = output_result.get_summary() return output_result, summary_result
def main(): stream = subunit.ByteStreamToStreamResult(sys.stdin, non_subunit_name='stdout') starts = Starts(sys.stdout) outcomes = testtools.StreamToDict( functools.partial(show_outcome, sys.stdout)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([starts, outcomes, summary]) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() print_summary(sys.stdout) return (0 if summary.wasSuccessful() else 1)
def _check_subunit(self, output_stream): stream = subunit_lib.ByteStreamToStreamResult(output_stream) starts = testtools.StreamResult() summary = testtools.StreamSummary() tests = [] def _add_dict(test): tests.append(test) outcomes = testtools.StreamToDict(functools.partial(_add_dict)) result = testtools.CopyStreamResult([starts, outcomes, summary]) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() self.assertThat(len(tests), testtools.matchers.GreaterThan(0))
def trace(stdin, stdout, print_failures=False, failonly=False, enable_diff=False, abbreviate=False, color=False, post_fails=False, no_summary=False): stream = subunit.ByteStreamToStreamResult(stdin, non_subunit_name='stdout') outcomes = testtools.StreamToDict( functools.partial(show_outcome, stdout, print_failures=print_failures, failonly=failonly, enable_diff=enable_diff, abbreviate=abbreviate, enable_color=color)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result = testtools.StreamResultRouter(result) cat = subunit.test_results.CatFiles(stdout) result.add_rule(cat, 'test_id', test_id=None) start_time = datetime.datetime.utcnow() result.startTestRun() try: stream.run(result) finally: result.stopTestRun() stop_time = datetime.datetime.utcnow() elapsed_time = stop_time - start_time if count_tests('status', '.*') == 0: print("The test run didn't actually run any tests") return 1 if post_fails: print_fails(stdout) if not no_summary: print_summary(stdout, elapsed_time) # NOTE(mtreinish): Ideally this should live in testtools streamSummary # this is just in place until the behavior lands there (if it ever does) if count_tests('status', '^success$') == 0: print("\nNo tests were successful during the run") return 1 return 0 if summary.wasSuccessful() else 1
def test_get_test_from_test_run(self): repo = self.repo_impl.initialise(self.sample_url) result = repo.get_inserter() legacy_result = testtools.ExtendedToStreamDecorator(result) legacy_result.startTestRun() make_test('testrepository.tests.test_repository.Case.method', True).run(legacy_result) legacy_result.stopTestRun() inserted = result.get_id() run = repo.get_test_run(inserted) test = run.get_test() result = testtools.StreamSummary() result.startTestRun() try: test.run(result) finally: result.stopTestRun() self.assertEqual(1, result.testsRun)
def assertRunExit(self, cmd, expected, subunit=False, stdin=None): if stdin: p = subprocess.Popen("%s" % cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate(stdin) else: p = subprocess.Popen("%s" % cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if not subunit: self.assertEqual(p.returncode, expected, "Stdout: %s; Stderr: %s" % (out, err)) return (out, err) else: self.assertEqual( p.returncode, expected, "Expected return code: %s doesn't match actual " "return code of: %s" % (expected, p.returncode)) output_stream = io.BytesIO(out) stream = subunit_lib.ByteStreamToStreamResult(output_stream) starts = testtools.StreamResult() summary = testtools.StreamSummary() tests = [] def _add_dict(test): tests.append(test) outcomes = testtools.StreamToDict(functools.partial(_add_dict)) result = testtools.CopyStreamResult([starts, outcomes, summary]) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() self.assertThat(len(tests), testtools.matchers.GreaterThan(0)) return (out, err)
def make_result(self, get_id, test_command, previous_run=None): if getattr(self.options, 'subunit', False): serializer = subunit.StreamResultToBytes(self._stdout) # By pass user transforms - just forward it all, result = serializer # and interpret everything as success. summary = testtools.StreamSummary() summary.startTestRun() summary.stopTestRun() return result, summary else: # Apply user defined transforms. filter_tags = test_command.get_filter_tags() output = CLITestResult(self, get_id, self._stdout, previous_run, filter_tags=filter_tags) summary = output._summary return output, summary
def main(): args = parse_args() stream = subunit.ByteStreamToStreamResult(sys.stdin, non_subunit_name='stdout') outcomes = testtools.StreamToDict( functools.partial(show_outcome, sys.stdout, print_failures=args.print_failures)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() if count_tests('status', '.*') == 0: print("The test run didn't actually run any tests") return 1 if args.post_fails: print_fails(sys.stdout) print_summary(sys.stdout) return (0 if summary.wasSuccessful() else 1)
def main(): parser = optparse.OptionParser(description=__doc__) parser.add_option( "--times", action="store_true", help="list the time each test took (requires a timestamped stream)", default=False) parser.add_option( "--exists", action="store_true", help="list tests that are reported as existing (as well as ran)", default=False) parser.add_option("--no-passthrough", action="store_true", help="Hide all non subunit input.", default=False, dest="no_passthrough") (options, args) = parser.parse_args() test = pysubunit.ByteStreamToStreamResult(filters.find_stream( sys.stdin, args), non_subunit_name="stdout") result = test_results.TestIdPrintingResult(sys.stdout, options.times, options.exists) if not options.no_passthrough: result = testtools.StreamResultRouter(result) cat = test_results.CatFiles(sys.stdout) result.add_rule(cat, 'test_id', test_id=None) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([result, summary]) result.startTestRun() test.run(result) result.stopTestRun() if summary.wasSuccessful(): exit_code = 0 else: exit_code = 1 sys.exit(exit_code)
def _load_case(inserter, repo, case, subunit_out, pretty_out, color, stdout, abbreviate, suppress_attachments, all_attachments): if subunit_out: output_result, summary_result = output.make_result(inserter.get_id, output=stdout) elif pretty_out: outcomes = testtools.StreamToDict( functools.partial(subunit_trace.show_outcome, stdout, enable_color=color, abbreviate=abbreviate, suppress_attachments=suppress_attachments, all_attachments=all_attachments)) summary_result = testtools.StreamSummary() output_result = testtools.CopyStreamResult([outcomes, summary_result]) output_result = testtools.StreamResultRouter(output_result) cat = subunit.test_results.CatFiles(stdout) output_result.add_rule(cat, 'test_id', test_id=None) else: try: previous_run = repo.get_latest_run() except KeyError: previous_run = None output_result = results.CLITestResult(inserter.get_id, stdout, previous_run) summary_result = output_result.get_summary() result = testtools.CopyStreamResult([inserter, output_result]) result.startTestRun() try:
def run_command(config='.stestr.conf', repo_type='file', repo_url=None, test_path=None, top_dir=None, group_regex=None, failing=False, serial=False, concurrency=0, load_list=None, partial=False, subunit_out=False, until_failure=False, analyze_isolation=False, isolated=False, worker_path=None, blacklist_file=None, whitelist_file=None, black_regex=None, no_discover=False, random=False, combine=False, filters=None, pretty_out=True, color=False, stdout=sys.stdout, abbreviate=False, suppress_attachments=False): """Function to execute the run command This function implements the run command. It will run the tests specified in the parameters based on the provided config file and/or arguments specified in the way specified by the arguments. The results will be printed to STDOUT and loaded into the repository. :param str config: The path to the stestr config file. Must be a string. :param str repo_type: This is the type of repository to use. Valid choices are 'file' and 'sql'. :param str repo_url: The url of the repository to use. :param str test_path: Set the test path to use for unittest discovery. If both this and the corresponding config file option are set, this value will be used. :param str top_dir: The top dir to use for unittest discovery. This takes precedence over the value in the config file. (if one is present in the config file) :param str group_regex: Set a group regex to use for grouping tests together in the stestr scheduler. If both this and the corresponding config file option are set this value will be used. :param bool failing: Run only tests known to be failing. :param bool serial: Run tests serially :param int concurrency: "How many processes to use. The default (0) autodetects your CPU count and uses that. :param str load_list: The path to a list of test_ids. If specified only tests listed in the named file will be run. :param bool partial: DEPRECATED: Only some tests will be run. Implied by `--failing`. This flag is deprecated because and doesn't do anything it will be removed in a future release. :param bool subunit_out: Display results in subunit format. :param bool until_failure: Repeat the run again and again until failure occurs. :param bool analyze_isolation: Search the last test run for 2-test test isolation interactions. :param bool isolated: Run each test id in a separate test runner. :param str worker_path: Optional path of a manual worker grouping file to use for the run. :param str blacklist_file: Path to a blacklist file, this file contains a separate regex exclude on each newline. :param str whitelist_file: Path to a whitelist file, this file contains a separate regex on each newline. :param str black_regex: Test rejection regex. If a test cases name matches on re.search() operation, it will be removed from the final test list. :param str no_discover: Takes in a single test_id to bypasses test discover and just execute the test specified. A file name may be used in place of a test name. :param bool random: Randomize the test order after they are partitioned into separate workers :param bool combine: Combine the results from the test run with the last run in the repository :param list filters: A list of string regex filters to initially apply on the test list. Tests that match any of the regexes will be used. (assuming any other filtering specified also uses it) :param bool pretty_out: Use the subunit-trace output filter :param bool color: Enable colorized output in subunit-trace :param file stdout: The file object to write all output to. By default this is sys.stdout :param bool abbreviate: Use abbreviated output if set true :param bool suppress_attachments: When set true attachments subunit_trace will not print attachments on successful test execution. :return return_code: The exit code for the command. 0 for success and > 0 for failures. :rtype: int """ if partial: warnings.warn('The partial flag is deprecated and has no effect ' 'anymore') try: repo = util.get_repo_open(repo_type, repo_url) # If a repo is not found, and there a testr config exists just create it except repository.RepositoryNotFound: if not os.path.isfile(config) and not test_path: msg = ("No config file found and --test-path not specified. " "Either create or specify a .stestr.conf or use " "--test-path ") stdout.write(msg) exit(1) repo = util.get_repo_initialise(repo_type, repo_url) combine_id = None if combine: latest_id = repo.latest_id() combine_id = six.text_type(latest_id) if no_discover: ids = no_discover if '::' in ids: ids = ids.replace('::', '.') if ids.find('/') != -1: root = ids.replace('.py', '') ids = root.replace('/', '.') run_cmd = 'python -m subunit.run ' + ids def run_tests(): run_proc = [('subunit', output.ReturnCodeToSubunit( subprocess.Popen(run_cmd, shell=True, stdout=subprocess.PIPE)))] return load.load(in_streams=run_proc, subunit_out=subunit_out, repo_type=repo_type, repo_url=repo_url, run_id=combine_id, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments) if not until_failure: return run_tests() else: while True: result = run_tests() # If we're using subunit output we want to make sure to check # the result from the repository because load() returns 0 # always on subunit output if subunit: summary = testtools.StreamSummary() last_run = repo.get_latest_run().get_subunit_stream() stream = subunit.ByteStreamToStreamResult(last_run) summary.startTestRun() try: stream.run(summary) finally: summary.stopTestRun() if not results.wasSuccessful(summary): result = 1 if result: return result if failing or analyze_isolation: ids = _find_failing(repo) else: ids = None if load_list: list_ids = set() # Should perhaps be text.. currently does its own decode. with open(load_list, 'rb') as list_file: list_ids = set(parse_list(list_file.read())) if ids is None: # Use the supplied list verbatim ids = list_ids else: # We have some already limited set of ids, just reduce to ids # that are both failing and listed. ids = list_ids.intersection(ids) conf = config_file.TestrConf(config) if not analyze_isolation: cmd = conf.get_run_command(ids, regexes=filters, group_regex=group_regex, repo_type=repo_type, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, blacklist_file=blacklist_file, whitelist_file=whitelist_file, black_regex=black_regex, top_dir=top_dir, test_path=test_path, randomize=random) if isolated: result = 0 cmd.setUp() try: ids = cmd.list_tests() finally: cmd.cleanUp() for test_id in ids: # TODO(mtreinish): add regex cmd = conf.get_run_command([test_id], filters, group_regex=group_regex, repo_type=repo_type, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, blacklist_file=blacklist_file, whitelist_file=whitelist_file, black_regex=black_regex, randomize=random, test_path=test_path, top_dir=top_dir) run_result = _run_tests( cmd, until_failure, subunit_out=subunit_out, combine_id=combine_id, repo_type=repo_type, repo_url=repo_url, pretty_out=pretty_out, color=color, abbreviate=abbreviate, stdout=stdout, suppress_attachments=suppress_attachments) if run_result > result: result = run_result return result else: return _run_tests(cmd, until_failure, subunit_out=subunit_out, combine_id=combine_id, repo_type=repo_type, repo_url=repo_url, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments) else: # Where do we source data about the cause of conflicts. latest_run = repo.get_latest_run() # Stage one: reduce the list of failing tests (possibly further # reduced by testfilters) to eliminate fails-on-own tests. spurious_failures = set() for test_id in ids: # TODO(mtrienish): Add regex cmd = conf.get_run_command([test_id], group_regex=group_regex, repo_type=repo_type, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, blacklist_file=blacklist_file, whitelist_file=whitelist_file, black_regex=black_regex, randomize=random, test_path=test_path, top_dir=top_dir) if not _run_tests(cmd, until_failure): # If the test was filtered, it won't have been run. if test_id in repo.get_test_ids(repo.latest_id()): spurious_failures.add(test_id) # This is arguably ugly, why not just tell the system that # a pass here isn't a real pass? [so that when we find a # test that is spuriously failing, we don't forget # that it is actually failing. # Alternatively, perhaps this is a case for data mining: # when a test starts passing, keep a journal, and allow # digging back in time to see that it was a failure, # what it failed with etc... # The current solution is to just let it get marked as # a pass temporarily. if not spurious_failures: # All done. return 0 bisect_runner = bisect_tests.IsolationAnalyzer(latest_run, conf, _run_tests, repo, test_path=test_path, top_dir=top_dir, group_regex=group_regex, repo_type=repo_type, repo_url=repo_url, serial=serial, concurrency=concurrency) # spurious-failure -> cause. return bisect_runner.bisect_tests(spurious_failures)
if args.list: data = file("test/testcases.js").read() for test in re.compile("(?<=').+(?=')").findall(data): print test[:-5] sys.exit(-1) if args.load_list: tests = list( set( x.split(':')[0].strip() + '.html' for x in args.load_list.readlines())) else: tests = [] # Collect summary of all the individual test runs summary = testtools.StreamSummary() # Output information to stdout if not args.subunit: # Human readable test output pertest = testtools.StreamToExtendedDecorator( testtools.MultiTestResult( # Individual test progress unittest.TextTestResult( unittest.runner._WritelnDecorator(sys.stdout), False, 2), # End of run, summary of failures. testtools.TextTestResult(sys.stdout), )) else: from subunit.v2 import StreamResultToBytes pertest = StreamResultToBytes(sys.stdout)
def run_command(config='.stestr.conf', repo_url=None, test_path=None, top_dir=None, group_regex=None, failing=False, serial=False, concurrency=0, load_list=None, subunit_out=False, until_failure=False, analyze_isolation=False, isolated=False, worker_path=None, exclude_list=None, include_list=None, exclude_regex=None, no_discover=False, random=False, combine=False, filters=None, pretty_out=True, color=False, stdout=sys.stdout, abbreviate=False, suppress_attachments=False, all_attachments=False, show_binary_attachments=True, pdb=False): """Function to execute the run command This function implements the run command. It will run the tests specified in the parameters based on the provided config file and/or arguments specified in the way specified by the arguments. The results will be printed to STDOUT and loaded into the repository. :param str config: The path to the stestr config file. Must be a string. :param str repo_url: The url of the repository to use. :param str test_path: Set the test path to use for unittest discovery. If both this and the corresponding config file option are set, this value will be used. :param str top_dir: The top dir to use for unittest discovery. This takes precedence over the value in the config file. (if one is present in the config file) :param str group_regex: Set a group regex to use for grouping tests together in the stestr scheduler. If both this and the corresponding config file option are set this value will be used. :param bool failing: Run only tests known to be failing. :param bool serial: Run tests serially :param int concurrency: "How many processes to use. The default (0) autodetects your CPU count and uses that. :param str load_list: The path to a list of test_ids. If specified only tests listed in the named file will be run. :param bool subunit_out: Display results in subunit format. :param bool until_failure: Repeat the run again and again until failure occurs. :param bool analyze_isolation: Search the last test run for 2-test test isolation interactions. :param bool isolated: Run each test id in a separate test runner. :param str worker_path: Optional path of a manual worker grouping file to use for the run. :param str exclude_list: Path to an exclusion list file, this file contains a separate regex exclude on each newline. :param str include_list: Path to a inclusion list file, this file contains a separate regex on each newline. :param str exclude_regex: Test rejection regex. If a test cases name matches on re.search() operation, it will be removed from the final test list. :param str no_discover: Takes in a single test_id to bypasses test discover and just execute the test specified. A file name may be used in place of a test name. :param bool random: Randomize the test order after they are partitioned into separate workers :param bool combine: Combine the results from the test run with the last run in the repository :param list filters: A list of string regex filters to initially apply on the test list. Tests that match any of the regexes will be used. (assuming any other filtering specified also uses it) :param bool pretty_out: Use the subunit-trace output filter :param bool color: Enable colorized output in subunit-trace :param file stdout: The file object to write all output to. By default this is sys.stdout :param bool abbreviate: Use abbreviated output if set true :param bool suppress_attachments: When set true attachments subunit_trace will not print attachments on successful test execution. :param bool all_attachments: When set true subunit_trace will print all text attachments on successful test execution. :param bool show_binary_attachments: When set to true, subunit_trace will print binary attachments in addition to text attachments. :param str pdb: Takes in a single test_id to bypasses test discover and just execute the test specified without launching any additional processes. A file name may be used in place of a test name. :return return_code: The exit code for the command. 0 for success and > 0 for failures. :rtype: int """ try: repo = util.get_repo_open(repo_url=repo_url) # If a repo is not found, and there a stestr config exists just create it except repository.RepositoryNotFound: if not os.path.isfile(config) and not test_path: # If there is no config and no test-path if os.path.isfile('tox.ini'): tox_conf = configparser.SafeConfigParser() tox_conf.read('tox.ini') if not tox_conf.has_section('stestr'): msg = ("No file found, --test-path not specified, and " "stestr section not found in tox.ini. Either " "create or specify a .stestr.conf, use " "--test-path, or add an stestr section to the " "tox.ini") stdout.write(msg) exit(1) else: msg = ("No config file found and --test-path not specified. " "Either create or specify a .stestr.conf or use " "--test-path ") stdout.write(msg) exit(1) try: repo = util.get_repo_initialise(repo_url=repo_url) except OSError as e: if e.errno != errno.EEXIST: raise repo_path = repo_url or './stestr' stdout.write('The specified repository directory %s already ' 'exists. Please check if the repository already ' 'exists or select a different path\n' % repo_path) return 1 combine_id = None concurrency = _to_int(concurrency) if concurrency and concurrency < 0: msg = ("The provided concurrency value: %s is not valid. An integer " ">= 0 must be used.\n" % concurrency) stdout.write(msg) return 2 if combine: latest_id = repo.latest_id() combine_id = str(latest_id) if no_discover and pdb: msg = ("--no-discover and --pdb are mutually exclusive options, " "only specify one at a time") stdout.write(msg) return 2 if pdb and until_failure: msg = ("pdb mode does not function with the --until-failure flag, " "only specify one at a time") stdout.write(msg) return 2 if no_discover: ids = no_discover if '::' in ids: ids = ids.replace('::', '.') if ids.find('/') != -1: root = ids.replace('.py', '') ids = root.replace('/', '.') stestr_python = sys.executable if os.environ.get('PYTHON'): python_bin = os.environ.get('PYTHON') elif stestr_python: python_bin = stestr_python else: raise RuntimeError("The Python interpreter was not found and " "PYTHON is not set") run_cmd = python_bin + ' -m stestr.subunit_runner.run ' + ids def run_tests(): run_proc = [('subunit', output.ReturnCodeToSubunit( subprocess.Popen(run_cmd, shell=True, stdout=subprocess.PIPE)))] return load.load(in_streams=run_proc, subunit_out=subunit_out, repo_url=repo_url, run_id=combine_id, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments) if not until_failure: return run_tests() else: while True: result = run_tests() # If we're using subunit output we want to make sure to check # the result from the repository because load() returns 0 # always on subunit output if subunit: summary = testtools.StreamSummary() last_run = repo.get_latest_run().get_subunit_stream() stream = subunit.ByteStreamToStreamResult(last_run) summary.startTestRun() try: stream.run(summary) finally: summary.stopTestRun() if not results.wasSuccessful(summary): result = 1 if result: return result if pdb: ids = pdb if '::' in ids: ids = ids.replace('::', '.') if ids.find('/') != -1: root = ids.replace('.py', '') ids = root.replace('/', '.') runner = subunit_run.SubunitTestRunner stream = io.BytesIO() program.TestProgram(module=None, argv=['stestr', ids], testRunner=functools.partial(runner, stdout=stream)) stream.seek(0) run_proc = [('subunit', stream)] return load.load(in_streams=run_proc, subunit_out=subunit_out, repo_url=repo_url, run_id=combine_id, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments) if failing or analyze_isolation: ids = _find_failing(repo) else: ids = None if load_list: list_ids = set() # Should perhaps be text.. currently does its own decode. with open(load_list, 'rb') as list_file: list_ids = set(parse_list(list_file.read())) if ids is None: # Use the supplied list verbatim ids = list_ids else: # We have some already limited set of ids, just reduce to ids # that are both failing and listed. ids = list_ids.intersection(ids) if config and os.path.isfile(config): conf = config_file.TestrConf(config) elif os.path.isfile('tox.ini'): conf = config_file.TestrConf('tox.ini', section='stestr') else: conf = config_file.TestrConf(config) if not analyze_isolation: cmd = conf.get_run_command(ids, regexes=filters, group_regex=group_regex, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, exclude_list=exclude_list, include_list=include_list, exclude_regex=exclude_regex, top_dir=top_dir, test_path=test_path, randomize=random) if isolated: result = 0 cmd.setUp() try: ids = cmd.list_tests() finally: cmd.cleanUp() for test_id in ids: # TODO(mtreinish): add regex cmd = conf.get_run_command([test_id], filters, group_regex=group_regex, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, exclude_list=exclude_list, include_list=include_list, exclude_regex=exclude_regex, randomize=random, test_path=test_path, top_dir=top_dir) run_result = _run_tests( cmd, until_failure, subunit_out=subunit_out, combine_id=combine_id, repo_url=repo_url, pretty_out=pretty_out, color=color, abbreviate=abbreviate, stdout=stdout, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments) if run_result > result: result = run_result return result else: return _run_tests(cmd, until_failure, subunit_out=subunit_out, combine_id=combine_id, repo_url=repo_url, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments) else: # Where do we source data about the cause of conflicts. latest_run = repo.get_latest_run() # Stage one: reduce the list of failing tests (possibly further # reduced by testfilters) to eliminate fails-on-own tests. spurious_failures = set() for test_id in ids: # TODO(mtrienish): Add regex cmd = conf.get_run_command([test_id], group_regex=group_regex, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, exclude_list=exclude_list, include_list=include_list, exclude_regex=exclude_regex, randomize=random, test_path=test_path, top_dir=top_dir) if not _run_tests(cmd, until_failure): # If the test was filtered, it won't have been run. if test_id in repo.get_test_ids(repo.latest_id()): spurious_failures.add(test_id) # This is arguably ugly, why not just tell the system that # a pass here isn't a real pass? [so that when we find a # test that is spuriously failing, we don't forget # that it is actually failing. # Alternatively, perhaps this is a case for data mining: # when a test starts passing, keep a journal, and allow # digging back in time to see that it was a failure, # what it failed with etc... # The current solution is to just let it get marked as # a pass temporarily. if not spurious_failures: # All done. return 0 bisect_runner = bisect_tests.IsolationAnalyzer(latest_run, conf, _run_tests, repo, test_path=test_path, top_dir=top_dir, group_regex=group_regex, repo_url=repo_url, serial=serial, concurrency=concurrency) # spurious-failure -> cause. return bisect_runner.bisect_tests(spurious_failures)