def run(args): _args = args[0] ids = None filters = None if args[1]: filters = args[1] conf = config_file.TestrConf(_args.config) cmd = conf.get_run_command(_args, ids, filters) not_filtered = filters is None and _args.blacklist_file is None\ and _args.whitelist_file is None and _args.black_regex is None try: cmd.setUp() # List tests if the fixture has not already needed to to filter. if not_filtered: ids = cmd.list_tests() else: ids = cmd.test_ids stream = BytesIO() for id in ids: stream.write(('%s\n' % id).encode('utf8')) stream.seek(0) output.output_stream(stream) return 0 finally: cmd.cleanUp()
def list_command(config='.stestr.conf', repo_type='file', repo_url=None, test_path=None, top_dir=None, group_regex=None, blacklist_file=None, whitelist_file=None, black_regex=None, filters=None, stdout=sys.stdout): """Print a list of test_ids for a project This function will print the test_ids for tests in a project. You can filter the output just like with the run command to see exactly what will be run. :param str config: The path to the stestr config file. Must be a string. :param str repo_type: This is the type of repository to use. Valid choices are 'file' and 'sql'. :param str repo_url: The url of the repository to use. :param str test_path: Set the test path to use for unittest discovery. If both this and the corresponding config file option are set, this value will be used. :param str top_dir: The top dir to use for unittest discovery. This takes precedence over the value in the config file. (if one is present in the config file) :param str group_regex: Set a group regex to use for grouping tests together in the stestr scheduler. If both this and the corresponding config file option are set this value will be used. :param str blacklist_file: Path to a blacklist file, this file contains a separate regex exclude on each newline. :param str whitelist_file: Path to a whitelist file, this file contains a separate regex on each newline. :param str black_regex: Test rejection regex. If a test cases name matches on re.search() operation, it will be removed from the final test list. :param list filters: A list of string regex filters to initially apply on the test list. Tests that match any of the regexes will be used. (assuming any other filtering specified also uses it) :param file stdout: The output file to write all output to. By default this is sys.stdout """ ids = None conf = config_file.TestrConf(config) cmd = conf.get_run_command( regexes=filters, repo_type=repo_type, repo_url=repo_url, group_regex=group_regex, blacklist_file=blacklist_file, whitelist_file=whitelist_file, black_regex=black_regex, test_path=test_path, top_dir=top_dir) not_filtered = filters is None and blacklist_file is None\ and whitelist_file is None and black_regex is None try: cmd.setUp() # List tests if the fixture has not already needed to to filter. if not_filtered: ids = cmd.list_tests() else: ids = cmd.test_ids stream = BytesIO() for id in ids: stream.write(('%s\n' % id).encode('utf8')) stream.seek(0) output.output_stream(stream, output=stdout) return 0 finally: cmd.cleanUp()
def discover(self, **kwargs): """Iterate over test_ids for a project This method will print the test_ids for tests in a project. You can filter the output just like with the run command to see exactly what will be run. """ params = dict(config=self.config, repo_type=self.repo_type, repo_url=self.repo_url, test_path=self.test_path, top_dir=self.top_dir, group_regex=self.group_regex, blacklist_file=self.blacklist_file, whitelist_file=self.whitelist_file, black_regex=self.black_regex, filters=self.filters) if kwargs: params.update(kwargs) ids = None config = params.pop('config') conf = config_file.TestrConf(config) filters = params.pop('filters') blacklist_file = params.pop('blacklist_file') whitelist_file = params.pop('whitelist_file') black_regex = params.pop('black_regex') cmd = conf.get_run_command( regexes=filters, repo_type=params['repo_type'], repo_url=params['repo_url'], group_regex=params['group_regex'], blacklist_file=blacklist_file, whitelist_file=whitelist_file, black_regex=black_regex, test_path=params['test_path'], top_dir=params['top_dir']) not_filtered = filters is None and blacklist_file is None\ and whitelist_file is None and black_regex is None try: cmd.setUp() # List tests if the fixture has not already needed to to filter. if not_filtered: ids = cmd.list_tests() else: ids = cmd.test_ids except SystemExit: msg = ("Error discovering test cases IDs with parameters: " "{!r}").format(params) raise RuntimeError(msg) finally: cmd.cleanUp() return sorted(ids)
def run_command(config='.stestr.conf', repo_type='file', repo_url=None, test_path=None, top_dir=None, group_regex=None, failing=False, serial=False, concurrency=0, load_list=None, partial=False, subunit_out=False, until_failure=False, analyze_isolation=False, isolated=False, worker_path=None, blacklist_file=None, whitelist_file=None, black_regex=None, no_discover=False, random=False, combine=False, filters=None, pretty_out=True, color=False, stdout=sys.stdout, abbreviate=False, suppress_attachments=False): """Function to execute the run command This function implements the run command. It will run the tests specified in the parameters based on the provided config file and/or arguments specified in the way specified by the arguments. The results will be printed to STDOUT and loaded into the repository. :param str config: The path to the stestr config file. Must be a string. :param str repo_type: This is the type of repository to use. Valid choices are 'file' and 'sql'. :param str repo_url: The url of the repository to use. :param str test_path: Set the test path to use for unittest discovery. If both this and the corresponding config file option are set, this value will be used. :param str top_dir: The top dir to use for unittest discovery. This takes precedence over the value in the config file. (if one is present in the config file) :param str group_regex: Set a group regex to use for grouping tests together in the stestr scheduler. If both this and the corresponding config file option are set this value will be used. :param bool failing: Run only tests known to be failing. :param bool serial: Run tests serially :param int concurrency: "How many processes to use. The default (0) autodetects your CPU count and uses that. :param str load_list: The path to a list of test_ids. If specified only tests listed in the named file will be run. :param bool partial: DEPRECATED: Only some tests will be run. Implied by `--failing`. This flag is deprecated because and doesn't do anything it will be removed in a future release. :param bool subunit_out: Display results in subunit format. :param bool until_failure: Repeat the run again and again until failure occurs. :param bool analyze_isolation: Search the last test run for 2-test test isolation interactions. :param bool isolated: Run each test id in a separate test runner. :param str worker_path: Optional path of a manual worker grouping file to use for the run. :param str blacklist_file: Path to a blacklist file, this file contains a separate regex exclude on each newline. :param str whitelist_file: Path to a whitelist file, this file contains a separate regex on each newline. :param str black_regex: Test rejection regex. If a test cases name matches on re.search() operation, it will be removed from the final test list. :param str no_discover: Takes in a single test_id to bypasses test discover and just execute the test specified. A file name may be used in place of a test name. :param bool random: Randomize the test order after they are partitioned into separate workers :param bool combine: Combine the results from the test run with the last run in the repository :param list filters: A list of string regex filters to initially apply on the test list. Tests that match any of the regexes will be used. (assuming any other filtering specified also uses it) :param bool pretty_out: Use the subunit-trace output filter :param bool color: Enable colorized output in subunit-trace :param file stdout: The file object to write all output to. By default this is sys.stdout :param bool abbreviate: Use abbreviated output if set true :param bool suppress_attachments: When set true attachments subunit_trace will not print attachments on successful test execution. :return return_code: The exit code for the command. 0 for success and > 0 for failures. :rtype: int """ if partial: warnings.warn('The partial flag is deprecated and has no effect ' 'anymore') try: repo = util.get_repo_open(repo_type, repo_url) # If a repo is not found, and there a testr config exists just create it except repository.RepositoryNotFound: if not os.path.isfile(config) and not test_path: msg = ("No config file found and --test-path not specified. " "Either create or specify a .stestr.conf or use " "--test-path ") stdout.write(msg) exit(1) repo = util.get_repo_initialise(repo_type, repo_url) combine_id = None if combine: latest_id = repo.latest_id() combine_id = six.text_type(latest_id) if no_discover: ids = no_discover if '::' in ids: ids = ids.replace('::', '.') if ids.find('/') != -1: root = ids.replace('.py', '') ids = root.replace('/', '.') run_cmd = 'python -m subunit.run ' + ids def run_tests(): run_proc = [('subunit', output.ReturnCodeToSubunit( subprocess.Popen(run_cmd, shell=True, stdout=subprocess.PIPE)))] return load.load(in_streams=run_proc, subunit_out=subunit_out, repo_type=repo_type, repo_url=repo_url, run_id=combine_id, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments) if not until_failure: return run_tests() else: while True: result = run_tests() # If we're using subunit output we want to make sure to check # the result from the repository because load() returns 0 # always on subunit output if subunit: summary = testtools.StreamSummary() last_run = repo.get_latest_run().get_subunit_stream() stream = subunit.ByteStreamToStreamResult(last_run) summary.startTestRun() try: stream.run(summary) finally: summary.stopTestRun() if not results.wasSuccessful(summary): result = 1 if result: return result if failing or analyze_isolation: ids = _find_failing(repo) else: ids = None if load_list: list_ids = set() # Should perhaps be text.. currently does its own decode. with open(load_list, 'rb') as list_file: list_ids = set(parse_list(list_file.read())) if ids is None: # Use the supplied list verbatim ids = list_ids else: # We have some already limited set of ids, just reduce to ids # that are both failing and listed. ids = list_ids.intersection(ids) conf = config_file.TestrConf(config) if not analyze_isolation: cmd = conf.get_run_command(ids, regexes=filters, group_regex=group_regex, repo_type=repo_type, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, blacklist_file=blacklist_file, whitelist_file=whitelist_file, black_regex=black_regex, top_dir=top_dir, test_path=test_path, randomize=random) if isolated: result = 0 cmd.setUp() try: ids = cmd.list_tests() finally: cmd.cleanUp() for test_id in ids: # TODO(mtreinish): add regex cmd = conf.get_run_command([test_id], filters, group_regex=group_regex, repo_type=repo_type, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, blacklist_file=blacklist_file, whitelist_file=whitelist_file, black_regex=black_regex, randomize=random, test_path=test_path, top_dir=top_dir) run_result = _run_tests( cmd, until_failure, subunit_out=subunit_out, combine_id=combine_id, repo_type=repo_type, repo_url=repo_url, pretty_out=pretty_out, color=color, abbreviate=abbreviate, stdout=stdout, suppress_attachments=suppress_attachments) if run_result > result: result = run_result return result else: return _run_tests(cmd, until_failure, subunit_out=subunit_out, combine_id=combine_id, repo_type=repo_type, repo_url=repo_url, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments) else: # Where do we source data about the cause of conflicts. latest_run = repo.get_latest_run() # Stage one: reduce the list of failing tests (possibly further # reduced by testfilters) to eliminate fails-on-own tests. spurious_failures = set() for test_id in ids: # TODO(mtrienish): Add regex cmd = conf.get_run_command([test_id], group_regex=group_regex, repo_type=repo_type, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, blacklist_file=blacklist_file, whitelist_file=whitelist_file, black_regex=black_regex, randomize=random, test_path=test_path, top_dir=top_dir) if not _run_tests(cmd, until_failure): # If the test was filtered, it won't have been run. if test_id in repo.get_test_ids(repo.latest_id()): spurious_failures.add(test_id) # This is arguably ugly, why not just tell the system that # a pass here isn't a real pass? [so that when we find a # test that is spuriously failing, we don't forget # that it is actually failing. # Alternatively, perhaps this is a case for data mining: # when a test starts passing, keep a journal, and allow # digging back in time to see that it was a failure, # what it failed with etc... # The current solution is to just let it get marked as # a pass temporarily. if not spurious_failures: # All done. return 0 bisect_runner = bisect_tests.IsolationAnalyzer(latest_run, conf, _run_tests, repo, test_path=test_path, top_dir=top_dir, group_regex=group_regex, repo_type=repo_type, repo_url=repo_url, serial=serial, concurrency=concurrency) # spurious-failure -> cause. return bisect_runner.bisect_tests(spurious_failures)
def setUp(self, mock_ConfigParser): super(TestTestrConf, self).setUp() self._testr_conf = config_file.TestrConf(mock.sentinel.config_file) self._testr_conf.parser = mock.Mock()
def run(arguments): args = arguments[0] filters = arguments[1] or None try: repo = util.get_repo_open(args.repo_type, args.repo_url) # If a repo is not found, and there a testr config exists just create it except repository.RepositoryNotFound: if not os.path.isfile(args.config): raise repo = util.get_repo_initialise(args.repo_type, args.repo_url) if args.no_discover: ids = args.no_discover if ids.find('/') != -1: root, _ = os.path.splitext(ids) ids = root.replace('/', '.') run_cmd = 'python -m subunit.run ' + ids def run_tests(): run_proc = [('subunit', output.ReturnCodeToSubunit( subprocess.Popen(run_cmd, shell=True, stdout=subprocess.PIPE)))] return load.load((None, None), in_streams=run_proc, partial=args.partial, subunit_out=args.subunit, repo_type=args.repo_type, repo_url=args.repo_url) if not args.until_failure: return run_tests() else: result = run_tests() while not result: result = run_tests() return result if args.failing or args.analyze_isolation: ids = _find_failing(repo) else: ids = None if args.load_list: list_ids = set() # Should perhaps be text.. currently does its own decode. with open(args.load_list, 'rb') as list_file: list_ids = set(parse_list(list_file.read())) if ids is None: # Use the supplied list verbatim ids = list_ids else: # We have some already limited set of ids, just reduce to ids # that are both failing and listed. ids = list_ids.intersection(ids) conf = config_file.TestrConf(args.config) if not args.analyze_isolation: cmd = conf.get_run_command(args, ids, filters) if args.isolated: result = 0 cmd.setUp() try: ids = cmd.list_tests() finally: cmd.cleanUp() for test_id in ids: # TODO(mtreinish): add regex cmd = conf.get_run_command(args, [test_id], filters) run_result = _run_tests(cmd, args.failing, args.analyze_isolation, args.isolated, args.until_failure, subunit_out=args.subunit) if run_result > result: result = run_result return result else: return _run_tests(cmd, args.failing, args.analyze_isolation, args.isolated, args.until_failure, subunit_out=args.subunit) else: # Where do we source data about the cause of conflicts. # XXX: Should instead capture the run id in with the failing test # data so that we can deal with failures split across many partial # runs. latest_run = repo.get_latest_run() # Stage one: reduce the list of failing tests (possibly further # reduced by testfilters) to eliminate fails-on-own tests. spurious_failures = set() for test_id in ids: # TODO(mtrienish): Add regex cmd = conf.get_run_command(args, [test_id]) if not _run_tests(cmd): # If the test was filtered, it won't have been run. if test_id in repo.get_test_ids(repo.latest_id()): spurious_failures.add(test_id) # This is arguably ugly, why not just tell the system that # a pass here isn't a real pass? [so that when we find a # test that is spuriously failing, we don't forget # that it is actually failng. # Alternatively, perhaps this is a case for data mining: # when a test starts passing, keep a journal, and allow # digging back in time to see that it was a failure, # what it failed with etc... # The current solution is to just let it get marked as # a pass temporarily. if not spurious_failures: # All done. return 0 # spurious-failure -> cause. test_conflicts = {} for spurious_failure in spurious_failures: candidate_causes = _prior_tests( latest_run, spurious_failure) bottom = 0 top = len(candidate_causes) width = top - bottom while width: check_width = int(ceil(width / 2.0)) # TODO(mtreinish): Add regex cmd = conf.get_run_command( args, candidate_causes[bottom:bottom + check_width] + [spurious_failure]) _run_tests(cmd) # check that the test we're probing still failed - still # awkward. found_fail = [] def find_fail(test_dict): if test_dict['id'] == spurious_failure: found_fail.append(True) checker = testtools.StreamToDict(find_fail) checker.startTestRun() try: repo.get_failing().get_test().run(checker) finally: checker.stopTestRun() if found_fail: # Our conflict is in bottom - clamp the range down. top = bottom + check_width if width == 1: # found the cause test_conflicts[ spurious_failure] = candidate_causes[bottom] width = 0 else: width = top - bottom else: # Conflict in the range we did not run: discard bottom. bottom = bottom + check_width if width == 1: # there will be no more to check, so we didn't # reproduce the failure. width = 0 else: width = top - bottom if spurious_failure not in test_conflicts: # Could not determine cause test_conflicts[spurious_failure] = 'unknown - no conflicts' if test_conflicts: table = [('failing test', 'caused by test')] for failure, causes in test_conflicts.items(): table.append((failure, causes)) output.output_table(table) return 3 return 0
def run_command(config='.stestr.conf', repo_url=None, test_path=None, top_dir=None, group_regex=None, failing=False, serial=False, concurrency=0, load_list=None, subunit_out=False, until_failure=False, analyze_isolation=False, isolated=False, worker_path=None, exclude_list=None, include_list=None, exclude_regex=None, no_discover=False, random=False, combine=False, filters=None, pretty_out=True, color=False, stdout=sys.stdout, abbreviate=False, suppress_attachments=False, all_attachments=False, show_binary_attachments=True, pdb=False): """Function to execute the run command This function implements the run command. It will run the tests specified in the parameters based on the provided config file and/or arguments specified in the way specified by the arguments. The results will be printed to STDOUT and loaded into the repository. :param str config: The path to the stestr config file. Must be a string. :param str repo_url: The url of the repository to use. :param str test_path: Set the test path to use for unittest discovery. If both this and the corresponding config file option are set, this value will be used. :param str top_dir: The top dir to use for unittest discovery. This takes precedence over the value in the config file. (if one is present in the config file) :param str group_regex: Set a group regex to use for grouping tests together in the stestr scheduler. If both this and the corresponding config file option are set this value will be used. :param bool failing: Run only tests known to be failing. :param bool serial: Run tests serially :param int concurrency: "How many processes to use. The default (0) autodetects your CPU count and uses that. :param str load_list: The path to a list of test_ids. If specified only tests listed in the named file will be run. :param bool subunit_out: Display results in subunit format. :param bool until_failure: Repeat the run again and again until failure occurs. :param bool analyze_isolation: Search the last test run for 2-test test isolation interactions. :param bool isolated: Run each test id in a separate test runner. :param str worker_path: Optional path of a manual worker grouping file to use for the run. :param str exclude_list: Path to an exclusion list file, this file contains a separate regex exclude on each newline. :param str include_list: Path to a inclusion list file, this file contains a separate regex on each newline. :param str exclude_regex: Test rejection regex. If a test cases name matches on re.search() operation, it will be removed from the final test list. :param str no_discover: Takes in a single test_id to bypasses test discover and just execute the test specified. A file name may be used in place of a test name. :param bool random: Randomize the test order after they are partitioned into separate workers :param bool combine: Combine the results from the test run with the last run in the repository :param list filters: A list of string regex filters to initially apply on the test list. Tests that match any of the regexes will be used. (assuming any other filtering specified also uses it) :param bool pretty_out: Use the subunit-trace output filter :param bool color: Enable colorized output in subunit-trace :param file stdout: The file object to write all output to. By default this is sys.stdout :param bool abbreviate: Use abbreviated output if set true :param bool suppress_attachments: When set true attachments subunit_trace will not print attachments on successful test execution. :param bool all_attachments: When set true subunit_trace will print all text attachments on successful test execution. :param bool show_binary_attachments: When set to true, subunit_trace will print binary attachments in addition to text attachments. :param str pdb: Takes in a single test_id to bypasses test discover and just execute the test specified without launching any additional processes. A file name may be used in place of a test name. :return return_code: The exit code for the command. 0 for success and > 0 for failures. :rtype: int """ try: repo = util.get_repo_open(repo_url=repo_url) # If a repo is not found, and there a stestr config exists just create it except repository.RepositoryNotFound: if not os.path.isfile(config) and not test_path: # If there is no config and no test-path if os.path.isfile('tox.ini'): tox_conf = configparser.SafeConfigParser() tox_conf.read('tox.ini') if not tox_conf.has_section('stestr'): msg = ("No file found, --test-path not specified, and " "stestr section not found in tox.ini. Either " "create or specify a .stestr.conf, use " "--test-path, or add an stestr section to the " "tox.ini") stdout.write(msg) exit(1) else: msg = ("No config file found and --test-path not specified. " "Either create or specify a .stestr.conf or use " "--test-path ") stdout.write(msg) exit(1) try: repo = util.get_repo_initialise(repo_url=repo_url) except OSError as e: if e.errno != errno.EEXIST: raise repo_path = repo_url or './stestr' stdout.write('The specified repository directory %s already ' 'exists. Please check if the repository already ' 'exists or select a different path\n' % repo_path) return 1 combine_id = None concurrency = _to_int(concurrency) if concurrency and concurrency < 0: msg = ("The provided concurrency value: %s is not valid. An integer " ">= 0 must be used.\n" % concurrency) stdout.write(msg) return 2 if combine: latest_id = repo.latest_id() combine_id = str(latest_id) if no_discover and pdb: msg = ("--no-discover and --pdb are mutually exclusive options, " "only specify one at a time") stdout.write(msg) return 2 if pdb and until_failure: msg = ("pdb mode does not function with the --until-failure flag, " "only specify one at a time") stdout.write(msg) return 2 if no_discover: ids = no_discover if '::' in ids: ids = ids.replace('::', '.') if ids.find('/') != -1: root = ids.replace('.py', '') ids = root.replace('/', '.') stestr_python = sys.executable if os.environ.get('PYTHON'): python_bin = os.environ.get('PYTHON') elif stestr_python: python_bin = stestr_python else: raise RuntimeError("The Python interpreter was not found and " "PYTHON is not set") run_cmd = python_bin + ' -m stestr.subunit_runner.run ' + ids def run_tests(): run_proc = [('subunit', output.ReturnCodeToSubunit( subprocess.Popen(run_cmd, shell=True, stdout=subprocess.PIPE)))] return load.load(in_streams=run_proc, subunit_out=subunit_out, repo_url=repo_url, run_id=combine_id, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments) if not until_failure: return run_tests() else: while True: result = run_tests() # If we're using subunit output we want to make sure to check # the result from the repository because load() returns 0 # always on subunit output if subunit: summary = testtools.StreamSummary() last_run = repo.get_latest_run().get_subunit_stream() stream = subunit.ByteStreamToStreamResult(last_run) summary.startTestRun() try: stream.run(summary) finally: summary.stopTestRun() if not results.wasSuccessful(summary): result = 1 if result: return result if pdb: ids = pdb if '::' in ids: ids = ids.replace('::', '.') if ids.find('/') != -1: root = ids.replace('.py', '') ids = root.replace('/', '.') runner = subunit_run.SubunitTestRunner stream = io.BytesIO() program.TestProgram(module=None, argv=['stestr', ids], testRunner=functools.partial(runner, stdout=stream)) stream.seek(0) run_proc = [('subunit', stream)] return load.load(in_streams=run_proc, subunit_out=subunit_out, repo_url=repo_url, run_id=combine_id, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments) if failing or analyze_isolation: ids = _find_failing(repo) else: ids = None if load_list: list_ids = set() # Should perhaps be text.. currently does its own decode. with open(load_list, 'rb') as list_file: list_ids = set(parse_list(list_file.read())) if ids is None: # Use the supplied list verbatim ids = list_ids else: # We have some already limited set of ids, just reduce to ids # that are both failing and listed. ids = list_ids.intersection(ids) if config and os.path.isfile(config): conf = config_file.TestrConf(config) elif os.path.isfile('tox.ini'): conf = config_file.TestrConf('tox.ini', section='stestr') else: conf = config_file.TestrConf(config) if not analyze_isolation: cmd = conf.get_run_command(ids, regexes=filters, group_regex=group_regex, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, exclude_list=exclude_list, include_list=include_list, exclude_regex=exclude_regex, top_dir=top_dir, test_path=test_path, randomize=random) if isolated: result = 0 cmd.setUp() try: ids = cmd.list_tests() finally: cmd.cleanUp() for test_id in ids: # TODO(mtreinish): add regex cmd = conf.get_run_command([test_id], filters, group_regex=group_regex, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, exclude_list=exclude_list, include_list=include_list, exclude_regex=exclude_regex, randomize=random, test_path=test_path, top_dir=top_dir) run_result = _run_tests( cmd, until_failure, subunit_out=subunit_out, combine_id=combine_id, repo_url=repo_url, pretty_out=pretty_out, color=color, abbreviate=abbreviate, stdout=stdout, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments) if run_result > result: result = run_result return result else: return _run_tests(cmd, until_failure, subunit_out=subunit_out, combine_id=combine_id, repo_url=repo_url, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments) else: # Where do we source data about the cause of conflicts. latest_run = repo.get_latest_run() # Stage one: reduce the list of failing tests (possibly further # reduced by testfilters) to eliminate fails-on-own tests. spurious_failures = set() for test_id in ids: # TODO(mtrienish): Add regex cmd = conf.get_run_command([test_id], group_regex=group_regex, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, exclude_list=exclude_list, include_list=include_list, exclude_regex=exclude_regex, randomize=random, test_path=test_path, top_dir=top_dir) if not _run_tests(cmd, until_failure): # If the test was filtered, it won't have been run. if test_id in repo.get_test_ids(repo.latest_id()): spurious_failures.add(test_id) # This is arguably ugly, why not just tell the system that # a pass here isn't a real pass? [so that when we find a # test that is spuriously failing, we don't forget # that it is actually failing. # Alternatively, perhaps this is a case for data mining: # when a test starts passing, keep a journal, and allow # digging back in time to see that it was a failure, # what it failed with etc... # The current solution is to just let it get marked as # a pass temporarily. if not spurious_failures: # All done. return 0 bisect_runner = bisect_tests.IsolationAnalyzer(latest_run, conf, _run_tests, repo, test_path=test_path, top_dir=top_dir, group_regex=group_regex, repo_url=repo_url, serial=serial, concurrency=concurrency) # spurious-failure -> cause. return bisect_runner.bisect_tests(spurious_failures)