def test_empty_with_pretty_out(self): stream = io.BytesIO() output = io.BytesIO() res = load.load(in_streams=[('subunit', stream)], pretty_out=True, stdout=output) self.assertEqual(1, res)
def run_tests(): run_proc = [('subunit', output.ReturnCodeToSubunit( subprocess.Popen(run_cmd, shell=True, stdout=subprocess.PIPE)))] return load.load((None, None), in_streams=run_proc, partial=args.partial, subunit_out=args.subunit, repo_type=args.repo_type, repo_url=args.repo_url)
def run_tests(): run_proc = [('subunit', output.ReturnCodeToSubunit( subprocess.Popen(run_cmd, shell=True, stdout=subprocess.PIPE)))] return load.load(in_streams=run_proc, subunit_out=subunit_out, repo_type=repo_type, repo_url=repo_url, run_id=combine_id, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments)
def run_tests(): run_procs = [('subunit', output.ReturnCodeToSubunit( proc)) for proc in cmd.run_tests()] partial = False if (failing or analyze_isolation or isolated): partial = True if not run_procs: print("The specified regex doesn't match with anything.") return 0 return load.load((None, None), in_streams=run_procs, partial=partial, subunit_out=subunit_out, repo_type=cmd.options.repo_type, repo_url=cmd.options.repo_url)
def run_tests(): run_procs = [('subunit', output.ReturnCodeToSubunit( proc)) for proc in cmd.run_tests()] if not run_procs: stdout.write("The specified regex doesn't match with anything") return 1 return load.load((None, None), in_streams=run_procs, subunit_out=subunit_out, repo_type=repo_type, repo_url=repo_url, run_id=combine_id, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments)
def run_command(config='.stestr.conf', repo_type='file', repo_url=None, test_path=None, top_dir=None, group_regex=None, failing=False, serial=False, concurrency=0, load_list=None, partial=False, subunit_out=False, until_failure=False, analyze_isolation=False, isolated=False, worker_path=None, blacklist_file=None, whitelist_file=None, black_regex=None, no_discover=False, random=False, combine=False, filters=None, pretty_out=True, color=False, stdout=sys.stdout, abbreviate=False, suppress_attachments=False, all_attachments=False, show_binary_attachments=True, pdb=False): """Function to execute the run command This function implements the run command. It will run the tests specified in the parameters based on the provided config file and/or arguments specified in the way specified by the arguments. The results will be printed to STDOUT and loaded into the repository. :param str config: The path to the stestr config file. Must be a string. :param str repo_type: This is the type of repository to use. Valid choices are 'file' and 'sql'. :param str repo_url: The url of the repository to use. :param str test_path: Set the test path to use for unittest discovery. If both this and the corresponding config file option are set, this value will be used. :param str top_dir: The top dir to use for unittest discovery. This takes precedence over the value in the config file. (if one is present in the config file) :param str group_regex: Set a group regex to use for grouping tests together in the stestr scheduler. If both this and the corresponding config file option are set this value will be used. :param bool failing: Run only tests known to be failing. :param bool serial: Run tests serially :param int concurrency: "How many processes to use. The default (0) autodetects your CPU count and uses that. :param str load_list: The path to a list of test_ids. If specified only tests listed in the named file will be run. :param bool partial: DEPRECATED: Only some tests will be run. Implied by `--failing`. This flag is deprecated because and doesn't do anything it will be removed in a future release. :param bool subunit_out: Display results in subunit format. :param bool until_failure: Repeat the run again and again until failure occurs. :param bool analyze_isolation: Search the last test run for 2-test test isolation interactions. :param bool isolated: Run each test id in a separate test runner. :param str worker_path: Optional path of a manual worker grouping file to use for the run. :param str blacklist_file: Path to a blacklist file, this file contains a separate regex exclude on each newline. :param str whitelist_file: Path to a whitelist file, this file contains a separate regex on each newline. :param str black_regex: Test rejection regex. If a test cases name matches on re.search() operation, it will be removed from the final test list. :param str no_discover: Takes in a single test_id to bypasses test discover and just execute the test specified. A file name may be used in place of a test name. :param bool random: Randomize the test order after they are partitioned into separate workers :param bool combine: Combine the results from the test run with the last run in the repository :param list filters: A list of string regex filters to initially apply on the test list. Tests that match any of the regexes will be used. (assuming any other filtering specified also uses it) :param bool pretty_out: Use the subunit-trace output filter :param bool color: Enable colorized output in subunit-trace :param file stdout: The file object to write all output to. By default this is sys.stdout :param bool abbreviate: Use abbreviated output if set true :param bool suppress_attachments: When set true attachments subunit_trace will not print attachments on successful test execution. :param bool all_attachments: When set true subunit_trace will print all text attachments on successful test execution. :param bool show_binary_attachments: When set to true, subunit_trace will print binary attachments in addition to text attachments. :param str pdb: Takes in a single test_id to bypasses test discover and just execute the test specified without launching any additional processes. A file name may be used in place of a test name. :return return_code: The exit code for the command. 0 for success and > 0 for failures. :rtype: int """ if partial: warnings.warn('The partial flag is deprecated and has no effect ' 'anymore') try: repo = util.get_repo_open(repo_type, repo_url) # If a repo is not found, and there a testr config exists just create it except repository.RepositoryNotFound: if not os.path.isfile(config) and not test_path: msg = ("No config file found and --test-path not specified. " "Either create or specify a .stestr.conf or use " "--test-path ") stdout.write(msg) exit(1) try: repo = util.get_repo_initialise(repo_type, repo_url) except OSError as e: if e.errno != errno.EEXIST: raise repo_path = repo_url or './stestr' stdout.write('The specified repository directory %s already ' 'exists. Please check if the repository already ' 'exists or select a different path\n' % repo_path) return 1 combine_id = None concurrency = _to_int(concurrency) if concurrency and concurrency < 0: msg = ("The provided concurrency value: %s is not valid. An integer " ">= 0 must be used.\n" % concurrency) stdout.write(msg) return 2 if combine: latest_id = repo.latest_id() combine_id = str(latest_id) if no_discover and pdb: msg = ("--no-discover and --pdb are mutually exclusive options, " "only specify one at a time") stdout.write(msg) return 2 if pdb and until_failure: msg = ("pdb mode does not function with the --until-failure flag, " "only specify one at a time") stdout.write(msg) return 2 if no_discover: ids = no_discover if '::' in ids: ids = ids.replace('::', '.') if ids.find('/') != -1: root = ids.replace('.py', '') ids = root.replace('/', '.') stestr_python = sys.executable if os.environ.get('PYTHON'): python_bin = os.environ.get('PYTHON') elif stestr_python: python_bin = stestr_python else: raise RuntimeError("The Python interpreter was not found and " "PYTHON is not set") run_cmd = python_bin + ' -m stestr.subunit_runner.run ' + ids def run_tests(): run_proc = [('subunit', output.ReturnCodeToSubunit( subprocess.Popen(run_cmd, shell=True, stdout=subprocess.PIPE)))] return load.load(in_streams=run_proc, subunit_out=subunit_out, repo_type=repo_type, repo_url=repo_url, run_id=combine_id, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments) if not until_failure: return run_tests() else: while True: result = run_tests() # If we're using subunit output we want to make sure to check # the result from the repository because load() returns 0 # always on subunit output if subunit: summary = testtools.StreamSummary() last_run = repo.get_latest_run().get_subunit_stream() stream = subunit.ByteStreamToStreamResult(last_run) summary.startTestRun() try: stream.run(summary) finally: summary.stopTestRun() if not results.wasSuccessful(summary): result = 1 if result: return result if pdb: ids = pdb if '::' in ids: ids = ids.replace('::', '.') if ids.find('/') != -1: root = ids.replace('.py', '') ids = root.replace('/', '.') runner = subunit_run.SubunitTestRunner stream = io.BytesIO() program.TestProgram(module=None, argv=['stestr', ids], testRunner=functools.partial(runner, stdout=stream)) stream.seek(0) run_proc = [('subunit', stream)] return load.load(in_streams=run_proc, subunit_out=subunit_out, repo_type=repo_type, repo_url=repo_url, run_id=combine_id, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments) if failing or analyze_isolation: ids = _find_failing(repo) else: ids = None if load_list: list_ids = set() # Should perhaps be text.. currently does its own decode. with open(load_list, 'rb') as list_file: list_ids = set(parse_list(list_file.read())) if ids is None: # Use the supplied list verbatim ids = list_ids else: # We have some already limited set of ids, just reduce to ids # that are both failing and listed. ids = list_ids.intersection(ids) conf = config_file.TestrConf(config) if not analyze_isolation: cmd = conf.get_run_command( ids, regexes=filters, group_regex=group_regex, repo_type=repo_type, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, blacklist_file=blacklist_file, whitelist_file=whitelist_file, black_regex=black_regex, top_dir=top_dir, test_path=test_path, randomize=random) if isolated: result = 0 cmd.setUp() try: ids = cmd.list_tests() finally: cmd.cleanUp() for test_id in ids: # TODO(mtreinish): add regex cmd = conf.get_run_command( [test_id], filters, group_regex=group_regex, repo_type=repo_type, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, blacklist_file=blacklist_file, whitelist_file=whitelist_file, black_regex=black_regex, randomize=random, test_path=test_path, top_dir=top_dir) run_result = _run_tests( cmd, until_failure, subunit_out=subunit_out, combine_id=combine_id, repo_type=repo_type, repo_url=repo_url, pretty_out=pretty_out, color=color, abbreviate=abbreviate, stdout=stdout, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments) if run_result > result: result = run_result return result else: return _run_tests(cmd, until_failure, subunit_out=subunit_out, combine_id=combine_id, repo_type=repo_type, repo_url=repo_url, pretty_out=pretty_out, color=color, stdout=stdout, abbreviate=abbreviate, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments) else: # Where do we source data about the cause of conflicts. latest_run = repo.get_latest_run() # Stage one: reduce the list of failing tests (possibly further # reduced by testfilters) to eliminate fails-on-own tests. spurious_failures = set() for test_id in ids: # TODO(mtrienish): Add regex cmd = conf.get_run_command( [test_id], group_regex=group_regex, repo_type=repo_type, repo_url=repo_url, serial=serial, worker_path=worker_path, concurrency=concurrency, blacklist_file=blacklist_file, whitelist_file=whitelist_file, black_regex=black_regex, randomize=random, test_path=test_path, top_dir=top_dir) if not _run_tests(cmd, until_failure): # If the test was filtered, it won't have been run. if test_id in repo.get_test_ids(repo.latest_id()): spurious_failures.add(test_id) # This is arguably ugly, why not just tell the system that # a pass here isn't a real pass? [so that when we find a # test that is spuriously failing, we don't forget # that it is actually failing. # Alternatively, perhaps this is a case for data mining: # when a test starts passing, keep a journal, and allow # digging back in time to see that it was a failure, # what it failed with etc... # The current solution is to just let it get marked as # a pass temporarily. if not spurious_failures: # All done. return 0 bisect_runner = bisect_tests.IsolationAnalyzer( latest_run, conf, _run_tests, repo, test_path=test_path, top_dir=top_dir, group_regex=group_regex, repo_type=repo_type, repo_url=repo_url, serial=serial, concurrency=concurrency) # spurious-failure -> cause. return bisect_runner.bisect_tests(spurious_failures)