def __init__(self, stream_file, attachments=False, attr_regex=None, targets=None, use_wall_time=False, non_subunit_name=None): if targets is None: targets = [] else: targets = targets[:] self.use_wall_time = use_wall_time self.stream_file = stream_file self.stream = subunit.ByteStreamToStreamResult( self.stream_file, non_subunit_name=non_subunit_name) starts = testtools.StreamResult() summary = testtools.StreamSummary() outcomes = testtools.StreamToDict(functools.partial( self.parse_outcome)) targets.extend([starts, outcomes, summary]) self.result = testtools.CopyStreamResult(targets) self.results = {} self.attachments = attachments if attr_regex: self.attr_regex = re.compile(attr_regex) # NOTE(mtreinish): Default to the previous implicit regex if None is # specified for backwards compat else: self.attr_regex = re.compile('\[(.*)\]')
def _prior_tests(self, run, failing_id): """Calculate what tests from the test run run ran before test_id. Tests that ran in a different worker are not included in the result. """ if not getattr(self, '_worker_to_test', False): case = run.get_test() # Use None if there is no worker-N tag # If there are multiple, map them all. # (worker-N -> [testid, ...]) worker_to_test = {} # (testid -> [workerN, ...]) test_to_worker = {} def map_test(test_dict): tags = test_dict['tags'] id = test_dict['id'] workers = [] for tag in tags: if tag.startswith('worker-'): workers.append(tag) if not workers: workers = [None] for worker in workers: worker_to_test.setdefault(worker, []).append(id) test_to_worker.setdefault(id, []).extend(workers) mapper = testtools.StreamToDict(map_test) mapper.startTestRun() try: case.run(mapper)
def startTestRun(self): self._subunit = BytesIO() serialiser = subunit.v2.StreamResultToBytes(self._subunit) self._hook = testtools.CopyStreamResult([ testtools.StreamToDict(self._handle_test), serialiser]) self._hook.startTestRun()
def _get_run_details(stream_file, stdout): stream = subunit.ByteStreamToStreamResult(stream_file, non_subunit_name='stdout') global start_times global stop_times start_times = [] stop_times = [] def collect_data(stream, test): global start_times global stop_times start_times.append(test['timestamps'][0]) stop_times.append(test['timestamps'][1]) outcomes = testtools.StreamToDict(functools.partial(collect_data, stdout)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result = testtools.StreamResultRouter(result) cat = subunit.test_results.CatFiles(stdout) result.add_rule(cat, 'test_id', test_id=None) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() successful = results.wasSuccessful(summary) if start_times and stop_times: start_time = min(start_times) stop_time = max(stop_times) run_time = subunit_trace.get_duration([start_time, stop_time]) else: run_time = '---' successful = '---' start_time = '---' return {'passed': successful, 'runtime': run_time, 'start': start_time}
def main(): args = parse_args() stream = subunit.ByteStreamToStreamResult(sys.stdin, non_subunit_name='stdout') outcomes = testtools.StreamToDict( functools.partial(show_outcome, sys.stdout, print_failures=args.print_failures, failonly=args.failonly)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result = testtools.StreamResultRouter(result) cat = subunit.test_results.CatFiles(sys.stdout) result.add_rule(cat, 'test_id', test_id=None) start_time = datetime.datetime.utcnow() result.startTestRun() try: stream.run(result) finally: result.stopTestRun() stop_time = datetime.datetime.utcnow() elapsed_time = stop_time - start_time if count_tests('status', '.*') == 0: print("The test run didn't actually run any tests") exit(1) if args.post_fails: print_fails(sys.stdout) print_summary(sys.stdout, elapsed_time) exit(0 if summary.wasSuccessful() else 1)
def trace(stdin, stdout, print_failures=False, failonly=False, enable_diff=False, abbreviate=False, color=False, post_fails=False, no_summary=False, suppress_attachments=False, all_attachments=False, show_binary_attachments=False): stream = subunit.ByteStreamToStreamResult(stdin, non_subunit_name='stdout') outcomes = testtools.StreamToDict( functools.partial(show_outcome, stdout, print_failures=print_failures, failonly=failonly, enable_diff=enable_diff, abbreviate=abbreviate, enable_color=color, suppress_attachments=suppress_attachments, all_attachments=all_attachments, show_binary_attachments=show_binary_attachments)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result = testtools.StreamResultRouter(result) cat = subunit.test_results.CatFiles(stdout) result.add_rule(cat, 'test_id', test_id=None) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() start_times = [] stop_times = [] for worker in RESULTS: start_times += [x['timestamps'][0] for x in RESULTS[worker]] stop_times += [x['timestamps'][1] for x in RESULTS[worker]] start_time = min(start_times) stop_time = max(stop_times) elapsed_time = stop_time - start_time if count_tests('status', '.*') == 0: print("The test run didn't actually run any tests", file=sys.stderr) return 1 if post_fails: print_fails(stdout) if not no_summary: print_summary(stdout, elapsed_time) # NOTE(mtreinish): Ideally this should live in testtools streamSummary # this is just in place until the behavior lands there (if it ever does) if count_tests('status', '^success$') == 0: print("\nNo tests were successful during the run", file=sys.stderr) return 1 return 0 if results.wasSuccessful(summary) else 1
def _find_failing(self, repo): run = repo.get_failing() case = run.get_test() ids = [] def gather_errors(test_dict): if test_dict['status'] == 'fail': ids.append(test_dict['id']) result = testtools.StreamToDict(gather_errors) result.startTestRun() try: case.run(result)
def trace(): global OSTREAM case = subunit.ByteStreamToStreamResult(sys.stdin, non_subunit_name='stdout') result = testtools.StreamToDict(handle_test) # Call handle_test # when test completes with open('/tmp/the-trace.json', 'w') as f: OSTREAM = f OSTREAM.write("[\n") result.startTestRun() case.run(result) result.stopTestRun() OSTREAM.write("]\n")
def main(): stream = subunit.ByteStreamToStreamResult(sys.stdin, non_subunit_name='stdout') starts = Starts(sys.stdout) outcomes = testtools.StreamToDict( functools.partial(show_outcome, sys.stdout)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([starts, outcomes, summary]) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() print_summary(sys.stdout) return (0 if summary.wasSuccessful() else 1)
def startTestRun(self): self._subunit = io.BytesIO() self.subunit_stream = subunit.v2.StreamResultToBytes(self._subunit) self.hook = testtools.CopyStreamResult( [testtools.StreamToDict(self._handle_test), self.subunit_stream]) self.hook.startTestRun() self.start_time = datetime.datetime.utcnow() session = self.session_factory() if not self._run_id: self.run = db_api.create_run(session=session) self._run_id = self.run.uuid else: int_id = db_api.get_run_id_from_uuid(self._run_id, session=session) self.run = db_api.get_run_by_id(int_id, session=session) session.close() self.totals = {}
def _check_subunit(self, output_stream): stream = subunit_lib.ByteStreamToStreamResult(output_stream) starts = testtools.StreamResult() summary = testtools.StreamSummary() tests = [] def _add_dict(test): tests.append(test) outcomes = testtools.StreamToDict(functools.partial(_add_dict)) result = testtools.CopyStreamResult([starts, outcomes, summary]) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() self.assertThat(len(tests), testtools.matchers.GreaterThan(0))
def trace(stdin, stdout, print_failures=False, failonly=False, enable_diff=False, abbreviate=False, color=False, post_fails=False, no_summary=False): stream = subunit.ByteStreamToStreamResult(stdin, non_subunit_name='stdout') outcomes = testtools.StreamToDict( functools.partial(show_outcome, stdout, print_failures=print_failures, failonly=failonly, enable_diff=enable_diff, abbreviate=abbreviate, enable_color=color)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result = testtools.StreamResultRouter(result) cat = subunit.test_results.CatFiles(stdout) result.add_rule(cat, 'test_id', test_id=None) start_time = datetime.datetime.utcnow() result.startTestRun() try: stream.run(result) finally: result.stopTestRun() stop_time = datetime.datetime.utcnow() elapsed_time = stop_time - start_time if count_tests('status', '.*') == 0: print("The test run didn't actually run any tests") return 1 if post_fails: print_fails(stdout) if not no_summary: print_summary(stdout, elapsed_time) # NOTE(mtreinish): Ideally this should live in testtools streamSummary # this is just in place until the behavior lands there (if it ever does) if count_tests('status', '^success$') == 0: print("\nNo tests were successful during the run") return 1 return 0 if summary.wasSuccessful() else 1
def __init__(self, repository, partial=False): # XXX: Perhaps should factor into a decorator and use an unaltered # TestProtocolClient. self._repository = repository fd, name = tempfile.mkstemp(dir=self._repository.base) self.fname = name stream = os.fdopen(fd, 'wb') self.partial = partial # The time take by each test, flushed at the end. self._times = {} self._test_start = None self._time = None subunit_client = testtools.StreamToExtendedDecorator( TestProtocolClient(stream)) self.hook = testtools.CopyStreamResult([ subunit_client, testtools.StreamToDict(self._handle_test)]) self._stream = stream
def assertRunExit(self, cmd, expected, subunit=False, stdin=None): if stdin: p = subprocess.Popen("%s" % cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate(stdin) else: p = subprocess.Popen("%s" % cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if not subunit: self.assertEqual(p.returncode, expected, "Stdout: %s; Stderr: %s" % (out, err)) return (out, err) else: self.assertEqual( p.returncode, expected, "Expected return code: %s doesn't match actual " "return code of: %s" % (expected, p.returncode)) output_stream = io.BytesIO(out) stream = subunit_lib.ByteStreamToStreamResult(output_stream) starts = testtools.StreamResult() summary = testtools.StreamSummary() tests = [] def _add_dict(test): tests.append(test) outcomes = testtools.StreamToDict(functools.partial(_add_dict)) result = testtools.CopyStreamResult([starts, outcomes, summary]) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() self.assertThat(len(tests), testtools.matchers.GreaterThan(0)) return (out, err)
def main(): args = parse_args() stream = subunit.ByteStreamToStreamResult(sys.stdin, non_subunit_name='stdout') outcomes = testtools.StreamToDict( functools.partial(show_outcome, sys.stdout, print_failures=args.print_failures)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() if count_tests('status', '.*') == 0: print("The test run didn't actually run any tests") return 1 if args.post_fails: print_fails(sys.stdout) print_summary(sys.stdout) return (0 if summary.wasSuccessful() else 1)
for line in res: line = line.encode('utf8') stream.write("%s\n" % line) stream.write('\n\n') ADDPROP_FAIL.append(test) break else: FAILS.append(test) elif status == 'success' or status == 'xfail': SUCCESS.append(test) elif status == 'skip': SKIPS.append(test) stream = subunit.ByteStreamToStreamResult(sys.stdin, non_subunit_name='stdout') outcome = testtools.StreamToDict(functools.partial(show_outcome, sys.stdout)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcome, summary]) result.startTestRun() try: stream.run(result) finally: result.stopTestRun() print("\n\n------------------------------------------------------------------") print("%s Tests Failed" % len(FAILS)) print("%s Tests Failed with AdditionalProperties" % len(ADDPROP_FAIL)) print("%s Tests Skipped" % len(SKIPS)) print("%s Tests Passed" % len(SUCCESS)) print("To see the full details run this subunit stream through subunit-trace")
color, stdout, abbreviate, suppress_attachments, all_attachments) return retval def _load_case(inserter, repo, case, subunit_out, pretty_out, color, stdout, abbreviate, suppress_attachments, all_attachments): if subunit_out: output_result, summary_result = output.make_result(inserter.get_id, output=stdout) elif pretty_out: outcomes = testtools.StreamToDict( functools.partial(subunit_trace.show_outcome, stdout, enable_color=color, abbreviate=abbreviate, suppress_attachments=suppress_attachments, all_attachments=all_attachments)) summary_result = testtools.StreamSummary() output_result = testtools.CopyStreamResult([outcomes, summary_result]) output_result = testtools.StreamResultRouter(output_result) cat = subunit.test_results.CatFiles(stdout) output_result.add_rule(cat, 'test_id', test_id=None) else: try: previous_run = repo.get_latest_run() except KeyError: previous_run = None output_result = results.CLITestResult(inserter.get_id, stdout, previous_run)
decorate = functools.partial(mktagger, pos) case = testtools.DecorateTestCaseResult(case, decorate) yield (case, str(pos)) case = testtools.ConcurrentStreamTestSuite(make_tests) if not run_id: inserter = repo.get_inserter() else: inserter = repo.get_inserter(run_id=run_id) if subunit_out: output_result, summary_result = output.make_result(inserter.get_id, output=stdout) elif pretty_out: outcomes = testtools.StreamToDict( functools.partial(subunit_trace.show_outcome, stdout, enable_color=color, abbreviate=abbreviate)) summary_result = testtools.StreamSummary() output_result = testtools.CopyStreamResult([outcomes, summary_result]) output_result = testtools.StreamResultRouter(output_result) cat = subunit.test_results.CatFiles(stdout) output_result.add_rule(cat, 'test_id', test_id=None) else: try: previous_run = repo.get_latest_run() except KeyError: previous_run = None output_result = results.CLITestResult(inserter.get_id, stdout, previous_run) summary_result = output_result.get_summary()
def bisect_tests(self, spurious_failures): test_conflicts = {} if not spurious_failures: raise ValueError('No failures provided to bisect the cause of') for spurious_failure in spurious_failures: candidate_causes = self._prior_tests(self.latest_run, spurious_failure) bottom = 0 top = len(candidate_causes) width = top - bottom while width: check_width = int(math.ceil(width / 2.0)) test_ids = candidate_causes[bottom:bottom + check_width] + [spurious_failure] cmd = self.conf.get_run_command(test_ids, group_regex=self.group_regex, repo_type=self.repo_type, repo_url=self.repo_url, serial=self.serial, concurrency=self.concurrency, test_path=self.test_path, top_dir=self.top_dir) self.run_func(cmd, False, True, False, False, pretty_out=False, repo_type=self.repo_type, repo_url=self.repo_url) # check that the test we're probing still failed - still # awkward. found_fail = [] def find_fail(test_dict): if test_dict['id'] == spurious_failure: found_fail.append(True) checker = testtools.StreamToDict(find_fail) checker.startTestRun() try: self.repo.get_failing().get_test().run(checker) finally: checker.stopTestRun() if found_fail: # Our conflict is in bottom - clamp the range down. top = bottom + check_width if width == 1: # found the cause test_conflicts[spurious_failure] = candidate_causes[ bottom] width = 0 else: width = top - bottom else: # Conflict in the range we did not run: discard bottom. bottom = bottom + check_width if width == 1: # there will be no more to check, so we didn't # reproduce the failure. width = 0 else: width = top - bottom if spurious_failure not in test_conflicts: # Could not determine cause test_conflicts[spurious_failure] = 'unknown - no conflicts' if test_conflicts: table = [('failing test', 'caused by test')] for failure in sorted(test_conflicts): causes = test_conflicts[failure] table.append((failure, causes)) output.output_table(table) return 3 return 0
def run(arguments): args = arguments[0] filters = arguments[1] or None try: repo = util.get_repo_open(args.repo_type, args.repo_url) # If a repo is not found, and there a testr config exists just create it except repository.RepositoryNotFound: if not os.path.isfile(args.config): raise repo = util.get_repo_initialise(args.repo_type, args.repo_url) if args.no_discover: ids = args.no_discover if ids.find('/') != -1: root, _ = os.path.splitext(ids) ids = root.replace('/', '.') run_cmd = 'python -m subunit.run ' + ids def run_tests(): run_proc = [('subunit', output.ReturnCodeToSubunit( subprocess.Popen(run_cmd, shell=True, stdout=subprocess.PIPE)))] return load.load((None, None), in_streams=run_proc, partial=args.partial, subunit_out=args.subunit, repo_type=args.repo_type, repo_url=args.repo_url) if not args.until_failure: return run_tests() else: result = run_tests() while not result: result = run_tests() return result if args.failing or args.analyze_isolation: ids = _find_failing(repo) else: ids = None if args.load_list: list_ids = set() # Should perhaps be text.. currently does its own decode. with open(args.load_list, 'rb') as list_file: list_ids = set(parse_list(list_file.read())) if ids is None: # Use the supplied list verbatim ids = list_ids else: # We have some already limited set of ids, just reduce to ids # that are both failing and listed. ids = list_ids.intersection(ids) conf = config_file.TestrConf(args.config) if not args.analyze_isolation: cmd = conf.get_run_command(args, ids, filters) if args.isolated: result = 0 cmd.setUp() try: ids = cmd.list_tests() finally: cmd.cleanUp() for test_id in ids: # TODO(mtreinish): add regex cmd = conf.get_run_command(args, [test_id], filters) run_result = _run_tests(cmd, args.failing, args.analyze_isolation, args.isolated, args.until_failure, subunit_out=args.subunit) if run_result > result: result = run_result return result else: return _run_tests(cmd, args.failing, args.analyze_isolation, args.isolated, args.until_failure, subunit_out=args.subunit) else: # Where do we source data about the cause of conflicts. # XXX: Should instead capture the run id in with the failing test # data so that we can deal with failures split across many partial # runs. latest_run = repo.get_latest_run() # Stage one: reduce the list of failing tests (possibly further # reduced by testfilters) to eliminate fails-on-own tests. spurious_failures = set() for test_id in ids: # TODO(mtrienish): Add regex cmd = conf.get_run_command(args, [test_id]) if not _run_tests(cmd): # If the test was filtered, it won't have been run. if test_id in repo.get_test_ids(repo.latest_id()): spurious_failures.add(test_id) # This is arguably ugly, why not just tell the system that # a pass here isn't a real pass? [so that when we find a # test that is spuriously failing, we don't forget # that it is actually failng. # Alternatively, perhaps this is a case for data mining: # when a test starts passing, keep a journal, and allow # digging back in time to see that it was a failure, # what it failed with etc... # The current solution is to just let it get marked as # a pass temporarily. if not spurious_failures: # All done. return 0 # spurious-failure -> cause. test_conflicts = {} for spurious_failure in spurious_failures: candidate_causes = _prior_tests( latest_run, spurious_failure) bottom = 0 top = len(candidate_causes) width = top - bottom while width: check_width = int(ceil(width / 2.0)) # TODO(mtreinish): Add regex cmd = conf.get_run_command( args, candidate_causes[bottom:bottom + check_width] + [spurious_failure]) _run_tests(cmd) # check that the test we're probing still failed - still # awkward. found_fail = [] def find_fail(test_dict): if test_dict['id'] == spurious_failure: found_fail.append(True) checker = testtools.StreamToDict(find_fail) checker.startTestRun() try: repo.get_failing().get_test().run(checker) finally: checker.stopTestRun() if found_fail: # Our conflict is in bottom - clamp the range down. top = bottom + check_width if width == 1: # found the cause test_conflicts[ spurious_failure] = candidate_causes[bottom] width = 0 else: width = top - bottom else: # Conflict in the range we did not run: discard bottom. bottom = bottom + check_width if width == 1: # there will be no more to check, so we didn't # reproduce the failure. width = 0 else: width = top - bottom if spurious_failure not in test_conflicts: # Could not determine cause test_conflicts[spurious_failure] = 'unknown - no conflicts' if test_conflicts: table = [('failing test', 'caused by test')] for failure, causes in test_conflicts.items(): table.append((failure, causes)) output.output_table(table) return 3 return 0
def run(self): try: repo = self.repository_factory.open(self.ui.here) except RepositoryNotFound: if self.ui.options.force_init: repo = self.repository_factory.initialise(self.ui.here) else: raise if self.ui.options.failing or self.ui.options.analyze_isolation: ids = self._find_failing(repo) else: ids = None if self.ui.options.load_list: list_ids = set() # Should perhaps be text.. currently does its own decode. with open(self.ui.options.load_list, 'rb') as list_file: list_ids = set(parse_list(list_file.read())) if ids is None: # Use the supplied list verbatim ids = list_ids else: # We have some already limited set of ids, just reduce to ids # that are both failing and listed. ids = list_ids.intersection(ids) if self.ui.arguments['testfilters']: filters = self.ui.arguments['testfilters'] else: filters = None testcommand = self.command_factory(self.ui, repo) testcommand.setUp() try: if not self.ui.options.analyze_isolation: cmd = testcommand.get_run_command( ids, self.ui.arguments['testargs'], test_filters=filters) if self.ui.options.isolated: result = 0 cmd.setUp() try: ids = cmd.list_tests() finally: cmd.cleanUp() for test_id in ids: cmd = testcommand.get_run_command( [test_id], self.ui.arguments['testargs'], test_filters=filters) run_result = self._run_tests(cmd) if run_result > result: result = run_result return result else: return self._run_tests(cmd) else: # Where do we source data about the cause of conflicts. # XXX: Should instead capture the run id in with the failing test # data so that we can deal with failures split across many partial # runs. latest_run = repo.get_latest_run() # Stage one: reduce the list of failing tests (possibly further # reduced by testfilters) to eliminate fails-on-own tests. spurious_failures = set() for test_id in ids: cmd = testcommand.get_run_command( [test_id], self.ui.arguments['testargs'], test_filters=filters) if not self._run_tests(cmd): # If the test was filtered, it won't have been run. if test_id in repo.get_test_ids(repo.latest_id()): spurious_failures.add(test_id) # This is arguably ugly, why not just tell the system that # a pass here isn't a real pass? [so that when we find a # test that is spuriously failing, we don't forget # that it is actually failng. # Alternatively, perhaps this is a case for data mining: # when a test starts passing, keep a journal, and allow # digging back in time to see that it was a failure, # what it failed with etc... # The current solution is to just let it get marked as # a pass temporarily. if not spurious_failures: # All done. return 0 # spurious-failure -> cause. test_conflicts = {} for spurious_failure in spurious_failures: candidate_causes = self._prior_tests( latest_run, spurious_failure) bottom = 0 top = len(candidate_causes) width = top - bottom while width: check_width = int(ceil(width / 2.0)) cmd = testcommand.get_run_command( candidate_causes[bottom:bottom + check_width] + [spurious_failure], self.ui.arguments['testargs']) self._run_tests(cmd) # check that the test we're probing still failed - still # awkward. found_fail = [] def find_fail(test_dict): if test_dict['id'] == spurious_failure: found_fail.append(True) checker = testtools.StreamToDict(find_fail) checker.startTestRun() try: repo.get_failing().get_test().run(checker) finally: checker.stopTestRun() if found_fail: # Our conflict is in bottom - clamp the range down. top = bottom + check_width if width == 1: # found the cause test_conflicts[ spurious_failure] = candidate_causes[ bottom] width = 0 else: width = top - bottom else: # Conflict in the range we did not run: discard bottom. bottom = bottom + check_width if width == 1: # there will be no more to check, so we didn't # reproduce the failure. width = 0 else: width = top - bottom if spurious_failure not in test_conflicts: # Could not determine cause test_conflicts[ spurious_failure] = 'unknown - no conflicts' if test_conflicts: table = [('failing test', 'caused by test')] for failure, causes in test_conflicts.items(): table.append((failure, causes)) self.ui.output_table(table) return 3 return 0 finally: testcommand.cleanUp()