def run_tests_from_stream(input_stream, result, passthrough_stream=None, forward_stream=None, protocol_version=1, passthrough_subunit=True): """Run tests from a subunit input stream through 'result'. Non-test events - top level file attachments - are expected to be dropped by v2 StreamResults at the present time (as all the analysis code is in ExtendedTestResult API's), so to implement passthrough_stream they are diverted and copied directly when that is set. :param input_stream: A stream containing subunit input. :param result: A TestResult that will receive the test events. NB: This should be an ExtendedTestResult for v1 and a StreamResult for v2. :param passthrough_stream: All non-subunit input received will be sent to this stream. If not provided, uses the ``TestProtocolServer`` default, which is ``sys.stdout``. :param forward_stream: All subunit input received will be forwarded to this stream. If not provided, uses the ``TestProtocolServer`` default, which is to not forward any input. Do not set this when transforming the stream - items would be double-reported. :param protocol_version: What version of the subunit protocol to expect. :param passthrough_subunit: If True, passthrough should be as subunit otherwise unwrap it. Only has effect when forward_stream is None. (when forwarding as subunit non-subunit input is always turned into subunit) """ if 1==protocol_version: test = ProtocolTestCase( input_stream, passthrough=passthrough_stream, forward=forward_stream) elif 2==protocol_version: # In all cases we encapsulate unknown inputs. if forward_stream is not None: # Send events to forward_stream as subunit. forward_result = StreamResultToBytes(forward_stream) # If we're passing non-subunit through, copy: if passthrough_stream is None: # Not passing non-test events - split them off to nothing. router = StreamResultRouter(forward_result) router.add_rule(StreamResult(), 'test_id', test_id=None) result = CopyStreamResult([router, result]) else: # otherwise, copy all events to forward_result result = CopyStreamResult([forward_result, result]) elif passthrough_stream is not None: if not passthrough_subunit: # Route non-test events to passthrough_stream, unwrapping them for # display. passthrough_result = CatFiles(passthrough_stream) else: passthrough_result = StreamResultToBytes(passthrough_stream) result = StreamResultRouter(result) result.add_rule(passthrough_result, 'test_id', test_id=None) test = ByteStreamToStreamResult(input_stream, non_subunit_name='stdout') else: raise Exception("Unknown protocol version.") result.startTestRun() test.run(result) result.stopTestRun()
def find_tests(argv): """Find tests to parallel run. :param argv: The argv given to the test runner, used to get the tests to run. :return: A list of test IDs. """ load_list = find_load_list(argv) if load_list: # just use the load_list with open(load_list, 'rt') as list_file: return [id for id in list_file.read().split('\n') if id] # run in --list-tests mode argv = prepare_argv(argv) + ['--list-tests', '--subunit'] process = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE) process.stdin.close() test = ProtocolTestCase(process.stdout) result = GatherIDs() test.run(result) process.wait() if process.returncode: raise Exception('error listing tests: %s' % err) return result.ids
def run(self, result): with tempfile.NamedTemporaryFile() as test_list_file: for test_id in self._test_ids: test_list_file.write(test_id + '\n') test_list_file.flush() argv = self._args + ['--subunit', '--load-list', test_list_file.name] process = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=1) try: # If it tries to read, give it EOF. process.stdin.close() ProtocolTestCase.__init__(self, process.stdout) ProtocolTestCase.run(self, result) finally: process.wait()
def _runTest(self, result): yeti = os.environ["YETI"] port = os.environ.get("YETI_PORT", "4422") cmd = yeti.split() + ["--formatter=subunit", "--quiet", "--solo=1", "--port=%s" % port] for base, dirs, files in os.walk(self.tests_directory): for filename in fnmatch.filter(files, "test_*.html"): cmd.append(os.path.join(base, filename)) proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) suite = ProtocolTestCase(proc.stdout) suite.run(result) proc.wait()
def run(self, result): with tempfile.NamedTemporaryFile() as test_list_file: for test_id in self._test_ids: test_list_file.write(test_id + '\n') test_list_file.flush() argv = self._args + [ '--subunit', '--load-list', test_list_file.name ] process = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=1) try: # If it tries to read, give it EOF. process.stdin.close() ProtocolTestCase.__init__(self, process.stdout) ProtocolTestCase.run(self, result) finally: process.wait()
def run_tests_from_stream(input_stream, result, passthrough_stream=None, forward_stream=None): """Run tests from a subunit input stream through 'result'. :param input_stream: A stream containing subunit input. :param result: A TestResult that will receive the test events. :param passthrough_stream: All non-subunit input received will be sent to this stream. If not provided, uses the ``TestProtocolServer`` default, which is ``sys.stdout``. :param forward_stream: All subunit input received will be forwarded to this stream. If not provided, uses the ``TestProtocolServer`` default, which is to not forward any input. """ test = ProtocolTestCase( input_stream, passthrough=passthrough_stream, forward=forward_stream) result.startTestRun() test.run(result) result.stopTestRun()
def do_fork(suite): """Take suite and start up multiple runners by forking (Unix only). :param suite: TestSuite object. :return: An iterable of TestCase-like objects which can each have run(result) called on them to feed tests to result. """ result = [] test_blocks = partition_tests(suite, concurrency_num) # Clear the tests from the original suite so it doesn't keep them alive suite._tests[:] = [] for process_tests in test_blocks: process_suite = unittest.TestSuite(process_tests) # Also clear each split list so new suite has only reference process_tests[:] = [] c2pread, c2pwrite = os.pipe() pid = os.fork() if pid == 0: try: stream = os.fdopen(c2pwrite, 'wb') os.close(c2pread) # Leave stderr and stdout open so we can see test noise # Close stdin so that the child goes away if it decides to # read from stdin (otherwise its a roulette to see what # child actually gets keystrokes for pdb etc). sys.stdin.close() subunit_result = AutoTimingTestResultDecorator( test_protocol_client_class(stream)) process_suite.run(subunit_result) except: # Try and report traceback on stream, but exit with error # even if stream couldn't be created or something else # goes wrong. The traceback is formatted to a string and # written in one go to avoid interleaving lines from # multiple failing children. try: stream.write(traceback.format_exc()) finally: os._exit(1) os._exit(0) else: os.close(c2pwrite) stream = os.fdopen(c2pread, 'rb') # If we don't pass the second argument here, it defaults # to sys.stdout.buffer down the line. But if we don't # pass it *now*, it may be resolved after sys.stdout is # replaced with a StringIO (to capture tests' outputs) # which doesn't have a buffer attribute and can end up # occasionally causing a 'broken-runner' error. test = ProtocolTestCase(stream, sys.stdout.buffer) result.append(test) return result
def fork_for_tests(concurrency_num, suite): result = [] if 'BUILDDIR' in os.environ: selftestdir = get_test_layer() test_blocks = partition_tests(suite, concurrency_num) # Clear the tests from the original suite so it doesn't keep them alive suite._tests[:] = [] totaltests = sum(len(x) for x in test_blocks) for process_tests in test_blocks: numtests = len(process_tests) process_suite = unittest.TestSuite(process_tests) # Also clear each split list so new suite has only reference process_tests[:] = [] c2pread, c2pwrite = os.pipe() # Clear buffers before fork to avoid duplicate output sys.stdout.flush() sys.stderr.flush() pid = os.fork() if pid == 0: ourpid = os.getpid() try: newbuilddir = None stream = os.fdopen(c2pwrite, 'wb', 1) os.close(c2pread) (builddir, newbuilddir) = suite.setupfunc("-st-" + str(ourpid), selftestdir, process_suite) # Leave stderr and stdout open so we can see test noise # Close stdin so that the child goes away if it decides to # read from stdin (otherwise its a roulette to see what # child actually gets keystrokes for pdb etc). newsi = os.open(os.devnull, os.O_RDWR) os.dup2(newsi, sys.stdin.fileno()) subunit_client = TestProtocolClient(stream) # Force buffering of stdout/stderr so the console doesn't get corrupted by test output # as per default in parent code subunit_client.buffer = True subunit_result = AutoTimingTestResultDecorator(subunit_client) process_suite.run( ExtraResultsEncoderTestResult(subunit_result)) if ourpid != os.getpid(): os._exit(0) if newbuilddir: removebuilddir(newbuilddir) except: # Don't do anything with process children if ourpid != os.getpid(): os._exit(1) # Try and report traceback on stream, but exit with error # even if stream couldn't be created or something else # goes wrong. The traceback is formatted to a string and # written in one go to avoid interleaving lines from # multiple failing children. try: stream.write(traceback.format_exc().encode('utf-8')) except: sys.stderr.write(traceback.format_exc()) finally: if newbuilddir: removebuilddir(newbuilddir) stream.flush() os._exit(1) stream.flush() os._exit(0) else: os.close(c2pwrite) stream = os.fdopen(c2pread, 'rb', 1) test = ProtocolTestCase(stream) result.append((test, numtests)) return result, totaltests
def run_tests_from_stream(input_stream, result, passthrough_stream=None, forward_stream=None, protocol_version=1, passthrough_subunit=True): """Run tests from a subunit input stream through 'result'. Non-test events - top level file attachments - are expected to be dropped by v2 StreamResults at the present time (as all the analysis code is in ExtendedTestResult API's), so to implement passthrough_stream they are diverted and copied directly when that is set. :param input_stream: A stream containing subunit input. :param result: A TestResult that will receive the test events. NB: This should be an ExtendedTestResult for v1 and a StreamResult for v2. :param passthrough_stream: All non-subunit input received will be sent to this stream. If not provided, uses the ``TestProtocolServer`` default, which is ``sys.stdout``. :param forward_stream: All subunit input received will be forwarded to this stream. If not provided, uses the ``TestProtocolServer`` default, which is to not forward any input. Do not set this when transforming the stream - items would be double-reported. :param protocol_version: What version of the subunit protocol to expect. :param passthrough_subunit: If True, passthrough should be as subunit otherwise unwrap it. Only has effect when forward_stream is None. (when forwarding as subunit non-subunit input is always turned into subunit) """ if 1 == protocol_version: test = ProtocolTestCase(input_stream, passthrough=passthrough_stream, forward=forward_stream) elif 2 == protocol_version: # In all cases we encapsulate unknown inputs. if forward_stream is not None: # Send events to forward_stream as subunit. forward_result = StreamResultToBytes(forward_stream) # If we're passing non-subunit through, copy: if passthrough_stream is None: # Not passing non-test events - split them off to nothing. router = StreamResultRouter(forward_result) router.add_rule(StreamResult(), 'test_id', test_id=None) result = CopyStreamResult([router, result]) else: # otherwise, copy all events to forward_result result = CopyStreamResult([forward_result, result]) elif passthrough_stream is not None: if not passthrough_subunit: # Route non-test events to passthrough_stream, unwrapping them for # display. passthrough_result = CatFiles(passthrough_stream) else: passthrough_result = StreamResultToBytes(passthrough_stream) result = StreamResultRouter(result) result.add_rule(passthrough_result, 'test_id', test_id=None) test = ByteStreamToStreamResult(input_stream, non_subunit_name='stdout') else: raise Exception("Unknown protocol version.") result.startTestRun() test.run(result) result.stopTestRun()
def fork_for_tests(concurrency_num, suite): result = [] if 'BUILDDIR' in os.environ: selftestdir = get_test_layer() test_blocks = partition_tests(suite, concurrency_num) # Clear the tests from the original suite so it doesn't keep them alive suite._tests[:] = [] totaltests = sum(len(x) for x in test_blocks) for process_tests in test_blocks: numtests = len(process_tests) process_suite = unittest.TestSuite(process_tests) # Also clear each split list so new suite has only reference process_tests[:] = [] c2pread, c2pwrite = os.pipe() # Clear buffers before fork to avoid duplicate output sys.stdout.flush() sys.stderr.flush() pid = os.fork() if pid == 0: ourpid = os.getpid() try: newbuilddir = None stream = os.fdopen(c2pwrite, 'wb', 1) os.close(c2pread) # Create a new separate BUILDDIR for each group of tests if 'BUILDDIR' in os.environ: builddir = os.environ['BUILDDIR'] newbuilddir = builddir + "-st-" + str(ourpid) newselftestdir = newbuilddir + "/meta-selftest" bb.utils.mkdirhier(newbuilddir) oe.path.copytree(builddir + "/conf", newbuilddir + "/conf") oe.path.copytree(builddir + "/cache", newbuilddir + "/cache") oe.path.copytree(selftestdir, newselftestdir) for e in os.environ: if builddir in os.environ[e]: os.environ[e] = os.environ[e].replace(builddir, newbuilddir) subprocess.check_output("git init; git add *; git commit -a -m 'initial'", cwd=newselftestdir, shell=True) # Tried to used bitbake-layers add/remove but it requires recipe parsing and hence is too slow subprocess.check_output("sed %s/conf/bblayers.conf -i -e 's#%s#%s#g'" % (newbuilddir, selftestdir, newselftestdir), cwd=newbuilddir, shell=True) os.chdir(newbuilddir) for t in process_suite: if not hasattr(t, "tc"): continue cp = t.tc.config_paths for p in cp: if selftestdir in cp[p] and newselftestdir not in cp[p]: cp[p] = cp[p].replace(selftestdir, newselftestdir) if builddir in cp[p] and newbuilddir not in cp[p]: cp[p] = cp[p].replace(builddir, newbuilddir) # Leave stderr and stdout open so we can see test noise # Close stdin so that the child goes away if it decides to # read from stdin (otherwise its a roulette to see what # child actually gets keystrokes for pdb etc). newsi = os.open(os.devnull, os.O_RDWR) os.dup2(newsi, sys.stdin.fileno()) subunit_client = TestProtocolClient(stream) # Force buffering of stdout/stderr so the console doesn't get corrupted by test output # as per default in parent code subunit_client.buffer = True subunit_result = AutoTimingTestResultDecorator(subunit_client) process_suite.run(subunit_result) if ourpid != os.getpid(): os._exit(0) if newbuilddir: removebuilddir(newbuilddir) except: # Don't do anything with process children if ourpid != os.getpid(): os._exit(1) # Try and report traceback on stream, but exit with error # even if stream couldn't be created or something else # goes wrong. The traceback is formatted to a string and # written in one go to avoid interleaving lines from # multiple failing children. try: stream.write(traceback.format_exc().encode('utf-8')) except: sys.stderr.write(traceback.format_exc()) finally: if newbuilddir: removebuilddir(newbuilddir) stream.flush() os._exit(1) stream.flush() os._exit(0) else: os.close(c2pwrite) stream = os.fdopen(c2pread, 'rb', 1) test = ProtocolTestCase(stream) result.append((test, numtests)) return result, totaltests