def run_tests_from_stream(input_stream, result, passthrough_stream=None,
forward_stream=None, protocol_version=1, passthrough_subunit=True):
"""Run tests from a subunit input stream through 'result'.
Non-test events - top level file attachments - are expected to be
dropped by v2 StreamResults at the present time (as all the analysis code
is in ExtendedTestResult API's), so to implement passthrough_stream they
are diverted and copied directly when that is set.
:param input_stream: A stream containing subunit input.
:param result: A TestResult that will receive the test events.
NB: This should be an ExtendedTestResult for v1 and a StreamResult for
v2.
:param passthrough_stream: All non-subunit input received will be
sent to this stream. If not provided, uses the ``TestProtocolServer``
default, which is ``sys.stdout``.
:param forward_stream: All subunit input received will be forwarded
to this stream. If not provided, uses the ``TestProtocolServer``
default, which is to not forward any input. Do not set this when
transforming the stream - items would be double-reported.
:param protocol_version: What version of the subunit protocol to expect.
:param passthrough_subunit: If True, passthrough should be as subunit
otherwise unwrap it. Only has effect when forward_stream is None.
(when forwarding as subunit non-subunit input is always turned into
subunit)
"""
if 1==protocol_version:
test = ProtocolTestCase(
input_stream, passthrough=passthrough_stream,
forward=forward_stream)
elif 2==protocol_version:
# In all cases we encapsulate unknown inputs.
if forward_stream is not None:
# Send events to forward_stream as subunit.
forward_result = StreamResultToBytes(forward_stream)
# If we're passing non-subunit through, copy:
if passthrough_stream is None:
# Not passing non-test events - split them off to nothing.
router = StreamResultRouter(forward_result)
router.add_rule(StreamResult(), 'test_id', test_id=None)
result = CopyStreamResult([router, result])
else:
# otherwise, copy all events to forward_result
result = CopyStreamResult([forward_result, result])
elif passthrough_stream is not None:
if not passthrough_subunit:
# Route non-test events to passthrough_stream, unwrapping them for
# display.
passthrough_result = CatFiles(passthrough_stream)
else:
passthrough_result = StreamResultToBytes(passthrough_stream)
result = StreamResultRouter(result)
result.add_rule(passthrough_result, 'test_id', test_id=None)
test = ByteStreamToStreamResult(input_stream,
non_subunit_name='stdout')
else:
raise Exception("Unknown protocol version.")
result.startTestRun()
test.run(result)
result.stopTestRun()
def find_tests(argv):
"""Find tests to parallel run.
:param argv: The argv given to the test runner, used to get the tests to
run.
:return: A list of test IDs.
"""
load_list = find_load_list(argv)
if load_list:
# just use the load_list
with open(load_list, 'rt') as list_file:
return [id for id in list_file.read().split('\n') if id]
# run in --list-tests mode
argv = prepare_argv(argv) + ['--list-tests', '--subunit']
process = subprocess.Popen(argv,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
process.stdin.close()
test = ProtocolTestCase(process.stdout)
result = GatherIDs()
test.run(result)
process.wait()
if process.returncode:
raise Exception('error listing tests: %s' % err)
return result.ids
def run(self, result):
with tempfile.NamedTemporaryFile() as test_list_file:
for test_id in self._test_ids:
test_list_file.write(test_id + '\n')
test_list_file.flush()
argv = self._args + ['--subunit', '--load-list', test_list_file.name]
process = subprocess.Popen(argv, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, bufsize=1)
try:
# If it tries to read, give it EOF.
process.stdin.close()
ProtocolTestCase.__init__(self, process.stdout)
ProtocolTestCase.run(self, result)
finally:
process.wait()
def _runTest(self, result):
yeti = os.environ["YETI"]
port = os.environ.get("YETI_PORT", "4422")
cmd = yeti.split() + ["--formatter=subunit",
"--quiet",
"--solo=1",
"--port=%s" % port]
for base, dirs, files in os.walk(self.tests_directory):
for filename in fnmatch.filter(files, "test_*.html"):
cmd.append(os.path.join(base, filename))
proc = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
suite = ProtocolTestCase(proc.stdout)
suite.run(result)
proc.wait()
def run(self, result):
with tempfile.NamedTemporaryFile() as test_list_file:
for test_id in self._test_ids:
test_list_file.write(test_id + '\n')
test_list_file.flush()
argv = self._args + [
'--subunit', '--load-list', test_list_file.name
]
process = subprocess.Popen(argv,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
bufsize=1)
try:
# If it tries to read, give it EOF.
process.stdin.close()
ProtocolTestCase.__init__(self, process.stdout)
ProtocolTestCase.run(self, result)
finally:
process.wait()
def run_tests_from_stream(input_stream, result, passthrough_stream=None,
forward_stream=None):
"""Run tests from a subunit input stream through 'result'.
:param input_stream: A stream containing subunit input.
:param result: A TestResult that will receive the test events.
:param passthrough_stream: All non-subunit input received will be
sent to this stream. If not provided, uses the ``TestProtocolServer``
default, which is ``sys.stdout``.
:param forward_stream: All subunit input received will be forwarded
to this stream. If not provided, uses the ``TestProtocolServer``
default, which is to not forward any input.
"""
test = ProtocolTestCase(
input_stream, passthrough=passthrough_stream,
forward=forward_stream)
result.startTestRun()
test.run(result)
result.stopTestRun()
def do_fork(suite):
"""Take suite and start up multiple runners by forking (Unix only).
:param suite: TestSuite object.
:return: An iterable of TestCase-like objects which can each have
run(result) called on them to feed tests to result.
"""
result = []
test_blocks = partition_tests(suite, concurrency_num)
# Clear the tests from the original suite so it doesn't keep them alive
suite._tests[:] = []
for process_tests in test_blocks:
process_suite = unittest.TestSuite(process_tests)
# Also clear each split list so new suite has only reference
process_tests[:] = []
c2pread, c2pwrite = os.pipe()
pid = os.fork()
if pid == 0:
try:
stream = os.fdopen(c2pwrite, 'wb')
os.close(c2pread)
# Leave stderr and stdout open so we can see test noise
# Close stdin so that the child goes away if it decides to
# read from stdin (otherwise its a roulette to see what
# child actually gets keystrokes for pdb etc).
sys.stdin.close()
subunit_result = AutoTimingTestResultDecorator(
test_protocol_client_class(stream))
process_suite.run(subunit_result)
except:
# Try and report traceback on stream, but exit with error
# even if stream couldn't be created or something else
# goes wrong. The traceback is formatted to a string and
# written in one go to avoid interleaving lines from
# multiple failing children.
try:
stream.write(traceback.format_exc())
finally:
os._exit(1)
os._exit(0)
else:
os.close(c2pwrite)
stream = os.fdopen(c2pread, 'rb')
# If we don't pass the second argument here, it defaults
# to sys.stdout.buffer down the line. But if we don't
# pass it *now*, it may be resolved after sys.stdout is
# replaced with a StringIO (to capture tests' outputs)
# which doesn't have a buffer attribute and can end up
# occasionally causing a 'broken-runner' error.
test = ProtocolTestCase(stream, sys.stdout.buffer)
result.append(test)
return result
def find_tests(argv):
"""Find tests to parallel run.
:param argv: The argv given to the test runner, used to get the tests to
run.
:return: A list of test IDs.
"""
load_list = find_load_list(argv)
if load_list:
# just use the load_list
with open(load_list, 'rt') as list_file:
return [id for id in list_file.read().split('\n') if id]
# run in --list-tests mode
argv = prepare_argv(argv) + ['--list-tests', '--subunit']
process = subprocess.Popen(argv, stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
process.stdin.close()
test = ProtocolTestCase(process.stdout)
result = GatherIDs()
test.run(result)
process.wait()
if process.returncode:
raise Exception('error listing tests: %s' % err)
return result.ids
def fork_for_tests(concurrency_num, suite):
result = []
if 'BUILDDIR' in os.environ:
selftestdir = get_test_layer()
test_blocks = partition_tests(suite, concurrency_num)
# Clear the tests from the original suite so it doesn't keep them alive
suite._tests[:] = []
totaltests = sum(len(x) for x in test_blocks)
for process_tests in test_blocks:
numtests = len(process_tests)
process_suite = unittest.TestSuite(process_tests)
# Also clear each split list so new suite has only reference
process_tests[:] = []
c2pread, c2pwrite = os.pipe()
# Clear buffers before fork to avoid duplicate output
sys.stdout.flush()
sys.stderr.flush()
pid = os.fork()
if pid == 0:
ourpid = os.getpid()
try:
newbuilddir = None
stream = os.fdopen(c2pwrite, 'wb', 1)
os.close(c2pread)
(builddir,
newbuilddir) = suite.setupfunc("-st-" + str(ourpid),
selftestdir, process_suite)
# Leave stderr and stdout open so we can see test noise
# Close stdin so that the child goes away if it decides to
# read from stdin (otherwise its a roulette to see what
# child actually gets keystrokes for pdb etc).
newsi = os.open(os.devnull, os.O_RDWR)
os.dup2(newsi, sys.stdin.fileno())
subunit_client = TestProtocolClient(stream)
# Force buffering of stdout/stderr so the console doesn't get corrupted by test output
# as per default in parent code
subunit_client.buffer = True
subunit_result = AutoTimingTestResultDecorator(subunit_client)
process_suite.run(
ExtraResultsEncoderTestResult(subunit_result))
if ourpid != os.getpid():
os._exit(0)
if newbuilddir:
removebuilddir(newbuilddir)
except:
# Don't do anything with process children
if ourpid != os.getpid():
os._exit(1)
# Try and report traceback on stream, but exit with error
# even if stream couldn't be created or something else
# goes wrong. The traceback is formatted to a string and
# written in one go to avoid interleaving lines from
# multiple failing children.
try:
stream.write(traceback.format_exc().encode('utf-8'))
except:
sys.stderr.write(traceback.format_exc())
finally:
if newbuilddir:
removebuilddir(newbuilddir)
stream.flush()
os._exit(1)
stream.flush()
os._exit(0)
else:
os.close(c2pwrite)
stream = os.fdopen(c2pread, 'rb', 1)
test = ProtocolTestCase(stream)
result.append((test, numtests))
return result, totaltests
def run_tests_from_stream(input_stream,
result,
passthrough_stream=None,
forward_stream=None,
protocol_version=1,
passthrough_subunit=True):
"""Run tests from a subunit input stream through 'result'.
Non-test events - top level file attachments - are expected to be
dropped by v2 StreamResults at the present time (as all the analysis code
is in ExtendedTestResult API's), so to implement passthrough_stream they
are diverted and copied directly when that is set.
:param input_stream: A stream containing subunit input.
:param result: A TestResult that will receive the test events.
NB: This should be an ExtendedTestResult for v1 and a StreamResult for
v2.
:param passthrough_stream: All non-subunit input received will be
sent to this stream. If not provided, uses the ``TestProtocolServer``
default, which is ``sys.stdout``.
:param forward_stream: All subunit input received will be forwarded
to this stream. If not provided, uses the ``TestProtocolServer``
default, which is to not forward any input. Do not set this when
transforming the stream - items would be double-reported.
:param protocol_version: What version of the subunit protocol to expect.
:param passthrough_subunit: If True, passthrough should be as subunit
otherwise unwrap it. Only has effect when forward_stream is None.
(when forwarding as subunit non-subunit input is always turned into
subunit)
"""
if 1 == protocol_version:
test = ProtocolTestCase(input_stream,
passthrough=passthrough_stream,
forward=forward_stream)
elif 2 == protocol_version:
# In all cases we encapsulate unknown inputs.
if forward_stream is not None:
# Send events to forward_stream as subunit.
forward_result = StreamResultToBytes(forward_stream)
# If we're passing non-subunit through, copy:
if passthrough_stream is None:
# Not passing non-test events - split them off to nothing.
router = StreamResultRouter(forward_result)
router.add_rule(StreamResult(), 'test_id', test_id=None)
result = CopyStreamResult([router, result])
else:
# otherwise, copy all events to forward_result
result = CopyStreamResult([forward_result, result])
elif passthrough_stream is not None:
if not passthrough_subunit:
# Route non-test events to passthrough_stream, unwrapping them for
# display.
passthrough_result = CatFiles(passthrough_stream)
else:
passthrough_result = StreamResultToBytes(passthrough_stream)
result = StreamResultRouter(result)
result.add_rule(passthrough_result, 'test_id', test_id=None)
test = ByteStreamToStreamResult(input_stream,
non_subunit_name='stdout')
else:
raise Exception("Unknown protocol version.")
result.startTestRun()
test.run(result)
result.stopTestRun()
def fork_for_tests(concurrency_num, suite):
result = []
if 'BUILDDIR' in os.environ:
selftestdir = get_test_layer()
test_blocks = partition_tests(suite, concurrency_num)
# Clear the tests from the original suite so it doesn't keep them alive
suite._tests[:] = []
totaltests = sum(len(x) for x in test_blocks)
for process_tests in test_blocks:
numtests = len(process_tests)
process_suite = unittest.TestSuite(process_tests)
# Also clear each split list so new suite has only reference
process_tests[:] = []
c2pread, c2pwrite = os.pipe()
# Clear buffers before fork to avoid duplicate output
sys.stdout.flush()
sys.stderr.flush()
pid = os.fork()
if pid == 0:
ourpid = os.getpid()
try:
newbuilddir = None
stream = os.fdopen(c2pwrite, 'wb', 1)
os.close(c2pread)
# Create a new separate BUILDDIR for each group of tests
if 'BUILDDIR' in os.environ:
builddir = os.environ['BUILDDIR']
newbuilddir = builddir + "-st-" + str(ourpid)
newselftestdir = newbuilddir + "/meta-selftest"
bb.utils.mkdirhier(newbuilddir)
oe.path.copytree(builddir + "/conf", newbuilddir + "/conf")
oe.path.copytree(builddir + "/cache", newbuilddir + "/cache")
oe.path.copytree(selftestdir, newselftestdir)
for e in os.environ:
if builddir in os.environ[e]:
os.environ[e] = os.environ[e].replace(builddir, newbuilddir)
subprocess.check_output("git init; git add *; git commit -a -m 'initial'", cwd=newselftestdir, shell=True)
# Tried to used bitbake-layers add/remove but it requires recipe parsing and hence is too slow
subprocess.check_output("sed %s/conf/bblayers.conf -i -e 's#%s#%s#g'" % (newbuilddir, selftestdir, newselftestdir), cwd=newbuilddir, shell=True)
os.chdir(newbuilddir)
for t in process_suite:
if not hasattr(t, "tc"):
continue
cp = t.tc.config_paths
for p in cp:
if selftestdir in cp[p] and newselftestdir not in cp[p]:
cp[p] = cp[p].replace(selftestdir, newselftestdir)
if builddir in cp[p] and newbuilddir not in cp[p]:
cp[p] = cp[p].replace(builddir, newbuilddir)
# Leave stderr and stdout open so we can see test noise
# Close stdin so that the child goes away if it decides to
# read from stdin (otherwise its a roulette to see what
# child actually gets keystrokes for pdb etc).
newsi = os.open(os.devnull, os.O_RDWR)
os.dup2(newsi, sys.stdin.fileno())
subunit_client = TestProtocolClient(stream)
# Force buffering of stdout/stderr so the console doesn't get corrupted by test output
# as per default in parent code
subunit_client.buffer = True
subunit_result = AutoTimingTestResultDecorator(subunit_client)
process_suite.run(subunit_result)
if ourpid != os.getpid():
os._exit(0)
if newbuilddir:
removebuilddir(newbuilddir)
except:
# Don't do anything with process children
if ourpid != os.getpid():
os._exit(1)
# Try and report traceback on stream, but exit with error
# even if stream couldn't be created or something else
# goes wrong. The traceback is formatted to a string and
# written in one go to avoid interleaving lines from
# multiple failing children.
try:
stream.write(traceback.format_exc().encode('utf-8'))
except:
sys.stderr.write(traceback.format_exc())
finally:
if newbuilddir:
removebuilddir(newbuilddir)
stream.flush()
os._exit(1)
stream.flush()
os._exit(0)
else:
os.close(c2pwrite)
stream = os.fdopen(c2pread, 'rb', 1)
test = ProtocolTestCase(stream)
result.append((test, numtests))
return result, totaltests
