def _ProcessResults(self, result, start_ms, duration_ms):
"""Translates a Java test result into a Python result for this test.
The TestRunner class that we use under the covers will return a test result
for that specific Java test. However, to make reporting clearer, we have
this method to abstract that detail and instead report that as a failure of
this particular test case while still including the Java stack trace.
Args:
result: TestResults with a single Java test result
start_ms: the time the test started
duration_ms: the length of the test
Returns:
A TestResults object containing a result for this Python test.
"""
test_results = TestResults()
# If our test is in broken, then it crashed/failed.
broken = result.GetAllBroken()
if broken:
# Since we have run only one test, take the first and only item.
single_result = broken[0]
log = single_result.log
if not log:
log = 'No logging information.'
short_error_msg = single_result.log.split('\n')[0]
# err_info is ostensibly for Sponge to consume; it's a short error
# message and a longer one.
err_info = (short_error_msg, log)
python_result = SingleTestResult(self.qualified_name, start_ms,
duration_ms,
PYTHON,
log,
err_info)
# Figure out where the test belonged. There's probably a cleaner way of
# doing this.
if single_result in result.crashed:
test_results.crashed = [python_result]
elif single_result in result.failed:
test_results.failed = [python_result]
elif single_result in result.unknown:
test_results.unknown = [python_result]
else:
python_result = SingleTestResult(self.qualified_name, start_ms,
duration_ms,
PYTHON)
test_results.ok = [python_result]
return test_results
def RunShardedTests(self):
"""Runs the tests in all connected devices.
Returns:
A TestResults object.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
final_results = TestResults()
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self.SetupSharding(self.tests)
test_runners = []
for index, device in enumerate(self.attached_devices):
logging.warning('*' * 80)
logging.warning('Creating shard %d for %s', index, device)
logging.warning('*' * 80)
test_runner = self.CreateShardedTestRunner(device, index)
test_runners += [test_runner]
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[BaseTestSharder.tests_container])
# map can't handle KeyboardInterrupt exception. It's a python bug.
# So use map_async instead.
async_results = pool.map_async(_ShardedTestRunnable, test_runners)
results_lists = async_results.get(999999)
test_results = TestResults.FromTestResults(results_lists)
# Re-check the attached devices for some devices may
# become offline
retry_devices = set(android_commands.GetAttachedDevices())
# Remove devices that had exceptions.
retry_devices -= TestResults.DeviceExceptions(results_lists)
# Retry on devices that didn't have any exception.
self.attached_devices = list(retry_devices)
if (retry == self.retries - 1 or
len(self.attached_devices) == 0):
all_passed = final_results.ok + test_results.ok
final_results = test_results
final_results.ok = all_passed
break
else:
final_results.ok += test_results.ok
self.tests = []
for t in test_results.GetAllBroken():
self.tests += [t.name]
if not self.tests:
break
self.OnTestsCompleted(test_runners, final_results)
return final_results
def RunShardedTests(self):
"""Runs the tests in all connected devices.
Returns:
A TestResults object.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
final_results = TestResults()
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self.SetupSharding(self.tests)
test_runners = []
for index, device in enumerate(self.attached_devices):
logging.warning('*' * 80)
logging.warning('Creating shard %d for %s', index, device)
logging.warning('*' * 80)
test_runner = self.CreateShardedTestRunner(device, index)
test_runners += [test_runner]
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[BaseTestSharder.tests_container])
# map can't handle KeyboardInterrupt exception. It's a python bug.
# So use map_async instead.
async_results = pool.map_async(_ShardedTestRunnable, test_runners)
results_lists = async_results.get(999999)
test_results = TestResults.FromTestResults(results_lists)
# Re-check the attached devices for some devices may
# become offline
retry_devices = set(android_commands.GetAttachedDevices())
# Remove devices that had exceptions.
retry_devices -= TestResults.DeviceExceptions(results_lists)
# Retry on devices that didn't have any exception.
self.attached_devices = list(retry_devices)
if (retry == self.retries - 1 or len(self.attached_devices) == 0):
all_passed = final_results.ok + test_results.ok
final_results = test_results
final_results.ok = all_passed
break
else:
final_results.ok += test_results.ok
self.tests = []
for t in test_results.GetAllBroken():
self.tests += [t.name]
if not self.tests:
break
self.OnTestsCompleted(test_runners, final_results)
return final_results
def RunShardedTests(self):
"""Runs tests in parallel using a pool of workers.
Returns:
A list of test results aggregated from all test runs.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
all_passed = []
test_results = TestResults()
tests_to_run = self.tests
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self._SetupSharding(self.tests)
test_runners = self._MakeTestRunners(self.attached_devices)
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[PythonTestSharder.tests_container])
# List of TestResults objects from each test execution.
try:
results_lists = pool.map(_DefaultRunnable, test_runners)
except Exception:
logging.exception('Unable to run tests. Something with the '
'PythonTestRunners has gone wrong.')
raise FatalTestException('PythonTestRunners were unable to run tests.')
test_results = TestResults.FromTestResults(results_lists)
# Accumulate passing results.
all_passed += test_results.ok
# If we have failed tests, map them to tests to retry.
failed_tests = test_results.GetAllBroken()
tests_to_run = self._GetTestsToRetry(self.tests,
failed_tests)
# Bail out early if we have no more tests. This can happen if all tests
# pass before we're out of retries, for example.
if not tests_to_run:
break
final_results = TestResults()
# all_passed has accumulated all passing test results.
# test_results will have the results from the most recent run, which could
# include a variety of failure modes (unknown, crashed, failed, etc).
final_results = test_results
final_results.ok = all_passed
return final_results
def RunShardedTests(self):
"""Runs tests in parallel using a pool of workers.
Returns:
A list of test results aggregated from all test runs.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
all_passed = []
test_results = TestResults()
tests_to_run = self.tests
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self._SetupSharding(self.tests)
test_runners = self._MakeTestRunners(self.attached_devices)
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[PythonTestSharder.tests_container])
# List of TestResults objects from each test execution.
try:
results_lists = pool.map(_DefaultRunnable, test_runners)
except Exception:
logging.exception('Unable to run tests. Something with the '
'PythonTestRunners has gone wrong.')
raise FatalTestException(
'PythonTestRunners were unable to run tests.')
test_results = TestResults.FromTestResults(results_lists)
# Accumulate passing results.
all_passed += test_results.ok
# If we have failed tests, map them to tests to retry.
failed_tests = test_results.GetAllBroken()
tests_to_run = self._GetTestsToRetry(self.tests, failed_tests)
# Bail out early if we have no more tests. This can happen if all tests
# pass before we're out of retries, for example.
if not tests_to_run:
break
final_results = TestResults()
# all_passed has accumulated all passing test results.
# test_results will have the results from the most recent run, which could
# include a variety of failure modes (unknown, crashed, failed, etc).
final_results = test_results
final_results.ok = all_passed
return final_results
def _ProcessResults(self, result, start_ms, duration_ms):
"""Translates a Java test result into a Python result for this test.
The TestRunner class that we use under the covers will return a test result
for that specific Java test. However, to make reporting clearer, we have
this method to abstract that detail and instead report that as a failure of
this particular test case while still including the Java stack trace.
Args:
result: TestResults with a single Java test result
start_ms: the time the test started
duration_ms: the length of the test
Returns:
A TestResults object containing a result for this Python test.
"""
test_results = TestResults()
# If our test is in broken, then it crashed/failed.
broken = result.GetAllBroken()
if broken:
# Since we have run only one test, take the first and only item.
single_result = broken[0]
log = single_result.log
if not log:
log = 'No logging information.'
short_error_msg = single_result.log.split('\n')[0]
# err_info is ostensibly for Sponge to consume; it's a short error
# message and a longer one.
err_info = (short_error_msg, log)
python_result = SingleTestResult(self.qualified_name, start_ms,
duration_ms, PYTHON, log,
err_info)
# Figure out where the test belonged. There's probably a cleaner way of
# doing this.
if single_result in result.crashed:
test_results.crashed = [python_result]
elif single_result in result.failed:
test_results.failed = [python_result]
elif single_result in result.unknown:
test_results.unknown = [python_result]
else:
python_result = SingleTestResult(self.qualified_name, start_ms,
duration_ms, PYTHON)
test_results.ok = [python_result]
return test_results
def RunShardedTests(self):
"""Runs the tests in all connected devices.
Returns:
A TestResults object.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
final_results = TestResults()
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self.SetupSharding(self.tests)
test_runners = []
for index, device in enumerate(self.attached_devices):
logging.warning('*' * 80)
logging.warning('Creating shard %d for %s', index, device)
logging.warning('*' * 80)
test_runner = self.CreateShardedTestRunner(device, index)
test_runners += [test_runner]
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[BaseTestSharder.tests_container])
results_lists = pool.map(_ShardedTestRunnable, test_runners)
test_results = TestResults.FromTestResults(results_lists)
if retry == self.retries - 1:
all_passed = final_results.ok + test_results.ok
final_results = test_results
final_results.ok = all_passed
break
else:
final_results.ok += test_results.ok
self.tests = []
for t in test_results.GetAllBroken():
self.tests += [t.name]
if not self.tests:
break
self.OnTestsCompleted(test_runners, final_results)
return final_results
def RunShardedTests(self):
"""Runs the tests in all connected devices.
Returns:
A TestResults object.
"""
logging.warning("*" * 80)
logging.warning("Sharding in " + str(len(self.attached_devices)) + " devices.")
logging.warning("Note that the output is not synchronized.")
logging.warning('Look for the "Final result" banner in the end.')
logging.warning("*" * 80)
final_results = TestResults()
for retry in xrange(self.retries):
logging.warning("Try %d of %d", retry + 1, self.retries)
self.SetupSharding(self.tests)
test_runners = []
for index, device in enumerate(self.attached_devices):
logging.warning("*" * 80)
logging.warning("Creating shard %d for %s", index, device)
logging.warning("*" * 80)
test_runner = self.CreateShardedTestRunner(device, index)
test_runners += [test_runner]
logging.warning("Starting...")
pool = multiprocessing.Pool(
len(self.attached_devices), SetTestsContainer, [BaseTestSharder.tests_container]
)
results_lists = pool.map(_ShardedTestRunnable, test_runners)
test_results = TestResults.FromTestResults(results_lists)
if retry == self.retries - 1:
all_passed = final_results.ok + test_results.ok
final_results = test_results
final_results.ok = all_passed
break
else:
final_results.ok += test_results.ok
self.tests = []
for t in test_results.GetAllBroken():
self.tests += [t.name]
if not self.tests:
break
self.OnTestsCompleted(test_runners, final_results)
return final_results
def RunShardedTests(self):
"""Runs the tests in all connected devices.
Returns:
A TestResults object.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
final_results = TestResults()
self._KillHostForwarder()
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self.SetupSharding(self.tests)
test_runners = []
# Try to create N shards, and retrying on failure.
try:
for index, device in enumerate(self.attached_devices):
logging.warning('*' * 80)
logging.warning('Creating shard %d for %s', index, device)
logging.warning('*' * 80)
test_runner = self.CreateShardedTestRunner(device, index)
test_runners += [test_runner]
except errors.DeviceUnresponsiveError as e:
logging.critical('****Failed to create a shard: [%s]', e)
self.attached_devices.remove(device)
continue
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[BaseTestSharder.tests_container])
# map can't handle KeyboardInterrupt exception. It's a python bug.
# So use map_async instead.
async_results = pool.map_async(_ShardedTestRunnable, test_runners)
try:
results_lists = async_results.get(999999)
except errors.DeviceUnresponsiveError as e:
logging.critical('****Failed to run test: [%s]', e)
self.attached_devices = android_commands.GetAttachedDevices()
continue
test_results = TestResults.FromTestResults(results_lists)
# Re-check the attached devices for some devices may
# become offline
retry_devices = set(android_commands.GetAttachedDevices())
# Remove devices that had exceptions.
retry_devices -= TestResults.DeviceExceptions(results_lists)
# Retry on devices that didn't have any exception.
self.attached_devices = list(retry_devices)
if (retry == self.retries - 1 or
len(self.attached_devices) == 0):
all_passed = final_results.ok + test_results.ok
final_results = test_results
final_results.ok = all_passed
break
else:
final_results.ok += test_results.ok
# Timed out tests are not reported in GetAllBroken().
if test_results.timed_out:
final_results.timed_out = True
self.tests = []
for t in test_results.GetAllBroken():
self.tests += [t.name]
if not self.tests:
break
else:
# We ran out retries, possibly out of healthy devices.
# There's no recovery at this point.
raise Exception('Unrecoverable error while retrying test runs.')
self.OnTestsCompleted(test_runners, final_results)
self._KillHostForwarder()
return final_results
def RunShardedTests(self):
"""Runs the tests in all connected devices.
Returns:
A TestResults object.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
final_results = TestResults()
self._KillHostForwarder()
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self.SetupSharding(self.tests)
test_runners = []
# Try to create N shards, and retrying on failure.
try:
for index, device in enumerate(self.attached_devices):
logging.warning('*' * 80)
logging.warning('Creating shard %d for %s', index, device)
logging.warning('*' * 80)
test_runner = self.CreateShardedTestRunner(device, index)
test_runners += [test_runner]
except errors.DeviceUnresponsiveError as e:
logging.critical('****Failed to create a shard: [%s]', e)
self.attached_devices.remove(device)
continue
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[BaseTestSharder.tests_container])
# map can't handle KeyboardInterrupt exception. It's a python bug.
# So use map_async instead.
async_results = pool.map_async(_ShardedTestRunnable, test_runners)
try:
results_lists = async_results.get(999999)
except errors.DeviceUnresponsiveError as e:
logging.critical('****Failed to run test: [%s]', e)
self.attached_devices = android_commands.GetAttachedDevices()
continue
test_results = TestResults.FromTestResults(results_lists)
# Re-check the attached devices for some devices may
# become offline
retry_devices = set(android_commands.GetAttachedDevices())
# Remove devices that had exceptions.
retry_devices -= TestResults.DeviceExceptions(results_lists)
# Retry on devices that didn't have any exception.
self.attached_devices = list(retry_devices)
if (retry == self.retries - 1 or len(self.attached_devices) == 0):
all_passed = final_results.ok + test_results.ok
final_results = test_results
final_results.ok = all_passed
break
else:
final_results.ok += test_results.ok
self.tests = []
for t in test_results.GetAllBroken():
self.tests += [t.name]
if not self.tests:
break
else:
# We ran out retries, possibly out of healthy devices.
# There's no recovery at this point.
raise Exception('Unrecoverable error while retrying test runs.')
self.OnTestsCompleted(test_runners, final_results)
self._KillHostForwarder()
return final_results
