def stub_out_methods(self, test_views):
"""Stub out methods in test_push module with given test results.
@param test_views: Desired test result views.
"""
self.mox.UnsetStubs()
response = StringIO.StringIO('some_value')
self.mox.StubOutWithMock(urllib2, 'urlopen')
urllib2.urlopen(mox.IgnoreArg()).AndReturn(response)
self.mox.StubOutWithMock(test_push, 'get_default_build')
test_push.get_default_build(
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn('stumpy-release/R36-5881-0.0')
test_push.get_default_build(
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn('quawks-release/R36-5881-0.0')
self.mox.StubOutWithMock(test_push, 'check_dut_image')
test_push.check_dut_image(mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(None)
self.mox.StubOutWithMock(test_push, 'do_run_suite')
test_push.do_run_suite(test_push.PUSH_TO_PROD_SUITE, mox.IgnoreArg(),
mox.IgnoreArg(), mox.IgnoreArg()).AndReturn((1))
self.mox.StubOutWithMock(site_utils, 'get_test_views_from_tko')
self.mox.StubOutWithMock(frontend_wrappers, 'RetryingTKO')
frontend_wrappers.RetryingTKO(timeout_min=0.1,
delay_sec=10).AndReturn(None)
site_utils.get_test_views_from_tko(1, None).AndReturn(test_views)
def stub_out_methods(self, test_views):
"""Stub out methods in test_push module with given test results.
@param test_views: Desired test result views.
"""
self.mox.UnsetStubs()
response = StringIO.StringIO('some_value')
self.mox.StubOutWithMock(urllib2, 'urlopen')
urllib2.urlopen(mox.IgnoreArg()).AndReturn(response)
self.mox.StubOutWithMock(test_push, 'check_dut_image')
test_push.check_dut_image(mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(None)
self.mox.StubOutWithMock(test_push, 'do_run_suite')
test_push.do_run_suite(test_push.PUSH_TO_PROD_SUITE, mox.IgnoreArg(),
mox.IgnoreArg(), mox.IgnoreArg()).AndReturn((1))
self.mox.StubOutWithMock(site_utils, 'get_test_views_from_tko')
site_utils.get_test_views_from_tko(1, None).AndReturn(test_views)
def verify_test_results(job_id, expected_results):
"""Verify the test results with the expected results.
@param job_id: id of the running jobs. For suite job, it is suite_job_id.
@param expected_results: A dictionary of test name to test result.
@raise TestPushException: If verify fails.
"""
print 'Comparing test results...'
test_views = site_utils.get_test_views_from_tko(job_id, TKO)
summary = test_push_common.summarize_push(test_views, expected_results,
_IGNORED_TESTS)
# Test link to log can be loaded.
job_name = '%s-%s' % (job_id, getpass.getuser())
log_link = URL_PATTERN % (rpc_client_lib.add_protocol(URL_HOST), job_name)
try:
urllib2.urlopen(log_link).read()
except urllib2.URLError:
summary.append('Failed to load page for link to log: %s.' % log_link)
if summary:
raise TestPushException('\n'.join(summary))
def verify_test_results(job_id, expected_results):
"""Verify the test results with the expected results.
@param job_id: id of the running jobs. For suite job, it is suite_job_id.
@param expected_results: A dictionary of test name to test result.
@raise TestPushException: If verify fails.
"""
print 'Comparing test results...'
test_views = site_utils.get_test_views_from_tko(job_id, TKO)
mismatch_errors = []
extra_test_errors = []
found_keys = set()
for test_name, test_status in test_views.items():
print "%s%s" % (test_name.ljust(30), test_status)
# platform_InstallTestImage test may exist in old builds.
if re.search('platform_InstallTestImage_SERVER_JOB$', test_name):
continue
test_found = False
for key, val in expected_results.items():
if re.search(key, test_name):
test_found = True
found_keys.add(key)
if val != test_status:
error = ('%s Expected: [%s], Actual: [%s]' %
(test_name, val, test_status))
mismatch_errors.append(error)
if not test_found:
extra_test_errors.append(test_name)
missing_test_errors = set(expected_results.keys()) - found_keys
for exception in IGNORE_MISSING_TESTS:
try:
missing_test_errors.remove(exception)
except KeyError:
pass
summary = []
if mismatch_errors:
summary.append(('Results of %d test(s) do not match expected '
'values:') % len(mismatch_errors))
summary.extend(mismatch_errors)
summary.append('\n')
if extra_test_errors:
summary.append('%d test(s) are not expected to be run:' %
len(extra_test_errors))
summary.extend(extra_test_errors)
summary.append('\n')
if missing_test_errors:
summary.append('%d test(s) are missing from the results:' %
len(missing_test_errors))
summary.extend(missing_test_errors)
summary.append('\n')
# Test link to log can be loaded.
job_name = '%s-%s' % (job_id, getpass.getuser())
log_link = URL_PATTERN % (URL_HOST, job_name)
try:
urllib2.urlopen(log_link).read()
except urllib2.URLError:
summary.append('Failed to load page for link to log: %s.' % log_link)
if summary:
raise TestPushException('\n'.join(summary))
def test_suite(suite_name, expected_results, arguments, use_shard=False,
create_and_return=False):
"""Call run_suite to start a suite job and verify results.
@param suite_name: Name of a suite, e.g., dummy
@param expected_results: A dictionary of test name to test result.
@param arguments: Arguments for run_suite command.
@param use_shard: If true, suite is scheduled for shard board.
@param create_and_return: If True, run_suite just creates the suite, print
the job id, then finish immediately.
"""
suite_job_id = do_run_suite(suite_name, arguments, use_shard,
create_and_return)
# Confirm all DUTs used for the suite are imaged to expected build.
# hqe.host_id for jobs running in shard is not synced back to master db,
# therefore, skip verifying dut build for jobs running in shard.
if suite_name != AU_SUITE and not use_shard:
check_dut_image(arguments.build, suite_job_id)
# Find all tests and their status
print 'Comparing test results...'
TKO = frontend_wrappers.RetryingTKO(timeout_min=0.1, delay_sec=10)
test_views = site_utils.get_test_views_from_tko(suite_job_id, TKO)
mismatch_errors = []
extra_test_errors = []
found_keys = set()
for test_name,test_status in test_views.items():
print "%s%s" % (test_name.ljust(30), test_status)
test_found = False
for key,val in expected_results.items():
if re.search(key, test_name):
test_found = True
found_keys.add(key)
# TODO(dshi): result for this test is ignored until servo is
# added to a host accessible by cbf server (crbug.com/277109).
if key == 'platform_InstallTestImage_SERVER_JOB$':
continue
if val != test_status:
error = ('%s Expected: [%s], Actual: [%s]' %
(test_name, val, test_status))
mismatch_errors.append(error)
if not test_found:
extra_test_errors.append(test_name)
missing_test_errors = set(expected_results.keys()) - found_keys
for exception in IGNORE_MISSING_TESTS:
try:
missing_test_errors.remove(exception)
except KeyError:
pass
summary = []
if mismatch_errors:
summary.append(('Results of %d test(s) do not match expected '
'values:') % len(mismatch_errors))
summary.extend(mismatch_errors)
summary.append('\n')
if extra_test_errors:
summary.append('%d test(s) are not expected to be run:' %
len(extra_test_errors))
summary.extend(extra_test_errors)
summary.append('\n')
if missing_test_errors:
summary.append('%d test(s) are missing from the results:' %
len(missing_test_errors))
summary.extend(missing_test_errors)
summary.append('\n')
# Test link to log can be loaded.
job_name = '%s-%s' % (suite_job_id, getpass.getuser())
log_link = URL_PATTERN % (URL_HOST, job_name)
try:
urllib2.urlopen(log_link).read()
except urllib2.URLError:
summary.append('Failed to load page for link to log: %s.' % log_link)
if summary:
raise TestPushException('\n'.join(summary))