def stub_out_methods(self, test_views): """Stub out methods in test_push module with given test results. @param test_views: Desired test result views. """ self.mox.UnsetStubs() response = StringIO.StringIO('some_value') self.mox.StubOutWithMock(urllib2, 'urlopen') urllib2.urlopen(mox.IgnoreArg()).AndReturn(response) self.mox.StubOutWithMock(test_push, 'get_default_build') test_push.get_default_build( mox.IgnoreArg(), mox.IgnoreArg()).AndReturn('stumpy-release/R36-5881-0.0') test_push.get_default_build( mox.IgnoreArg(), mox.IgnoreArg()).AndReturn('quawks-release/R36-5881-0.0') self.mox.StubOutWithMock(test_push, 'check_dut_image') test_push.check_dut_image(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(None) self.mox.StubOutWithMock(test_push, 'do_run_suite') test_push.do_run_suite(test_push.PUSH_TO_PROD_SUITE, mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()).AndReturn((1)) self.mox.StubOutWithMock(site_utils, 'get_test_views_from_tko') self.mox.StubOutWithMock(frontend_wrappers, 'RetryingTKO') frontend_wrappers.RetryingTKO(timeout_min=0.1, delay_sec=10).AndReturn(None) site_utils.get_test_views_from_tko(1, None).AndReturn(test_views)
def stub_out_methods(self, test_views): """Stub out methods in test_push module with given test results. @param test_views: Desired test result views. """ self.mox.UnsetStubs() response = StringIO.StringIO('some_value') self.mox.StubOutWithMock(urllib2, 'urlopen') urllib2.urlopen(mox.IgnoreArg()).AndReturn(response) self.mox.StubOutWithMock(test_push, 'check_dut_image') test_push.check_dut_image(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(None) self.mox.StubOutWithMock(test_push, 'do_run_suite') test_push.do_run_suite(test_push.PUSH_TO_PROD_SUITE, mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()).AndReturn((1)) self.mox.StubOutWithMock(site_utils, 'get_test_views_from_tko') site_utils.get_test_views_from_tko(1, None).AndReturn(test_views)
def verify_test_results(job_id, expected_results): """Verify the test results with the expected results. @param job_id: id of the running jobs. For suite job, it is suite_job_id. @param expected_results: A dictionary of test name to test result. @raise TestPushException: If verify fails. """ print 'Comparing test results...' test_views = site_utils.get_test_views_from_tko(job_id, TKO) summary = test_push_common.summarize_push(test_views, expected_results, _IGNORED_TESTS) # Test link to log can be loaded. job_name = '%s-%s' % (job_id, getpass.getuser()) log_link = URL_PATTERN % (rpc_client_lib.add_protocol(URL_HOST), job_name) try: urllib2.urlopen(log_link).read() except urllib2.URLError: summary.append('Failed to load page for link to log: %s.' % log_link) if summary: raise TestPushException('\n'.join(summary))
def verify_test_results(job_id, expected_results): """Verify the test results with the expected results. @param job_id: id of the running jobs. For suite job, it is suite_job_id. @param expected_results: A dictionary of test name to test result. @raise TestPushException: If verify fails. """ print 'Comparing test results...' test_views = site_utils.get_test_views_from_tko(job_id, TKO) mismatch_errors = [] extra_test_errors = [] found_keys = set() for test_name, test_status in test_views.items(): print "%s%s" % (test_name.ljust(30), test_status) # platform_InstallTestImage test may exist in old builds. if re.search('platform_InstallTestImage_SERVER_JOB$', test_name): continue test_found = False for key, val in expected_results.items(): if re.search(key, test_name): test_found = True found_keys.add(key) if val != test_status: error = ('%s Expected: [%s], Actual: [%s]' % (test_name, val, test_status)) mismatch_errors.append(error) if not test_found: extra_test_errors.append(test_name) missing_test_errors = set(expected_results.keys()) - found_keys for exception in IGNORE_MISSING_TESTS: try: missing_test_errors.remove(exception) except KeyError: pass summary = [] if mismatch_errors: summary.append(('Results of %d test(s) do not match expected ' 'values:') % len(mismatch_errors)) summary.extend(mismatch_errors) summary.append('\n') if extra_test_errors: summary.append('%d test(s) are not expected to be run:' % len(extra_test_errors)) summary.extend(extra_test_errors) summary.append('\n') if missing_test_errors: summary.append('%d test(s) are missing from the results:' % len(missing_test_errors)) summary.extend(missing_test_errors) summary.append('\n') # Test link to log can be loaded. job_name = '%s-%s' % (job_id, getpass.getuser()) log_link = URL_PATTERN % (URL_HOST, job_name) try: urllib2.urlopen(log_link).read() except urllib2.URLError: summary.append('Failed to load page for link to log: %s.' % log_link) if summary: raise TestPushException('\n'.join(summary))
def test_suite(suite_name, expected_results, arguments, use_shard=False, create_and_return=False): """Call run_suite to start a suite job and verify results. @param suite_name: Name of a suite, e.g., dummy @param expected_results: A dictionary of test name to test result. @param arguments: Arguments for run_suite command. @param use_shard: If true, suite is scheduled for shard board. @param create_and_return: If True, run_suite just creates the suite, print the job id, then finish immediately. """ suite_job_id = do_run_suite(suite_name, arguments, use_shard, create_and_return) # Confirm all DUTs used for the suite are imaged to expected build. # hqe.host_id for jobs running in shard is not synced back to master db, # therefore, skip verifying dut build for jobs running in shard. if suite_name != AU_SUITE and not use_shard: check_dut_image(arguments.build, suite_job_id) # Find all tests and their status print 'Comparing test results...' TKO = frontend_wrappers.RetryingTKO(timeout_min=0.1, delay_sec=10) test_views = site_utils.get_test_views_from_tko(suite_job_id, TKO) mismatch_errors = [] extra_test_errors = [] found_keys = set() for test_name,test_status in test_views.items(): print "%s%s" % (test_name.ljust(30), test_status) test_found = False for key,val in expected_results.items(): if re.search(key, test_name): test_found = True found_keys.add(key) # TODO(dshi): result for this test is ignored until servo is # added to a host accessible by cbf server (crbug.com/277109). if key == 'platform_InstallTestImage_SERVER_JOB$': continue if val != test_status: error = ('%s Expected: [%s], Actual: [%s]' % (test_name, val, test_status)) mismatch_errors.append(error) if not test_found: extra_test_errors.append(test_name) missing_test_errors = set(expected_results.keys()) - found_keys for exception in IGNORE_MISSING_TESTS: try: missing_test_errors.remove(exception) except KeyError: pass summary = [] if mismatch_errors: summary.append(('Results of %d test(s) do not match expected ' 'values:') % len(mismatch_errors)) summary.extend(mismatch_errors) summary.append('\n') if extra_test_errors: summary.append('%d test(s) are not expected to be run:' % len(extra_test_errors)) summary.extend(extra_test_errors) summary.append('\n') if missing_test_errors: summary.append('%d test(s) are missing from the results:' % len(missing_test_errors)) summary.extend(missing_test_errors) summary.append('\n') # Test link to log can be loaded. job_name = '%s-%s' % (suite_job_id, getpass.getuser()) log_link = URL_PATTERN % (URL_HOST, job_name) try: urllib2.urlopen(log_link).read() except urllib2.URLError: summary.append('Failed to load page for link to log: %s.' % log_link) if summary: raise TestPushException('\n'.join(summary))