def testStopLookingBackIfFindTheFirstBuild(self, mock_fn): master_name = 'm' builder_name = 'b' build_number = 2 self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number, analysis_status.RUNNING) # Setup build data for builds: mock_fn.side_effect = [ self._GetBuildData(master_name, builder_name, 2), self._GetBuildData(master_name, builder_name, 1), self._GetBuildData(master_name, builder_name, 0) ] pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'a_tests': { 'current_failure': 2, 'first_failure': 0 }, 'unit_tests': { 'current_failure': 2, 'first_failure': 0 } } self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testStopLookingBackIfAllFailedStepsPassedInLastBuild(self, mock_fn): master_name = 'm' builder_name = 'b' build_number = 124 self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number, analysis_status.RUNNING) # Setup build data for builds: mock_fn.side_effect = [ self._GetBuildData(master_name, builder_name, 124), self._GetBuildData(master_name, builder_name, 123) ] pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'a': { 'last_pass': 123, 'current_failure': 124, 'first_failure': 124 } } self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testFirstFailureLastPassUpdating(self): """last pass always should just be updated once.""" master_name = 'm' builder_name = 'b' build_number = 100 self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number, wf_analysis_status.ANALYZING) # Setup build data for builds: # 100: net_unitests failed, unit_tests failed. # 99: net_unitests passed, unit_tests failed. # 98: net_unitests passed, unit_tests failed. # 97: net_unitests failed, unit_tests failed. # 96: net_unitests passed, unit_tests passed. for i in range(5): self._MockUrlfetchWithBuildData(master_name, builder_name, 100 - i) pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'net_unittests': { 'last_pass': 99, 'current_failure': 100, 'first_failure': 100 }, 'unit_tests': { 'last_pass': 96, 'current_failure': 100, 'first_failure': 97 } } self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testStopLookingBackIfFindTheFirstBuild(self): master_name = 'm' builder_name = 'b' build_number = 2 self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number, wf_analysis_status.ANALYZING) # Setup build data for builds: self._MockUrlfetchWithBuildData(master_name, builder_name, 2) self._MockUrlfetchWithBuildData(master_name, builder_name, 1) self._MockUrlfetchWithBuildData(master_name, builder_name, 0) pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'a_tests': { 'current_failure': 2, 'first_failure': 0 }, 'unit_tests': { 'current_failure': 2, 'first_failure': 0 } } self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testStopLookingBackIfAllFailedStepsPassedInLastBuild(self): master_name = 'm' builder_name = 'b' build_number = 124 self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number, wf_analysis_status.ANALYZING) # Setup build data for builds: self._MockUrlfetchWithBuildData(master_name, builder_name, 124) self._MockUrlfetchWithBuildData(master_name, builder_name, 123, build_data=None, archive=True) self._MockUrlfetchWithBuildData(master_name, builder_name, 122, build_data='Blow up if used!') pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'a': { 'last_pass': 123, 'current_failure': 124, 'first_failure': 124 } } self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testStopLookingBackIfFindTheFirstBuild(self): master_name = 'm' builder_name = 'b' build_number = 2 self._CreateAndSaveWfAnanlysis( master_name, builder_name, build_number, wf_analysis_status.ANALYZING) # Setup build data for builds: self._MockUrlfetchWithBuildData(master_name, builder_name, 2) self._MockUrlfetchWithBuildData(master_name, builder_name, 1) self._MockUrlfetchWithBuildData(master_name, builder_name, 0) pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'a_tests': { 'current_failure': 2, 'first_failure': 0 }, 'unit_tests': { 'current_failure': 2, 'first_failure': 0 } } self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testStopLookingBackIfAllFailedStepsPassedInLastBuild(self): master_name = 'm' builder_name = 'b' build_number = 124 self._CreateAndSaveWfAnanlysis( master_name, builder_name, build_number, wf_analysis_status.ANALYZING) # Setup build data for builds: self._MockUrlfetchWithBuildData(master_name, builder_name, 124) self._MockUrlfetchWithBuildData(master_name, builder_name, 123, build_data=None, archive=True) self._MockUrlfetchWithBuildData( master_name, builder_name, 122, build_data='Blow up if used!') pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'a': { 'last_pass': 123, 'current_failure': 124, 'first_failure': 124 } } self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testFirstFailureLastPassUpdating(self): """last pass always should just be updated once.""" master_name = 'm' builder_name = 'b' build_number = 100 self._CreateAndSaveWfAnanlysis( master_name, builder_name, build_number, wf_analysis_status.ANALYZING) # Setup build data for builds: # 100: net_unitests failed, unit_tests failed. # 99: net_unitests passed, unit_tests failed. # 98: net_unitests passed, unit_tests failed. # 97: net_unitests failed, unit_tests failed. # 96: net_unitests passed, unit_tests passed. for i in range(5): self._MockUrlfetchWithBuildData(master_name, builder_name, 100 - i) pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'net_unittests': { 'last_pass': 99, 'current_failure': 100, 'first_failure': 100 }, 'unit_tests': { 'last_pass': 96, 'current_failure': 100, 'first_failure': 97 } } self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testAnalyzeSuccessfulBuild(self, mock_fn): master_name = 'm' builder_name = 'b' build_number = 121 self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number, analysis_status.RUNNING) # Setup build data for builds: mock_fn.return_value = self._GetBuildData(master_name, builder_name, 121) pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) self.assertFalse(failure_info['failed'])
def testAnalyzeInfraExceptionBuild(self, mock_fn): master_name = 'm' builder_name = 'b' build_number = 120 self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number, analysis_status.RUNNING) # Setup build data for builds: mock_fn.return_value = self._GetBuildData(master_name, builder_name, 120) pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) self.assertEqual(failure_info['failure_type'], failure_type.INFRA)
def testAnalyzeSuccessfulBuild(self): master_name = 'm' builder_name = 'b' build_number = 121 self._CreateAndSaveWfAnanlysis( master_name, builder_name, build_number, wf_analysis_status.ANALYZING) # Setup build data for builds: self._MockUrlfetchWithBuildData(master_name, builder_name, 121) self._MockUrlfetchWithBuildData( master_name, builder_name, 120, build_data='Blow up if used!') pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) self.assertFalse(failure_info['failed'])
def testLookBackUntilGreenBuild(self): master_name = 'm' builder_name = 'b' build_number = 123 self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number, wf_analysis_status.ANALYZING) # Setup build data for builds: # 123: mock urlfetch to ensure it is fetched. self._MockUrlfetchWithBuildData(master_name, builder_name, 123) # 122: mock a build in datastore to ensure it is not fetched again. build = WfBuild.Create(master_name, builder_name, 122) build.data = self._GetBuildData(master_name, builder_name, 122) build.completed = True build.put() self._MockUrlfetchWithBuildData(master_name, builder_name, 122, build_data='Blow up if used!') # 121: mock a build in datastore to ensure it is updated. build = WfBuild.Create(master_name, builder_name, 121) build.data = 'Blow up if used!' build.last_crawled_time = self._TimeBeforeNowBySeconds(7200) build.completed = False build.put() self._MockUrlfetchWithBuildData(master_name, builder_name, 121) pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'net_unittests': { 'last_pass': 122, 'current_failure': 123, 'first_failure': 123 }, 'unit_tests': { 'last_pass': 121, 'current_failure': 123, 'first_failure': 122 } } self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testRunPipelineForCompileFailure(self): def _MockExtractBuildInfo(*_): build_info = BuildInfo('m', 'b', 25409) build_info.failed_steps = { 'compile': { 'last_pass': '******', 'current_failure': '25409', 'first_failure': '25409' } } return build_info self.mock(DetectFirstFailurePipeline, '_ExtractBuildInfo', _MockExtractBuildInfo) self._CreateAndSaveWfAnanlysis('m', 'b', 25409, analysis_status.RUNNING) pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run('m', 'b', 25409) expected_failure_info = { 'failed': True, 'master_name': 'm', 'builder_name': 'b', 'build_number': 25409, 'chromium_revision': None, 'builds': { 25409: { 'blame_list': [], 'chromium_revision': None } }, 'failed_steps': { 'compile': { 'current_failure': 25409, 'first_failure': 25409 } }, 'failure_type': failure_type.COMPILE, 'parent_mastername': None, 'parent_buildername': None, } self.assertEqual(failure_info, expected_failure_info)
def testLookBackUntilGreenBuild(self): master_name = 'm' builder_name = 'b' build_number = 123 self._CreateAndSaveWfAnanlysis( master_name, builder_name, build_number, wf_analysis_status.ANALYZING) # Setup build data for builds: # 123: mock urlfetch to ensure it is fetched. self._MockUrlfetchWithBuildData(master_name, builder_name, 123) # 122: mock a build in datastore to ensure it is not fetched again. build = WfBuild.Create(master_name, builder_name, 122) build.data = self._GetBuildData(master_name, builder_name, 122) build.completed = True build.put() self._MockUrlfetchWithBuildData( master_name, builder_name, 122, build_data='Blow up if used!') # 121: mock a build in datastore to ensure it is updated. build = WfBuild.Create(master_name, builder_name, 121) build.data = 'Blow up if used!' build.last_crawled_time = self._TimeBeforeNowBySeconds(7200) build.completed = False build.put() self._MockUrlfetchWithBuildData(master_name, builder_name, 121) pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'net_unittests': { 'last_pass': 122, 'current_failure': 123, 'first_failure': 123 }, 'unit_tests': { 'last_pass': 121, 'current_failure': 123, 'first_failure': 122 } } self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testAnalyzeSuccessfulBuild(self): master_name = 'm' builder_name = 'b' build_number = 121 self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number, wf_analysis_status.ANALYZING) # Setup build data for builds: self._MockUrlfetchWithBuildData(master_name, builder_name, 121) self._MockUrlfetchWithBuildData(master_name, builder_name, 120, build_data='Blow up if used!') pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) self.assertFalse(failure_info['failed'])
def testFirstFailureLastPassUpdating(self, mock_fn): """last pass always should just be updated once.""" master_name = 'm' builder_name = 'b' build_number = 100 self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number, analysis_status.RUNNING) # Setup build data for builds: # 100: net_unitests failed, unit_tests failed. # 99: net_unitests passed, unit_tests failed. # 98: net_unitests passed, unit_tests failed. # 97: net_unitests failed, unit_tests failed. # 96: net_unitests passed, unit_tests passed. side_effects = [] for i in range(5): side_effects.append( self._GetBuildData(master_name, builder_name, 100 - i)) mock_fn.side_effect = side_effects pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'net_unittests': { 'last_pass': 99, 'current_failure': 100, 'first_failure': 100 }, 'unit_tests': { 'last_pass': 96, 'current_failure': 100, 'first_failure': 97 } } self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testTestLevelFailedInfo(self): master_name = 'm' builder_name = 'b' build_number = 223 self._CreateAndSaveWfAnanlysis( master_name, builder_name, build_number, analysis_status.RUNNING) # Mock data for retrieving data from swarming server for a build. self._MockUrlFetchWithSwarmingData(master_name, builder_name, 223) for n in xrange(223, 219, -1): # pragma: no cover # Setup build data for builds: self._MockUrlfetchWithBuildData(master_name, builder_name, n) if n == 220: break # Mock data for retrieving data from swarming server for a single step. self._MockUrlFetchWithSwarmingData( master_name, builder_name, n, 'abc_test') # Mock data for retrieving hash to output.json from isolated server. isolated_data = { 'isolatedserver': 'https://isolateserver.appspot.com', 'namespace': { 'namespace': 'default-gzip' }, 'digest': 'isolatedhashabctest-%d' % n } self._MockUrlfetchWithIsolatedData( isolated_data, build_number=n) # Mock data for retrieving url to output.json from isolated server. file_hash_data = { 'isolatedserver': 'https://isolateserver.appspot.com', 'namespace': { 'namespace': 'default-gzip' }, 'digest': 'abctestoutputjsonhash-%d' % n } self._MockUrlfetchWithIsolatedData( file_hash_data, build_number=n) # Mock data for downloading output.json from isolated server. self._MockUrlfetchWithIsolatedData( None, ('https://isolateserver.storage.googleapis.com/default-gzip/' 'm_b_%d_abc_test' % n), '%s_%s_%d_%s.json' % (master_name, builder_name, n, 'abc_test')) step_221 = WfStep.Create(master_name, builder_name, 221, 'abc_test') step_221.isolated = True step_221.log_data = ( '{"Unittest3.Subtest3": "YS9iL3UzczIuY2M6MTEwOiBGYWlsdXJlCg=="}') step_221.put() pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'compile': { 'current_failure': 223, 'first_failure': 221, 'last_pass': 220 }, 'abc_test': { 'current_failure': 223, 'first_failure': 222, 'last_pass': 221, 'list_isolated_data': [ { 'isolatedserver': 'https://isolateserver.appspot.com', 'namespace': 'default-gzip', 'digest': 'isolatedhashabctest-223' } ], 'tests': { 'Unittest2.Subtest1': { 'current_failure': 223, 'first_failure': 222, 'last_pass': 221 }, 'Unittest3.Subtest2': { 'current_failure': 223, 'first_failure': 222, 'last_pass': 221 } } } } expected_step_log_data = { 223: ('{"Unittest2.Subtest1": "RVJST1I6eF90ZXN0LmNjOjEyMzRcbmEvYi91Mn' 'MxLmNjOjU2NzogRmFpbHVyZVxuRVJST1I6WzJdOiAyNTk0NzM1MDAwIGJvZ28tb' 'Wljcm9zZWNvbmRzXG5FUlJPUjp4X3Rlc3QuY2M6MTIzNAphL2IvdTJzMS5jYzo1' 'Njc6IEZhaWx1cmUK", ' '"Unittest3.Subtest2": "YS9iL3UzczIuY2M6MTEwOiBGYWlsdXJlCg=="}'), 222: ('{"Unittest2.Subtest1": "RVJST1I6eF90ZXN0LmNjOjEyMzRcbmEvYi91Mn' 'MxLmNjOjU2NzogRmFpbHVyZVxuRVJST1I6WzJdOiAyNTk0NzM1MDAwIGJvZ28tb' 'Wljcm9zZWNvbmRzXG5FUlJPUjp4X3Rlc3QuY2M6MTIzNAphL2IvdTJzMS5jYzo1' 'Njc6IEZhaWx1cmUK", ' '"Unittest3.Subtest2": "YS9iL3UzczIuY2M6MTEwOiBGYWlsdXJlCg=="}'), 221: '{"Unittest3.Subtest3": "YS9iL3UzczIuY2M6MTEwOiBGYWlsdXJlCg=="}' } for n in xrange(223, 220, -1): step = WfStep.Get(master_name, builder_name, n, 'abc_test') self.assertIsNotNone(step) self.assertTrue(step.isolated) self.assertEqual(expected_step_log_data[n], step.log_data) self.assertEqual(expected_failed_steps, failure_info['failed_steps'])
def testTestLevelFailedInfo(self, mock_fn): master_name = 'm' builder_name = 'b' build_number = 223 self._CreateAndSaveWfAnanlysis(master_name, builder_name, build_number, analysis_status.RUNNING) # Mock data for retrieving data from swarming server for a build. self._MockUrlFetchWithSwarmingData(master_name, builder_name, 223) mock_fn.side_effect = [ self._GetBuildData(master_name, builder_name, 223), self._GetBuildData(master_name, builder_name, 222), self._GetBuildData(master_name, builder_name, 221), self._GetBuildData(master_name, builder_name, 220) ] for n in xrange(223, 219, -1): # pragma: no branch. # Setup build data for builds: if n == 220: break # Mock data for retrieving data from swarming server for a single step. self._MockUrlFetchWithSwarmingData(master_name, builder_name, n, 'abc_test') # Mock data for retrieving hash to output.json from isolated server. isolated_data = { 'isolatedserver': 'https://isolateserver.appspot.com', 'namespace': { 'namespace': 'default-gzip' }, 'digest': 'isolatedhashabctest-%d' % n } self._MockUrlfetchWithIsolatedData(isolated_data, build_number=n) # Mock data for retrieving url to output.json from isolated server. file_hash_data = { 'isolatedserver': 'https://isolateserver.appspot.com', 'namespace': { 'namespace': 'default-gzip' }, 'digest': 'abctestoutputjsonhash-%d' % n } self._MockUrlfetchWithIsolatedData(file_hash_data, build_number=n) # Mock data for downloading output.json from isolated server. self._MockUrlfetchWithIsolatedData( None, ('https://isolateserver.storage.googleapis.com/default-gzip/' 'm_b_%d_abc_test' % n), '%s_%s_%d_%s.json' % (master_name, builder_name, n, 'abc_test')) step_221 = WfStep.Create(master_name, builder_name, 221, 'abc_test') step_221.isolated = True step_221.log_data = ( '{"Unittest3.Subtest3": "YS9iL3UzczIuY2M6MTEwOiBGYWlsdXJlCg=="}') step_221.put() pipeline = DetectFirstFailurePipeline() failure_info = pipeline.run(master_name, builder_name, build_number) expected_failed_steps = { 'abc_test': { 'current_failure': 223, 'first_failure': 222, 'last_pass': 221, 'list_isolated_data': [{ 'isolatedserver': 'https://isolateserver.appspot.com', 'namespace': 'default-gzip', 'digest': 'isolatedhashabctest-223' }], 'tests': { 'Unittest2.Subtest1': { 'current_failure': 223, 'first_failure': 222, 'last_pass': 221, 'base_test_name': 'Unittest2.Subtest1' }, 'Unittest3.Subtest2': { 'current_failure': 223, 'first_failure': 222, 'last_pass': 221, 'base_test_name': 'Unittest3.Subtest2' } } } } expected_step_log_data = { 223: ('{"Unittest2.Subtest1": "RVJST1I6eF90ZXN0LmNjOjEyMzRcbmEvYi91Mn' 'MxLmNjOjU2NzogRmFpbHVyZVxuRVJST1I6WzJdOiAyNTk0NzM1MDAwIGJvZ28tb' 'Wljcm9zZWNvbmRzXG5FUlJPUjp4X3Rlc3QuY2M6MTIzNAphL2IvdTJzMS5jYzo1' 'Njc6IEZhaWx1cmUK", ' '"Unittest3.Subtest2": "YS9iL3UzczIuY2M6MTEwOiBGYWlsdXJlCg=="}'), 222: ('{"Unittest2.Subtest1": "RVJST1I6eF90ZXN0LmNjOjEyMzRcbmEvYi91Mn' 'MxLmNjOjU2NzogRmFpbHVyZVxuRVJST1I6WzJdOiAyNTk0NzM1MDAwIGJvZ28tb' 'Wljcm9zZWNvbmRzXG5FUlJPUjp4X3Rlc3QuY2M6MTIzNAphL2IvdTJzMS5jYzo1' 'Njc6IEZhaWx1cmUK", ' '"Unittest3.Subtest2": "YS9iL3UzczIuY2M6MTEwOiBGYWlsdXJlCg=="}'), 221: '{"Unittest3.Subtest3": "YS9iL3UzczIuY2M6MTEwOiBGYWlsdXJlCg=="}' } for n in xrange(223, 220, -1): step = WfStep.Get(master_name, builder_name, n, 'abc_test') self.assertIsNotNone(step) self.assertTrue(step.isolated) self.assertEqual(expected_step_log_data[n], step.log_data) self.assertEqual(expected_failed_steps, failure_info['failed_steps'])