def testTriggerTryJob(self, mock_module):
master_name = 'm'
builder_name = 'b'
build_number = 1
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = {
'properties': {
'parent_mastername': 'pm',
'parent_buildername': 'pb'
}
}
build.put()
response = {
'build': {
'id': '1',
'url': 'url',
'status': 'SCHEDULED',
}
}
results = [(None,
buildbucket_client.BuildbucketBuild(response['build']))]
mock_module.TriggerTryJobs.return_value = results
build_id = ScheduleFlakeTryJobPipeline()._TriggerTryJob(
master_name, builder_name, {}, [],
failure_type.GetDescriptionForFailureType(failure_type.FLAKY_TEST),
None, None)
self.assertEqual(build_id, '1')
def testSuccessfullyScheduleNewTryJobForCompileWithSuspectedRevisions(
self, mock_module):
master_name = 'm'
builder_name = 'b'
build_number = 223
good_revision = 'rev1'
bad_revision = 'rev2'
build_id = '1'
url = 'url'
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = {
'properties': {
'parent_mastername': 'pm',
'parent_buildername': 'pb'
}
}
build.put()
response = {
'build': {
'id': build_id,
'url': url,
'status': 'SCHEDULED',
}
}
results = [(None,
buildbucket_client.BuildbucketBuild(response['build']))]
mock_module.TriggerTryJobs.return_value = results
WfTryJob.Create(master_name, builder_name, build_number).put()
try_job_pipeline = ScheduleCompileTryJobPipeline()
try_job_id = try_job_pipeline.run(master_name, builder_name,
build_number, good_revision,
bad_revision, failure_type.COMPILE,
None, ['r5'], None, None)
try_job = WfTryJob.Get(master_name, builder_name, build_number)
try_job_data = WfTryJobData.Get(build_id)
expected_try_job_id = '1'
self.assertEqual(expected_try_job_id, try_job_id)
self.assertEqual(expected_try_job_id,
try_job.compile_results[-1]['try_job_id'])
self.assertTrue(expected_try_job_id in try_job.try_job_ids)
self.assertIsNotNone(try_job_data)
self.assertEqual(try_job_data.master_name, master_name)
self.assertEqual(try_job_data.builder_name, builder_name)
self.assertEqual(try_job_data.build_number, build_number)
self.assertEqual(
try_job_data.try_job_type,
failure_type.GetDescriptionForFailureType(failure_type.COMPILE))
self.assertFalse(try_job_data.has_compile_targets)
self.assertTrue(try_job_data.has_heuristic_results)
def testUpdateTryJobMetadata(self):
try_job_id = '1'
url = 'url'
build_data = {
'id': try_job_id,
'url': url,
'status': 'COMPLETED',
'completed_ts': '1454367574000000',
'created_ts': '1454367570000000',
}
report = {
'result': {
'rev1': 'passed',
'rev2': 'failed'
},
'metadata': {
'regression_range_size': 2
}
}
build = buildbucket_client.BuildbucketBuild(build_data)
expected_error_dict = {
'message':
'Try job monitoring was abandoned.',
'reason':
('Timeout after %s hours' %
waterfall_config.GetTryJobSettings().get('job_timeout_hours'))
}
try_job_data = WfTryJobData.Create(try_job_id)
try_job_data.try_job_key = WfTryJob.Create('m', 'b', 123).key
monitor_try_job_pipeline._UpdateTryJobMetadata(try_job_data,
failure_type.COMPILE,
build, None, False,
report)
try_job_data = WfTryJobData.Get(try_job_id)
self.assertIsNone(try_job_data.error)
self.assertEqual(try_job_data.regression_range_size, 2)
self.assertEqual(try_job_data.number_of_commits_analyzed, 2)
self.assertEqual(try_job_data.end_time,
datetime(2016, 2, 1, 22, 59, 34))
self.assertEqual(try_job_data.request_time,
datetime(2016, 2, 1, 22, 59, 30))
self.assertEqual(try_job_data.try_job_url, url)
monitor_try_job_pipeline._UpdateTryJobMetadata(try_job_data,
failure_type.COMPILE,
build, None, True)
self.assertEqual(try_job_data.error, expected_error_dict)
self.assertEqual(try_job_data.error_code, try_job_error.TIMEOUT)
def testScheduleFlakeTryJob(self, mock_module):
master_name = 'm'
builder_name = 'b'
build_number = 1
step_name = 's'
test_name = 't'
git_hash = 'a1b2c3d4'
build_id = '1'
url = 'url'
analysis_key = ndb.Key('key', 1)
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = {
'properties': {
'parent_mastername': 'pm',
'parent_buildername': 'pb'
}
}
build.put()
response = {
'build': {
'id': build_id,
'url': url,
'status': 'SCHEDULED',
}
}
results = [(None,
buildbucket_client.BuildbucketBuild(response['build']))]
mock_module.TriggerTryJobs.return_value = results
FlakeTryJob.Create(master_name, builder_name, step_name, test_name,
git_hash).put()
try_job_pipeline = ScheduleFlakeTryJobPipeline()
try_job_id = try_job_pipeline.run(master_name, builder_name, step_name,
test_name, git_hash,
analysis_key.urlsafe(), None, None)
try_job = FlakeTryJob.Get(master_name, builder_name, step_name,
test_name, git_hash)
try_job_data = FlakeTryJobData.Get(build_id)
self.assertEqual(build_id, try_job_id)
self.assertEqual(build_id, try_job.flake_results[-1]['try_job_id'])
self.assertTrue(build_id in try_job.try_job_ids)
self.assertEqual(try_job_data.try_job_key, try_job.key)
self.assertEqual(analysis_key, try_job_data.analysis_key)
def testSuccessfullyScheduleNewTryJobForTest(self, mock_module):
master_name = 'm'
builder_name = 'b'
build_number = 223
good_revision = 'rev1'
bad_revision = 'rev2'
targeted_tests = ['a on platform', ['a', ['test1', 'test2']]]
build_id = '1'
build = WfBuild.Create(master_name, builder_name, build_number)
build.data = {'properties': {'parent_mastername': 'pm',
'parent_buildername': 'pb'}}
build.put()
response = {
'build': {
'id': build_id,
'url': 'url',
'status': 'SCHEDULED',
}
}
results = [(None, buildbucket_client.BuildbucketBuild(response['build']))]
mock_module.TriggerTryJobs.return_value = results
WfTryJob.Create(master_name, builder_name, build_number).put()
try_job_pipeline = ScheduleTestTryJobPipeline()
try_job_id = try_job_pipeline.run(
master_name, builder_name, build_number, good_revision, bad_revision,
failure_type.TEST, None, None, None, targeted_tests)
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(try_job_id, build_id)
self.assertEqual(try_job.test_results[-1]['try_job_id'], build_id)
try_job_data = WfTryJobData.Get(try_job_id)
self.assertIsNotNone(try_job_data)
self.assertEqual(try_job_data.master_name, master_name)
self.assertEqual(try_job_data.builder_name, builder_name)
self.assertEqual(try_job_data.build_number, build_number)
self.assertEqual(
try_job_data.try_job_type,
failure_type.GetDescriptionForFailureType(failure_type.TEST))
self.assertFalse(try_job_data.has_compile_targets)
self.assertFalse(try_job_data.has_heuristic_results)
def testGetTryJobsForCompileSuccessSwarming(self, mock_buildbucket,
mock_report):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
try_job_url = 'https://luci-milo.appspot.com/swarming/task/3595be5002f4bc10'
regression_range_size = 2
try_job = WfTryJob.Create(master_name, builder_name, build_number)
try_job_data = WfTryJobData.Create(try_job_id)
try_job_data.try_job_key = try_job.key
try_job_data.try_job_url = try_job_url
try_job_data.put()
try_job.compile_results = [{
'report': None,
'url': try_job_url,
'try_job_id': '1',
}]
try_job.status = analysis_status.RUNNING
try_job.put()
build_response = {
'id': '1',
'url': try_job_url,
'status': 'COMPLETED',
}
report = {
'result': {
'rev1': 'passed',
'rev2': 'failed'
},
'metadata': {
'regression_range_size': 2
}
}
mock_buildbucket.GetTryJobs.return_value = [
(None, buildbucket_client.BuildbucketBuild(build_response))
]
mock_report.return_value = json.dumps(report)
pipeline = MonitorTryJobPipeline()
pipeline.start_test()
pipeline.run(try_job.key.urlsafe(), failure_type.COMPILE, try_job_id)
pipeline.callback(callback_params=pipeline.last_params)
# Reload from ID to get all internal properties in sync.
pipeline = MonitorTryJobPipeline.from_id(pipeline.pipeline_id)
pipeline.finalized()
compile_result = pipeline.outputs.default.value
expected_compile_result = {
'report': {
'result': {
'rev1': 'passed',
'rev2': 'failed'
},
'metadata': {
'regression_range_size': regression_range_size
}
},
'url': try_job_url,
'try_job_id': '1',
}
self.assertEqual(expected_compile_result, compile_result)
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(expected_compile_result, try_job.compile_results[-1])
self.assertEqual(analysis_status.RUNNING, try_job.status)
try_job_data = WfTryJobData.Get(try_job_id)
self.assertEqual(try_job_data.regression_range_size,
regression_range_size)
def testGetTryJobsForCompileSuccessBackwardCompatibleCallback(
self, mock_buildbucket, mock_report):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '1'
regression_range_size = 2
try_job = WfTryJob.Create(master_name, builder_name, build_number)
try_job_data = WfTryJobData.Create(try_job_id)
try_job_data.try_job_key = try_job.key
try_job_data.try_job_url = (
'https://build.chromium.org/p/m/builders/b/builds/1234')
try_job_data.put()
try_job.compile_results = [{
'report': None,
'url': 'https://build.chromium.org/p/m/builders/b/builds/1234',
'try_job_id': '1',
}]
try_job.status = analysis_status.RUNNING
try_job.put()
report = {
'result': {
'rev1': 'passed',
'rev2': 'failed'
},
'metadata': {
'regression_range_size': 2
}
}
build_response = {
'id': '1',
'url': 'https://build.chromium.org/p/m/builders/b/builds/1234',
'status': 'COMPLETED',
'completed_ts': '1454367574000000',
'created_ts': '1454367570000000',
'updated_ts': '1454367574000000',
}
mock_buildbucket.GetTryJobs.return_value = [
(None, buildbucket_client.BuildbucketBuild(build_response))
]
mock_report.return_value = json.dumps(report)
pipeline = MonitorTryJobPipeline(try_job.key.urlsafe(),
failure_type.COMPILE, try_job_id)
pipeline.start_test()
pipeline.run(try_job.key.urlsafe(), failure_type.COMPILE, try_job_id)
pipeline.callback(**pipeline.last_params)
# Reload from ID to get all internal properties in sync.
pipeline = MonitorTryJobPipeline.from_id(pipeline.pipeline_id)
pipeline.finalized()
compile_result = pipeline.outputs.default.value
expected_compile_result = {
'report': {
'result': {
'rev1': 'passed',
'rev2': 'failed'
},
'metadata': {
'regression_range_size': regression_range_size
}
},
'url': 'https://build.chromium.org/p/m/builders/b/builds/1234',
'try_job_id': '1',
}
self.assertEqual(expected_compile_result, compile_result)
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(expected_compile_result, try_job.compile_results[-1])
self.assertEqual(analysis_status.RUNNING, try_job.status)
try_job_data = WfTryJobData.Get(try_job_id)
self.assertEqual(try_job_data.regression_range_size,
regression_range_size)
self.assertIsInstance(try_job_data.start_time, datetime)
def testGetTryJobsForFlakeSuccess(self, mock_buildbucket, mock_report):
master_name = 'm'
builder_name = 'b'
step_name = 's'
test_name = 't'
git_hash = 'a1b2c3d4'
try_job_id = '1'
try_job = FlakeTryJob.Create(master_name, builder_name, step_name,
test_name, git_hash)
try_job.flake_results = [{
'report': None,
'url': 'https://build.chromium.org/p/m/builders/b/builds/1234',
'try_job_id': '1',
}]
try_job.status = analysis_status.RUNNING
try_job.put()
try_job_data = FlakeTryJobData.Create(try_job_id)
try_job_data.try_job_key = try_job.key
try_job_data.try_job_url = (
'https://build.chromium.org/p/m/builders/b/builds/1234')
try_job_data.put()
build_response = {
'id': '1',
'url': 'https://build.chromium.org/p/m/builders/b/builds/1234',
'status': 'COMPLETED',
}
report = {
'result': {
'r0': {
'gl_tests': {
'status': 'passed',
'valid': True,
'pass_fail_counts': {
'Test.One': {
'pass_count': 100,
'fail_count': 0
}
}
}
}
}
}
mock_buildbucket.GetTryJobs.return_value = [
(None, buildbucket_client.BuildbucketBuild(build_response))
]
mock_report.return_value = json.dumps(report)
pipeline = MonitorTryJobPipeline()
pipeline.start_test()
pipeline.run(try_job.key.urlsafe(), failure_type.FLAKY_TEST,
try_job_id)
pipeline.callback(callback_params=pipeline.last_params)
# Reload from ID to get all internal properties in sync.
pipeline = MonitorTryJobPipeline.from_id(pipeline.pipeline_id)
pipeline.finalized()
flake_result = pipeline.outputs.default.value
expected_flake_result = {
'report': {
'result': {
'r0': {
'gl_tests': {
'status': 'passed',
'valid': True,
'pass_fail_counts': {
'Test.One': {
'pass_count': 100,
'fail_count': 0
}
}
}
}
}
},
'url': 'https://build.chromium.org/p/m/builders/b/builds/1234',
'try_job_id': '1',
}
self.assertEqual(expected_flake_result, flake_result)
try_job = FlakeTryJob.Get(master_name, builder_name, step_name,
test_name, git_hash)
self.assertEqual(expected_flake_result, try_job.flake_results[-1])
self.assertEqual(analysis_status.RUNNING, try_job.status)
try_job_data = FlakeTryJobData.Get(try_job_id)
self.assertEqual(try_job_data.last_buildbucket_response,
build_response)
def testGetTryJobsForTestSuccess(self, mock_buildbucket, mock_report):
master_name = 'm'
builder_name = 'b'
build_number = 1
try_job_id = '3'
try_job = WfTryJob.Create(master_name, builder_name, build_number)
try_job.test_results = [{
'report': None,
'url': 'https://build.chromium.org/p/m/builders/b/builds/1234',
'try_job_id': try_job_id,
}]
try_job.status = analysis_status.RUNNING
try_job.put()
try_job_data = WfTryJobData.Create(try_job_id)
try_job_data.try_job_key = try_job.key
try_job_data.try_job_url = (
'https://build.chromium.org/p/m/builders/b/builds/1234')
try_job_data.put()
data = [{
'build': {
'id': '3',
'url': 'https://build.chromium.org/p/m/builders/b/builds/1234',
'status': 'STARTED'
}
}, {
'error': {
'reason': 'BUILD_NOT_FOUND',
'message': 'message',
}
}, {
'build': {
'id': '3',
'url': 'https://build.chromium.org/p/m/builders/b/builds/1234',
'status': 'STARTED'
}
}, {
'error': {
'reason': 'BUILD_NOT_FOUND',
'message': 'message',
}
}, {
'build': {
'id': '3',
'url': 'https://build.chromium.org/p/m/builders/b/builds/1234',
'status': 'COMPLETED',
}
}]
report = {
'result': {
'rev1': {
'a_test': {
'status': 'passed',
'valid': True
}
},
'rev2': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['test1', 'test2']
}
}
}
}
get_tryjobs_responses = [
[(None, buildbucket_client.BuildbucketBuild(data[0]['build']))],
[(buildbucket_client.BuildbucketError(data[1]['error']), None)],
[(None, buildbucket_client.BuildbucketBuild(data[2]['build']))],
[(buildbucket_client.BuildbucketError(data[3]['error']), None)],
[(None, buildbucket_client.BuildbucketBuild(data[4]['build']))],
]
mock_buildbucket.GetTryJobs.side_effect = get_tryjobs_responses
mock_report.return_value = json.dumps(report)
pipeline = MonitorTryJobPipeline()
pipeline.start_test()
pipeline.run(try_job.key.urlsafe(), failure_type.TEST, try_job_id)
pipeline.run(try_job.key.urlsafe(), failure_type.TEST, try_job_id)
# Since run() calls callback() immediately, we use -1.
for _ in range(len(get_tryjobs_responses) - 1):
pipeline.callback(callback_params=pipeline.last_params)
# Reload from ID to get all internal properties in sync.
pipeline = MonitorTryJobPipeline.from_id(pipeline.pipeline_id)
pipeline.finalized()
test_result = pipeline.outputs.default.value
expected_test_result = {
'report': {
'result': {
'rev1': {
'a_test': {
'status': 'passed',
'valid': True
}
},
'rev2': {
'a_test': {
'status': 'failed',
'valid': True,
'failures': ['test1', 'test2']
}
}
}
},
'url': 'https://build.chromium.org/p/m/builders/b/builds/1234',
'try_job_id': '3',
}
self.assertEqual(expected_test_result, test_result)
try_job = WfTryJob.Get(master_name, builder_name, build_number)
self.assertEqual(expected_test_result, try_job.test_results[-1])
self.assertEqual(analysis_status.RUNNING, try_job.status)
