def testCollectSwarmingTaskResultsError(self, mock_monitoring, *_):
master_name = 'm'
builder_name = 'b'
build_number = 15
params = CollectSwarmingTaskResultsInputs(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
build_completed=True)
steps = ['step1', 'step2']
task1 = WfSwarmingTask.Create(master_name, builder_name, build_number,
'step1')
task1.status = analysis_status.ERROR
task1.put()
task2 = WfSwarmingTask.Create(master_name, builder_name, build_number,
'step2')
task2.status = analysis_status.COMPLETED
task2.tests_statuses = {
'TestSuite1.test1': {
'total_run': 2,
'SUCCESS': 2
},
'TestSuite1.test2': {
'total_run': 4,
'SUCCESS': 2,
'FAILURE': 2
},
'TestSuite1.test3': {
'total_run': 6,
'FAILURE': 6
},
'TestSuite1.test4': {
'total_run': 6,
'SKIPPED': 6
},
'TestSuite1.test5': {
'total_run': 6,
'UNKNOWN': 6
}
}
task2.put()
expected_result_json = {
'consistent_failures': {
'step2': ['TestSuite1.test3', 'TestSuite1.test4']
}
}
self.assertEqual(
CollectSwarmingTaskResultsOutputs.FromSerializable(
expected_result_json),
test_swarming.GetConsistentFailuresWhenAllTasksComplete(
params, steps))
mock_monitoring.assert_called_once_with('step2', 'step2', 'skip', 1)
def testWaitingForTheTaskId(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
step_name = 's'
swarming_task = WfSwarmingTask.Create(master_name, builder_name,
build_number, step_name)
swarming_task.status = analysis_status.PENDING
swarming_task.put()
def MockedSleep(*_):
swarming_task = WfSwarmingTask.Get(master_name, builder_name,
build_number, step_name)
self.assertEqual(analysis_status.PENDING, swarming_task.status)
swarming_task.status = analysis_status.RUNNING
swarming_task.task_id = 'task_id'
swarming_task.put()
self.mock(time, 'sleep', MockedSleep)
self.assertEqual(
'task_id',
swarmed_test_util.GetTaskIdFromSwarmingTaskEntity(
swarming_task.key.urlsafe()))
def testGetTaskIdFromSwarmingTaskEntityNoTask(self):
swarming_task = WfSwarmingTask.Create('m', 'b', 200, 's')
swarming_task.put()
key = swarming_task.key.urlsafe()
swarming_task.key.delete()
with self.assertRaises(Exception):
swarmed_test_util.GetTaskIdFromSwarmingTaskEntity(key)
def testOnSwarmingTaskTimeout(self, mock_mon, _):
master_name = 'm'
builder_name = 'b'
build_number = 16
step_name = 's'
swarming_task = WfSwarmingTask.Create(master_name, builder_name,
build_number, step_name)
swarming_task.put()
parameters = RunSwarmingTaskParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
step_name=step_name,
tests=['tests'])
test_swarming.OnSwarmingTaskTimeout(parameters, 'task_id')
swarming_task = WfSwarmingTask.Get(master_name, builder_name,
build_number, step_name)
self.assertEqual(analysis_status.ERROR, swarming_task.status)
self.assertEqual(
{
'code': swarming_task_error.RUNNER_TIMEOUT,
'message': 'Runner to run swarming task timed out'
}, swarming_task.error)
mock_mon.assert_called_once_with(master_name, builder_name,
build_number, step_name,
analysis_status.ERROR,
analysis_approach_type.SWARMING)
def testOnSwarmingTaskCompleted(self, mock_mon, *_):
master_name = 'm'
builder_name = 'b'
build_number = 13
step_name = 's'
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
data = {
'state': constants.STATE_COMPLETED,
'created_ts': '2015-07-30T18:11:16.743220',
'started_ts': '2015-07-30T18:12:16.743220',
'completed_ts': '2015-07-30T18:15:16.743220'
}
test_swarming.OnSwarmingTaskCompleted(master_name, builder_name,
build_number, step_name, data,
'output_json')
swarming_task = WfSwarmingTask.Get(master_name, builder_name,
build_number, step_name)
self.assertEqual({}, swarming_task.tests_statuses)
self.assertEqual(analysis_status.COMPLETED, swarming_task.status)
self.assertEqual(datetime.datetime(2015, 7, 30, 18, 11, 16, 743220),
swarming_task.created_time)
self.assertEqual(datetime.datetime(2015, 7, 30, 18, 12, 16, 743220),
swarming_task.started_time)
self.assertEqual(datetime.datetime(2015, 7, 30, 18, 15, 16, 743220),
swarming_task.completed_time)
mock_mon.assert_called_once_with(master_name, builder_name,
build_number, step_name,
analysis_status.COMPLETED,
analysis_approach_type.SWARMING)
def testProcessSwarmingTaskResultPipelineSerializedCallback(self, _):
# End to end test.
task = WfSwarmingTask.Create(self.master_name, self.builder_name,
self.build_number, self.step_name)
task.task_id = 'task_id1'
task.put()
pipeline = ProcessSwarmingTaskResultPipeline()
pipeline.start_test()
pipeline.run(self.master_name, self.builder_name, self.build_number,
self.step_name)
pipeline.callback(callback_params=json.dumps(pipeline.last_params))
# Reload from ID to get all internal properties in sync.
pipeline = ProcessSwarmingTaskResultPipeline.from_id(
pipeline.pipeline_id)
pipeline.finalized()
step_name, task_info = pipeline.outputs.default.value
self.assertEqual(self.step_name, step_name)
self.assertEqual('abc_tests', task_info[0])
self.assertEqual(_EXPECTED_CLASSIFIED_TESTS['reliable_tests'],
task_info[1])
task = WfSwarmingTask.Get(self.master_name, self.builder_name,
self.build_number, self.step_name)
self.assertEqual(analysis_status.COMPLETED, task.status)
self.assertEqual(_EXPECTED_TESTS_STATUS, task.tests_statuses)
self.assertEqual(_EXPECTED_CLASSIFIED_TESTS, task.classified_tests)
self.assertEqual(datetime.datetime(2016, 2, 10, 18, 32, 6, 538220),
task.created_time)
self.assertEqual(datetime.datetime(2016, 2, 10, 18, 32, 9, 90550),
task.started_time)
self.assertEqual(datetime.datetime(2016, 2, 10, 18, 33, 9),
task.completed_time)
def testSwarmingPushMissingCallback(self, logging_mock):
task = WfSwarmingTask.Create('m', 'b', 1, 'test')
task.task_id = '12345'
task.put()
# This should not break, so that pubsub does not keep retrying. We'll only
# log a message.
self.test_app.post(
'/pubsub/swarmingpush',
params={
'data':
json.dumps({
'message': {
'attributes': {
'auth_token':
pubsub_callback.GetVerificationToken(),
},
'data':
base64.b64encode(
json.dumps({
'task_id':
'12345',
'userdata':
json.dumps({
'Message-Type':
'SwarmingTaskStatusChange',
}),
})),
},
}),
'format':
'json',
})
self.assertTrue(logging_mock.called)
def testTriggerFlakeAnalysesPipeline(self):
master_name = 'm'
builder_name = 'b'
build_number = 2
step_name = 'a_tests'
test_name = 'Unittest1.Subtest1'
analysis = WfAnalysis.Create(master_name, builder_name, build_number)
analysis.failure_result_map = {
step_name: {
test_name:
'%s/%s/%s' % (master_name, builder_name, build_number)
}
}
analysis.put()
swarming_task = WfSwarmingTask.Create(master_name, builder_name,
build_number, step_name)
swarming_task.tests_statuses = {test_name: {'SUCCESS': 1}}
swarming_task.put()
with mock.patch.object(flake_analysis_service,
'ScheduleAnalysisForFlake') as (
mocked_ScheduleAnalysisForFlake):
pipeline = TriggerFlakeAnalysesPipeline()
pipeline.run(master_name, builder_name, build_number)
mocked_ScheduleAnalysisForFlake.assert_called_once()
def testProcessSwarmingTaskResultPipelineTaskNotRunning(self):
task = WfSwarmingTask.Create(self.master_name, self.builder_name,
self.build_number, self.step_name)
task.task_id = 'task_id2'
task.put()
pipeline = ProcessSwarmingTaskResultPipeline()
pipeline.start_test()
pipeline.run(self.master_name, self.builder_name, self.build_number,
self.step_name)
pipeline.callback(callback_params=pipeline.last_params)
# Reload from ID to get all internal properties in sync.
pipeline = ProcessSwarmingTaskResultPipeline.from_id(
pipeline.pipeline_id)
step_name, task_info = pipeline.outputs.default.value
self.assertEqual(self.step_name, step_name)
self.assertIsNone(task_info[0])
self.assertEqual([], task_info[1])
task = WfSwarmingTask.Get(self.master_name, self.builder_name,
self.build_number, self.step_name)
self.assertEqual(analysis_status.ERROR, task.status)
self.assertEqual({}, task.tests_statuses)
self.assertEqual({}, task.classified_tests)
def testTriggerSwarmingTask(self, mock_ref, mock_create, mock_trigger, _):
runner_id = 'runner_id'
master_name = 'm'
builder_name = 'b'
build_number = 6
step_name = 's'
tests = ['test']
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
parameters = RunSwarmingTaskParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
step_name=step_name,
tests=tests)
new_request = SwarmingTaskRequest.FromSerializable(
_SAMPLE_REQUEST_JSON)
mock_create.return_value = new_request
task_id = test_swarming.TriggerSwarmingTask(parameters, runner_id)
self.assertEqual('new_task_id', task_id)
mock_ref.assert_called_once_with(master_name, builder_name,
build_number, step_name, None)
mock_create.assert_called_once_with('runner_id', 'task_id',
'ref_request', master_name,
builder_name, step_name, tests, 10)
mock_trigger.assert_called_once_with('chromium-swarm.appspot.com',
new_request, None)
def testSwarmingPush(self):
task = WfSwarmingTask.Create('m', 'b', 1, 'test')
task.task_id = '12345'
task.callback_url = '/callback?pipeline_id=f9f89162ef32c7fb7'
task.put()
with mock.patch('google.appengine.api.taskqueue.add') as mock_queue:
self.test_app.post(
'/pubsub/swarmingpush',
params={
'data':
json.dumps({
'message': {
'attributes': {
'auth_token':
pubsub_callback.GetVerificationToken(),
},
'data':
base64.b64encode(
json.dumps({
'task_id':
'12345',
'userdata':
json.dumps({
'Message-Type':
'SwarmingTaskStatusChange',
}),
})),
},
}),
'format':
'json',
})
mock_queue.assert_called_once()
def testStepName(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
expected_step_name = 's'
task = WfSwarmingTask.Create(master_name, builder_name, build_number,
expected_step_name)
self.assertEqual(expected_step_name, task.step_name)
def testGetStatusAndCulpritFromTryJobSwarmingTaskIsRunning(self):
swarming_task = WfSwarmingTask.Create('m', 'b', 123, 'step')
swarming_task.put()
status, culprit = findit_api.FindItApi(
)._GetStatusAndCulpritFromTryJob(None, swarming_task, None, 'step',
None)
self.assertEqual(status, findit_api._TryJobStatus.RUNNING)
self.assertIsNone(culprit)
def testGetTaskIdFromSwarmingTaskEntity(self):
swarming_task = WfSwarmingTask.Create('m', 'b', 123, 's')
swarming_task.task_id = 'task_id'
swarming_task.put()
self.assertEqual(
'task_id',
swarmed_test_util.GetTaskIdFromSwarmingTaskEntity(
swarming_task.key.urlsafe()))
def testNeedANewSwarmingTaskForce(self):
swarming_task = WfSwarmingTask.Create('m', 'b', 2, 's')
swarming_task.status = analysis_status.ERROR
swarming_task.put()
need, urlsafe_task_key = test_swarming.NeedANewSwarmingTask(
'm', 'b', 2, 's', True)
self.assertTrue(need)
swarming_task = ndb.Key(urlsafe=urlsafe_task_key).get()
self.assertEqual(analysis_status.PENDING, swarming_task.status)
def testCollectSwarmingTaskResultsTaskRunning(self, mock_monitoring, _):
master_name = 'm'
builder_name = 'b'
build_number = 15
params = CollectSwarmingTaskResultsInputs(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
build_completed=True)
steps = ['step1', 'step2']
task1 = WfSwarmingTask.Create(master_name, builder_name, build_number,
'step2')
task1.status = analysis_status.COMPLETED
task1.put()
WfSwarmingTask.Create(master_name, builder_name, build_number,
'step1').put()
self.assertIsNone(
test_swarming.GetConsistentFailuresWhenAllTasksComplete(
params, steps))
self.assertFalse(mock_monitoring.called)
def testMonitorSwarmingTaskGetSwarmingTaskFailureLogError(self, _):
task = WfSwarmingTask.Create(self.master_name, self.builder_name,
self.build_number, self.step_name)
task.task_id = 'task_id1'
task.put()
pipeline = ProcessSwarmingTaskResultPipeline()
pipeline.start_test()
pipeline.run(self.master_name, self.builder_name, self.build_number,
self.step_name)
self.assertEqual(analysis_status.ERROR, task.status)
self.assertEqual(task.error, {'code': 1, 'message': 'error'})
def testOnSwarmingTaskStateChangedError(self, *_):
master_name = 'm'
builder_name = 'b'
build_number = 10
step_name = 's'
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
parameters = RunSwarmingTaskParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
step_name=step_name,
tests=['test'])
self.assertFalse(
test_swarming.OnSwarmingTaskStateChanged(parameters, 'task_id'))
def testGetSwarmingTask(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
task = TriggerSwarmingTaskPipeline()._GetSwarmingTask(
master_name, builder_name, build_number, step_name)
self.assertEqual(master_name, task.master_name)
self.assertEqual(builder_name, task.builder_name)
self.assertEqual(build_number, task.build_number)
self.assertEqual(step_name, task.step_name)
def testOnSwarmingTaskErrorShouldNotCompletePipeline(self, mock_mon):
master_name = 'm'
builder_name = 'b'
build_number = 12
step_name = 's'
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
error = {'code': 1, 'message': 'error'}
test_swarming.OnSwarmingTaskError(
master_name, builder_name, build_number, step_name,
SwarmingTaskError.FromSerializable(error), False)
swarming_task = WfSwarmingTask.Get(master_name, builder_name,
build_number, step_name)
self.assertEqual(error, swarming_task.error)
self.assertEqual(analysis_status.PENDING, swarming_task.status)
self.assertFalse(mock_mon.called)
def testNoNewSwarmingTaskIsNeeded(self):
master_name = 'm'
builder_name = 'b'
build_number = 1
step_name = 's'
tests = ['a.b']
swarming_task = WfSwarmingTask.Create(master_name, builder_name,
build_number, step_name)
swarming_task.status = analysis_status.RUNNING
swarming_task.task_id = 'task_id'
swarming_task.put()
pipeline = TriggerSwarmingTaskPipeline()
task_id = pipeline.run(master_name, builder_name, build_number,
step_name, tests)
self.assertEqual('task_id', task_id)
def testMonitorSwarmingTaskWhereNoTaskOutputs(self):
task = WfSwarmingTask.Create(self.master_name, self.builder_name,
self.build_number, self.step_name)
task.task_id = 'task_id4'
task.put()
pipeline = ProcessSwarmingTaskResultPipeline()
pipeline.start_test()
pipeline.run(self.master_name, self.builder_name, self.build_number,
self.step_name)
self.assertEqual(analysis_status.ERROR, task.status)
self.assertEqual(
task.error, {
'code': swarming_util.NO_TASK_OUTPUTS,
'message': 'outputs_ref is None'
})
def testMonitorSwarmingTaskWhereTaskFailed(self):
task = WfSwarmingTask.Create(self.master_name, self.builder_name,
self.build_number, self.step_name)
task.task_id = 'task_id3'
task.put()
pipeline = ProcessSwarmingTaskResultPipeline()
pipeline.start_test()
pipeline.run(self.master_name, self.builder_name, self.build_number,
self.step_name)
self.assertEqual(analysis_status.ERROR, task.status)
self.assertEqual(
task.error, {
'code':
swarming_util.TASK_FAILED,
'message':
swarming_util.EXIT_CODE_DESCRIPTIONS[swarming_util.TASK_FAILED]
})
def testCollectSwarmingTaskResultsNoResult(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 16
params = CollectSwarmingTaskResultsInputs(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
build_completed=True)
steps = ['step1']
task1 = WfSwarmingTask.Create(master_name, builder_name, build_number,
'step1')
task1.status = analysis_status.COMPLETED
task1.put()
self.assertEqual(
CollectSwarmingTaskResultsOutputs.FromSerializable({}),
test_swarming.GetConsistentFailuresWhenAllTasksComplete(
params, steps))
def testCollectSwarmingTaskResultsAllFlaky(self, mock_monitoring,
mock_update_analysis,
mock_save_flakes, _):
master_name = 'm'
builder_name = 'b'
build_number = 17
params = CollectSwarmingTaskResultsInputs(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
build_completed=True)
steps = ['step2']
task2 = WfSwarmingTask.Create(master_name, builder_name, build_number,
'step2')
task2.status = analysis_status.COMPLETED
task2.tests_statuses = {
'TestSuite1.test1': {
'total_run': 2,
'SUCCESS': 2
},
'TestSuite1.test2': {
'total_run': 4,
'SUCCESS': 2,
'FAILURE': 2
},
}
task2.put()
self.assertEqual(
CollectSwarmingTaskResultsOutputs.FromSerializable({}),
test_swarming.GetConsistentFailuresWhenAllTasksComplete(
params, steps))
flake_tests = {'step2': ['TestSuite1.test2']}
mock_update_analysis.assert_called_once_with(master_name, builder_name,
build_number, flake_tests)
mock_monitoring.assert_called_once_with('step2', 'step2', 'skip', 1)
mock_save_flakes.assert_called_once_with(master_name, builder_name,
build_number, flake_tests)
def testOnSwarmingTaskStateChangedCompleted(self, mock_complete, *_):
master_name = 'm'
builder_name = 'b'
build_number = 8
step_name = 's'
task = WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name)
task.put()
parameters = RunSwarmingTaskParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
step_name=step_name,
tests=['test'])
result = test_swarming.OnSwarmingTaskStateChanged(
parameters, 'task_id')
self.assertTrue(result)
mock_complete.assert_called_once_with(
master_name, builder_name, build_number, step_name,
{'state': constants.STATE_COMPLETED}, 'content')
def testOnSwarmingTaskStateChangedNoTaskData(self, mock_error, _):
master_name = 'm'
builder_name = 'b'
build_number = 11
step_name = 's'
task = WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name)
task.put()
parameters = RunSwarmingTaskParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
step_name=step_name,
tests=['test'])
result = test_swarming.OnSwarmingTaskStateChanged(
parameters, 'task_id')
self.assertIsNone(result)
mock_error.assert_called_once_with(master_name, builder_name,
build_number, step_name, 'error',
False)
def testOnSwarmingTaskErrorShouldCompletePipeline(self, mock_mon):
master_name = 'm'
builder_name = 'b'
build_number = 11
step_name = 's'
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
error = {'code': 1, 'message': 'error'}
self.assertFalse(
test_swarming.OnSwarmingTaskError(
master_name, builder_name, build_number, step_name,
SwarmingTaskError.FromSerializable(error)))
swarming_task = WfSwarmingTask.Get(master_name, builder_name,
build_number, step_name)
self.assertEqual(error, swarming_task.error)
self.assertEqual(analysis_status.ERROR, swarming_task.status)
mock_mon.assert_called_once_with(master_name, builder_name,
build_number, step_name,
analysis_status.ERROR,
analysis_approach_type.SWARMING)
def NeedANewSwarmingTask(master_name, builder_name, build_number, step_name,
force):
"""Checks if a WfSwarmingTask for the given params exists, or creates it."""
swarming_task = WfSwarmingTask.Get(master_name, builder_name, build_number,
step_name)
if not swarming_task:
swarming_task = WfSwarmingTask.Create(master_name, builder_name,
build_number, step_name)
swarming_task.status = analysis_status.PENDING
swarming_task.put()
return True, swarming_task.key.urlsafe()
if force:
swarming_task.Reset()
swarming_task.put()
return True, swarming_task.key.urlsafe()
# TODO(http://crbug.com/585676): Rerun the Swarming task if it runs into
# unexpected infra errors.
return False, swarming_task.key.urlsafe()
def testClassifiedTestsLegacy(self):
task = WfSwarmingTask.Create('m', 'b', 121, 'browser_tests')
task.tests_statuses = {
'TestSuite1.test1': {
'total_run': 2,
'SUCCESS': 2
},
'TestSuite1.test2': {
'total_run': 4,
'SUCCESS': 2,
'FAILURE': 2
},
'TestSuite1.test3': {
'total_run': 6,
'FAILURE': 6
},
'TestSuite1.test4': {
'total_run': 6,
'SKIPPED': 6
},
'TestSuite1.test5': {
'total_run': 6,
'UNKNOWN': 6
}
}
expected_classified_tests = {
'flaky_tests': ['TestSuite1.test2', 'TestSuite1.test1'],
'reliable_tests': ['TestSuite1.test3', 'TestSuite1.test4'],
'unknown_tests': ['TestSuite1.test5']
}
expected_reproducible_flaky_tests = ['TestSuite1.test2']
self.assertEqual(expected_classified_tests, task.classified_tests)
self.assertEqual(expected_classified_tests['reliable_tests'],
task.reliable_tests)
self.assertEqual(expected_classified_tests['flaky_tests'], task.flaky_tests)
self.assertEqual(expected_reproducible_flaky_tests,
task.reproducible_flaky_tests)