def _ParseFlakeSwarmingTaskOutput(task_data, output_json, error, parameters):
"""Returns swarming task results as a FlakeswarmingTaskOutput object.
Assumption: only 1 test was run in the Swarming task.
"""
assert task_data
iterations = parameters.iterations
if output_json:
# Gets the total numbers of runs and number of successful runs from
# test results
tries, successes = flake_test_results.GetCountsFromSwarmingRerun(
output_json)
if tries is None or successes is None:
# Something went wrong preventing even a single test from being processed
# which counts as an error.
error = error or SwarmingTaskError.GenerateError(
code=swarming_task_error.UNKNOWN)
tries = None
successes = None
elif (tries == 1 and task_data['state'] == constants.STATE_COMPLETED
and not task_data.get('failure')
and not task_data.get('infra_failure')):
# webkit_layout_tests special case: test results will be combined into
# one if all results are the same.
# Use iterations instead assuming the test repeated that many times.
# Currently only do this if task completes successfully.
tries = iterations
successes = iterations * successes
return FlakeSwarmingTaskOutput(
completed_time=time_util.DatetimeFromString(
task_data.get('completed_ts')),
error=error,
iterations=tries,
pass_count=successes,
started_time=time_util.DatetimeFromString(
task_data.get('started_ts')),
task_id=task_data['task_id'])
else:
return FlakeSwarmingTaskOutput(
completed_time=time_util.DatetimeFromString(
task_data.get('completed_ts')),
error=error or
SwarmingTaskError.GenerateError(code=swarming_task_error.UNKNOWN),
iterations=None,
pass_count=None,
started_time=time_util.DatetimeFromString(
task_data.get('started_ts')),
task_id=task_data['task_id'])
def GetIsolatedOuptputFileToHashMap(digest, name_space, isolated_server,
http_client):
"""Gets the mapping of all files and their hashes and other info.
Args:
digest(str): Hash to file for retrieve request.
name_space(str): Name space info for retrieve request.
isolated_server(str): Host to isolate server.
http_client(RetryHttpClient): Http client to send the request.
Returns:
(dict): Mapping from file names to hashes.
"""
content, error = isolate_util.FetchFileFromIsolatedServer(
digest, name_space, isolated_server, http_client)
if not content:
return None, error
file_hash_mapping = {}
content_json = json.loads(content)
if not content_json.get('files'):
return None, SwarmingTaskError.GenerateError(
swarming_task_error.NO_ISOLATED_FILES)
for file_name, info in content_json['files'].iteritems():
file_hash_mapping[file_name] = info.get('h')
return file_hash_mapping, None
def testOnSwarmingTaskTimeout(self, mocked_parse, mocked_result):
error = SwarmingTaskError.GenerateError(
swarming_task_error.RUNNER_TIMEOUT)
task_id = 'task_id'
task_data = {
'state': 'COMPLETE',
}
mocked_result.return_value = task_data, 'content', None
commit_position = 1000
isolate_sha = 'sha1'
iterations = 50
timeout_seconds = 1200
parameters = RunFlakeSwarmingTaskInput(builder_name='b',
commit_position=commit_position,
isolate_sha=isolate_sha,
iterations=iterations,
master_name='m',
reference_build_number=123,
step_name='s',
test_name='t',
timeout_seconds=timeout_seconds)
flake_swarming.OnSwarmingTaskTimeout(parameters, task_id)
mocked_parse.assert_called_once_with(task_data, 'content', error,
parameters)
def testParseFlakeSwarmingTaskOutputNoOutput(self):
task_data = {
'created_ts': '2018-04-02T18:32:06.538220',
'started_ts': '2018-04-02T19:32:06.538220',
'completed_ts': '2018-04-02T20:32:06.538220',
'task_id': 'task_id'
}
error = SwarmingTaskError(code=1, message='m')
commit_position = 1000
isolate_sha = 'sha1'
iterations = 50
timeout_seconds = 1200
parameters = RunFlakeSwarmingTaskInput(builder_name='b',
commit_position=commit_position,
isolate_sha=isolate_sha,
iterations=iterations,
master_name='m',
reference_build_number=123,
step_name='s',
test_name='t',
timeout_seconds=timeout_seconds)
expected_result = FlakeSwarmingTaskOutput(
task_id='task_id',
started_time=datetime(2018, 4, 2, 19, 32, 6, 538220),
completed_time=datetime(2018, 4, 2, 20, 32, 6, 538220),
iterations=None,
error=error,
pass_count=None)
self.assertEqual(
expected_result,
flake_swarming._ParseFlakeSwarmingTaskOutput(
task_data, None, error, parameters))
def OnSwarmingTaskError(task_id, error):
"""Returns a FlakeSwarmingTaskOutput object representing a failed task."""
return FlakeSwarmingTaskOutput(
completed_time=None,
error=error
or SwarmingTaskError.GenerateError(code=swarming_task_error.UNKNOWN),
iterations=None,
pass_count=None,
started_time=None,
task_id=task_id)
def testGetSwarmingTaskDataAndResultFailedState(self, mock_fn):
data = {'state': 'BOT_DIED', 'outputs_ref': 'outputs_ref'}
mock_fn.return_value = (data, None)
error = SwarmingTaskError.FromSerializable({
'code':
swarming_task_error.BOT_DIED,
'message':
'BOT_DIED'
})
self.assertEqual(
(data, None, error),
swarmed_test_util.GetSwarmingTaskDataAndResult('task_id', None))
def testOnSwarmingTaskError(self):
task_id = 'task_id'
error = SwarmingTaskError(code=1000, message='Unknown error')
expected_result = FlakeSwarmingTaskOutput(task_id=task_id,
started_time=None,
completed_time=None,
iterations=None,
error=error,
pass_count=None)
self.assertEqual(expected_result,
flake_swarming.OnSwarmingTaskError(task_id, error))
def testUpdateAnalysisDataPointsExistingDataPointWithErrorSalvagable(
self, _):
commit_position = 1000
revision = 'r1000'
iterations = 100
pass_count = 50
completed_time = datetime(2018, 1, 1, 0, 1, 0)
error = SwarmingTaskError(code=1, message='m')
started_time = datetime(2018, 1, 1, 0, 0, 0)
task_id_1 = 'task_1'
task_id_2 = 'task_2'
build_url = 'url'
try_job_url = None
swarming_task_output = FlakeSwarmingTaskOutput(
completed_time=completed_time,
error=error,
iterations=iterations,
pass_count=pass_count,
started_time=started_time,
task_id=task_id_2)
initial_flakiness = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=60,
error=None,
failed_swarming_task_attempts=0,
iterations=50,
pass_rate=0.5,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable([task_id_1]))
expected_flakiness = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=120,
error=None, # Only set error if no more retries.
failed_swarming_task_attempts=0, # Task was salvaged.
iterations=150,
pass_rate=0.5,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable([task_id_1, task_id_2]))
resulting_flakiness = flakiness_util.UpdateFlakiness(
initial_flakiness, swarming_task_output)
self.assertEqual(expected_flakiness, resulting_flakiness)
def testCalculateRunParametersForSwarmingTaskWithError(self):
expected_iterations_to_run_after_timeout = 10
flakiness = Flakiness(commit_position=1000,
pass_rate=1.0,
iterations=1,
total_test_run_seconds=400)
self.assertEqual(
(expected_iterations_to_run_after_timeout, 3600),
run_swarming_util.CalculateRunParametersForSwarmingTask(
flakiness,
SwarmingTaskError(code=swarming_task_error.TIMED_OUT,
message='m')))
def testGetSwarmingTaskDataAndResultNoOutputRef(self, mock_fn):
data = {'state': constants.STATE_COMPLETED}
mock_fn.return_value = (data, None)
error = SwarmingTaskError.FromSerializable({
'code':
swarming_task_error.NO_TASK_OUTPUTS,
'message':
'outputs_ref is None'
})
self.assertEqual(
(data, None, error),
swarmed_test_util.GetSwarmingTaskDataAndResult(None, None))
def testOnSwarmingTaskTimeoutNoTaskId(self, mocked_result):
error = SwarmingTaskError(
code=350, message='Runner to run swarming task timed out')
mocked_result.return_value = None, None, error
expected_result = FlakeSwarmingTaskOutput(task_id=None,
started_time=None,
completed_time=None,
iterations=None,
error=error,
pass_count=None)
self.assertEqual(expected_result,
flake_swarming.OnSwarmingTaskTimeout(None, None))
def testReportSwarmingTaskError(self):
error_code = 1
error_message = 'e'
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.start_time = datetime(2018, 1, 23, 1, 0, 0)
analysis.end_time = datetime(2018, 1, 23, 1, 1, 0)
analysis.Save()
run_swarming_util.ReportSwarmingTaskError(
analysis, SwarmingTaskError(code=error_code,
message=error_message))
self.assertEqual(error_code, analysis.error['code'])
self.assertEqual(error_message, analysis.error['message'])
def testOnSwarmingTaskTimeoutNoData(self, mocked_result):
error = SwarmingTaskError(code=1000, message='Unknown error')
mocked_result.return_value = None, None, error
task_id = 'task_id'
expected_result = FlakeSwarmingTaskOutput(task_id=task_id,
started_time=None,
completed_time=None,
iterations=None,
error=error,
pass_count=None)
self.assertEqual(expected_result,
flake_swarming.OnSwarmingTaskTimeout(None, task_id))
def testUpdateFlakinessWithErrorUnsalvagable(self, _):
commit_position = 1000
completed_time = datetime(2018, 1, 1, 1, 0, 0)
error = SwarmingTaskError(code=1, message='message')
iterations = None
pass_count = None
revision = 'r1000'
started_time = datetime(2018, 1, 1, 0, 0, 0)
task_id = 'task_id'
build_url = 'url'
try_job_url = None
swarming_task_output = FlakeSwarmingTaskOutput(
completed_time=completed_time,
error=error,
iterations=iterations,
pass_count=pass_count,
started_time=started_time,
task_id=task_id)
flakiness_to_update = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=0,
error=None,
failed_swarming_task_attempts=0,
iterations=0,
pass_rate=None,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable([]))
expected_flakiness = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=0,
error=None,
failed_swarming_task_attempts=1,
iterations=0,
pass_rate=None,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable([task_id]))
resulting_flakiness = flakiness_util.UpdateFlakiness(
flakiness_to_update, swarming_task_output)
self.assertEqual(expected_flakiness, resulting_flakiness)
def GetSwarmingTaskDataAndResult(task_id, http_client=_FINDIT_HTTP_CLIENT):
"""Gets information about a swarming task.
Returns:
(str, dict, dict): The state, test result and error for a swarming task.
"""
data, error = swarming_util.GetSwarmingTaskResultById(swarming.SwarmingHost(),
task_id, http_client)
error = SwarmingTaskError.FromSerializable(error)
if not data:
return None, None, error
task_state = data['state']
output_json = None
if task_state not in constants.STATE_NOT_STOP:
if task_state == constants.STATE_COMPLETED:
outputs_ref = data.get('outputs_ref')
# If swarming task aborted because of errors in request arguments,
# it's possible that there is no outputs_ref.
if not outputs_ref:
error = error or SwarmingTaskError.GenerateError(
swarming_task_error.NO_TASK_OUTPUTS)
else:
output_json, error = GetOutputJsonByOutputsRef(outputs_ref, http_client)
if not output_json:
error = error or SwarmingTaskError.GenerateError(
swarming_task_error.NO_OUTPUT_JSON)
else:
# The swarming task did not complete successfully.
logging.error('Swarming task %s stopped with status: %s', task_id,
task_state)
error = SwarmingTaskError.GenerateError(
swarming_task_error.STATES_NOT_RUNNING_TO_ERROR_CODES[task_state])
return data, output_json, error
def testOnSwarmingTaskErrorShouldNotCompletePipeline(self, mock_mon):
master_name = 'm'
builder_name = 'b'
build_number = 12
step_name = 's'
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
error = {'code': 1, 'message': 'error'}
test_swarming.OnSwarmingTaskError(
master_name, builder_name, build_number, step_name,
SwarmingTaskError.FromSerializable(error), False)
swarming_task = WfSwarmingTask.Get(master_name, builder_name,
build_number, step_name)
self.assertEqual(error, swarming_task.error)
self.assertEqual(analysis_status.PENDING, swarming_task.status)
self.assertFalse(mock_mon.called)
def OnSwarmingTaskTimeout(parameters, task_id):
"""To be called when waiting for a swarming task times out."""
timeout_error = SwarmingTaskError.GenerateError(
swarming_task_error.RUNNER_TIMEOUT)
if not task_id:
# The pipeline timedout without successfully triggering a task.
return OnSwarmingTaskError(None, timeout_error)
task_data, output_json, error = (
swarmed_test_util.GetSwarmingTaskDataAndResult(task_id))
if not task_data or not task_data.get('state'):
return OnSwarmingTaskError(task_id, error or timeout_error)
# Attempt to salvage whatever is available, but still report an error.
return _ParseFlakeSwarmingTaskOutput(task_data, output_json, error
or timeout_error, parameters)
def testOnSwarmingTaskErrorShouldCompletePipeline(self, mock_mon):
master_name = 'm'
builder_name = 'b'
build_number = 11
step_name = 's'
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
error = {'code': 1, 'message': 'error'}
self.assertFalse(
test_swarming.OnSwarmingTaskError(
master_name, builder_name, build_number, step_name,
SwarmingTaskError.FromSerializable(error)))
swarming_task = WfSwarmingTask.Get(master_name, builder_name,
build_number, step_name)
self.assertEqual(error, swarming_task.error)
self.assertEqual(analysis_status.ERROR, swarming_task.status)
mock_mon.assert_called_once_with(master_name, builder_name,
build_number, step_name,
analysis_status.ERROR,
analysis_approach_type.SWARMING)
def OnSwarmingTaskTimeout(run_swarming_task_params, task_id):
master_name, builder_name, build_number = (
run_swarming_task_params.build_key.GetParts())
step_name = run_swarming_task_params.step_name
error = SwarmingTaskError.GenerateError(swarming_task_error.RUNNER_TIMEOUT)
data, output_json, _ = swarmed_test_util.GetSwarmingTaskDataAndResult(
task_id)
if output_json and test_results_util.IsTestResultsValid(output_json):
test_results = test_results_util.GetTestResultObject(output_json)
classified_test_results = (test_results.GetClassifiedTestResults()
if test_results else {})
_UpdateSwarmingTaskEntity(
master_name,
builder_name,
build_number,
step_name,
status=analysis_status.COMPLETED,
error=error,
classified_test_results=classified_test_results,
created_ts=data.get('created_ts'),
started_ts=data.get('started_ts'),
completed_ts=data.get('completed_ts'))
_RecordSwarmingTaskStateChange(master_name, builder_name, build_number,
step_name, analysis_status.COMPLETED,
analysis_approach_type.SWARMING)
else:
_UpdateSwarmingTaskEntity(master_name,
builder_name,
build_number,
step_name,
status=analysis_status.ERROR,
error=error)
_RecordSwarmingTaskStateChange(master_name, builder_name, build_number,
step_name, analysis_status.ERROR,
analysis_approach_type.SWARMING)
def testGenerateError(self):
expected_error = {'code': 1000, 'message': 'Unknown error'}
self.assertEqual(
expected_error,
SwarmingTaskError.GenerateError(
swarming_task_error.UNKNOWN).ToSerializable())
def testDetermineApproximatePassRateMaximumRetriesPerSwarmingTaskReached(
self, _):
master_name = 'm'
builder_name = 'b'
reference_build_number = 123
step_name = 's'
test_name = 't'
commit_position = 1000
incoming_pass_count = 15
iterations = 30
incoming_pass_rate = float(incoming_pass_count / iterations)
isolate_sha = 'sha1'
revision = 'r1000'
task_id = 'task_id_2'
started_time = datetime(2018, 1, 1, 0, 0, 0)
completed_time = datetime(2018, 1, 1, 1, 0, 0)
build_url = 'url'
try_job_url = None
swarming_task_error = SwarmingTaskError(code=1, message='error')
isolate_sha_output = GetIsolateShaOutput(
build_number=None,
build_url=build_url,
isolate_sha=isolate_sha,
try_job_url=try_job_url)
flakiness_thus_far = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=3600,
error=None,
failed_swarming_task_attempts=0,
iterations=iterations,
pass_rate=incoming_pass_rate,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable(['task_id_1']))
expected_flakiness_thus_far = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=3600,
error=swarming_task_error,
failed_swarming_task_attempts=0,
iterations=iterations,
pass_rate=incoming_pass_rate,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable(['task_id_1']))
incoming_flake_swarming_task_output = FlakeSwarmingTaskOutput(
completed_time=completed_time,
error=swarming_task_error,
pass_count=incoming_pass_count,
iterations=iterations,
started_time=started_time,
task_id=task_id)
determine_approximate_pass_rate_input = DetermineApproximatePassRateInput(
builder_name=builder_name,
commit_position=commit_position,
flakiness_thus_far=flakiness_thus_far,
get_isolate_sha_output=isolate_sha_output,
master_name=master_name,
previous_swarming_task_output=incoming_flake_swarming_task_output,
reference_build_number=reference_build_number,
revision=revision,
step_name=step_name,
test_name=test_name)
pipeline_job = DetermineApproximatePassRatePipeline(
determine_approximate_pass_rate_input)
pipeline_job.start()
self.execute_queued_tasks()
pipeline_job = pipelines.pipeline.Pipeline.from_id(pipeline_job.pipeline_id)
self.assertEqual(expected_flakiness_thus_far.ToSerializable(),
pipeline_job.outputs.default.value)
def testGetSwarmingTaskDataAndResultNoData(self, mock_fn):
error = {'code': 1, 'message': 'error'}
mock_fn.return_value = (None, error)
self.assertEqual(
(None, None, SwarmingTaskError.FromSerializable(error)),
swarmed_test_util.GetSwarmingTaskDataAndResult('task_id', None))
class TestSwarmingTest(wf_testcase.WaterfallTestCase):
def testNeedANewSwarmingTaskCreateANewOne(self):
need, urlsafe_task_key = test_swarming.NeedANewSwarmingTask(
'm', 'b', 1, 's', False)
self.assertTrue(need)
swarming_task = ndb.Key(urlsafe=urlsafe_task_key).get()
self.assertIsNotNone(swarming_task)
def testNeedANewSwarmingTaskForce(self):
swarming_task = WfSwarmingTask.Create('m', 'b', 2, 's')
swarming_task.status = analysis_status.ERROR
swarming_task.put()
need, urlsafe_task_key = test_swarming.NeedANewSwarmingTask(
'm', 'b', 2, 's', True)
self.assertTrue(need)
swarming_task = ndb.Key(urlsafe=urlsafe_task_key).get()
self.assertEqual(analysis_status.PENDING, swarming_task.status)
def testNotNeedANewSwarmingTask(self):
swarming_task = WfSwarmingTask.Create('m', 'b', 3, 's')
swarming_task.status = analysis_status.ERROR
swarming_task.put()
need, _ = test_swarming.NeedANewSwarmingTask('m', 'b', 3, 's', False)
self.assertFalse(need)
@mock.patch.object(test_swarming, 'FinditHttpClient', return_value=None)
@mock.patch.object(swarming_util,
'TriggerSwarmingTask',
return_value=('new_task_id', None))
@mock.patch.object(swarming, 'CreateNewSwarmingTaskRequestTemplate')
@mock.patch.object(swarming,
'GetReferredSwarmingTaskRequestInfo',
return_value=('task_id', 'ref_request'))
def testTriggerSwarmingTask(self, mock_ref, mock_create, mock_trigger, _):
runner_id = 'runner_id'
master_name = 'm'
builder_name = 'b'
build_number = 6
step_name = 's'
tests = ['test']
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
parameters = RunSwarmingTaskParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
step_name=step_name,
tests=tests)
new_request = SwarmingTaskRequest.FromSerializable(
_SAMPLE_REQUEST_JSON)
mock_create.return_value = new_request
task_id = test_swarming.TriggerSwarmingTask(parameters, runner_id)
self.assertEqual('new_task_id', task_id)
mock_ref.assert_called_once_with(master_name, builder_name,
build_number, step_name, None)
mock_create.assert_called_once_with('runner_id', 'task_id',
'ref_request', master_name,
builder_name, step_name, tests, 10)
mock_trigger.assert_called_once_with('chromium-swarm.appspot.com',
new_request, None)
def testOnSwarmingTaskTriggered(self):
master_name = 'm'
builder_name = 'b'
build_number = 6
step_name = 's'
task = WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name)
task.put()
task_id = 'task_id'
tests = ['t']
iterations = 100
new_request = {
'expiration_secs':
'3600',
'name':
'findit/ref_task_id/ref_task_id/2018-03-15 00:00:00 000000',
'parent_task_id':
'',
'priority':
'25',
'properties': {
'command':
'cmd',
'dimensions': [{
'key': 'k',
'value': 'v'
}],
'env': [
{
'key': 'a',
'value': '1'
},
],
'execution_timeout_secs':
'10',
'extra_args': [
'--flag=value',
'--gtest_filter=a.b:a.c',
'--gtest_repeat=30',
'--test-launcher-retry-limit=0',
'--gtest_also_run_disabled_tests',
],
'grace_period_secs':
'30',
'idempotent':
False,
'inputs_ref': {
'isolatedserver': 'isolatedserver',
'isolated': 'sha'
},
'io_timeout_secs':
'1200',
},
'tags': [
'ref_master:m',
'ref_buildername:b',
'ref_buildnumber:4',
'ref_stepname:s',
'ref_name:test',
],
'user':
'',
'pubsub_auth_token':
'auth_token',
'pubsub_topic':
'projects/app-id/topics/swarming',
'pubsub_userdata':
json.dumps({'runner_id': 'runner_id'}),
}
test_swarming.OnSwarmingTaskTriggered(
master_name, builder_name, build_number, step_name, tests,
'task_id', iterations,
SwarmingTaskRequest.FromSerializable(new_request))
task = WfSwarmingTask.Get(master_name, builder_name, build_number,
step_name)
self.assertEqual(task.task_id, task_id)
@mock.patch.object(_GTEST_RESULTS,
'GetClassifiedTestResults',
return_value={})
@mock.patch.object(test_results_util,
'IsTestResultsValid',
return_value=True)
@mock.patch.object(test_results_util,
'GetTestResultObject',
return_value=_GTEST_RESULTS)
@mock.patch.object(swarmed_test_util,
'GetSwarmingTaskDataAndResult',
return_value=({
'state': constants.STATE_COMPLETED
}, 'content', None))
@mock.patch.object(test_swarming, '_RecordSwarmingTaskStateChange')
def testOnSwarmingTaskTimeoutGotResult(self, mock_mon, *_):
master_name = 'm'
builder_name = 'b'
build_number = 15
step_name = 's'
swarming_task = WfSwarmingTask.Create(master_name, builder_name,
build_number, step_name)
swarming_task.put()
parameters = RunSwarmingTaskParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
step_name=step_name,
tests=['tests'])
test_swarming.OnSwarmingTaskTimeout(parameters, 'task_id')
swarming_task = WfSwarmingTask.Get(master_name, builder_name,
build_number, step_name)
self.assertEqual(analysis_status.COMPLETED, swarming_task.status)
self.assertEqual({}, swarming_task.tests_statuses)
self.assertEqual(
{
'code': swarming_task_error.RUNNER_TIMEOUT,
'message': 'Runner to run swarming task timed out'
}, swarming_task.error)
mock_mon.assert_called_once_with(master_name, builder_name,
build_number, step_name,
analysis_status.COMPLETED,
analysis_approach_type.SWARMING)
@mock.patch.object(swarmed_test_util,
'GetSwarmingTaskDataAndResult',
return_value=(None, None, 'error'))
@mock.patch.object(test_swarming, '_RecordSwarmingTaskStateChange')
def testOnSwarmingTaskTimeout(self, mock_mon, _):
master_name = 'm'
builder_name = 'b'
build_number = 16
step_name = 's'
swarming_task = WfSwarmingTask.Create(master_name, builder_name,
build_number, step_name)
swarming_task.put()
parameters = RunSwarmingTaskParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
step_name=step_name,
tests=['tests'])
test_swarming.OnSwarmingTaskTimeout(parameters, 'task_id')
swarming_task = WfSwarmingTask.Get(master_name, builder_name,
build_number, step_name)
self.assertEqual(analysis_status.ERROR, swarming_task.status)
self.assertEqual(
{
'code': swarming_task_error.RUNNER_TIMEOUT,
'message': 'Runner to run swarming task timed out'
}, swarming_task.error)
mock_mon.assert_called_once_with(master_name, builder_name,
build_number, step_name,
analysis_status.ERROR,
analysis_approach_type.SWARMING)
@mock.patch.object(_GTEST_RESULTS,
'GetClassifiedTestResults',
return_value={})
@mock.patch.object(test_results_util,
'GetTestResultObject',
return_value=_GTEST_RESULTS)
@mock.patch.object(test_swarming, '_RecordSwarmingTaskStateChange')
def testOnSwarmingTaskCompleted(self, mock_mon, *_):
master_name = 'm'
builder_name = 'b'
build_number = 13
step_name = 's'
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
data = {
'state': constants.STATE_COMPLETED,
'created_ts': '2015-07-30T18:11:16.743220',
'started_ts': '2015-07-30T18:12:16.743220',
'completed_ts': '2015-07-30T18:15:16.743220'
}
test_swarming.OnSwarmingTaskCompleted(master_name, builder_name,
build_number, step_name, data,
'output_json')
swarming_task = WfSwarmingTask.Get(master_name, builder_name,
build_number, step_name)
self.assertEqual({}, swarming_task.tests_statuses)
self.assertEqual(analysis_status.COMPLETED, swarming_task.status)
self.assertEqual(datetime.datetime(2015, 7, 30, 18, 11, 16, 743220),
swarming_task.created_time)
self.assertEqual(datetime.datetime(2015, 7, 30, 18, 12, 16, 743220),
swarming_task.started_time)
self.assertEqual(datetime.datetime(2015, 7, 30, 18, 15, 16, 743220),
swarming_task.completed_time)
mock_mon.assert_called_once_with(master_name, builder_name,
build_number, step_name,
analysis_status.COMPLETED,
analysis_approach_type.SWARMING)
@mock.patch.object(test_results_util,
'IsTestResultsValid',
return_value=True)
@mock.patch.object(swarmed_test_util,
'GetSwarmingTaskDataAndResult',
return_value=({
'state': constants.STATE_COMPLETED
}, 'content', None))
@mock.patch.object(test_swarming,
'OnSwarmingTaskCompleted',
return_value=True)
def testOnSwarmingTaskStateChangedCompleted(self, mock_complete, *_):
master_name = 'm'
builder_name = 'b'
build_number = 8
step_name = 's'
task = WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name)
task.put()
parameters = RunSwarmingTaskParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
step_name=step_name,
tests=['test'])
result = test_swarming.OnSwarmingTaskStateChanged(
parameters, 'task_id')
self.assertTrue(result)
mock_complete.assert_called_once_with(
master_name, builder_name, build_number, step_name,
{'state': constants.STATE_COMPLETED}, 'content')
@mock.patch.object(swarmed_test_util,
'GetSwarmingTaskDataAndResult',
return_value=({
'state': constants.STATE_RUNNING
}, None, None))
@mock.patch.object(test_swarming, '_UpdateSwarmingTaskEntity')
def testOnSwarmingTaskStateChangedRunning(self, mock_update, _):
master_name = 'm'
builder_name = 'b'
build_number = 9
step_name = 's'
task = WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name)
task.put()
parameters = RunSwarmingTaskParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
step_name=step_name,
tests=['test'])
result = test_swarming.OnSwarmingTaskStateChanged(
parameters, 'task_id')
self.assertIsNone(result)
mock_update.assert_called_once_with(master_name,
builder_name,
build_number,
step_name,
status=analysis_status.RUNNING)
@mock.patch.object(test_swarming, '_RecordSwarmingTaskStateChange')
@mock.patch.object(swarmed_test_util,
'GetSwarmingTaskDataAndResult',
return_value=({
'state': constants.STATE_COMPLETED
}, None,
SwarmingTaskError.FromSerializable({
'code':
1,
'message':
'message'
})))
def testOnSwarmingTaskStateChangedError(self, *_):
master_name = 'm'
builder_name = 'b'
build_number = 10
step_name = 's'
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
parameters = RunSwarmingTaskParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
step_name=step_name,
tests=['test'])
self.assertFalse(
test_swarming.OnSwarmingTaskStateChanged(parameters, 'task_id'))
@mock.patch.object(swarmed_test_util,
'GetSwarmingTaskDataAndResult',
return_value=(None, None, 'error'))
@mock.patch.object(test_swarming, 'OnSwarmingTaskError')
def testOnSwarmingTaskStateChangedNoTaskData(self, mock_error, _):
master_name = 'm'
builder_name = 'b'
build_number = 11
step_name = 's'
task = WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name)
task.put()
parameters = RunSwarmingTaskParameters(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
step_name=step_name,
tests=['test'])
result = test_swarming.OnSwarmingTaskStateChanged(
parameters, 'task_id')
self.assertIsNone(result)
mock_error.assert_called_once_with(master_name, builder_name,
build_number, step_name, 'error',
False)
@mock.patch.object(test_swarming, '_RecordSwarmingTaskStateChange')
def testOnSwarmingTaskErrorShouldCompletePipeline(self, mock_mon):
master_name = 'm'
builder_name = 'b'
build_number = 11
step_name = 's'
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
error = {'code': 1, 'message': 'error'}
self.assertFalse(
test_swarming.OnSwarmingTaskError(
master_name, builder_name, build_number, step_name,
SwarmingTaskError.FromSerializable(error)))
swarming_task = WfSwarmingTask.Get(master_name, builder_name,
build_number, step_name)
self.assertEqual(error, swarming_task.error)
self.assertEqual(analysis_status.ERROR, swarming_task.status)
mock_mon.assert_called_once_with(master_name, builder_name,
build_number, step_name,
analysis_status.ERROR,
analysis_approach_type.SWARMING)
@mock.patch.object(test_swarming, '_RecordSwarmingTaskStateChange')
def testOnSwarmingTaskErrorShouldNotCompletePipeline(self, mock_mon):
master_name = 'm'
builder_name = 'b'
build_number = 12
step_name = 's'
WfSwarmingTask.Create(master_name, builder_name, build_number,
step_name).put()
error = {'code': 1, 'message': 'error'}
test_swarming.OnSwarmingTaskError(
master_name, builder_name, build_number, step_name,
SwarmingTaskError.FromSerializable(error), False)
swarming_task = WfSwarmingTask.Get(master_name, builder_name,
build_number, step_name)
self.assertEqual(error, swarming_task.error)
self.assertEqual(analysis_status.PENDING, swarming_task.status)
self.assertFalse(mock_mon.called)
@mock.patch.object(test_failure_analysis,
'GetFirstTimeFailedSteps',
return_value=['step'])
def testGetStepsToCollectSwarmingTaskResults(self, mock_fn):
master_name = 'm'
builder_name = 'b'
build_number = 13
params = CollectSwarmingTaskResultsInputs(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
build_completed=True)
self.assertEqual(
['step'],
test_swarming.GetStepsToCollectSwarmingTaskResults(params))
mock_fn.assert_called_once_with(master_name, builder_name,
build_number)
@mock.patch.object(test_failure_analysis, 'GetFirstTimeFailedSteps')
def testGetStepsToCollectSwarmingTaskResultsBuildNotComplete(
self, mock_fn):
master_name = 'm'
builder_name = 'b'
build_number = 14
params = CollectSwarmingTaskResultsInputs(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
build_completed=False)
self.assertEqual(
[], test_swarming.GetStepsToCollectSwarmingTaskResults(params))
self.assertFalse(mock_fn.called)
@mock.patch.object(test_failure_analysis,
'UpdateAnalysisWithFlakesFoundBySwarmingReruns')
@mock.patch.object(monitoring, 'OnFlakeIdentified')
def testCollectSwarmingTaskResultsTaskRunning(self, mock_monitoring, _):
master_name = 'm'
builder_name = 'b'
build_number = 15
params = CollectSwarmingTaskResultsInputs(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
build_completed=True)
steps = ['step1', 'step2']
task1 = WfSwarmingTask.Create(master_name, builder_name, build_number,
'step2')
task1.status = analysis_status.COMPLETED
task1.put()
WfSwarmingTask.Create(master_name, builder_name, build_number,
'step1').put()
self.assertIsNone(
test_swarming.GetConsistentFailuresWhenAllTasksComplete(
params, steps))
self.assertFalse(mock_monitoring.called)
@mock.patch.object(test_failure_analysis,
'UpdateAnalysisWithFlakesFoundBySwarmingReruns')
@mock.patch.object(step_util,
'GetStepMetadata',
return_value={
'canonical_step_name': 'step2',
'isolate_target_name': 'step2'
})
@mock.patch.object(monitoring, 'OnFlakeIdentified')
def testCollectSwarmingTaskResultsError(self, mock_monitoring, *_):
master_name = 'm'
builder_name = 'b'
build_number = 15
params = CollectSwarmingTaskResultsInputs(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
build_completed=True)
steps = ['step1', 'step2']
task1 = WfSwarmingTask.Create(master_name, builder_name, build_number,
'step1')
task1.status = analysis_status.ERROR
task1.put()
task2 = WfSwarmingTask.Create(master_name, builder_name, build_number,
'step2')
task2.status = analysis_status.COMPLETED
task2.tests_statuses = {
'TestSuite1.test1': {
'total_run': 2,
'SUCCESS': 2
},
'TestSuite1.test2': {
'total_run': 4,
'SUCCESS': 2,
'FAILURE': 2
},
'TestSuite1.test3': {
'total_run': 6,
'FAILURE': 6
},
'TestSuite1.test4': {
'total_run': 6,
'SKIPPED': 6
},
'TestSuite1.test5': {
'total_run': 6,
'UNKNOWN': 6
}
}
task2.put()
expected_result_json = {
'consistent_failures': {
'step2': ['TestSuite1.test3', 'TestSuite1.test4']
}
}
self.assertEqual(
CollectSwarmingTaskResultsOutputs.FromSerializable(
expected_result_json),
test_swarming.GetConsistentFailuresWhenAllTasksComplete(
params, steps))
mock_monitoring.assert_called_once_with('step2', 'step2', 'skip', 1)
@mock.patch.object(test_failure_analysis,
'UpdateAnalysisWithFlakesFoundBySwarmingReruns')
def testCollectSwarmingTaskResultsNoResult(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 16
params = CollectSwarmingTaskResultsInputs(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
build_completed=True)
steps = ['step1']
task1 = WfSwarmingTask.Create(master_name, builder_name, build_number,
'step1')
task1.status = analysis_status.COMPLETED
task1.put()
self.assertEqual(
CollectSwarmingTaskResultsOutputs.FromSerializable({}),
test_swarming.GetConsistentFailuresWhenAllTasksComplete(
params, steps))
@mock.patch.object(step_util,
'GetStepMetadata',
return_value={
'canonical_step_name': 'step2',
'isolate_target_name': 'step2'
})
@mock.patch.object(detect_flake_occurrences, 'StoreDetectedCIFlakes')
@mock.patch.object(test_failure_analysis,
'UpdateAnalysisWithFlakesFoundBySwarmingReruns')
@mock.patch.object(monitoring, 'OnFlakeIdentified')
def testCollectSwarmingTaskResultsAllFlaky(self, mock_monitoring,
mock_update_analysis,
mock_save_flakes, _):
master_name = 'm'
builder_name = 'b'
build_number = 17
params = CollectSwarmingTaskResultsInputs(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
build_completed=True)
steps = ['step2']
task2 = WfSwarmingTask.Create(master_name, builder_name, build_number,
'step2')
task2.status = analysis_status.COMPLETED
task2.tests_statuses = {
'TestSuite1.test1': {
'total_run': 2,
'SUCCESS': 2
},
'TestSuite1.test2': {
'total_run': 4,
'SUCCESS': 2,
'FAILURE': 2
},
}
task2.put()
self.assertEqual(
CollectSwarmingTaskResultsOutputs.FromSerializable({}),
test_swarming.GetConsistentFailuresWhenAllTasksComplete(
params, steps))
flake_tests = {'step2': ['TestSuite1.test2']}
mock_update_analysis.assert_called_once_with(master_name, builder_name,
build_number, flake_tests)
mock_monitoring.assert_called_once_with('step2', 'step2', 'skip', 1)
mock_save_flakes.assert_called_once_with(master_name, builder_name,
build_number, flake_tests)
@mock.patch.object(test_swarming,
'NeedANewSwarmingTask',
side_effect=[True, False])
@mock.patch.object(test_failure_analysis,
'GetsFirstFailureAtTestLevel',
return_value={
'step': ['test'],
'step1': ['test1']
})
def testGetFirstTimeTestFailuresToRunSwarmingTasks(self, mock_fn, _):
master_name = 'm'
builder_name = 'b'
build_number = 13
step_name = 'step'
failure_info_json = {
'failure_type': failure_type.TEST,
'failed_steps': {
step_name: {}
}
}
failure_info = TestFailureInfo.FromSerializable(failure_info_json)
heuristic_result = TestHeuristicAnalysisOutput(
failure_info=failure_info,
heuristic_result=TestHeuristicResult.FromSerializable({}))
params = RunSwarmingTasksInput(build_key=BuildKey(
master_name=master_name,
builder_name=builder_name,
build_number=build_number),
heuristic_result=heuristic_result,
force=False)
self.assertEqual(
{'step': ['test']},
test_swarming.GetFirstTimeTestFailuresToRunSwarmingTasks(params))
mock_fn.assert_called_once_with(master_name, builder_name,
build_number, failure_info, False)
@mock.patch.object(test_failure_analysis, 'GetsFirstFailureAtTestLevel')
def testGetFirstTimeTestFailuresToRunSwarmingTasksBailOut(self, mock_fn):
master_name = 'm'
builder_name = 'b'
build_number = 14
params = RunSwarmingTasksInput(
build_key=BuildKey(master_name=master_name,
builder_name=builder_name,
build_number=build_number),
heuristic_result=TestHeuristicAnalysisOutput.FromSerializable({}),
force=False)
self.assertEqual(
{},
test_swarming.GetFirstTimeTestFailuresToRunSwarmingTasks(params))
self.assertFalse(mock_fn.called)
