def testGetCountsFromSwarmingRerunTestNotExistYet(self):
test_results = {
'all_tests': [],
'per_iteration_data': [],
}
self.assertEqual(
(0, 0),
flake_test_results.GetCountsFromSwarmingRerun(test_results))
def testGetCountsFromSwarmingRerunWebkitLayoutTest(self):
test_results = {
'seconds_since_epoch': 1522268603,
'tests': {
'bluetooth': {
'requestDevice': {
'chooser': {
'new-scan-device-added.html': {
'expected': 'PASS',
'time': 1.1,
'actual':
'CRASH FAIL CRASH FAIL TEXT PASS PASS',
'has_stderr': True,
'is_unexpected': True
}
},
}
}
},
'skipped': 1,
'build_number': 'DUMMY_BUILD_NUMBER',
'num_failures_by_type': {
'SLOW': 0,
'CRASH': 1,
'MISSING': 0,
'SKIP': 1,
'IMAGE': 0,
'LEAK': 0,
'IMAGE+TEXT': 0,
'FAIL': 0,
'TEXT': 0,
'TIMEOUT': 0,
'PASS': 3,
'REBASELINE': 0,
'WONTFIX': 0,
'AUDIO': 0,
'NEEDSMANUALREBASELINE': 0
},
'interrupted': False,
'path_delimiter': '/',
'layout_tests_dir': '/b/s/w/ir/third_party/blink/web_tests',
'flag_name': None,
'version': 3,
'chromium_revision': '',
'num_passes': 3,
'pixel_tests_enabled': True,
'num_regressions': 1,
'fixable': 0,
'num_flaky': 0,
'random_order_seed': 4,
'builder_name': ''
}
self.assertEqual(
(7, 2),
flake_test_results.GetCountsFromSwarmingRerun(test_results))
def _ParseFlakeSwarmingTaskOutput(task_data, output_json, error, parameters):
"""Returns swarming task results as a FlakeswarmingTaskOutput object.
Assumption: only 1 test was run in the Swarming task.
"""
assert task_data
iterations = parameters.iterations
if output_json:
# Gets the total numbers of runs and number of successful runs from
# test results
tries, successes = flake_test_results.GetCountsFromSwarmingRerun(
output_json)
if tries is None or successes is None:
# Something went wrong preventing even a single test from being processed
# which counts as an error.
error = error or SwarmingTaskError.GenerateError(
code=swarming_task_error.UNKNOWN)
tries = None
successes = None
elif (tries == 1 and task_data['state'] == constants.STATE_COMPLETED
and not task_data.get('failure')
and not task_data.get('infra_failure')):
# webkit_layout_tests special case: test results will be combined into
# one if all results are the same.
# Use iterations instead assuming the test repeated that many times.
# Currently only do this if task completes successfully.
tries = iterations
successes = iterations * successes
return FlakeSwarmingTaskOutput(
completed_time=time_util.DatetimeFromString(
task_data.get('completed_ts')),
error=error,
iterations=tries,
pass_count=successes,
started_time=time_util.DatetimeFromString(
task_data.get('started_ts')),
task_id=task_data['task_id'])
else:
return FlakeSwarmingTaskOutput(
completed_time=time_util.DatetimeFromString(
task_data.get('completed_ts')),
error=error or
SwarmingTaskError.GenerateError(code=swarming_task_error.UNKNOWN),
iterations=None,
pass_count=None,
started_time=time_util.DatetimeFromString(
task_data.get('started_ts')),
task_id=task_data['task_id'])
def testGetCountsFromSwarmingRerun(self):
test_results = {
'all_tests': [
'Unittest1.Subtest1', 'Unittest1.PRE_PRE_Subtest1',
'Unittest1.PRE_Subtest1'
],
'per_iteration_data': [{
'Unittest1.Subtest1': [{
'status':
'SUCCESS',
'output_snippet_base64':
'WyAgICAgICBPSyBdCg=='
}],
'Unittest1.PRE_PRE_Subtest1': [{
'status':
'FAILURE',
'output_snippet_base64':
'YS9iL3UxczIuY2M6MTIzNDog'
}, {
'status':
'SUCCESS',
'output_snippet_base64':
'YS9iL3UxczIuY2M6MTIzNDog'
}, {
'status':
'SUCCESS',
'output_snippet_base64':
'WyAgICAgICBPSyBdCg=='
}],
'Unittest1.PRE_Subtest1': [{
'status':
'FAILURE',
'output_snippet_base64':
'RVJST1I6eF90ZXN0LmNjOjEyMz'
}, {
'status':
'SUCCESS',
'output_snippet_base64':
'WyAgICAgICBPSyBdCg=='
}]
}]
}
self.assertEqual(
(3, 1),
flake_test_results.GetCountsFromSwarmingRerun(test_results))
def testGetCountsFromSwarmingRerunInvalidResults(self):
test_results = 'invalid'
self.assertEqual(
(None, None),
flake_test_results.GetCountsFromSwarmingRerun(test_results))
def ResultsForBatch(experiment_id, batch):
"""Retrieve swarming results for each task and aggregate them.
This function makes the same assumptions about the swarming task as flake
analysis reruns.
Args:
batch (dict): Entry with an isolate hash, and swarming task ids, as well as,
possibly, partial results.
Returns:
A dict with counts for tries, passes, incomplete, errored and successful
tasks.
"""
batch['results'] = batch.get('results', {})
batch['task_results'] = batch.get('task_results', {})
total_tries = batch['results'].get('total_tries', 0)
total_passes = batch['results'].get('total_passes', 0)
incomplete_tasks = 0
errored_tasks = batch['results'].get('errored_tasks', 0)
successful_tasks = batch['results'].get('successful_tasks', 0)
error_bots = set(batch['results'].get('error_bots', []))
not_all_pass_bots = set(batch['task_results'].get('not_all_pass_bots', []))
all_pass_bots = set(batch['task_results'].get('all_pass_bots', []))
for task_id in ListTasks(experiment_id, batch['isolate_hash']):
if task_id not in batch['task_results']:
batch['task_results'][task_id] = {}
elif 'complete' in batch['task_results'][task_id]:
continue
task_data, task_output, error = GetSwarmingTaskDataAndResult(task_id)
if task_data.get('state') == constants.STATE_COMPLETED:
batch['task_results'][task_id]['complete'] = True
batch['task_results'][task_id]['bot'] = task_data.get('bot_id')
tries, passes = flake_test_results.GetCountsFromSwarmingRerun(
task_output)
if tries is None or passes is None:
errored_tasks += 1
error_bots.add(task_data.get('bot_id'))
batch['task_results'][task_id]['errored'] = True
print >> sys.stderr, (error
or 'Tries or passes info missing for ' +
task_id)
continue
total_tries += tries or 0
total_passes += passes or 0
successful_tasks += 1
if passes == tries:
all_pass_bots.add(task_data.get('bot_id'))
else:
not_all_pass_bots.add(task_data.get('bot_id'))
batch['task_results'][task_id]['passes'] = passes
batch['task_results'][task_id]['failures'] = tries - passes
else:
incomplete_tasks += 1
batch['results'].update({
'total_tries': total_tries,
'total_passes': total_passes,
'incomplete_tasks': incomplete_tasks,
'errored_tasks': errored_tasks,
'successful_tasks': successful_tasks,
'total_tasks': len(batch['task_results']),
'error_bots': list(error_bots),
'all_pass_bots': list(all_pass_bots),
'not_all_pass_bots': list(not_all_pass_bots),
})
return batch