def _StartTestLevelCheckForFirstFailure(master_name, builder_name,
build_number, step_name, failed_step,
http_client):
"""Downloads test results and initiates first failure info at test level."""
list_isolated_data = failed_step.list_isolated_data
list_isolated_data = (list_isolated_data.ToSerializable()
if list_isolated_data else [])
result_log = swarmed_test_util.RetrieveShardedTestResultsFromIsolatedServer(
list_isolated_data, http_client)
test_results_object = test_results_util.GetTestResultObject(result_log)
if not test_results_object or not step_util.IsStepSupportedByFindit(
test_results_object,
step_util.GetCanonicalStepName(master_name, builder_name,
build_number, step_name),
master_name):
return False
failed_test_log, reliable_failed_tests = (
test_results_service.GetFailedTestsInformationFromTestResult(
test_results_object))
_SaveIsolatedResultToStep(master_name, builder_name, build_number,
step_name, failed_test_log)
return _InitiateTestLevelFirstFailure(reliable_failed_tests, failed_step)
def GetSwarmingTaskIdForTryJob(report, revision, step_name, test_name):
"""Check json output for each task and return id of the one with test result.
Args:
report (dict): A dict in the format:
{
'result': {
revision: {
step_name: {
'valid': (bool),
'step_metadata': (dict),
'pass_fail_counts': {
test_name: {
'pass_count': (int),
'fail_count': (int),
}
}
}
}
}
}
revision (str): The git hash the try job ran.
step_name (str): The name of the step the flaky test was found on.
test_name (str): The name of the flaky test.
Returns:
The swarming task id (str) that the try job ran to determine the pass rate,
or None if not found.
"""
if not report:
return None
http_client = FinditHttpClient()
step_result = report.get('result', {}).get(revision, {}).get(step_name, {})
pass_fail_counts = step_result.get('pass_fail_counts', {}).get(test_name)
task_ids = step_result.get('step_metadata', {}).get('swarm_task_ids', [])
if len(task_ids) == 1:
return task_ids[0]
if not pass_fail_counts: # Test doesn't exist.
return task_ids[0] if task_ids else None
for task_id in task_ids:
output_json = swarmed_test_util.GetTestResultForSwarmingTask(
task_id, http_client)
test_results = test_results_util.GetTestResultObject(
output_json, partial_result=True)
if output_json and test_results and test_results.IsTestResultUseful():
return task_id
return None
def IsTestEnabled(test_name, tasks):
"""Returns True if the test is enabled, False otherwise."""
# Get the isolated outputs from the tests that were just run.
for task in tasks:
test_results_log = GetTestResultForSwarmingTask(task.task_id,
_FINDIT_HTTP_CLIENT)
test_result_object = test_results_util.GetTestResultObject(
test_results_log, partial_result=True)
if test_result_object.DoesTestExist(test_name):
return test_result_object.IsTestEnabled(test_name)
elif test_result_object.contains_all_tests:
# Has checked all tests and the test doesn't exist.
return False
return False
def _GetTestLevelLogForAStep(master_name, builder_name, build_number,
step_name, http_client):
"""Downloads swarming test results for a step from a build and returns logs
for failed tests.
Returns:
A dict of failure logs for each failed test.
"""
step = WfStep.Get(master_name, builder_name, build_number, step_name)
if (step and step.isolated and step.log_data
and step.log_data != constants.TOO_LARGE_LOG):
# Test level log has been saved for this step.
try:
if step.log_data == constants.FLAKY_FAILURE_LOG:
return {}
return json.loads(step.log_data)
except ValueError:
logging.error(
'log_data %s of step %s/%s/%d/%s is not json loadable.' %
(step.log_data, master_name, builder_name, build_number,
step_name))
return None
# Sends request to swarming server for isolated data.
step_isolated_data = swarming.GetIsolatedDataForStep(
master_name, builder_name, build_number, step_name, http_client)
if not step_isolated_data:
logging.warning(
'Failed to get step_isolated_data for build %s/%s/%d/%s.' %
(master_name, builder_name, build_number, step_name))
return None
result_log = swarmed_test_util.RetrieveShardedTestResultsFromIsolatedServer(
step_isolated_data, http_client)
test_results = test_results_util.GetTestResultObject(result_log)
if not test_results:
logging.warning(
'Failed to get swarming test results for build %s/%s/%d/%s.' %
(master_name, builder_name, build_number, step_name))
return None
failed_test_log, _ = (
test_results_service.GetFailedTestsInformationFromTestResult(
test_results))
return failed_test_log
def RetrieveShardedTestResultsFromIsolatedServer(list_isolated_data,
http_client):
"""Gets test results from isolated server and merge the results."""
shard_results = []
for isolated_data in list_isolated_data:
test_result_log, _ = isolate.DownloadFileFromIsolatedServer(
isolated_data, http_client, 'output.json')
if not test_result_log:
return None
shard_results.append(json.loads(test_result_log))
if not shard_results:
return []
test_results = test_results_util.GetTestResultObject(shard_results[0])
return test_results.GetMergedTestResults(
shard_results) if test_results else None
def OnSwarmingTaskCompleted(master_name, builder_name, build_number, step_name,
data, output_json):
test_results = test_results_util.GetTestResultObject(output_json)
classified_test_results = (test_results.GetClassifiedTestResults()
if test_results else {})
_UpdateSwarmingTaskEntity(master_name,
builder_name,
build_number,
step_name,
status=analysis_status.COMPLETED,
classified_test_results=classified_test_results,
created_ts=data.get('created_ts'),
started_ts=data.get('started_ts'),
completed_ts=data.get('completed_ts'))
_RecordSwarmingTaskStateChange(master_name, builder_name, build_number,
step_name, analysis_status.COMPLETED,
analysis_approach_type.SWARMING)
return True
def GetTestLocation(task_id, test_name):
"""Gets the filepath and line number of a test from swarming.
Args:
task_id (str): The swarming task id to query.
test_name (str): The name of the test whose location to return.
Returns:
(TestLocation): The file path and line number of the test, or None
if the test location was not be retrieved.
"""
test_results_log = GetTestResultForSwarmingTask(task_id, _FINDIT_HTTP_CLIENT)
test_results = test_results_util.GetTestResultObject(test_results_log)
test_location, error = test_results.GetTestLocation(
test_name) if test_results else (None, constants.WRONG_FORMAT_LOG)
if error:
logging.info('Failed to get test location for task %s: %s', task_id, error)
return None
return TestLocation.FromSerializable(test_location or {})
def GetCountsFromSwarmingRerun(test_results_json):
"""Gets the total number of runs and passes from the test result of a swarming
rerun determining pass_rate.
Assumption: only one test is run.
When getting pass counts for a swarming rerun for a flaky test to
determine pass rate, always assume the test is expected to pass,
regardless what's currently expected for the test in test results.
So counts results based on pass/fail only, rather than expected/unexpected.
Args:
test_results_json(dict): Test results log.
Returns:
(int, int) total number of tries and number of successful runs.
"""
test_results_obj = test_results_util.GetTestResultObject(test_results_json)
if not test_results_obj:
return None, None
classified_test_results = test_results_obj.GetClassifiedTestResults()
num_tests = len(classified_test_results)
if num_tests == 0: # The test doesn't exist yet, i.e. newly-added tests.
return 0, 0
# There should be exactly 1 test that was run.
assert num_tests == 1, 'Expecting 1 test in results, but got {}'.format(
num_tests)
test_result = classified_test_results.values()[0]
tries = test_result.total_run
passes = test_result.results.passes
successes = sum(passes.values())
return tries, successes
def OnSwarmingTaskTimeout(run_swarming_task_params, task_id):
master_name, builder_name, build_number = (
run_swarming_task_params.build_key.GetParts())
step_name = run_swarming_task_params.step_name
error = SwarmingTaskError.GenerateError(swarming_task_error.RUNNER_TIMEOUT)
data, output_json, _ = swarmed_test_util.GetSwarmingTaskDataAndResult(
task_id)
if output_json and test_results_util.IsTestResultsValid(output_json):
test_results = test_results_util.GetTestResultObject(output_json)
classified_test_results = (test_results.GetClassifiedTestResults()
if test_results else {})
_UpdateSwarmingTaskEntity(
master_name,
builder_name,
build_number,
step_name,
status=analysis_status.COMPLETED,
error=error,
classified_test_results=classified_test_results,
created_ts=data.get('created_ts'),
started_ts=data.get('started_ts'),
completed_ts=data.get('completed_ts'))
_RecordSwarmingTaskStateChange(master_name, builder_name, build_number,
step_name, analysis_status.COMPLETED,
analysis_approach_type.SWARMING)
else:
_UpdateSwarmingTaskEntity(master_name,
builder_name,
build_number,
step_name,
status=analysis_status.ERROR,
error=error)
_RecordSwarmingTaskStateChange(master_name, builder_name, build_number,
step_name, analysis_status.ERROR,
analysis_approach_type.SWARMING)
def testGetTestResultObject(self, _):
test_results = test_results_util.GetTestResultObject('log')
self.assertTrue(isinstance(test_results, GtestTestResults))
def testGetTestResultObjectNoMatch(self, _):
self.assertIsNone(test_results_util.GetTestResultObject('log'))
def ExtractSignalsForTestFailure(failure_info, http_client):
signals = {}
master_name = failure_info.master_name
builder_name = failure_info.builder_name
build_number = failure_info.build_number
failed_steps = failure_info.failed_steps or {}
for step_name in failed_steps:
failure_log = None
if not failed_steps[step_name].supported:
# Bail out if the step is not supported.
continue
# 1. Tries to get stored failure log from step.
step = (WfStep.Get(master_name, builder_name, build_number, step_name)
or WfStep.Create(master_name, builder_name, build_number,
step_name))
if step.log_data and step.log_data != constants.TOO_LARGE_LOG:
failure_log = step.log_data
else:
json_formatted_log = True
# 2. Gets test results.
list_isolated_data = failed_steps[step_name].list_isolated_data
list_isolated_data = (list_isolated_data.ToSerializable()
if list_isolated_data else [])
merged_test_results = (
swarmed_test_util.RetrieveShardedTestResultsFromIsolatedServer(
list_isolated_data, http_client))
if merged_test_results:
test_results = test_results_util.GetTestResultObject(
merged_test_results)
if test_results:
failure_log, _ = (
test_results_service.
GetFailedTestsInformationFromTestResult(test_results))
failure_log = json.dumps(
failure_log
) if failure_log else constants.FLAKY_FAILURE_LOG
else:
failure_log = constants.WRONG_FORMAT_LOG
if not merged_test_results or failure_log in [
constants.INVALID_FAILURE_LOG, constants.WRONG_FORMAT_LOG
]:
# 3. Gets stdout log.
json_formatted_log = False
failure_log = extract_signal.GetStdoutLog(
master_name, builder_name, build_number, step_name,
http_client)
try:
if not failure_log:
raise extract_signal.FailedToGetFailureLogError(
'Failed to pull failure log (stdio or ninja output) of step %s of'
' %s/%s/%d' %
(step_name, master_name, builder_name, build_number))
except extract_signal.FailedToGetFailureLogError:
return {}
# Save step log in datastore and avoid downloading again during retry.
step.log_data = extract_signal.ExtractStorablePortionOfLog(
failure_log, json_formatted_log
) if step.log_data != constants.TOO_LARGE_LOG else step.log_data
step.isolated = step.isolated or json_formatted_log
try:
step.put()
except Exception as e: # pragma: no cover
# Sometimes, the step log is too large to save in datastore.
logging.exception(e)
if step.isolated:
try:
json_failure_log = (json.loads(failure_log) if
failure_log != constants.FLAKY_FAILURE_LOG
else {})
except ValueError:
json_failure_log = {}
logging.warning('failure_log %s is not valid JSON.' %
failure_log)
signals[step_name] = {'tests': {}}
step_signal = FailureSignal()
for test_name, test_failure_log in json_failure_log.iteritems():
signals[step_name]['tests'][
test_name] = extractors.ExtractSignal(
master_name, builder_name, step_name, test_name,
base64.b64decode(test_failure_log)).ToDict()
# Save signals in test failure log to step level.
step_signal.MergeFrom(signals[step_name]['tests'][test_name])
signals[step_name]['files'] = step_signal.files
else:
signals[step_name] = extractors.ExtractSignal(
master_name, builder_name, step_name, None,
failure_log).ToDict()
return signals
def FindMatchingWaterfallStep(build_step, test_name):
"""Finds the matching Waterfall step and checks whether it is supported.
Only Swarmed and gtest-based steps are supported at the moment.
Args:
build_step (BuildStep): A build step on Waterfall or Commit Queue. It
will be updated with the matching Waterfall step and whether it is
Swarmed and supported.
test_name (str): The name of the test.
"""
build_step.swarmed = False
build_step.supported = False
http_client = FinditHttpClient()
if build_step.on_cq:
wf_master_name, wf_builder_name, wf_build_number, wf_step_name, metadata = (
_GetMatchingWaterfallBuildStep(build_step, http_client))
build_step.wf_master_name = wf_master_name
build_step.wf_builder_name = wf_builder_name
build_step.wf_build_number = wf_build_number
build_step.wf_step_name = wf_step_name
if not build_step.has_matching_waterfall_step:
return
else:
build_step.wf_master_name = build_step.master_name
build_step.wf_builder_name = build_step.builder_name
build_step.wf_build_number = build_step.build_number
build_step.wf_step_name = build_step.step_name
metadata = build_step.step_metadata or step_util.GetStepMetadata(
build_step.master_name, build_step.builder_name,
build_step.build_number, build_step.step_name)
if not metadata:
logging.error('Couldn\'t get step_metadata')
return
build_step.step_metadata = metadata
# Query Swarming for isolated data.
build_step.swarmed = True if metadata.get('swarm_task_ids') else False
if build_step.swarmed:
need_to_continue = False
for task_id in metadata['swarm_task_ids']:
output = swarmed_test_util.GetTestResultForSwarmingTask(
task_id, http_client)
if output:
# Guess from the format.
test_result_object = test_results_util.GetTestResultObject(
output, partial_result=True)
if not test_result_object:
build_step.supported = False
elif not step_util.IsStepSupportedByFindit(
test_result_object,
metadata.get('canonical_step_name')
or build_step.step_name, build_step.wf_master_name):
build_step.supported = False
else:
test_exists = test_result_object.DoesTestExist(test_name)
if test_exists:
build_step.supported = True
elif test_result_object.contains_all_tests:
# There is no such test for sure.
build_step.supported = False
else:
# Test is not in this task, but cannot determine if it's in other
# tasks.
need_to_continue = True
if not need_to_continue:
break
