def _StartTestLevelCheckForFirstFailure(master_name, builder_name, build_number, step_name, failed_step, http_client): """Downloads test results and initiates first failure info at test level.""" list_isolated_data = failed_step.list_isolated_data list_isolated_data = (list_isolated_data.ToSerializable() if list_isolated_data else []) result_log = swarmed_test_util.RetrieveShardedTestResultsFromIsolatedServer( list_isolated_data, http_client) test_results_object = test_results_util.GetTestResultObject(result_log) if not test_results_object or not step_util.IsStepSupportedByFindit( test_results_object, step_util.GetCanonicalStepName(master_name, builder_name, build_number, step_name), master_name): return False failed_test_log, reliable_failed_tests = ( test_results_service.GetFailedTestsInformationFromTestResult( test_results_object)) _SaveIsolatedResultToStep(master_name, builder_name, build_number, step_name, failed_test_log) return _InitiateTestLevelFirstFailure(reliable_failed_tests, failed_step)
def GetSwarmingTaskIdForTryJob(report, revision, step_name, test_name): """Check json output for each task and return id of the one with test result. Args: report (dict): A dict in the format: { 'result': { revision: { step_name: { 'valid': (bool), 'step_metadata': (dict), 'pass_fail_counts': { test_name: { 'pass_count': (int), 'fail_count': (int), } } } } } } revision (str): The git hash the try job ran. step_name (str): The name of the step the flaky test was found on. test_name (str): The name of the flaky test. Returns: The swarming task id (str) that the try job ran to determine the pass rate, or None if not found. """ if not report: return None http_client = FinditHttpClient() step_result = report.get('result', {}).get(revision, {}).get(step_name, {}) pass_fail_counts = step_result.get('pass_fail_counts', {}).get(test_name) task_ids = step_result.get('step_metadata', {}).get('swarm_task_ids', []) if len(task_ids) == 1: return task_ids[0] if not pass_fail_counts: # Test doesn't exist. return task_ids[0] if task_ids else None for task_id in task_ids: output_json = swarmed_test_util.GetTestResultForSwarmingTask( task_id, http_client) test_results = test_results_util.GetTestResultObject( output_json, partial_result=True) if output_json and test_results and test_results.IsTestResultUseful(): return task_id return None
def IsTestEnabled(test_name, tasks): """Returns True if the test is enabled, False otherwise.""" # Get the isolated outputs from the tests that were just run. for task in tasks: test_results_log = GetTestResultForSwarmingTask(task.task_id, _FINDIT_HTTP_CLIENT) test_result_object = test_results_util.GetTestResultObject( test_results_log, partial_result=True) if test_result_object.DoesTestExist(test_name): return test_result_object.IsTestEnabled(test_name) elif test_result_object.contains_all_tests: # Has checked all tests and the test doesn't exist. return False return False
def _GetTestLevelLogForAStep(master_name, builder_name, build_number, step_name, http_client): """Downloads swarming test results for a step from a build and returns logs for failed tests. Returns: A dict of failure logs for each failed test. """ step = WfStep.Get(master_name, builder_name, build_number, step_name) if (step and step.isolated and step.log_data and step.log_data != constants.TOO_LARGE_LOG): # Test level log has been saved for this step. try: if step.log_data == constants.FLAKY_FAILURE_LOG: return {} return json.loads(step.log_data) except ValueError: logging.error( 'log_data %s of step %s/%s/%d/%s is not json loadable.' % (step.log_data, master_name, builder_name, build_number, step_name)) return None # Sends request to swarming server for isolated data. step_isolated_data = swarming.GetIsolatedDataForStep( master_name, builder_name, build_number, step_name, http_client) if not step_isolated_data: logging.warning( 'Failed to get step_isolated_data for build %s/%s/%d/%s.' % (master_name, builder_name, build_number, step_name)) return None result_log = swarmed_test_util.RetrieveShardedTestResultsFromIsolatedServer( step_isolated_data, http_client) test_results = test_results_util.GetTestResultObject(result_log) if not test_results: logging.warning( 'Failed to get swarming test results for build %s/%s/%d/%s.' % (master_name, builder_name, build_number, step_name)) return None failed_test_log, _ = ( test_results_service.GetFailedTestsInformationFromTestResult( test_results)) return failed_test_log
def RetrieveShardedTestResultsFromIsolatedServer(list_isolated_data, http_client): """Gets test results from isolated server and merge the results.""" shard_results = [] for isolated_data in list_isolated_data: test_result_log, _ = isolate.DownloadFileFromIsolatedServer( isolated_data, http_client, 'output.json') if not test_result_log: return None shard_results.append(json.loads(test_result_log)) if not shard_results: return [] test_results = test_results_util.GetTestResultObject(shard_results[0]) return test_results.GetMergedTestResults( shard_results) if test_results else None
def OnSwarmingTaskCompleted(master_name, builder_name, build_number, step_name, data, output_json): test_results = test_results_util.GetTestResultObject(output_json) classified_test_results = (test_results.GetClassifiedTestResults() if test_results else {}) _UpdateSwarmingTaskEntity(master_name, builder_name, build_number, step_name, status=analysis_status.COMPLETED, classified_test_results=classified_test_results, created_ts=data.get('created_ts'), started_ts=data.get('started_ts'), completed_ts=data.get('completed_ts')) _RecordSwarmingTaskStateChange(master_name, builder_name, build_number, step_name, analysis_status.COMPLETED, analysis_approach_type.SWARMING) return True
def GetTestLocation(task_id, test_name): """Gets the filepath and line number of a test from swarming. Args: task_id (str): The swarming task id to query. test_name (str): The name of the test whose location to return. Returns: (TestLocation): The file path and line number of the test, or None if the test location was not be retrieved. """ test_results_log = GetTestResultForSwarmingTask(task_id, _FINDIT_HTTP_CLIENT) test_results = test_results_util.GetTestResultObject(test_results_log) test_location, error = test_results.GetTestLocation( test_name) if test_results else (None, constants.WRONG_FORMAT_LOG) if error: logging.info('Failed to get test location for task %s: %s', task_id, error) return None return TestLocation.FromSerializable(test_location or {})
def GetCountsFromSwarmingRerun(test_results_json): """Gets the total number of runs and passes from the test result of a swarming rerun determining pass_rate. Assumption: only one test is run. When getting pass counts for a swarming rerun for a flaky test to determine pass rate, always assume the test is expected to pass, regardless what's currently expected for the test in test results. So counts results based on pass/fail only, rather than expected/unexpected. Args: test_results_json(dict): Test results log. Returns: (int, int) total number of tries and number of successful runs. """ test_results_obj = test_results_util.GetTestResultObject(test_results_json) if not test_results_obj: return None, None classified_test_results = test_results_obj.GetClassifiedTestResults() num_tests = len(classified_test_results) if num_tests == 0: # The test doesn't exist yet, i.e. newly-added tests. return 0, 0 # There should be exactly 1 test that was run. assert num_tests == 1, 'Expecting 1 test in results, but got {}'.format( num_tests) test_result = classified_test_results.values()[0] tries = test_result.total_run passes = test_result.results.passes successes = sum(passes.values()) return tries, successes
def OnSwarmingTaskTimeout(run_swarming_task_params, task_id): master_name, builder_name, build_number = ( run_swarming_task_params.build_key.GetParts()) step_name = run_swarming_task_params.step_name error = SwarmingTaskError.GenerateError(swarming_task_error.RUNNER_TIMEOUT) data, output_json, _ = swarmed_test_util.GetSwarmingTaskDataAndResult( task_id) if output_json and test_results_util.IsTestResultsValid(output_json): test_results = test_results_util.GetTestResultObject(output_json) classified_test_results = (test_results.GetClassifiedTestResults() if test_results else {}) _UpdateSwarmingTaskEntity( master_name, builder_name, build_number, step_name, status=analysis_status.COMPLETED, error=error, classified_test_results=classified_test_results, created_ts=data.get('created_ts'), started_ts=data.get('started_ts'), completed_ts=data.get('completed_ts')) _RecordSwarmingTaskStateChange(master_name, builder_name, build_number, step_name, analysis_status.COMPLETED, analysis_approach_type.SWARMING) else: _UpdateSwarmingTaskEntity(master_name, builder_name, build_number, step_name, status=analysis_status.ERROR, error=error) _RecordSwarmingTaskStateChange(master_name, builder_name, build_number, step_name, analysis_status.ERROR, analysis_approach_type.SWARMING)
def testGetTestResultObject(self, _): test_results = test_results_util.GetTestResultObject('log') self.assertTrue(isinstance(test_results, GtestTestResults))
def testGetTestResultObjectNoMatch(self, _): self.assertIsNone(test_results_util.GetTestResultObject('log'))
def ExtractSignalsForTestFailure(failure_info, http_client): signals = {} master_name = failure_info.master_name builder_name = failure_info.builder_name build_number = failure_info.build_number failed_steps = failure_info.failed_steps or {} for step_name in failed_steps: failure_log = None if not failed_steps[step_name].supported: # Bail out if the step is not supported. continue # 1. Tries to get stored failure log from step. step = (WfStep.Get(master_name, builder_name, build_number, step_name) or WfStep.Create(master_name, builder_name, build_number, step_name)) if step.log_data and step.log_data != constants.TOO_LARGE_LOG: failure_log = step.log_data else: json_formatted_log = True # 2. Gets test results. list_isolated_data = failed_steps[step_name].list_isolated_data list_isolated_data = (list_isolated_data.ToSerializable() if list_isolated_data else []) merged_test_results = ( swarmed_test_util.RetrieveShardedTestResultsFromIsolatedServer( list_isolated_data, http_client)) if merged_test_results: test_results = test_results_util.GetTestResultObject( merged_test_results) if test_results: failure_log, _ = ( test_results_service. GetFailedTestsInformationFromTestResult(test_results)) failure_log = json.dumps( failure_log ) if failure_log else constants.FLAKY_FAILURE_LOG else: failure_log = constants.WRONG_FORMAT_LOG if not merged_test_results or failure_log in [ constants.INVALID_FAILURE_LOG, constants.WRONG_FORMAT_LOG ]: # 3. Gets stdout log. json_formatted_log = False failure_log = extract_signal.GetStdoutLog( master_name, builder_name, build_number, step_name, http_client) try: if not failure_log: raise extract_signal.FailedToGetFailureLogError( 'Failed to pull failure log (stdio or ninja output) of step %s of' ' %s/%s/%d' % (step_name, master_name, builder_name, build_number)) except extract_signal.FailedToGetFailureLogError: return {} # Save step log in datastore and avoid downloading again during retry. step.log_data = extract_signal.ExtractStorablePortionOfLog( failure_log, json_formatted_log ) if step.log_data != constants.TOO_LARGE_LOG else step.log_data step.isolated = step.isolated or json_formatted_log try: step.put() except Exception as e: # pragma: no cover # Sometimes, the step log is too large to save in datastore. logging.exception(e) if step.isolated: try: json_failure_log = (json.loads(failure_log) if failure_log != constants.FLAKY_FAILURE_LOG else {}) except ValueError: json_failure_log = {} logging.warning('failure_log %s is not valid JSON.' % failure_log) signals[step_name] = {'tests': {}} step_signal = FailureSignal() for test_name, test_failure_log in json_failure_log.iteritems(): signals[step_name]['tests'][ test_name] = extractors.ExtractSignal( master_name, builder_name, step_name, test_name, base64.b64decode(test_failure_log)).ToDict() # Save signals in test failure log to step level. step_signal.MergeFrom(signals[step_name]['tests'][test_name]) signals[step_name]['files'] = step_signal.files else: signals[step_name] = extractors.ExtractSignal( master_name, builder_name, step_name, None, failure_log).ToDict() return signals
def FindMatchingWaterfallStep(build_step, test_name): """Finds the matching Waterfall step and checks whether it is supported. Only Swarmed and gtest-based steps are supported at the moment. Args: build_step (BuildStep): A build step on Waterfall or Commit Queue. It will be updated with the matching Waterfall step and whether it is Swarmed and supported. test_name (str): The name of the test. """ build_step.swarmed = False build_step.supported = False http_client = FinditHttpClient() if build_step.on_cq: wf_master_name, wf_builder_name, wf_build_number, wf_step_name, metadata = ( _GetMatchingWaterfallBuildStep(build_step, http_client)) build_step.wf_master_name = wf_master_name build_step.wf_builder_name = wf_builder_name build_step.wf_build_number = wf_build_number build_step.wf_step_name = wf_step_name if not build_step.has_matching_waterfall_step: return else: build_step.wf_master_name = build_step.master_name build_step.wf_builder_name = build_step.builder_name build_step.wf_build_number = build_step.build_number build_step.wf_step_name = build_step.step_name metadata = build_step.step_metadata or step_util.GetStepMetadata( build_step.master_name, build_step.builder_name, build_step.build_number, build_step.step_name) if not metadata: logging.error('Couldn\'t get step_metadata') return build_step.step_metadata = metadata # Query Swarming for isolated data. build_step.swarmed = True if metadata.get('swarm_task_ids') else False if build_step.swarmed: need_to_continue = False for task_id in metadata['swarm_task_ids']: output = swarmed_test_util.GetTestResultForSwarmingTask( task_id, http_client) if output: # Guess from the format. test_result_object = test_results_util.GetTestResultObject( output, partial_result=True) if not test_result_object: build_step.supported = False elif not step_util.IsStepSupportedByFindit( test_result_object, metadata.get('canonical_step_name') or build_step.step_name, build_step.wf_master_name): build_step.supported = False else: test_exists = test_result_object.DoesTestExist(test_name) if test_exists: build_step.supported = True elif test_result_object.contains_all_tests: # There is no such test for sure. build_step.supported = False else: # Test is not in this task, but cannot determine if it's in other # tasks. need_to_continue = True if not need_to_continue: break