def _mark_as_fixed(testcase, revision): """Mark bug as fixed.""" testcase.open = False # Bisection not available for external reproduction infrastructure. Assume # range (crash revision : current revision). testcase.fixed = f'{testcase.crash_revision}:{revision}' data_handler.update_progression_completion_metadata( testcase, revision, message=f'fixed in r{revision}')
def _mark_errored(testcase, revision, error): """Mark testcase as errored out.""" message = 'Received error from external infra, marking testcase as NA.' logs.log_warn(message, error=error, testcase_id=testcase.key.id()) testcase.fixed = 'NA' testcase.open = False data_handler.update_progression_completion_metadata(testcase, revision, message=message)
def handle_update(testcase, revision, stacktrace, error): """Handle update.""" logs.log('Got external update for testcase.', testcase_id=testcase.key.id()) if error: _mark_errored(testcase, revision, error) return last_tested_revision = (testcase.get_metadata('last_tested_revision') or testcase.crash_revision) if revision < last_tested_revision: logs.log_warn(f'Revision {revision} less than previously tested ' f'revision {last_tested_revision}.') return fuzz_target = testcase.get_fuzz_target() if fuzz_target: fuzz_target_name = fuzz_target.binary else: fuzz_target_name = None # Record use of fuzz target to avoid garbage collection (since fuzz_task does # not run). data_handler.record_fuzz_target(fuzz_target.engine, fuzz_target.binary, testcase.job_type) state = stack_analyzer.get_crash_data(stacktrace, fuzz_target=fuzz_target_name, symbolize_flag=False, already_symbolized=True, detect_ooms_and_hangs=True) crash_comparer = CrashComparer(state.crash_state, testcase.crash_state) if not crash_comparer.is_similar(): logs.log(f'State no longer similar (' f'testcase_id={testcase.key.id()}, ' f'old_state={testcase.crash_state}, ' f'new_state={state.crash_state})') _mark_as_fixed(testcase, revision) return is_security = crash_analyzer.is_security_issue(state.crash_stacktrace, state.crash_type, state.crash_address) if is_security != testcase.security_flag: logs.log(f'Security flag for {testcase.key.id()} no longer matches.') _mark_as_fixed(testcase, revision) return logs.log(f'{testcase.key.id()} still crashes.') testcase.last_tested_crash_stacktrace = stacktrace data_handler.update_progression_completion_metadata(testcase, revision, is_crash=True)
def _save_fixed_range(testcase_id, min_revision, max_revision, testcase_file_path): """Update a test case and other metadata with a fixed range.""" testcase = data_handler.get_testcase_by_id(testcase_id) testcase.fixed = '%d:%d' % (min_revision, max_revision) testcase.open = False data_handler.update_progression_completion_metadata( testcase, max_revision, message='fixed in range r%s' % testcase.fixed) _write_to_bigquery(testcase, min_revision, max_revision) _store_testcase_for_regression_testing(testcase, testcase_file_path)
def _check_fixed_for_custom_binary(testcase, job_type, testcase_file_path): """Simplified fixed check for test cases using custom binaries.""" revision = environment.get_value('APP_REVISION') # Update comments to reflect bot information and clean up old comments. testcase_id = testcase.key.id() testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) build_manager.setup_build() if not build_manager.check_app_path(): testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment( testcase, data_types.TaskState.ERROR, 'Build setup failed for custom binary') build_fail_wait = environment.get_value('FAIL_WAIT') tasks.add_task( 'progression', testcase_id, job_type, wait_time=build_fail_wait) return test_timeout = environment.get_value('TEST_TIMEOUT', 10) result = testcase_manager.test_for_crash_with_retries( testcase, testcase_file_path, test_timeout, http_flag=testcase.http_flag) _log_output(revision, result) # Re-fetch to finalize testcase updates in branches below. testcase = data_handler.get_testcase_by_id(testcase.key.id()) # If this still crashes on the most recent build, it's not fixed. The task # will be rescheduled by a cron job and re-attempted eventually. if result.is_crash(): app_path = environment.get_value('APP_PATH') command = testcase_manager.get_command_line_for_application( testcase_file_path, app_path=app_path, needs_http=testcase.http_flag) symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True) unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False) stacktrace = utils.get_crash_stacktrace_output( command, symbolized_crash_stacktrace, unsymbolized_crash_stacktrace) testcase.last_tested_crash_stacktrace = data_handler.filter_stacktrace( stacktrace) data_handler.update_progression_completion_metadata( testcase, revision, is_crash=True, message='still crashes on latest custom build') return if result.unexpected_crash: testcase.set_metadata( 'crashes_on_unexpected_state', True, update_testcase=False) else: testcase.delete_metadata( 'crashes_on_unexpected_state', update_testcase=False) # Retry once on another bot to confirm our results and in case this bot is in # a bad state which we didn't catch through our usual means. if data_handler.is_first_retry_for_task(testcase, reset_after_retry=True): tasks.add_task('progression', testcase_id, job_type) data_handler.update_progression_completion_metadata(testcase, revision) return # The bug is fixed. testcase.fixed = 'Yes' testcase.open = False data_handler.update_progression_completion_metadata( testcase, revision, message='fixed on latest custom build')
def find_fixed_range(testcase_id, job_type): """Attempt to find the revision range where a testcase was fixed.""" deadline = tasks.get_task_completion_deadline() testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return if testcase.fixed: logs.log_error('Fixed range is already set as %s, skip.' % testcase.fixed) return # Setup testcase and its dependencies. file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type) if not file_list: return # Set a flag to indicate we are running progression task. This shows pending # status on testcase report page and avoid conflicting testcase updates by # triage cron. testcase.set_metadata('progression_pending', True) # Custom binaries are handled as special cases. if build_manager.is_custom_binary(): _check_fixed_for_custom_binary(testcase, job_type, testcase_file_path) return build_bucket_path = build_manager.get_primary_bucket_path() revision_list = build_manager.get_revisions_list( build_bucket_path, testcase=testcase) if not revision_list: data_handler.close_testcase_with_error(testcase_id, 'Failed to fetch revision list') return # Use min, max_index to mark the start and end of revision list that is used # for bisecting the progression range. Set start to the revision where noticed # the crash. Set end to the trunk revision. Also, use min, max from past run # if it timed out. min_revision = testcase.get_metadata('last_progression_min') max_revision = testcase.get_metadata('last_progression_max') if min_revision or max_revision: # Clear these to avoid using them in next run. If this run fails, then we # should try next run without them to see it succeeds. If this run succeeds, # we should still clear them to avoid capping max revision in next run. testcase = data_handler.get_testcase_by_id(testcase_id) testcase.delete_metadata('last_progression_min', update_testcase=False) testcase.delete_metadata('last_progression_max', update_testcase=False) testcase.put() last_tested_revision = testcase.get_metadata('last_tested_crash_revision') known_crash_revision = last_tested_revision or testcase.crash_revision if not min_revision: min_revision = known_crash_revision if not max_revision: max_revision = revisions.get_last_revision_in_list(revision_list) min_index = revisions.find_min_revision_index(revision_list, min_revision) if min_index is None: raise errors.BuildNotFoundError(min_revision, job_type) max_index = revisions.find_max_revision_index(revision_list, max_revision) if max_index is None: raise errors.BuildNotFoundError(max_revision, job_type) testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED, 'r%d' % max_revision) # Check to see if this testcase is still crashing now. If it is, then just # bail out. result = _testcase_reproduces_in_revision( testcase, testcase_file_path, job_type, max_revision, update_metadata=True) if result.is_crash(): logs.log('Found crash with same signature on latest revision r%d.' % max_revision) app_path = environment.get_value('APP_PATH') command = testcase_manager.get_command_line_for_application( testcase_file_path, app_path=app_path, needs_http=testcase.http_flag) symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True) unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False) stacktrace = utils.get_crash_stacktrace_output( command, symbolized_crash_stacktrace, unsymbolized_crash_stacktrace) testcase = data_handler.get_testcase_by_id(testcase_id) testcase.last_tested_crash_stacktrace = data_handler.filter_stacktrace( stacktrace) data_handler.update_progression_completion_metadata( testcase, max_revision, is_crash=True, message='still crashes on latest revision r%s' % max_revision) # Since we've verified that the test case is still crashing, clear out any # metadata indicating potential flake from previous runs. task_creation.mark_unreproducible_if_flaky(testcase, False) # For chromium project, save latest crash information for later upload # to chromecrash/. state = result.get_symbolized_data() crash_uploader.save_crash_info_if_needed(testcase_id, max_revision, job_type, state.crash_type, state.crash_address, state.frames) return if result.unexpected_crash: testcase.set_metadata('crashes_on_unexpected_state', True) else: testcase.delete_metadata('crashes_on_unexpected_state') # Don't burden NFS server with caching these random builds. environment.set_value('CACHE_STORE', False) # Verify that we do crash in the min revision. This is assumed to be true # while we are doing the bisect. result = _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, min_revision) if result and not result.is_crash(): testcase = data_handler.get_testcase_by_id(testcase_id) # Retry once on another bot to confirm our result. if data_handler.is_first_retry_for_task(testcase, reset_after_retry=True): tasks.add_task('progression', testcase_id, job_type) error_message = ( 'Known crash revision %d did not crash, will retry on another bot to ' 'confirm result' % known_crash_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) data_handler.update_progression_completion_metadata( testcase, max_revision) return data_handler.clear_progression_pending(testcase) error_message = ( 'Known crash revision %d did not crash' % known_crash_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) task_creation.mark_unreproducible_if_flaky(testcase, True) return # Start a binary search to find last non-crashing revision. At this point, we # know that we do crash in the min_revision, and do not crash in max_revision. while time.time() < deadline: min_revision = revision_list[min_index] max_revision = revision_list[max_index] # If the min and max revisions are one apart this is as much as we can # narrow the range. if max_index - min_index == 1: _save_fixed_range(testcase_id, min_revision, max_revision, testcase_file_path) return # Occasionally, we get into this bad state. It seems to be related to test # cases with flaky stacks, but the exact cause is unknown. if max_index - min_index < 1: testcase = data_handler.get_testcase_by_id(testcase_id) testcase.fixed = 'NA' testcase.open = False message = ('Fixed testing errored out (min and max revisions ' 'are both %d)' % min_revision) data_handler.update_progression_completion_metadata( testcase, max_revision, message=message) # Let the bisection service know about the NA status. bisection.request_bisection(testcase) return # Test the middle revision of our range. middle_index = (min_index + max_index) // 2 middle_revision = revision_list[middle_index] testcase = data_handler.get_testcase_by_id(testcase_id) log_message = 'Testing r%d (current range %d:%d)' % ( middle_revision, min_revision, max_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.WIP, log_message) try: result = _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, middle_revision) except errors.BadBuildError: # Skip this revision. del revision_list[middle_index] max_index -= 1 continue if result.is_crash(): min_index = middle_index else: max_index = middle_index _save_current_fixed_range_indices(testcase_id, revision_list[min_index], revision_list[max_index]) # If we've broken out of the loop, we've exceeded the deadline. Recreate the # task to pick up where we left off. testcase = data_handler.get_testcase_by_id(testcase_id) error_message = ('Timed out, current range r%d:r%d' % (revision_list[min_index], revision_list[max_index])) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task('progression', testcase_id, job_type)
def handle_update(testcase, revision, stacktraces, error, protocol_version): """Handle update.""" def is_still_crashing(st_index, stacktrace): """Check if the the given stackstrace indicates the testcase is still crashing""" state = stack_analyzer.get_crash_data(stacktrace, fuzz_target=fuzz_target_name, symbolize_flag=False, already_symbolized=True, detect_ooms_and_hangs=True) crash_comparer = CrashComparer(state.crash_state, testcase.crash_state) if not crash_comparer.is_similar(): return False logs.log(f'State for trial {st_index} of {testcase_id} ' f'remains similar' f'(old_state={testcase.crash_state}, ' f'new_state={state.crash_state}).') is_security = crash_analyzer.is_security_issue(state.crash_stacktrace, state.crash_type, state.crash_address) if is_security != testcase.security_flag: return False logs.log(f'Security flag for trial {st_index} of {testcase_id} ' f'still matches' f'({testcase.security_flag}).') return True testcase_id = testcase.key.id() logs.log('Got external update for testcase.', testcase_id=testcase_id) if error: _mark_errored(testcase, revision, error) return last_tested_revision = (testcase.get_metadata('last_tested_revision') or testcase.crash_revision) if revision < last_tested_revision: logs.log_warn(f'Revision {revision} less than previously tested ' f'revision {last_tested_revision}.') return if protocol_version not in [OLD_PROTOCOL, NEW_PROTOCOL]: logs.log_error(f'Invalid protocol_version provided: ' f'{protocol_version} ' f'is not one of {{{OLD_PROTOCOL, NEW_PROTOCOL}}} ' f'(testcase_id={testcase_id}).') return if not stacktraces: logs.log_error(f'Empty JSON stacktrace list provided ' f'(testcase_id={testcase_id}).') return fuzz_target = testcase.get_fuzz_target() if fuzz_target: fuzz_target_name = fuzz_target.binary else: fuzz_target_name = None # Record use of fuzz target to avoid garbage collection (since fuzz_task does # not run). data_handler.record_fuzz_target(fuzz_target.engine, fuzz_target.binary, testcase.job_type) for st_index, stacktrace in enumerate(stacktraces): if is_still_crashing(st_index, stacktrace): logs.log(f'stacktrace {st_index} of {testcase_id} still crashes.') testcase.last_tested_crash_stacktrace = stacktrace data_handler.update_progression_completion_metadata(testcase, revision, is_crash=True) return # All trials resulted in a non-crash. Close the testcase. logs.log(f'No matching crash detected in {testcase_id} ' f'over {len(stacktraces)} trials, marking as fixed.') _mark_as_fixed(testcase, revision)