def handle_unreproducible(): # Be more lenient with marking testcases as unreproducible when this is a # job override. if is_overriden_job: _skip_minimization(testcase, 'Unreproducible on overridden job.') else: task_creation.mark_unreproducible_if_flaky(testcase, True)
def find_regression_range(testcase_id, job_type): """Attempt to find when the testcase regressed.""" deadline = tasks.get_task_completion_deadline() testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return if testcase.regression: logs.log_error('Regression range is already set as %s, skip.' % testcase.regression) return # This task is not applicable for custom binaries. if build_manager.is_custom_binary(): testcase.regression = 'NA' data_handler.update_testcase_comment( testcase, data_types.TaskState.ERROR, 'Not applicable for custom binaries') return data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) # Setup testcase and its dependencies. file_list, _, testcase_file_path = setup.setup_testcase(testcase) if not file_list: testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, 'Failed to setup testcase') tasks.add_task('regression', testcase_id, job_type) return release_build_bucket_path = environment.get_value( 'RELEASE_BUILD_BUCKET_PATH') revision_list = build_manager.get_revisions_list(release_build_bucket_path, testcase=testcase) if not revision_list: testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, 'Failed to fetch revision list') tasks.add_task('regression', testcase_id, job_type) return # Don't burden NFS server with caching these random builds. environment.set_value('CACHE_STORE', False) # Pick up where left off in a previous run if necessary. min_revision = testcase.get_metadata('last_regression_min') max_revision = testcase.get_metadata('last_regression_max') first_run = not min_revision and not max_revision if not min_revision: min_revision = revisions.get_first_revision_in_list(revision_list) if not max_revision: max_revision = testcase.crash_revision min_index = revisions.find_min_revision_index(revision_list, min_revision) if min_index is None: raise errors.BuildNotFoundError(min_revision, job_type) max_index = revisions.find_max_revision_index(revision_list, max_revision) if max_index is None: raise errors.BuildNotFoundError(max_revision, job_type) # Make sure that the revision where we noticed the crash, still crashes at # that revision. Otherwise, our binary search algorithm won't work correctly. max_revision = revision_list[max_index] crashes_in_max_revision = _testcase_reproduces_in_revision( testcase, testcase_file_path, job_type, max_revision, should_log=False) if not crashes_in_max_revision: testcase = data_handler.get_testcase_by_id(testcase_id) error_message = ('Known crash revision %d did not crash' % max_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) task_creation.mark_unreproducible_if_flaky(testcase, True) return # If we've made it this far, the test case appears to be reproducible. Clear # metadata from previous runs had it been marked as potentially flaky. task_creation.mark_unreproducible_if_flaky(testcase, False) # On the first run, check to see if we regressed near either the min or max # revision. if first_run and found_regression_near_extreme_revisions( testcase, testcase_file_path, job_type, revision_list, min_index, max_index): return while time.time() < deadline: min_revision = revision_list[min_index] max_revision = revision_list[max_index] # If the min and max revisions are one apart (or the same, if we only have # one build), this is as much as we can narrow the range. if max_index - min_index <= 1: # Verify that the regression range seems correct, and save it if so. if not validate_regression_range(testcase, testcase_file_path, job_type, revision_list, min_index): return save_regression_range(testcase_id, min_revision, max_revision) return middle_index = (min_index + max_index) / 2 middle_revision = revision_list[middle_index] try: is_crash = _testcase_reproduces_in_revision( testcase, testcase_file_path, job_type, middle_revision, min_revision=min_revision, max_revision=max_revision) except errors.BadBuildError: # Skip this revision. del revision_list[middle_index] max_index -= 1 continue if is_crash: max_index = middle_index else: min_index = middle_index _save_current_regression_range_indices(testcase_id, revision_list[min_index], revision_list[max_index]) # If we've broken out of the above loop, we timed out. We'll finish by # running another regression task and picking up from this point. testcase = data_handler.get_testcase_by_id(testcase_id) error_message = 'Timed out, current range r%d:r%d' % ( revision_list[min_index], revision_list[max_index]) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task('regression', testcase_id, job_type)
def find_fixed_range(testcase_id, job_type): """Attempt to find the revision range where a testcase was fixed.""" deadline = tasks.get_task_completion_deadline() testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return if testcase.fixed: logs.log_error('Fixed range is already set as %s, skip.' % testcase.fixed) return # Setup testcase and its dependencies. file_list, _, testcase_file_path = setup.setup_testcase(testcase) if not file_list: return # Set a flag to indicate we are running progression task. This shows pending # status on testcase report page and avoid conflicting testcase updates by # triage cron. testcase.set_metadata('progression_pending', True) # Custom binaries are handled as special cases. if build_manager.is_custom_binary(): _check_fixed_for_custom_binary(testcase, job_type, testcase_file_path) return release_build_bucket_path = environment.get_value( 'RELEASE_BUILD_BUCKET_PATH') revision_list = build_manager.get_revisions_list(release_build_bucket_path, testcase=testcase) if not revision_list: testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, 'Failed to fetch revision list') tasks.add_task('progression', testcase_id, job_type) return # Use min, max_index to mark the start and end of revision list that is used # for bisecting the progression range. Set start to the revision where noticed # the crash. Set end to the trunk revision. Also, use min, max from past run # if it timed out. min_revision = testcase.get_metadata('last_progression_min') max_revision = testcase.get_metadata('last_progression_max') last_tested_revision = testcase.get_metadata('last_tested_crash_revision') known_crash_revision = last_tested_revision or testcase.crash_revision if not min_revision: min_revision = known_crash_revision if not max_revision: max_revision = revisions.get_last_revision_in_list(revision_list) min_index = revisions.find_min_revision_index(revision_list, min_revision) if min_index is None: raise errors.BuildNotFoundError(min_revision, job_type) max_index = revisions.find_max_revision_index(revision_list, max_revision) if max_index is None: raise errors.BuildNotFoundError(max_revision, job_type) testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED, 'r%d' % max_revision) # Check to see if this testcase is still crashing now. If it is, then just # bail out. result = _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, max_revision) if result.is_crash(): logs.log('Found crash with same signature on latest revision r%d.' % max_revision) app_path = environment.get_value('APP_PATH') command = testcase_manager.get_command_line_for_application( testcase_file_path, app_path=app_path, needs_http=testcase.http_flag) symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True) unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False) stacktrace = utils.get_crash_stacktrace_output( command, symbolized_crash_stacktrace, unsymbolized_crash_stacktrace) testcase = data_handler.get_testcase_by_id(testcase_id) testcase.last_tested_crash_stacktrace = data_handler.filter_stacktrace( stacktrace) _update_completion_metadata( testcase, max_revision, is_crash=True, message='still crashes on latest revision r%s' % max_revision) # Since we've verified that the test case is still crashing, clear out any # metadata indicating potential flake from previous runs. task_creation.mark_unreproducible_if_flaky(testcase, False) # For chromium project, save latest crash information for later upload # to chromecrash/. state = result.get_symbolized_data() crash_uploader.save_crash_info_if_needed(testcase_id, max_revision, job_type, state.crash_type, state.crash_address, state.frames) return # Don't burden NFS server with caching these random builds. environment.set_value('CACHE_STORE', False) # Verify that we do crash in the min revision. This is assumed to be true # while we are doing the bisect. result = _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, min_revision) if result and not result.is_crash(): testcase = data_handler.get_testcase_by_id(testcase_id) # Retry once on another bot to confirm our result. if data_handler.is_first_retry_for_task(testcase, reset_after_retry=True): tasks.add_task('progression', testcase_id, job_type) error_message = ( 'Known crash revision %d did not crash, will retry on another bot to ' 'confirm result' % known_crash_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) _update_completion_metadata(testcase, max_revision) return _clear_progression_pending(testcase) error_message = ('Known crash revision %d did not crash' % known_crash_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) task_creation.mark_unreproducible_if_flaky(testcase, True) return # Start a binary search to find last non-crashing revision. At this point, we # know that we do crash in the min_revision, and do not crash in max_revision. while time.time() < deadline: min_revision = revision_list[min_index] max_revision = revision_list[max_index] # If the min and max revisions are one apart this is as much as we can # narrow the range. if max_index - min_index == 1: _save_fixed_range(testcase_id, min_revision, max_revision) return # Test the middle revision of our range. middle_index = (min_index + max_index) // 2 middle_revision = revision_list[middle_index] testcase = data_handler.get_testcase_by_id(testcase_id) log_message = 'Testing r%d (current range %d:%d)' % ( middle_revision, min_revision, max_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.WIP, log_message) try: result = _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, middle_revision) except errors.BadBuildError: # Skip this revision. del revision_list[middle_index] max_index -= 1 continue if result.is_crash(): min_index = middle_index else: max_index = middle_index _save_current_fixed_range_indices(testcase_id, revision_list[min_index], revision_list[max_index]) # If we've broken out of the loop, we've exceeded the deadline. Recreate the # task to pick up where we left off. testcase = data_handler.get_testcase_by_id(testcase_id) error_message = ('Timed out, current range r%d:r%d' % (revision_list[min_index], revision_list[max_index])) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task('progression', testcase_id, job_type)
def execute_task(testcase_id, job_type): """Attempt to minimize a given testcase.""" # Get deadline to finish this task. deadline = tasks.get_task_completion_deadline() # Locate the testcase associated with the id. testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return # Update comments to reflect bot information. data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) # Setup testcase and its dependencies. file_list, input_directory, testcase_file_path = setup.setup_testcase( testcase) if not file_list: return # Initialize variables. max_timeout = environment.get_value('TEST_TIMEOUT', 10) app_arguments = environment.get_value('APP_ARGS') # Set up a custom or regular build based on revision. last_tested_crash_revision = testcase.get_metadata( 'last_tested_crash_revision') crash_revision = last_tested_crash_revision or testcase.crash_revision build_manager.setup_build(crash_revision) # Check if we have an application path. If not, our build failed # to setup correctly. app_path = environment.get_value('APP_PATH') if not app_path: logs.log_error('Unable to setup build for minimization.') build_fail_wait = environment.get_value('FAIL_WAIT') if environment.get_value('ORIGINAL_JOB_NAME'): _skip_minimization(testcase, 'Failed to setup build for overridden job.') else: # Only recreate task if this isn't an overriden job. It's possible that a # revision exists for the original job, but doesn't exist for the # overriden job. tasks.add_task( 'minimize', testcase_id, job_type, wait_time=build_fail_wait) return if environment.is_libfuzzer_job(): do_libfuzzer_minimization(testcase, testcase_file_path) return max_threads = utils.maximum_parallel_processes_allowed() # Prepare the test case runner. crash_retries = environment.get_value('CRASH_RETRIES') warmup_timeout = environment.get_value('WARMUP_TIMEOUT') required_arguments = environment.get_value('REQUIRED_APP_ARGS', '') # Add any testcase-specific required arguments if needed. additional_required_arguments = testcase.get_metadata( 'additional_required_app_args') if additional_required_arguments: required_arguments = '%s %s' % (required_arguments, additional_required_arguments) test_runner = TestRunner(testcase, testcase_file_path, file_list, input_directory, app_arguments, required_arguments, max_threads, deadline) # Verify the crash with a long timeout. warmup_crash_occurred = False result = test_runner.run(timeout=warmup_timeout, log_command=True) if result.is_crash(): warmup_crash_occurred = True logs.log('Warmup crash occurred in %d seconds.' % result.crash_time) saved_unsymbolized_crash_state, flaky_stack, crash_times = ( check_for_initial_crash(test_runner, crash_retries, testcase)) # If the warmup crash occurred but we couldn't reproduce this in with # multiple processes running in parallel, try to minimize single threaded. if (len(crash_times) < tests.REPRODUCIBILITY_FACTOR * crash_retries and warmup_crash_occurred and max_threads > 1): logs.log('Attempting to continue single-threaded.') max_threads = 1 test_runner = TestRunner(testcase, testcase_file_path, file_list, input_directory, app_arguments, required_arguments, max_threads, deadline) saved_unsymbolized_crash_state, flaky_stack, crash_times = ( check_for_initial_crash(test_runner, crash_retries, testcase)) if flaky_stack: testcase.flaky_stack = flaky_stack testcase.put() if len(crash_times) < tests.REPRODUCIBILITY_FACTOR * crash_retries: if not crash_times: # We didn't crash at all, so try again. This might be a legitimately # unreproducible test case, so it will get marked as such after being # retried on other bots. testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, 'Unable to reproduce crash') task_creation.mark_unreproducible_if_flaky(testcase, True) else: # We reproduced this crash at least once. It's too flaky to minimize, but # maybe we'll have more luck in the other jobs. testcase = data_handler.get_testcase_by_id(testcase_id) testcase.minimized_keys = 'NA' error_message = ( 'Unable to reproduce crash reliably, skipping ' 'minimization (crashed %d/%d)' % (len(crash_times), crash_retries)) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) create_additional_tasks(testcase) return # If we've made it this far, the test case appears to be reproducible. Clear # metadata from previous runs had it been marked as potentially flaky. task_creation.mark_unreproducible_if_flaky(testcase, False) test_runner.set_test_expectations(testcase.security_flag, flaky_stack, saved_unsymbolized_crash_state) # Use the max crash time unless this would be greater than the max timeout. test_timeout = min(max(crash_times), max_timeout) + 1 logs.log('Using timeout %d (was %d)' % (test_timeout, max_timeout)) test_runner.timeout = test_timeout logs.log('Starting minimization.') if should_attempt_phase(testcase, MinimizationPhase.GESTURES): gestures = minimize_gestures(test_runner, testcase) # We can't call check_deadline_exceeded_and_store_partial_minimized_testcase # at this point because we do not have a test case to store. testcase = data_handler.get_testcase_by_id(testcase.key.id()) if testcase.security_flag and len(testcase.gestures) != len(gestures): # Re-run security severity analysis since gestures affect the severity. testcase.security_severity = severity_analyzer.get_security_severity( testcase.crash_type, data_handler.get_stacktrace(testcase), job_type, bool(gestures)) testcase.gestures = gestures testcase.set_metadata('minimization_phase', MinimizationPhase.MAIN_FILE) if time.time() > test_runner.deadline: tasks.add_task('minimize', testcase.key.id(), job_type) return # Minimize the main file. data = utils.get_file_contents_with_fatal_error_on_failure(testcase_file_path) if should_attempt_phase(testcase, MinimizationPhase.MAIN_FILE): data = minimize_main_file(test_runner, testcase_file_path, data) if check_deadline_exceeded_and_store_partial_minimized_testcase( deadline, testcase_id, job_type, input_directory, file_list, data, testcase_file_path): return testcase.set_metadata('minimization_phase', MinimizationPhase.FILE_LIST) # Minimize the file list. if should_attempt_phase(testcase, MinimizationPhase.FILE_LIST): if environment.get_value('MINIMIZE_FILE_LIST', True): file_list = minimize_file_list(test_runner, file_list, input_directory, testcase_file_path) if check_deadline_exceeded_and_store_partial_minimized_testcase( deadline, testcase_id, job_type, input_directory, file_list, data, testcase_file_path): return else: logs.log('Skipping minimization of file list.') testcase.set_metadata('minimization_phase', MinimizationPhase.RESOURCES) # Minimize any files remaining in the file list. if should_attempt_phase(testcase, MinimizationPhase.RESOURCES): if environment.get_value('MINIMIZE_RESOURCES', True): for dependency in file_list: minimize_resource(test_runner, dependency, input_directory, testcase_file_path) if check_deadline_exceeded_and_store_partial_minimized_testcase( deadline, testcase_id, job_type, input_directory, file_list, data, testcase_file_path): return else: logs.log('Skipping minimization of resources.') testcase.set_metadata('minimization_phase', MinimizationPhase.ARGUMENTS) if should_attempt_phase(testcase, MinimizationPhase.ARGUMENTS): app_arguments = minimize_arguments(test_runner, app_arguments) # Arguments must be stored here in case we time out below. testcase.minimized_arguments = app_arguments testcase.put() if check_deadline_exceeded_and_store_partial_minimized_testcase( deadline, testcase_id, job_type, input_directory, file_list, data, testcase_file_path): return command = tests.get_command_line_for_application( testcase_file_path, app_args=app_arguments, needs_http=testcase.http_flag) last_crash_result = test_runner.last_failing_result store_minimized_testcase(testcase, input_directory, file_list, data, testcase_file_path) finalize_testcase( testcase_id, command, last_crash_result, flaky_stack=flaky_stack)
def do_libfuzzer_minimization(testcase, testcase_file_path): """Use libFuzzer's built-in minimizer where appropriate.""" is_overriden_job = bool(environment.get_value('ORIGINAL_JOB_NAME')) def handle_unreproducible(): # Be more lenient with marking testcases as unreproducible when this is a # job override. if is_overriden_job: _skip_minimization(testcase, 'Unreproducible on overridden job.') else: task_creation.mark_unreproducible_if_flaky(testcase, True) timeout = environment.get_value('LIBFUZZER_MINIMIZATION_TIMEOUT', 180) rounds = environment.get_value('LIBFUZZER_MINIMIZATION_ROUNDS', 10) current_testcase_path = testcase_file_path last_crash_result = None # Get initial crash state. initial_crash_result = _run_libfuzzer_testcase(testcase, testcase_file_path) if not initial_crash_result.is_crash(): logs.log_warn('Did not crash. Output:\n' + initial_crash_result.get_stacktrace(symbolized=True)) handle_unreproducible() return if testcase.security_flag != initial_crash_result.is_security_issue(): logs.log_warn('Security flag does not match.') handle_unreproducible() return task_creation.mark_unreproducible_if_flaky(testcase, False) expected_state = initial_crash_result.get_symbolized_data() logs.log('Initial crash state: %s\n' % expected_state.crash_state) # We attempt minimization multiple times in case one round results in an # incorrect state, or runs into another issue such as a slow unit. for round_number in range(1, rounds + 1): logs.log('Minimizing round %d.' % round_number) output_file_path, crash_result = _run_libfuzzer_tool( 'minimize', testcase, current_testcase_path, timeout, expected_state.crash_state, set_dedup_flags=True) if output_file_path: last_crash_result = crash_result current_testcase_path = output_file_path if not last_crash_result: repro_command = tests.get_command_line_for_application( file_to_run=testcase_file_path, needs_http=testcase.http_flag) _skip_minimization( testcase, 'LibFuzzer minimization failed.', crash_result=initial_crash_result, command=repro_command) return logs.log('LibFuzzer minimization succeeded.') if utils.is_oss_fuzz(): # Scrub the testcase of non-essential data. cleansed_testcase_path = do_libfuzzer_cleanse( testcase, current_testcase_path, expected_state.crash_state) if cleansed_testcase_path: current_testcase_path = cleansed_testcase_path # Finalize the test case if we were able to reproduce it. repro_command = tests.get_command_line_for_application( file_to_run=current_testcase_path, needs_http=testcase.http_flag) finalize_testcase(testcase.key.id(), repro_command, last_crash_result) # Clean up after we're done. shell.clear_testcase_directories()