def handle_unreproducible():
# Be more lenient with marking testcases as unreproducible when this is a
# job override.
if is_overriden_job:
_skip_minimization(testcase, 'Unreproducible on overridden job.')
else:
task_creation.mark_unreproducible_if_flaky(testcase, True)
def find_regression_range(testcase_id, job_type):
"""Attempt to find when the testcase regressed."""
deadline = tasks.get_task_completion_deadline()
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
if testcase.regression:
logs.log_error('Regression range is already set as %s, skip.' %
testcase.regression)
return
# This task is not applicable for custom binaries.
if build_manager.is_custom_binary():
testcase.regression = 'NA'
data_handler.update_testcase_comment(
testcase, data_types.TaskState.ERROR,
'Not applicable for custom binaries')
return
data_handler.update_testcase_comment(testcase,
data_types.TaskState.STARTED)
# Setup testcase and its dependencies.
file_list, _, testcase_file_path = setup.setup_testcase(testcase)
if not file_list:
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
'Failed to setup testcase')
tasks.add_task('regression', testcase_id, job_type)
return
release_build_bucket_path = environment.get_value(
'RELEASE_BUILD_BUCKET_PATH')
revision_list = build_manager.get_revisions_list(release_build_bucket_path,
testcase=testcase)
if not revision_list:
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
'Failed to fetch revision list')
tasks.add_task('regression', testcase_id, job_type)
return
# Don't burden NFS server with caching these random builds.
environment.set_value('CACHE_STORE', False)
# Pick up where left off in a previous run if necessary.
min_revision = testcase.get_metadata('last_regression_min')
max_revision = testcase.get_metadata('last_regression_max')
first_run = not min_revision and not max_revision
if not min_revision:
min_revision = revisions.get_first_revision_in_list(revision_list)
if not max_revision:
max_revision = testcase.crash_revision
min_index = revisions.find_min_revision_index(revision_list, min_revision)
if min_index is None:
raise errors.BuildNotFoundError(min_revision, job_type)
max_index = revisions.find_max_revision_index(revision_list, max_revision)
if max_index is None:
raise errors.BuildNotFoundError(max_revision, job_type)
# Make sure that the revision where we noticed the crash, still crashes at
# that revision. Otherwise, our binary search algorithm won't work correctly.
max_revision = revision_list[max_index]
crashes_in_max_revision = _testcase_reproduces_in_revision(
testcase, testcase_file_path, job_type, max_revision, should_log=False)
if not crashes_in_max_revision:
testcase = data_handler.get_testcase_by_id(testcase_id)
error_message = ('Known crash revision %d did not crash' %
max_revision)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
task_creation.mark_unreproducible_if_flaky(testcase, True)
return
# If we've made it this far, the test case appears to be reproducible. Clear
# metadata from previous runs had it been marked as potentially flaky.
task_creation.mark_unreproducible_if_flaky(testcase, False)
# On the first run, check to see if we regressed near either the min or max
# revision.
if first_run and found_regression_near_extreme_revisions(
testcase, testcase_file_path, job_type, revision_list, min_index,
max_index):
return
while time.time() < deadline:
min_revision = revision_list[min_index]
max_revision = revision_list[max_index]
# If the min and max revisions are one apart (or the same, if we only have
# one build), this is as much as we can narrow the range.
if max_index - min_index <= 1:
# Verify that the regression range seems correct, and save it if so.
if not validate_regression_range(testcase, testcase_file_path,
job_type, revision_list,
min_index):
return
save_regression_range(testcase_id, min_revision, max_revision)
return
middle_index = (min_index + max_index) / 2
middle_revision = revision_list[middle_index]
try:
is_crash = _testcase_reproduces_in_revision(
testcase,
testcase_file_path,
job_type,
middle_revision,
min_revision=min_revision,
max_revision=max_revision)
except errors.BadBuildError:
# Skip this revision.
del revision_list[middle_index]
max_index -= 1
continue
if is_crash:
max_index = middle_index
else:
min_index = middle_index
_save_current_regression_range_indices(testcase_id,
revision_list[min_index],
revision_list[max_index])
# If we've broken out of the above loop, we timed out. We'll finish by
# running another regression task and picking up from this point.
testcase = data_handler.get_testcase_by_id(testcase_id)
error_message = 'Timed out, current range r%d:r%d' % (
revision_list[min_index], revision_list[max_index])
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
error_message)
tasks.add_task('regression', testcase_id, job_type)
def find_fixed_range(testcase_id, job_type):
"""Attempt to find the revision range where a testcase was fixed."""
deadline = tasks.get_task_completion_deadline()
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
if testcase.fixed:
logs.log_error('Fixed range is already set as %s, skip.' %
testcase.fixed)
return
# Setup testcase and its dependencies.
file_list, _, testcase_file_path = setup.setup_testcase(testcase)
if not file_list:
return
# Set a flag to indicate we are running progression task. This shows pending
# status on testcase report page and avoid conflicting testcase updates by
# triage cron.
testcase.set_metadata('progression_pending', True)
# Custom binaries are handled as special cases.
if build_manager.is_custom_binary():
_check_fixed_for_custom_binary(testcase, job_type, testcase_file_path)
return
release_build_bucket_path = environment.get_value(
'RELEASE_BUILD_BUCKET_PATH')
revision_list = build_manager.get_revisions_list(release_build_bucket_path,
testcase=testcase)
if not revision_list:
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
'Failed to fetch revision list')
tasks.add_task('progression', testcase_id, job_type)
return
# Use min, max_index to mark the start and end of revision list that is used
# for bisecting the progression range. Set start to the revision where noticed
# the crash. Set end to the trunk revision. Also, use min, max from past run
# if it timed out.
min_revision = testcase.get_metadata('last_progression_min')
max_revision = testcase.get_metadata('last_progression_max')
last_tested_revision = testcase.get_metadata('last_tested_crash_revision')
known_crash_revision = last_tested_revision or testcase.crash_revision
if not min_revision:
min_revision = known_crash_revision
if not max_revision:
max_revision = revisions.get_last_revision_in_list(revision_list)
min_index = revisions.find_min_revision_index(revision_list, min_revision)
if min_index is None:
raise errors.BuildNotFoundError(min_revision, job_type)
max_index = revisions.find_max_revision_index(revision_list, max_revision)
if max_index is None:
raise errors.BuildNotFoundError(max_revision, job_type)
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.STARTED,
'r%d' % max_revision)
# Check to see if this testcase is still crashing now. If it is, then just
# bail out.
result = _testcase_reproduces_in_revision(testcase, testcase_file_path,
job_type, max_revision)
if result.is_crash():
logs.log('Found crash with same signature on latest revision r%d.' %
max_revision)
app_path = environment.get_value('APP_PATH')
command = testcase_manager.get_command_line_for_application(
testcase_file_path,
app_path=app_path,
needs_http=testcase.http_flag)
symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True)
unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False)
stacktrace = utils.get_crash_stacktrace_output(
command, symbolized_crash_stacktrace,
unsymbolized_crash_stacktrace)
testcase = data_handler.get_testcase_by_id(testcase_id)
testcase.last_tested_crash_stacktrace = data_handler.filter_stacktrace(
stacktrace)
_update_completion_metadata(
testcase,
max_revision,
is_crash=True,
message='still crashes on latest revision r%s' % max_revision)
# Since we've verified that the test case is still crashing, clear out any
# metadata indicating potential flake from previous runs.
task_creation.mark_unreproducible_if_flaky(testcase, False)
# For chromium project, save latest crash information for later upload
# to chromecrash/.
state = result.get_symbolized_data()
crash_uploader.save_crash_info_if_needed(testcase_id, max_revision,
job_type, state.crash_type,
state.crash_address,
state.frames)
return
# Don't burden NFS server with caching these random builds.
environment.set_value('CACHE_STORE', False)
# Verify that we do crash in the min revision. This is assumed to be true
# while we are doing the bisect.
result = _testcase_reproduces_in_revision(testcase, testcase_file_path,
job_type, min_revision)
if result and not result.is_crash():
testcase = data_handler.get_testcase_by_id(testcase_id)
# Retry once on another bot to confirm our result.
if data_handler.is_first_retry_for_task(testcase,
reset_after_retry=True):
tasks.add_task('progression', testcase_id, job_type)
error_message = (
'Known crash revision %d did not crash, will retry on another bot to '
'confirm result' % known_crash_revision)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
_update_completion_metadata(testcase, max_revision)
return
_clear_progression_pending(testcase)
error_message = ('Known crash revision %d did not crash' %
known_crash_revision)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
task_creation.mark_unreproducible_if_flaky(testcase, True)
return
# Start a binary search to find last non-crashing revision. At this point, we
# know that we do crash in the min_revision, and do not crash in max_revision.
while time.time() < deadline:
min_revision = revision_list[min_index]
max_revision = revision_list[max_index]
# If the min and max revisions are one apart this is as much as we can
# narrow the range.
if max_index - min_index == 1:
_save_fixed_range(testcase_id, min_revision, max_revision)
return
# Test the middle revision of our range.
middle_index = (min_index + max_index) // 2
middle_revision = revision_list[middle_index]
testcase = data_handler.get_testcase_by_id(testcase_id)
log_message = 'Testing r%d (current range %d:%d)' % (
middle_revision, min_revision, max_revision)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.WIP,
log_message)
try:
result = _testcase_reproduces_in_revision(testcase,
testcase_file_path,
job_type,
middle_revision)
except errors.BadBuildError:
# Skip this revision.
del revision_list[middle_index]
max_index -= 1
continue
if result.is_crash():
min_index = middle_index
else:
max_index = middle_index
_save_current_fixed_range_indices(testcase_id,
revision_list[min_index],
revision_list[max_index])
# If we've broken out of the loop, we've exceeded the deadline. Recreate the
# task to pick up where we left off.
testcase = data_handler.get_testcase_by_id(testcase_id)
error_message = ('Timed out, current range r%d:r%d' %
(revision_list[min_index], revision_list[max_index]))
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
error_message)
tasks.add_task('progression', testcase_id, job_type)
def execute_task(testcase_id, job_type):
"""Attempt to minimize a given testcase."""
# Get deadline to finish this task.
deadline = tasks.get_task_completion_deadline()
# Locate the testcase associated with the id.
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
# Update comments to reflect bot information.
data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED)
# Setup testcase and its dependencies.
file_list, input_directory, testcase_file_path = setup.setup_testcase(
testcase)
if not file_list:
return
# Initialize variables.
max_timeout = environment.get_value('TEST_TIMEOUT', 10)
app_arguments = environment.get_value('APP_ARGS')
# Set up a custom or regular build based on revision.
last_tested_crash_revision = testcase.get_metadata(
'last_tested_crash_revision')
crash_revision = last_tested_crash_revision or testcase.crash_revision
build_manager.setup_build(crash_revision)
# Check if we have an application path. If not, our build failed
# to setup correctly.
app_path = environment.get_value('APP_PATH')
if not app_path:
logs.log_error('Unable to setup build for minimization.')
build_fail_wait = environment.get_value('FAIL_WAIT')
if environment.get_value('ORIGINAL_JOB_NAME'):
_skip_minimization(testcase, 'Failed to setup build for overridden job.')
else:
# Only recreate task if this isn't an overriden job. It's possible that a
# revision exists for the original job, but doesn't exist for the
# overriden job.
tasks.add_task(
'minimize', testcase_id, job_type, wait_time=build_fail_wait)
return
if environment.is_libfuzzer_job():
do_libfuzzer_minimization(testcase, testcase_file_path)
return
max_threads = utils.maximum_parallel_processes_allowed()
# Prepare the test case runner.
crash_retries = environment.get_value('CRASH_RETRIES')
warmup_timeout = environment.get_value('WARMUP_TIMEOUT')
required_arguments = environment.get_value('REQUIRED_APP_ARGS', '')
# Add any testcase-specific required arguments if needed.
additional_required_arguments = testcase.get_metadata(
'additional_required_app_args')
if additional_required_arguments:
required_arguments = '%s %s' % (required_arguments,
additional_required_arguments)
test_runner = TestRunner(testcase, testcase_file_path, file_list,
input_directory, app_arguments, required_arguments,
max_threads, deadline)
# Verify the crash with a long timeout.
warmup_crash_occurred = False
result = test_runner.run(timeout=warmup_timeout, log_command=True)
if result.is_crash():
warmup_crash_occurred = True
logs.log('Warmup crash occurred in %d seconds.' % result.crash_time)
saved_unsymbolized_crash_state, flaky_stack, crash_times = (
check_for_initial_crash(test_runner, crash_retries, testcase))
# If the warmup crash occurred but we couldn't reproduce this in with
# multiple processes running in parallel, try to minimize single threaded.
if (len(crash_times) < tests.REPRODUCIBILITY_FACTOR * crash_retries and
warmup_crash_occurred and max_threads > 1):
logs.log('Attempting to continue single-threaded.')
max_threads = 1
test_runner = TestRunner(testcase, testcase_file_path, file_list,
input_directory, app_arguments, required_arguments,
max_threads, deadline)
saved_unsymbolized_crash_state, flaky_stack, crash_times = (
check_for_initial_crash(test_runner, crash_retries, testcase))
if flaky_stack:
testcase.flaky_stack = flaky_stack
testcase.put()
if len(crash_times) < tests.REPRODUCIBILITY_FACTOR * crash_retries:
if not crash_times:
# We didn't crash at all, so try again. This might be a legitimately
# unreproducible test case, so it will get marked as such after being
# retried on other bots.
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
'Unable to reproduce crash')
task_creation.mark_unreproducible_if_flaky(testcase, True)
else:
# We reproduced this crash at least once. It's too flaky to minimize, but
# maybe we'll have more luck in the other jobs.
testcase = data_handler.get_testcase_by_id(testcase_id)
testcase.minimized_keys = 'NA'
error_message = (
'Unable to reproduce crash reliably, skipping '
'minimization (crashed %d/%d)' % (len(crash_times), crash_retries))
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
error_message)
create_additional_tasks(testcase)
return
# If we've made it this far, the test case appears to be reproducible. Clear
# metadata from previous runs had it been marked as potentially flaky.
task_creation.mark_unreproducible_if_flaky(testcase, False)
test_runner.set_test_expectations(testcase.security_flag, flaky_stack,
saved_unsymbolized_crash_state)
# Use the max crash time unless this would be greater than the max timeout.
test_timeout = min(max(crash_times), max_timeout) + 1
logs.log('Using timeout %d (was %d)' % (test_timeout, max_timeout))
test_runner.timeout = test_timeout
logs.log('Starting minimization.')
if should_attempt_phase(testcase, MinimizationPhase.GESTURES):
gestures = minimize_gestures(test_runner, testcase)
# We can't call check_deadline_exceeded_and_store_partial_minimized_testcase
# at this point because we do not have a test case to store.
testcase = data_handler.get_testcase_by_id(testcase.key.id())
if testcase.security_flag and len(testcase.gestures) != len(gestures):
# Re-run security severity analysis since gestures affect the severity.
testcase.security_severity = severity_analyzer.get_security_severity(
testcase.crash_type, data_handler.get_stacktrace(testcase), job_type,
bool(gestures))
testcase.gestures = gestures
testcase.set_metadata('minimization_phase', MinimizationPhase.MAIN_FILE)
if time.time() > test_runner.deadline:
tasks.add_task('minimize', testcase.key.id(), job_type)
return
# Minimize the main file.
data = utils.get_file_contents_with_fatal_error_on_failure(testcase_file_path)
if should_attempt_phase(testcase, MinimizationPhase.MAIN_FILE):
data = minimize_main_file(test_runner, testcase_file_path, data)
if check_deadline_exceeded_and_store_partial_minimized_testcase(
deadline, testcase_id, job_type, input_directory, file_list, data,
testcase_file_path):
return
testcase.set_metadata('minimization_phase', MinimizationPhase.FILE_LIST)
# Minimize the file list.
if should_attempt_phase(testcase, MinimizationPhase.FILE_LIST):
if environment.get_value('MINIMIZE_FILE_LIST', True):
file_list = minimize_file_list(test_runner, file_list, input_directory,
testcase_file_path)
if check_deadline_exceeded_and_store_partial_minimized_testcase(
deadline, testcase_id, job_type, input_directory, file_list, data,
testcase_file_path):
return
else:
logs.log('Skipping minimization of file list.')
testcase.set_metadata('minimization_phase', MinimizationPhase.RESOURCES)
# Minimize any files remaining in the file list.
if should_attempt_phase(testcase, MinimizationPhase.RESOURCES):
if environment.get_value('MINIMIZE_RESOURCES', True):
for dependency in file_list:
minimize_resource(test_runner, dependency, input_directory,
testcase_file_path)
if check_deadline_exceeded_and_store_partial_minimized_testcase(
deadline, testcase_id, job_type, input_directory, file_list, data,
testcase_file_path):
return
else:
logs.log('Skipping minimization of resources.')
testcase.set_metadata('minimization_phase', MinimizationPhase.ARGUMENTS)
if should_attempt_phase(testcase, MinimizationPhase.ARGUMENTS):
app_arguments = minimize_arguments(test_runner, app_arguments)
# Arguments must be stored here in case we time out below.
testcase.minimized_arguments = app_arguments
testcase.put()
if check_deadline_exceeded_and_store_partial_minimized_testcase(
deadline, testcase_id, job_type, input_directory, file_list, data,
testcase_file_path):
return
command = tests.get_command_line_for_application(
testcase_file_path, app_args=app_arguments, needs_http=testcase.http_flag)
last_crash_result = test_runner.last_failing_result
store_minimized_testcase(testcase, input_directory, file_list, data,
testcase_file_path)
finalize_testcase(
testcase_id, command, last_crash_result, flaky_stack=flaky_stack)
def do_libfuzzer_minimization(testcase, testcase_file_path):
"""Use libFuzzer's built-in minimizer where appropriate."""
is_overriden_job = bool(environment.get_value('ORIGINAL_JOB_NAME'))
def handle_unreproducible():
# Be more lenient with marking testcases as unreproducible when this is a
# job override.
if is_overriden_job:
_skip_minimization(testcase, 'Unreproducible on overridden job.')
else:
task_creation.mark_unreproducible_if_flaky(testcase, True)
timeout = environment.get_value('LIBFUZZER_MINIMIZATION_TIMEOUT', 180)
rounds = environment.get_value('LIBFUZZER_MINIMIZATION_ROUNDS', 10)
current_testcase_path = testcase_file_path
last_crash_result = None
# Get initial crash state.
initial_crash_result = _run_libfuzzer_testcase(testcase, testcase_file_path)
if not initial_crash_result.is_crash():
logs.log_warn('Did not crash. Output:\n' +
initial_crash_result.get_stacktrace(symbolized=True))
handle_unreproducible()
return
if testcase.security_flag != initial_crash_result.is_security_issue():
logs.log_warn('Security flag does not match.')
handle_unreproducible()
return
task_creation.mark_unreproducible_if_flaky(testcase, False)
expected_state = initial_crash_result.get_symbolized_data()
logs.log('Initial crash state: %s\n' % expected_state.crash_state)
# We attempt minimization multiple times in case one round results in an
# incorrect state, or runs into another issue such as a slow unit.
for round_number in range(1, rounds + 1):
logs.log('Minimizing round %d.' % round_number)
output_file_path, crash_result = _run_libfuzzer_tool(
'minimize',
testcase,
current_testcase_path,
timeout,
expected_state.crash_state,
set_dedup_flags=True)
if output_file_path:
last_crash_result = crash_result
current_testcase_path = output_file_path
if not last_crash_result:
repro_command = tests.get_command_line_for_application(
file_to_run=testcase_file_path, needs_http=testcase.http_flag)
_skip_minimization(
testcase,
'LibFuzzer minimization failed.',
crash_result=initial_crash_result,
command=repro_command)
return
logs.log('LibFuzzer minimization succeeded.')
if utils.is_oss_fuzz():
# Scrub the testcase of non-essential data.
cleansed_testcase_path = do_libfuzzer_cleanse(
testcase, current_testcase_path, expected_state.crash_state)
if cleansed_testcase_path:
current_testcase_path = cleansed_testcase_path
# Finalize the test case if we were able to reproduce it.
repro_command = tests.get_command_line_for_application(
file_to_run=current_testcase_path, needs_http=testcase.http_flag)
finalize_testcase(testcase.key.id(), repro_command, last_crash_result)
# Clean up after we're done.
shell.clear_testcase_directories()
