def test(self):
"""Test that ASan instrumentation can be successfully set up on device."""
adb.revert_asan_device_setup_if_needed()
environment.reset_current_memory_tool_options()
sanitizer.setup_asan_if_needed()
self.assertEqual(0, self.mock.log_error.call_count)
self.assertTrue(adb.file_exists("/system/bin/asanwrapper"))
def test_doesnt_mutate_options(self):
"""Test that calling get_memory_tool_options followed by
set_memory_tool_options does not mutate sanitizer options unless we
do so explicitly."""
# Make environment module use the Windows symbolizer, since its path is
# hard to get right.
test_helpers.patch_environ(self)
os.environ['JOB_NAME'] = 'windows_libfuzzer_chrome_asan'
test_helpers.patch(self, [
'system.environment.platform',
'system.environment.get_llvm_symbolizer_path'
])
self.mock.platform.return_value = 'WINDOWS'
windows_symbolizer_path = (
r'c:\clusterfuzz\resources\platform\windows\llvm-symbolizer.exe')
self.mock.get_llvm_symbolizer_path.return_value = windows_symbolizer_path
environment.reset_current_memory_tool_options()
memory_tool_var = 'ASAN_OPTIONS'
first_asan_options_dict = environment.get_memory_tool_options(
memory_tool_var)
environment.set_memory_tool_options(memory_tool_var,
first_asan_options_dict)
second_asan_options_dict = environment.get_memory_tool_options(
memory_tool_var)
self.assertDictEqual(first_asan_options_dict, second_asan_options_dict)
def prepare_environment_for_testcase(testcase):
"""Set various environment variables based on the test case."""
# Setup memory debugging tool environment.
environment.reset_current_memory_tool_options(
redzone_size=testcase.redzone)
# Setup environment variable for windows size and location properties.
# Explicitly use empty string to indicate use of default window properties.
if hasattr(testcase, 'window_argument'):
environment.set_value('WINDOW_ARG', testcase.window_argument)
# Adjust timeout based on the stored multiplier (if available).
if hasattr(testcase, 'timeout_multiplier') and testcase.timeout_multiplier:
test_timeout = environment.get_value('TEST_TIMEOUT')
environment.set_value('TEST_TIMEOUT',
int(test_timeout * testcase.timeout_multiplier))
# Override APP_ARGS with minimized arguments (if available).
if (hasattr(testcase, 'minimized_arguments')
and testcase.minimized_arguments):
environment.set_value('APP_ARGS', testcase.minimized_arguments)
# Add FUZZ_TARGET to environment if this is a fuzz target testcase.
fuzz_target = testcase.get_metadata('fuzzer_binary_name')
if fuzz_target:
environment.set_value('FUZZ_TARGET', fuzz_target)
def run(self, timeout):
"""Merge testcases from corpus from other fuzz targets."""
if not shell.get_directory_file_count(self.context.shared_corpus_path):
logs.log('No files found in shared corpus, skip merge.')
return
# Run pruning on the shared corpus and log the result in case of error.
logs.log('Merging shared corpus...')
environment.reset_current_memory_tool_options(
redzone_size=DEFAULT_REDZONE)
self.runner.process_sanitizer_options()
additional_args = self.runner.get_libfuzzer_flags()
result = self.runner.merge([
self.context.minimized_corpus_path, self.context.shared_corpus_path
],
timeout,
artifact_prefix=self.context.bad_units_path,
tmp_dir=self.context.merge_tmp_dir,
additional_args=additional_args)
symbolized_output = stack_symbolizer.symbolize_stacktrace(
result.output)
if result.timed_out:
logs.log_error(
'Corpus pruning timed out while merging shared corpus: %s.' %
symbolized_output)
elif result.return_code:
logs.log_error(
'Corpus pruning failed to merge shared corpus: %s.' %
symbolized_output)
else:
logs.log('Shared corpus merge finished successfully.',
output=symbolized_output)
def run(self, timeout):
"""Merge testcases from corpus from other fuzz targets."""
if not shell.get_directory_file_count(self.context.shared_corpus_path):
logs.log("No files found in shared corpus, skip merge.")
return
# Run pruning on the shared corpus and log the result in case of error.
logs.log("Merging shared corpus...")
environment.reset_current_memory_tool_options(
redzone_size=DEFAULT_REDZONE)
self.runner.process_sanitizer_options()
additional_args = self.runner.get_libfuzzer_flags()
try:
result = self.runner.minimize_corpus(
additional_args,
[self.context.shared_corpus_path],
self.context.minimized_corpus_path,
self.context.bad_units_path,
timeout,
)
symbolized_output = stack_symbolizer.symbolize_stacktrace(
result.logs)
logs.log("Shared corpus merge finished successfully.",
output=symbolized_output)
except engine.TimeoutError as e:
logs.log_error(
"Corpus pruning timed out while merging shared corpus: " +
e.message)
except engine.Error as e:
raise CorpusPruningException(
"Corpus pruning failed to merge shared corpus\n" + e.message)
def prepare_environment_for_testcase(testcase):
"""Set various environment variables based on the test case."""
# Setup memory debugging tool environment.
environment.reset_current_memory_tool_options(
redzone_size=testcase.redzone)
# Setup environment variable for windows size and location properties.
# Explicit override to avoid using the default one from job definition since
# that contains unsubsituted vars like $WIDTH, etc.
environment.set_value('WINDOW_ARG', testcase.window_argument)
# Adjust timeout based on the stored multiplier (if available).
if testcase.timeout_multiplier:
test_timeout = environment.get_value('TEST_TIMEOUT')
environment.set_value('TEST_TIMEOUT',
int(test_timeout * testcase.timeout_multiplier))
# Add FUZZ_TARGET to environment if this is a fuzz target testcase.
fuzz_target = testcase.get_metadata('fuzzer_binary_name')
if fuzz_target:
environment.set_value('FUZZ_TARGET', fuzz_target)
# Override APP_ARGS with minimized arguments (if available). Don't do this
# for variant task since other job types can have its own set of required
# arguments, so use the full set of arguments of that job.
task_name = environment.get_value('TASK_NAME')
app_args = _get_application_arguments(testcase, task_name)
if app_args:
environment.set_value('APP_ARGS', app_args)
def process_bad_units(self, bad_units_path, quarantine_corpus_path, crashes):
"""Process bad units found during merge."""
# TODO(ochang): A lot of this function is similar to parts of fuzz_task.
# Ideally fuzz_task can be refactored in a way that lets us share the common
# code.
environment.reset_current_memory_tool_options(redzone_size=DEFAULT_REDZONE)
self.runner.process_sanitizer_options()
logs.log('Processing bad units.')
corpus_file_paths = _get_corpus_file_paths(bad_units_path)
num_bad_units = 0
# Run each corpus item individually.
for i, unit_path in enumerate(corpus_file_paths, 1):
if i % 100 == 0:
logs.log('Up to %d' % i)
unit_name = os.path.basename(unit_path)
if unit_name.startswith('timeout-') or unit_name.startswith('oom-'):
# Don't waste time re-running timeout or oom testcases.
unit_path = self._quarantine_unit(unit_path, quarantine_corpus_path)
num_bad_units += 1
continue
result = self._run_single_unit(unit_path)
if (not result.timed_out and
not crash_analyzer.is_memory_tool_crash(result.output)):
# Didn't crash or time out.
continue
if result.timed_out:
# Slow unit. Quarantine it.
unit_path = self._quarantine_unit(unit_path, quarantine_corpus_path)
num_bad_units += 1
continue
# Get memory tool crash information.
state = stack_analyzer.get_crash_data(result.output, symbolize_flag=True)
# Crashed or caused a leak. Quarantine it.
unit_path = self._quarantine_unit(unit_path, quarantine_corpus_path)
num_bad_units += 1
if crash_analyzer.ignore_stacktrace(state.crash_stacktrace):
continue
# Local de-duplication.
if state.crash_state not in crashes:
security_flag = crash_analyzer.is_security_issue(
state.crash_stacktrace, state.crash_type, state.crash_address)
crashes[state.crash_state] = CorpusCrash(
state.crash_state, state.crash_type, state.crash_address,
state.crash_stacktrace, unit_path, security_flag)
logs.log('Found %d bad units, %d unique crashes.' % (num_bad_units,
len(crashes)))
def test_ubsan_disabled(self):
"""Test reset_current_memory_tool_options when ubsan is disabled."""
os.environ['JOB_NAME'] = 'libfuzzer_chrome_asan'
os.environ['UBSAN'] = 'True'
environment.reset_current_memory_tool_options(disable_ubsan=True)
self.assertDictEqual(
{
'halt_on_error': 0,
'print_stacktrace': 0,
'print_suppressions': 0
}, environment.get_memory_tool_options('UBSAN_OPTIONS'))
def test_ubsan_disabled(self):
"""Test reset_current_memory_tool_options when ubsan is disabled."""
os.environ["JOB_NAME"] = "libfuzzer_chrome_asan"
os.environ["UBSAN"] = "True"
environment.reset_current_memory_tool_options(disable_ubsan=True)
self.assertDictEqual(
{
"halt_on_error": 0,
"print_stacktrace": 0,
"print_suppressions": 0
},
environment.get_memory_tool_options("UBSAN_OPTIONS"),
)
def _setup_memory_tools_environment(testcase):
"""Set up environment for various memory tools used."""
env = testcase.get_metadata('env')
if not env:
environment.reset_current_memory_tool_options(
redzone_size=testcase.redzone)
return
for options_name, options_value in six.iteritems(env):
if not options_value:
environment.remove_key(options_name)
continue
environment.set_memory_tool_options(options_name, options_value)
def setUp(self):
android_helpers.AndroidTest.setUp(self)
BaseIntegrationTest.setUp(self)
if android.settings.get_sanitizer_tool_name() != "hwasan":
raise Exception("Device is not set up with HWASan.")
environment.set_value("BUILD_DIR", ANDROID_DATA_DIR)
environment.set_value("JOB_NAME", "libfuzzer_hwasan_android_device")
environment.reset_current_memory_tool_options()
self.crash_dir = TEMP_DIR
self.adb_path = android.adb.get_adb_path()
self.hwasan_options = 'HWASAN_OPTIONS="%s"' % quote(
environment.get_value("HWASAN_OPTIONS"))
def test_windows_symbolizer(self):
"""Test that the reset_current_memory_tool_options returns the expected path
to the llvm symbolizer on Windows."""
os.environ['JOB_NAME'] = 'windows_libfuzzer_chrome_asan'
test_helpers.patch(self, [
'system.environment.platform',
'system.environment.get_llvm_symbolizer_path'
])
self.mock.platform.return_value = 'WINDOWS'
windows_symbolizer_path = (
r'c:\clusterfuzz\resources\platform\windows\llvm-symbolizer.exe')
self.mock.get_llvm_symbolizer_path.return_value = windows_symbolizer_path
environment.reset_current_memory_tool_options()
self.assertIn(
'external_symbolizer_path="%s"' % windows_symbolizer_path,
os.environ['ASAN_OPTIONS'])
def cleanup_task_state(): """Cleans state before and after a task is executed.""" # Cleanup stale processes. process_handler.cleanup_stale_processes() # Clear build urls, temp and testcase directories. shell.clear_build_urls_directory() shell.clear_crash_stacktraces_directory() shell.clear_testcase_directories() shell.clear_temp_directory() shell.clear_system_temp_directory() shell.clear_device_temp_directories() # Reset memory tool environment variables. environment.reset_current_memory_tool_options() # Call python's garbage collector. utils.python_gc()
def setUp(self):
android_helpers.AndroidTest.setUp(self)
BaseIntegrationTest.setUp(self)
if android.settings.get_sanitizer_tool_name() != 'hwasan':
raise Exception('Device is not set up with HWASan.')
environment.set_value('BUILD_DIR', ANDROID_DATA_DIR)
environment.set_value('JOB_NAME', 'libfuzzer_hwasan_android_device')
environment.reset_current_memory_tool_options()
self.crash_dir = TEMP_DIR
self.adb_path = android.adb.get_adb_path()
self.hwasan_options = 'HWASAN_OPTIONS="%s"' % quote(
environment.get_value('HWASAN_OPTIONS'))
self.ld_library_path = (
'LD_LIBRARY_PATH=' +
android.sanitizer.get_ld_library_path_for_sanitizers())
def run(self, initial_corpus_path, minimized_corpus_path, bad_units_path):
"""Run corpus pruning. Output result to directory."""
if not shell.get_directory_file_count(initial_corpus_path):
# Empty corpus, nothing to do.
return
# Set memory tool options and fuzzer arguments.
engine_common.unpack_seed_corpus_if_needed(self.runner.target_path,
initial_corpus_path,
force_unpack=True)
environment.reset_current_memory_tool_options(redzone_size=MIN_REDZONE,
leaks=True)
self.runner.process_sanitizer_options()
additional_args = self.runner.get_libfuzzer_flags()
# Execute fuzzer with arguments for corpus pruning.
logs.log("Running merge...")
try:
result = self.runner.minimize_corpus(
additional_args,
[initial_corpus_path],
minimized_corpus_path,
bad_units_path,
CORPUS_PRUNING_TIMEOUT,
)
except engine.TimeoutError as e:
raise CorpusPruningException(
"Corpus pruning timed out while minimizing corpus\n" +
e.message)
except engine.Error as e:
raise CorpusPruningException(
"Corpus pruning failed to minimize corpus\n" + e.message)
symbolized_output = stack_symbolizer.symbolize_stacktrace(result.logs)
# Sanity check that there are files in minimized corpus after merging.
if not shell.get_directory_file_count(minimized_corpus_path):
raise CorpusPruningException(
"Corpus pruning failed to minimize corpus\n" +
symbolized_output)
logs.log("Corpus merge finished successfully.",
output=symbolized_output)
def test_ubsan_enabled(self):
"""Test reset_current_memory_tool_options when ubsan is enabled."""
os.environ['JOB_NAME'] = 'libfuzzer_chrome_asan'
os.environ['UBSAN'] = 'True'
environment.reset_current_memory_tool_options(disable_ubsan=False)
self.assertDictEqual(
{
'halt_on_error': 1,
'handle_abort': 1,
'handle_segv': 1,
'handle_sigbus': 1,
'handle_sigfpe': 1,
'handle_sigill': 1,
'print_stacktrace': 1,
'print_summary': 1,
'print_suppressions': 0,
'silence_unsigned_overflow': 1,
'use_sigaltstack': 1
}, environment.get_memory_tool_options('UBSAN_OPTIONS'))
def test_windows_symbolizer(self):
"""Test that the reset_current_memory_tool_options returns the expected path
to the llvm symbolizer on Windows."""
os.environ["JOB_NAME"] = "windows_libfuzzer_chrome_asan"
test_helpers.patch(
self,
[
"system.environment.platform",
"system.environment.get_llvm_symbolizer_path",
],
)
self.mock.platform.return_value = "WINDOWS"
windows_symbolizer_path = (
r"c:\clusterfuzz\resources\platform\windows\llvm-symbolizer.exe")
self.mock.get_llvm_symbolizer_path.return_value = windows_symbolizer_path
environment.reset_current_memory_tool_options()
self.assertIn(
'external_symbolizer_path="%s"' % windows_symbolizer_path,
os.environ["ASAN_OPTIONS"],
)
def run(self, initial_corpus_path, minimized_corpus_path, bad_units_path):
"""Run corpus pruning. Output result to directory."""
if not shell.get_directory_file_count(initial_corpus_path):
# Empty corpus, nothing to do.
return
# Set memory tool options and fuzzer arguments.
engine_common.unpack_seed_corpus_if_needed(self.runner.fuzzer_path,
initial_corpus_path,
force_unpack=True)
environment.reset_current_memory_tool_options(redzone_size=MIN_REDZONE,
leaks=True)
self.runner.process_sanitizer_options()
additional_args = self.runner.get_libfuzzer_flags()
# Execute fuzzer with arguments for corpus pruning.
logs.log('Running merge...')
result = self.runner.merge(
[minimized_corpus_path, initial_corpus_path],
CORPUS_PRUNING_TIMEOUT,
artifact_prefix=bad_units_path,
tmp_dir=self.context.merge_tmp_dir,
additional_args=additional_args)
# Sanity check that we didn't time out.
symbolized_output = stack_symbolizer.symbolize_stacktrace(
result.output)
if result.timed_out:
raise CorpusPruningException(
'Corpus pruning timed out while merging corpus: %s.' %
symbolized_output)
# Sanity check that we didn't error out and there are files in minimized
# corpus after merging.
if (result.return_code
or not shell.get_directory_file_count(minimized_corpus_path)):
raise CorpusPruningException(
'Corpus pruning failed to merge corpus: %s.' %
symbolized_output)
logs.log('Corpus merge finished successfully.',
output=symbolized_output)
def test_ubsan_enabled(self):
"""Test reset_current_memory_tool_options when ubsan is enabled."""
os.environ["JOB_NAME"] = "libfuzzer_chrome_asan"
os.environ["UBSAN"] = "True"
environment.reset_current_memory_tool_options(disable_ubsan=False)
self.assertDictEqual(
{
"halt_on_error": 1,
"handle_abort": 1,
"handle_segv": 1,
"handle_sigbus": 1,
"handle_sigfpe": 1,
"handle_sigill": 1,
"print_stacktrace": 1,
"print_summary": 1,
"print_suppressions": 0,
"silence_unsigned_overflow": 1,
"use_sigaltstack": 1,
},
environment.get_memory_tool_options("UBSAN_OPTIONS"),
)
def run(self, timeout):
"""Merge testcases from corpus from other fuzz targets."""
if not shell.get_directory_file_count(self.context.shared_corpus_path):
logs.log('No files found in shared corpus, skip merge.')
return None
# Run pruning on the shared corpus and log the result in case of error.
logs.log('Merging shared corpus...')
environment.reset_current_memory_tool_options(
redzone_size=DEFAULT_REDZONE)
self.runner.process_sanitizer_options()
additional_args = self.runner.get_libfuzzer_flags()
try:
result = self.runner.minimize_corpus(
additional_args, [self.context.shared_corpus_path],
self.context.minimized_corpus_path,
self.context.bad_units_path, timeout)
symbolized_output = stack_symbolizer.symbolize_stacktrace(
result.logs)
logs.log('Shared corpus merge finished successfully.',
output=symbolized_output)
except engine.TimeoutError as e:
# Other cross pollinated fuzzer corpuses can have unexpected test cases
# that time us out. This is expected, so bail out.
logs.log_warn(
'Corpus pruning timed out while merging shared corpus\n' +
repr(e))
return None
except engine.Error as e:
# Other cross pollinated fuzzer corpuses can be large, so we can run out
# of disk space and exception out. This is expected, so bail out.
logs.log_warn('Corpus pruning failed to merge shared corpus\n' +
repr(e))
return None
return result.stats
def execute_task(testcase_id, job_type):
"""Run analyze task."""
# Reset redzones.
environment.reset_current_memory_tool_options(redzone_size=128)
# Unset window location size and position properties so as to use default.
environment.set_value('WINDOW_ARG', '')
# Locate the testcase associated with the id.
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED)
metadata = data_types.TestcaseUploadMetadata.query(
data_types.TestcaseUploadMetadata.testcase_id == int(testcase_id)).get()
if not metadata:
logs.log_error(
'Testcase %s has no associated upload metadata.' % testcase_id)
testcase.key.delete()
return
is_lsan_enabled = environment.get_value('LSAN')
if is_lsan_enabled:
# Creates empty local blacklist so all leaks will be visible to uploader.
leak_blacklist.create_empty_local_blacklist()
# Store the bot name and timestamp in upload metadata.
bot_name = environment.get_value('BOT_NAME')
metadata.bot_name = bot_name
metadata.timestamp = datetime.datetime.utcnow()
metadata.put()
# Adjust the test timeout, if user has provided one.
if metadata.timeout:
environment.set_value('TEST_TIMEOUT', metadata.timeout)
# Adjust the number of retries, if user has provided one.
if metadata.retries is not None:
environment.set_value('CRASH_RETRIES', metadata.retries)
# Setup testcase and get absolute testcase path.
file_list, _, testcase_file_path = setup.setup_testcase(testcase)
if not file_list:
return
# Set up a custom or regular build based on revision.
build_manager.setup_build(testcase.crash_revision)
# Check if we have an application path. If not, our build failed
# to setup correctly.
app_path = environment.get_value('APP_PATH')
if not app_path:
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
'Build setup failed')
if data_handler.is_first_retry_for_task(testcase):
build_fail_wait = environment.get_value('FAIL_WAIT')
tasks.add_task(
'analyze', testcase_id, job_type, wait_time=build_fail_wait)
else:
close_invalid_testcase_and_update_status(testcase, metadata,
'Build setup failed')
return
# Update initial testcase information.
testcase.absolute_path = testcase_file_path
testcase.job_type = job_type
testcase.binary_flag = utils.is_binary_file(testcase_file_path)
testcase.queue = tasks.default_queue()
testcase.crash_state = ''
# Set initial testcase metadata fields (e.g. build url, etc).
data_handler.set_initial_testcase_metadata(testcase)
# Update minimized arguments and use ones provided during user upload.
if not testcase.minimized_arguments:
minimized_arguments = environment.get_value('APP_ARGS') or ''
additional_command_line_flags = testcase.get_metadata(
'uploaded_additional_args')
if additional_command_line_flags:
minimized_arguments += ' %s' % additional_command_line_flags
environment.set_value('APP_ARGS', minimized_arguments)
testcase.minimized_arguments = minimized_arguments
# Update other fields not set at upload time.
testcase.crash_revision = environment.get_value('APP_REVISION')
data_handler.set_initial_testcase_metadata(testcase)
testcase.put()
# Initialize some variables.
gestures = testcase.gestures
http_flag = testcase.http_flag
test_timeout = environment.get_value('TEST_TIMEOUT')
# Get the crash output.
result = testcase_manager.test_for_crash_with_retries(
testcase,
testcase_file_path,
test_timeout,
http_flag=http_flag,
compare_crash=False)
# If we don't get a crash, try enabling http to see if we can get a crash.
# Skip engine fuzzer jobs (e.g. libFuzzer, AFL) for which http testcase paths
# are not applicable.
if (not result.is_crash() and not http_flag and
not environment.is_engine_fuzzer_job()):
result_with_http = testcase_manager.test_for_crash_with_retries(
testcase,
testcase_file_path,
test_timeout,
http_flag=True,
compare_crash=False)
if result_with_http.is_crash():
logs.log('Testcase needs http flag for crash.')
http_flag = True
result = result_with_http
# Refresh our object.
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
# Set application command line with the correct http flag.
application_command_line = (
testcase_manager.get_command_line_for_application(
testcase_file_path, needs_http=http_flag))
# Get the crash data.
crashed = result.is_crash()
crash_time = result.get_crash_time()
state = result.get_symbolized_data()
unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False)
# Get crash info object with minidump info. Also, re-generate unsymbolized
# stacktrace if needed.
crash_info, _ = (
crash_uploader.get_crash_info_and_stacktrace(
application_command_line, state.crash_stacktrace, gestures))
if crash_info:
testcase.minidump_keys = crash_info.store_minidump()
if not crashed:
# Could not reproduce the crash.
log_message = (
'Testcase didn\'t crash in %d seconds (with retries)' % test_timeout)
data_handler.update_testcase_comment(
testcase, data_types.TaskState.FINISHED, log_message)
# For an unreproducible testcase, retry once on another bot to confirm
# our results and in case this bot is in a bad state which we didn't catch
# through our usual means.
if data_handler.is_first_retry_for_task(testcase):
testcase.status = 'Unreproducible, retrying'
testcase.put()
tasks.add_task('analyze', testcase_id, job_type)
return
# In the general case, we will not attempt to symbolize if we do not detect
# a crash. For user uploads, we should symbolize anyway to provide more
# information about what might be happening.
crash_stacktrace_output = utils.get_crash_stacktrace_output(
application_command_line, state.crash_stacktrace,
unsymbolized_crash_stacktrace)
testcase.crash_stacktrace = data_handler.filter_stacktrace(
crash_stacktrace_output)
close_invalid_testcase_and_update_status(testcase, metadata,
'Unreproducible')
# A non-reproducing testcase might still impact production branches.
# Add the impact task to get that information.
task_creation.create_impact_task_if_needed(testcase)
return
# Update http flag and re-run testcase to store dependencies (for bundled
# archives only).
testcase.http_flag = http_flag
if not store_testcase_dependencies_from_bundled_testcase_archive(
metadata, testcase, testcase_file_path):
return
# Update testcase crash parameters.
testcase.crash_type = state.crash_type
testcase.crash_address = state.crash_address
testcase.crash_state = state.crash_state
# Try to guess if the bug is security or not.
security_flag = crash_analyzer.is_security_issue(
state.crash_stacktrace, state.crash_type, state.crash_address)
testcase.security_flag = security_flag
# If it is, guess the severity.
if security_flag:
testcase.security_severity = severity_analyzer.get_security_severity(
state.crash_type, state.crash_stacktrace, job_type, bool(gestures))
log_message = ('Testcase crashed in %d seconds (r%d)' %
(crash_time, testcase.crash_revision))
data_handler.update_testcase_comment(testcase, data_types.TaskState.FINISHED,
log_message)
# See if we have to ignore this crash.
if crash_analyzer.ignore_stacktrace(state.crash_stacktrace):
close_invalid_testcase_and_update_status(testcase, metadata, 'Irrelavant')
return
# Test for reproducibility.
one_time_crasher_flag = not testcase_manager.test_for_reproducibility(
testcase_file_path, state.crash_state, security_flag, test_timeout,
http_flag, gestures)
testcase.one_time_crasher_flag = one_time_crasher_flag
# Check to see if this is a duplicate.
project_name = data_handler.get_project_name(job_type)
existing_testcase = data_handler.find_testcase(
project_name, state.crash_type, state.crash_state, security_flag)
if existing_testcase:
# If the existing test case is unreproducible and we are, replace the
# existing test case with this one.
if existing_testcase.one_time_crasher_flag and not one_time_crasher_flag:
duplicate_testcase = existing_testcase
original_testcase = testcase
else:
duplicate_testcase = testcase
original_testcase = existing_testcase
metadata.status = 'Duplicate'
metadata.duplicate_of = existing_testcase.key.id()
duplicate_testcase.status = 'Duplicate'
duplicate_testcase.duplicate_of = original_testcase.key.id()
duplicate_testcase.put()
# Set testcase and metadata status if not set already.
if testcase.status != 'Duplicate':
testcase.status = 'Processed'
metadata.status = 'Confirmed'
# Add new leaks to global blacklist to avoid detecting duplicates.
# Only add if testcase has a direct leak crash and if it's reproducible.
if is_lsan_enabled:
leak_blacklist.add_crash_to_global_blacklist_if_needed(testcase)
# Add application specific information in the trace.
crash_stacktrace_output = utils.get_crash_stacktrace_output(
application_command_line, state.crash_stacktrace,
unsymbolized_crash_stacktrace)
testcase.crash_stacktrace = data_handler.filter_stacktrace(
crash_stacktrace_output)
# Update the testcase values.
testcase.put()
# Update the upload metadata.
metadata.security_flag = security_flag
metadata.put()
# Create tasks to
# 1. Minimize testcase (minimize).
# 2. Find regression range (regression).
# 3. Find testcase impact on production branches (impact).
# 4. Check whether testcase is fixed (progression).
# 5. Get second stacktrace from another job in case of
# one-time crashers (stack).
task_creation.create_tasks(testcase)
def setup_testcase(testcase):
"""Sets up the testcase and needed dependencies like fuzzer,
data bundle, etc."""
fuzzer_name = testcase.fuzzer_name
job_type = testcase.job_type
task_name = environment.get_value('TASK_NAME')
testcase_fail_wait = environment.get_value('FAIL_WAIT')
testcase_id = testcase.key.id()
# Clear testcase directories.
shell.clear_testcase_directories()
# Setup memory debugging tool environment.
environment.reset_current_memory_tool_options(
redzone_size=testcase.redzone)
# Adjust the test timeout value if this is coming from an user uploaded
# testcase.
_set_timeout_value_from_user_upload(testcase_id)
if task_name == 'minimize':
# Allow minimizing with a different fuzzer set up.
minimize_fuzzer_override = environment.get_value(
'MINIMIZE_FUZZER_OVERRIDE')
fuzzer_name = minimize_fuzzer_override or fuzzer_name
# Update the fuzzer if necessary in order to get the updated data bundle.
if fuzzer_name:
try:
update_successful = update_fuzzer_and_data_bundles(fuzzer_name)
except errors.InvalidFuzzerError:
# Close testcase and don't recreate tasks if this fuzzer is invalid.
testcase.open = False
testcase.fixed = 'NA'
testcase.set_metadata('fuzzer_was_deleted', True)
logs.log_error('Closed testcase %d with invalid fuzzer %s.' %
(testcase_id, fuzzer_name))
error_message = 'Fuzzer %s no longer exists.' % fuzzer_name
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
return None, None, None
if not update_successful:
error_message = 'Unable to setup fuzzer %s.' % fuzzer_name
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
tasks.add_task(task_name,
testcase_id,
job_type,
wait_time=testcase_fail_wait)
return None, None, None
# Extract the testcase and any of its resources to the input directory.
file_list, input_directory, testcase_file_path = unpack_testcase(testcase)
if not file_list:
error_message = 'Unable to setup testcase %s.' % testcase_file_path
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
tasks.add_task(task_name,
testcase_id,
job_type,
wait_time=testcase_fail_wait)
return None, None, None
# For Android/Fuchsia, we need to sync our local testcases directory with the
# one on the device.
if environment.platform() == 'ANDROID':
_copy_testcase_to_device_and_setup_environment(testcase,
testcase_file_path)
if environment.platform() == 'FUCHSIA':
fuchsia.device.copy_testcase_to_device(testcase_file_path)
# Push testcases to worker.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.push_testcases_to_worker()
# Copy global blacklist into local blacklist.
is_lsan_enabled = environment.get_value('LSAN')
if is_lsan_enabled:
# Get local blacklist without this testcase's entry.
leak_blacklist.copy_global_to_local_blacklist(
excluded_testcase=testcase)
# Setup environment variable for windows size and location properties.
# Explicitly use empty string to indicate use of default window properties.
if hasattr(testcase, 'window_argument'):
environment.set_value('WINDOW_ARG', testcase.window_argument)
# Adjust timeout based on the stored multiplier (if available).
if hasattr(testcase, 'timeout_multiplier') and testcase.timeout_multiplier:
test_timeout = environment.get_value('TEST_TIMEOUT')
environment.set_value('TEST_TIMEOUT',
int(test_timeout * testcase.timeout_multiplier))
# Override APP_ARGS with minimized arguments (if available).
if (hasattr(testcase, 'minimized_arguments')
and testcase.minimized_arguments):
environment.set_value('APP_ARGS', testcase.minimized_arguments)
# Add FUZZ_TARGET to environment if this is a fuzz target testcase.
fuzz_target = testcase.get_metadata('fuzzer_binary_name')
if fuzz_target:
environment.set_value('FUZZ_TARGET', fuzz_target)
return file_list, input_directory, testcase_file_path
def execute_task(testcase_id, job_type):
"""Execute a symbolize command."""
# Locate the testcase associated with the id.
testcase = data_handler.get_testcase_by_id(testcase_id)
# We should atleast have a symbolized debug or release build.
if not build_manager.has_symbolized_builds():
return
data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED)
# Setup testcase and its dependencies.
file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type)
if not file_list:
return
# Initialize variables.
build_fail_wait = environment.get_value("FAIL_WAIT")
old_crash_stacktrace = data_handler.get_stacktrace(testcase)
sym_crash_type = testcase.crash_type
sym_crash_address = testcase.crash_address
sym_crash_state = testcase.crash_state
sym_redzone = DEFAULT_REDZONE
warmup_timeout = environment.get_value("WARMUP_TIMEOUT")
# Decide which build revision to use.
if testcase.crash_stacktrace == "Pending":
# This usually happen when someone clicked the 'Update stacktrace from
# trunk' button on the testcase details page. In this case, we are forced
# to use trunk. No revision -> trunk build.
build_revision = None
else:
build_revision = testcase.crash_revision
# Set up a custom or regular build based on revision.
build_manager.setup_build(build_revision)
# Get crash revision used in setting up build.
crash_revision = environment.get_value("APP_REVISION")
if not build_manager.check_app_path():
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
"Build setup failed")
tasks.add_task(
"symbolize", testcase_id, job_type, wait_time=build_fail_wait)
return
# ASAN tool settings (if the tool is used).
# See if we can get better stacks with higher redzone sizes.
# A UAF might actually turn out to be OOB read/write with a bigger redzone.
if environment.tool_matches("ASAN", job_type) and testcase.security_flag:
redzone = MAX_REDZONE
while redzone >= MIN_REDZONE:
environment.reset_current_memory_tool_options(
redzone_size=testcase.redzone, disable_ubsan=testcase.disable_ubsan)
process_handler.terminate_stale_application_instances()
command = testcase_manager.get_command_line_for_application(
testcase_file_path, needs_http=testcase.http_flag)
return_code, crash_time, output = process_handler.run_process(
command, timeout=warmup_timeout, gestures=testcase.gestures)
crash_result = CrashResult(return_code, crash_time, output)
if crash_result.is_crash() and "AddressSanitizer" in output:
state = crash_result.get_symbolized_data()
security_flag = crash_result.is_security_issue()
if (not crash_analyzer.ignore_stacktrace(state.crash_stacktrace) and
security_flag == testcase.security_flag and
state.crash_type == testcase.crash_type and
(state.crash_type != sym_crash_type or
state.crash_state != sym_crash_state)):
logs.log("Changing crash parameters.\nOld : %s, %s, %s" %
(sym_crash_type, sym_crash_address, sym_crash_state))
sym_crash_type = state.crash_type
sym_crash_address = state.crash_address
sym_crash_state = state.crash_state
sym_redzone = redzone
old_crash_stacktrace = state.crash_stacktrace
logs.log("\nNew : %s, %s, %s" %
(sym_crash_type, sym_crash_address, sym_crash_state))
break
redzone /= 2
# We should have atleast a symbolized debug or a release build.
symbolized_builds = build_manager.setup_symbolized_builds(crash_revision)
if not symbolized_builds or (
not build_manager.check_app_path() and
not build_manager.check_app_path("APP_PATH_DEBUG")):
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
"Build setup failed")
tasks.add_task(
"symbolize", testcase_id, job_type, wait_time=build_fail_wait)
return
# Increase malloc_context_size to get all stack frames. Default is 30.
environment.reset_current_memory_tool_options(
redzone_size=sym_redzone,
malloc_context_size=STACK_FRAME_COUNT,
symbolize_inline_frames=True,
disable_ubsan=testcase.disable_ubsan,
)
# TSAN tool settings (if the tool is used).
if environment.tool_matches("TSAN", job_type):
environment.set_tsan_max_history_size()
# Do the symbolization if supported by this application.
result, sym_crash_stacktrace = get_symbolized_stacktraces(
testcase_file_path, testcase, old_crash_stacktrace, sym_crash_state)
# Update crash parameters.
testcase = data_handler.get_testcase_by_id(testcase_id)
testcase.crash_type = sym_crash_type
testcase.crash_address = sym_crash_address
testcase.crash_state = sym_crash_state
testcase.crash_stacktrace = data_handler.filter_stacktrace(
sym_crash_stacktrace)
if not result:
data_handler.update_testcase_comment(
testcase,
data_types.TaskState.ERROR,
"Unable to reproduce crash, skipping "
"stacktrace update",
)
else:
# Switch build url to use the less-optimized symbolized build with better
# stacktrace.
build_url = environment.get_value("BUILD_URL")
if build_url:
testcase.set_metadata("build_url", build_url, update_testcase=False)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.FINISHED)
testcase.symbolized = True
testcase.crash_revision = crash_revision
testcase.put()
# We might have updated the crash state. See if we need to marked as duplicate
# based on other testcases.
data_handler.handle_duplicate_entry(testcase)
task_creation.create_blame_task_if_needed(testcase)
# Switch current directory before builds cleanup.
root_directory = environment.get_value("ROOT_DIR")
os.chdir(root_directory)
# Cleanup symbolized builds which are space-heavy.
symbolized_builds.delete()
def check_for_bad_build(job_type, crash_revision):
"""Return true if the build is bad, i.e. crashes on startup."""
# Check the bad build check flag to see if we want do this.
if not environment.get_value('BAD_BUILD_CHECK'):
return False
# Do not detect leaks while checking for bad builds.
environment.reset_current_memory_tool_options(leaks=False)
# Create a blank command line with no file to run and no http.
command = get_command_line_for_application(file_to_run='',
needs_http=False)
# When checking for bad builds, we use the default window size.
# We don't want to pick a custom size since it can potentially cause a
# startup crash and cause a build to be detected incorrectly as bad.
default_window_argument = environment.get_value('WINDOW_ARG', '')
if default_window_argument:
command = command.replace(' %s' % default_window_argument, '')
# Warmup timeout.
fast_warmup_timeout = environment.get_value('FAST_WARMUP_TIMEOUT')
# TSAN is slow, and boots slow on first startup. Increase the warmup
# timeout for this case.
if environment.tool_matches('TSAN', job_type):
fast_warmup_timeout = environment.get_value('WARMUP_TIMEOUT')
# Initialize helper variables.
is_bad_build = False
build_run_console_output = ''
output = ''
app_directory = environment.get_value('APP_DIR')
# Check if the build is bad.
process_handler.terminate_stale_application_instances()
exit_code, _, output = process_handler.run_process(
command,
timeout=fast_warmup_timeout,
current_working_directory=app_directory)
output = utils.decode_to_unicode(output)
if crash_analyzer.is_crash(exit_code, output):
is_bad_build = True
build_run_console_output = (
'%s\n\n%s\n\n%s' %
(command, stack_symbolizer.symbolize_stacktrace(output), output))
logs.log('Bad build for %s detected at r%d.' %
(job_type, crash_revision),
output=build_run_console_output)
# Exit all running instances.
process_handler.terminate_stale_application_instances()
# Any of the conditions below indicate that bot is in a bad state and it is
# not caused by the build itself. In that case, just exit.
build_state = data_handler.get_build_state(job_type, crash_revision)
if (is_bad_build and ('cannot open display' in output
or 'logging service has stopped' in output
or 'Maximum number of clients reached' in output)):
logs.log_fatal_and_exit('Bad bot environment detected, exiting.',
output=build_run_console_output)
# If none of the other bots have added information about this build,
# then add it now.
if build_state == data_types.BuildState.UNMARKED:
data_handler.add_build_metadata(job_type, crash_revision, is_bad_build,
build_run_console_output)
# Reset memory tool options.
environment.reset_current_memory_tool_options()
return is_bad_build
