def prepare(self, corpus_dir, target_path, build_dir):
"""Prepare for a fuzzing session, by generating options. Returns a
FuzzOptions object.
Args:
corpus_dir: The main corpus directory.
target_path: Path to the target.
build_dir: Path to the build directory.
Returns:
A FuzzOptions object.
"""
del build_dir
arguments = fuzzer.get_arguments(target_path)
grammar = fuzzer.get_grammar(target_path)
if self.do_strategies:
strategy_pool = strategy_selection.generate_weighted_strategy_pool(
strategy_list=strategy.LIBFUZZER_STRATEGY_LIST,
use_generator=True,
engine_name=self.name)
else:
strategy_pool = strategy_selection.StrategyPool()
strategy_info = libfuzzer.pick_strategies(strategy_pool, target_path,
corpus_dir, arguments,
grammar)
arguments.extend(strategy_info.arguments)
# Check for seed corpus and add it into corpus directory.
engine_common.unpack_seed_corpus_if_needed(target_path, corpus_dir)
# Pick a few testcases from our corpus to use as the initial corpus.
subset_size = engine_common.random_choice(
engine_common.CORPUS_SUBSET_NUM_TESTCASES)
if (not strategy_info.use_dataflow_tracing
and strategy_pool.do_strategy(strategy.CORPUS_SUBSET_STRATEGY)
and shell.get_directory_file_count(corpus_dir) > subset_size):
# Copy |subset_size| testcases into 'subset' directory.
corpus_subset_dir = self._create_temp_corpus_dir('subset')
libfuzzer.copy_from_corpus(corpus_subset_dir, corpus_dir,
subset_size)
strategy_info.fuzzing_strategies.append(
strategy.CORPUS_SUBSET_STRATEGY.name + '_' + str(subset_size))
strategy_info.additional_corpus_dirs.append(corpus_subset_dir)
else:
strategy_info.additional_corpus_dirs.append(corpus_dir)
# Check dict argument to make sure that it's valid.
dict_path = fuzzer_utils.extract_argument(arguments,
constants.DICT_FLAG,
remove=False)
if dict_path and not os.path.exists(dict_path):
logs.log_error('Invalid dict %s for %s.' %
(dict_path, target_path))
fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG)
# If there's no dict argument, check for %target_binary_name%.dict file.
dict_path = fuzzer_utils.extract_argument(arguments,
constants.DICT_FLAG,
remove=False)
if not dict_path:
dict_path = dictionary_manager.get_default_dictionary_path(
target_path)
if os.path.exists(dict_path):
arguments.append(constants.DICT_FLAG + dict_path)
# If we have a dictionary, correct any items that are not formatted properly
# (e.g. quote items that are missing them).
dictionary_manager.correct_if_needed(dict_path)
strategies = stats.process_strategies(strategy_info.fuzzing_strategies,
name_modifier=lambda x: x)
return LibFuzzerOptions(corpus_dir, arguments, strategies,
strategy_info.additional_corpus_dirs,
strategy_info.extra_env,
strategy_info.use_dataflow_tracing,
strategy_info.is_mutations_run)
def _do_run_testcase_and_return_result_in_queue(crash_queue,
thread_index,
file_path,
gestures,
env_copy,
upload_output=False):
"""Run a single testcase and return crash results in the crash queue."""
try:
# Run testcase and check whether a crash occurred or not.
return_code, crash_time, output = run_testcase(thread_index, file_path,
gestures, env_copy)
# Pull testcase directory to host to get any stats files.
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
file_host.pull_testcases_from_worker()
# Analyze the crash.
crash_output = _get_crash_output(output)
crash_result = CrashResult(return_code, crash_time, crash_output)
# To provide consistency between stats and logs, we use timestamp taken
# from stats when uploading logs and testcase.
if upload_output:
log_time = _get_testcase_time(file_path)
if crash_result.is_crash():
# Initialize resource list with the testcase path.
resource_list = [file_path]
resource_list += get_resource_paths(crash_output)
# Store the crash stack file in the crash stacktrace directory
# with filename as the hash of the testcase path.
crash_stacks_directory = environment.get_value(
'CRASH_STACKTRACES_DIR')
stack_file_path = os.path.join(crash_stacks_directory,
utils.string_hash(file_path))
utils.write_data_to_file(crash_output, stack_file_path)
# Put crash/no-crash results in the crash queue.
crash_queue.put(
Crash(file_path=file_path,
crash_time=crash_time,
return_code=return_code,
resource_list=resource_list,
gestures=gestures,
stack_file_path=stack_file_path))
# Don't upload uninteresting testcases (no crash) or if there is no log to
# correlate it with (not upload_output).
if upload_output:
upload_testcase(file_path, log_time)
if upload_output:
# Include full output for uploaded logs (crash output, merge output, etc).
crash_result_full = CrashResult(return_code, crash_time, output)
log = prepare_log_for_upload(crash_result_full.get_stacktrace(),
return_code)
upload_log(log, log_time)
except Exception:
logs.log_error('Exception occurred while running '
'run_testcase_and_return_result_in_queue.')
def fuzz(self, target_path, options, reproducers_dir, max_time):
"""Run a fuzz session.
Args:
target_path: Path to the target.
options: The FuzzOptions object returned by prepare().
reproducers_dir: The directory to put reproducers in when crashes
are found.
max_time: Maximum allowed time for the fuzzing to run.
Returns:
A FuzzResult object.
"""
profiler.start_if_needed('libfuzzer_fuzz')
runner = libfuzzer.get_runner(target_path)
libfuzzer.set_sanitizer_options(target_path, fuzz_options=options)
# Directory to place new units.
if options.merge_back_new_testcases:
new_corpus_dir = self._create_temp_corpus_dir('new')
corpus_directories = [new_corpus_dir] + options.fuzz_corpus_dirs
else:
corpus_directories = options.fuzz_corpus_dirs
fuzz_result = runner.fuzz(corpus_directories,
fuzz_timeout=max_time,
additional_args=options.arguments,
artifact_prefix=reproducers_dir,
extra_env=options.extra_env)
project_qualified_fuzzer_name = (
engine_common.get_project_qualified_fuzzer_name(target_path))
dict_error_match = DICT_PARSING_FAILED_REGEX.search(fuzz_result.output)
if dict_error_match:
logs.log_error(
'Dictionary parsing failed '
f'(target={project_qualified_fuzzer_name}, '
f'line={dict_error_match.group(1)}).',
engine_output=fuzz_result.output)
elif (not environment.get_value('USE_MINIJAIL') and
fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE):
# Minijail returns 1 if the exit code is nonzero.
# Otherwise: we can assume that a return code of 1 means that libFuzzer
# itself ran into an error.
logs.log_error(ENGINE_ERROR_MESSAGE +
f' (target={project_qualified_fuzzer_name}).',
engine_output=fuzz_result.output)
log_lines = fuzz_result.output.splitlines()
# Output can be large, so save some memory by removing reference to the
# original output which is no longer needed.
fuzz_result.output = None
# Check if we crashed, and get the crash testcase path.
crash_testcase_file_path = runner.get_testcase_path(log_lines)
# If we exited with a non-zero return code with no crash file in output from
# libFuzzer, this is most likely a startup crash. Use an empty testcase to
# to store it as a crash.
if (not crash_testcase_file_path and fuzz_result.return_code
not in constants.NONCRASH_RETURN_CODES):
crash_testcase_file_path = self._create_empty_testcase_file(
reproducers_dir)
# Parse stats information based on libFuzzer output.
parsed_stats = libfuzzer.parse_log_stats(log_lines)
# Extend parsed stats by additional performance features.
parsed_stats.update(
stats.parse_performance_features(log_lines, options.strategies,
options.arguments))
# Set some initial stat overrides.
timeout_limit = fuzzer_utils.extract_argument(options.arguments,
constants.TIMEOUT_FLAG,
remove=False)
actual_duration = int(fuzz_result.time_executed)
fuzzing_time_percent = 100 * actual_duration / float(max_time)
parsed_stats.update({
'timeout_limit': int(timeout_limit),
'expected_duration': int(max_time),
'actual_duration': actual_duration,
'fuzzing_time_percent': fuzzing_time_percent,
})
# Remove fuzzing arguments before merge and dictionary analysis step.
non_fuzz_arguments = options.arguments.copy()
libfuzzer.remove_fuzzing_arguments(non_fuzz_arguments, is_merge=True)
if options.merge_back_new_testcases:
self._merge_new_units(target_path, options.corpus_dir,
new_corpus_dir, options.fuzz_corpus_dirs,
non_fuzz_arguments, parsed_stats)
fuzz_logs = '\n'.join(log_lines)
crashes = []
if crash_testcase_file_path:
reproduce_arguments = options.arguments[:]
libfuzzer.remove_fuzzing_arguments(reproduce_arguments)
# Use higher timeout for reproduction.
libfuzzer.fix_timeout_argument_for_reproduction(
reproduce_arguments)
# Write the new testcase.
# Copy crash testcase contents into the main testcase path.
crashes.append(
engine.Crash(crash_testcase_file_path, fuzz_logs,
reproduce_arguments, actual_duration))
if options.analyze_dictionary:
libfuzzer.analyze_and_update_recommended_dictionary(
runner, project_qualified_fuzzer_name, log_lines,
options.corpus_dir, non_fuzz_arguments)
return engine.FuzzResult(fuzz_logs, fuzz_result.command, crashes,
parsed_stats, fuzz_result.time_executed)
def get_crash_info(output):
"""Parse crash output to get (local) minidump path and any other information
useful for crash uploading, and store in a CrashReportInfo object."""
crash_stacks_directory = environment.get_value('CRASH_STACKTRACES_DIR')
output_lines = output.splitlines()
num_lines = len(output_lines)
is_android = environment.is_android()
for i, line in enumerate(output_lines):
if is_android:
# If we are on Android, the dump extraction is more complicated.
# The location placed in the crash-stacktrace is of the dump itself but
# in fact only the MIME of the dump exists, and will have a different
# extension. We need to pull the MIME and process it.
match = re.match(CRASH_DUMP_PATH_MARKER, line)
if not match:
continue
minidump_mime_filename_base = None
for j in range(i + 1, num_lines):
line = output_lines[j]
match = re.match(r'(.*)\.dmp', line)
if match:
minidump_mime_filename_base = os.path.basename(
match.group(1).strip())
break
if not minidump_mime_filename_base:
logs.log_error(
'Minidump marker was found, but no path in stacktrace.')
return None
# Look for MIME. If none found, bail.
# We might not have copied over the crash dumps yet (copying is buffered),
# so we want to search both the original directory and the one to which
# the minidumps should later be copied.
device_directories_to_search = [
constants.CRASH_DUMPS_DIR,
os.path.dirname(line.strip())
]
device_minidump_search_paths = []
device_minidump_mime_path = None
for device_directory in device_directories_to_search:
device_minidump_mime_potential_paths = adb.run_shell_command(
['ls', '"%s"' % device_directory], root=True).splitlines()
device_minidump_search_paths += device_minidump_mime_potential_paths
for potential_path in device_minidump_mime_potential_paths:
# Check that we actually found a file, and the right one (not logcat).
if 'No such file or directory' in potential_path:
continue
if minidump_mime_filename_base not in potential_path:
continue
if '.up' in potential_path or '.dmp' in potential_path:
device_minidump_mime_path = os.path.join(
device_directory, potential_path)
break
# Break if we found a path.
if device_minidump_mime_path is not None:
break
# If we still didn't find a minidump path, bail.
if device_minidump_mime_path is None:
logs.log_error('Could not get MIME path from ls:\n%s' %
str(device_minidump_search_paths))
return None
# Pull out MIME and parse to minidump file and MIME parameters.
minidump_mime_filename = '%s.mime' % minidump_mime_filename_base
local_minidump_mime_path = os.path.join(crash_stacks_directory,
minidump_mime_filename)
adb.run_command([
'pull',
'"%s"' % device_minidump_mime_path, local_minidump_mime_path
])
if not os.path.exists(local_minidump_mime_path):
logs.log_error(
'Could not pull MIME from %s to %s.' %
(device_minidump_mime_path, local_minidump_mime_path))
return None
crash_info = parse_mime_to_crash_report_info(
local_minidump_mime_path)
if crash_info is None:
return None
crash_info.unsymbolized_stacktrace = output
return crash_info
# Other platforms are not currently supported.
logs.log_error('Unable to fetch crash information for this platform.')
return None
# Could not find dump location, bail out. This could also happen when we don't
# have a minidump location in stack at all, e.g. when testcase does not crash
# during minimization.
return None
# Check if the build succeeded based on the existence of the
# local archive file.
if os.path.exists(archive_path_local):
# Build success. Now, copy it to google cloud storage and make it
# public.
os.system('gsutil cp %s %s' % (archive_path_local, archive_path_remote))
os.system('gsutil acl set public-read %s' % archive_path_remote)
logs.log('Build succeeded, created %s.' % archive_filename)
else:
LAST_BUILD[tool_and_build_type] = ''
logs.log_error('Build failed, unable to create %s.' % archive_filename)
logs.log('Completed cycle, waiting for %d secs.' % wait_time)
time.sleep(wait_time)
if __name__ == '__main__':
# Make sure environment is correctly configured.
logs.configure('run_bot')
environment.set_bot_environment()
fail_wait = environment.get_value('FAIL_WAIT')
# Continue this forever.
while True:
try:
main()
except Exception:
logs.log_error('Failed to create build.')
time.sleep(fail_wait)
def update_data_bundle(fuzzer, data_bundle):
"""Updates a data bundle to the latest version."""
# This module can't be in the global imports due to appengine issues
# with multiprocessing and psutil imports.
from clusterfuzz._internal.google_cloud_utils import gsutil
# If we are using a data bundle on NFS, it is expected that our testcases
# will usually be large enough that we would fill up our tmpfs directory
# pretty quickly. So, change it to use an on-disk directory.
if not data_bundle.is_local:
testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK')
environment.set_value('FUZZ_INPUTS', testcase_disk_directory)
data_bundle_directory = get_data_bundle_directory(fuzzer.name)
if not data_bundle_directory:
logs.log_error('Failed to setup data bundle %s.' % data_bundle.name)
return False
if not shell.create_directory(data_bundle_directory,
create_intermediates=True):
logs.log_error('Failed to create data bundle %s directory.' %
data_bundle.name)
return False
# Check if data bundle is up to date. If yes, skip the update.
if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
logs.log('Data bundle was recently synced, skip.')
return True
# Fetch lock for this data bundle.
if not _fetch_lock_for_data_bundle_update(data_bundle):
logs.log_error('Failed to lock data bundle %s.' % data_bundle.name)
return False
# Re-check if another bot did the sync already. If yes, skip.
if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
logs.log('Another bot finished the sync, skip.')
_release_lock_for_data_bundle_update(data_bundle)
return True
time_before_sync_start = time.time()
# No need to sync anything if this is a search index data bundle. In that
# case, the fuzzer will generate testcases from a gcs bucket periodically.
if not _is_search_index_data_bundle(data_bundle.name):
bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from clusterfuzz._internal.bot.untrusted_runner import corpus_manager
from clusterfuzz._internal.bot.untrusted_runner import file_host
worker_data_bundle_directory = file_host.rebase_to_worker_root(
data_bundle_directory)
file_host.create_directory(worker_data_bundle_directory,
create_intermediates=True)
result = corpus_manager.RemoteGSUtilRunner().rsync(
bucket_url, worker_data_bundle_directory, delete=False)
else:
result = gsutil.GSUtilRunner().rsync(bucket_url,
data_bundle_directory,
delete=False)
if result.return_code != 0:
logs.log_error('Failed to sync data bundle %s: %s.' %
(data_bundle.name, result.output))
_release_lock_for_data_bundle_update(data_bundle)
return False
# Update the testcase list file.
testcase_manager.create_testcase_list_file(data_bundle_directory)
# Write last synced time in the sync file.
sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory)
utils.write_data_to_file(time_before_sync_start, sync_file_path)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from clusterfuzz._internal.bot.untrusted_runner import file_host
worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path)
file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path)
# Release acquired lock.
_release_lock_for_data_bundle_update(data_bundle)
return True
def archive_testcase_and_dependencies_in_gcs(resource_list, testcase_path):
"""Archive testcase and its dependencies, and store in blobstore."""
if not os.path.exists(testcase_path):
logs.log_error('Unable to find testcase %s.' % testcase_path)
return None, None, None, None
absolute_filename = testcase_path
archived = False
zip_filename = None
zip_path = None
if not resource_list:
resource_list = []
# Add resource dependencies based on testcase path. These include
# stuff like extensions directory, dependency files, etc.
resource_list.extend(
testcase_manager.get_resource_dependencies(testcase_path))
# Filter out duplicates, directories, and files that do not exist.
resource_list = utils.filter_file_list(resource_list)
logs.log('Testcase and related files :\n%s' % str(resource_list))
if len(resource_list) <= 1:
# If this does not have any resources, just save the testcase.
# TODO(flowerhack): Update this when we teach CF how to download testcases.
try:
file_handle = open(testcase_path, 'rb')
except IOError:
logs.log_error('Unable to open testcase %s.' % testcase_path)
return None, None, None, None
else:
# If there are resources, create an archive.
# Find the common root directory for all of the resources.
# Assumption: resource_list[0] is the testcase path.
base_directory_list = resource_list[0].split(os.path.sep)
for list_index in range(1, len(resource_list)):
current_directory_list = resource_list[list_index].split(
os.path.sep)
length = min(len(base_directory_list), len(current_directory_list))
for directory_index in range(length):
if (current_directory_list[directory_index] !=
base_directory_list[directory_index]):
base_directory_list = base_directory_list[
0:directory_index]
break
base_directory = os.path.sep.join(base_directory_list)
logs.log('Subresource common base directory: %s' % base_directory)
if base_directory:
# Common parent directory, archive sub-paths only.
base_len = len(base_directory) + len(os.path.sep)
else:
# No common parent directory, archive all paths as it-is.
base_len = 0
# Prepare the filename for the archive.
zip_filename, _ = os.path.splitext(os.path.basename(testcase_path))
zip_filename += _TESTCASE_ARCHIVE_EXTENSION
# Create the archive.
zip_path = os.path.join(environment.get_value('INPUT_DIR'),
zip_filename)
zip_file = zipfile.ZipFile(zip_path, 'w')
for file_name in resource_list:
if os.path.exists(file_name):
relative_filename = file_name[base_len:]
zip_file.write(file_name, relative_filename,
zipfile.ZIP_DEFLATED)
zip_file.close()
try:
file_handle = open(zip_path, 'rb')
except IOError:
logs.log_error('Unable to open testcase archive %s.' % zip_path)
return None, None, None, None
archived = True
absolute_filename = testcase_path[base_len:]
fuzzed_key = blobs.write_blob(file_handle)
file_handle.close()
# Don't need the archive after writing testcase to blobstore.
if zip_path:
shell.remove_file(zip_path)
return fuzzed_key, archived, absolute_filename, zip_filename
def execute_task(testcase_id, job_type):
"""Run analyze task."""
# Reset redzones.
environment.reset_current_memory_tool_options(redzone_size=128)
# Unset window location size and position properties so as to use default.
environment.set_value('WINDOW_ARG', '')
# Locate the testcase associated with the id.
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
data_handler.update_testcase_comment(testcase,
data_types.TaskState.STARTED)
metadata = data_types.TestcaseUploadMetadata.query(
data_types.TestcaseUploadMetadata.testcase_id == int(
testcase_id)).get()
if not metadata:
logs.log_error('Testcase %s has no associated upload metadata.' %
testcase_id)
testcase.key.delete()
return
is_lsan_enabled = environment.get_value('LSAN')
if is_lsan_enabled:
# Creates empty local blacklist so all leaks will be visible to uploader.
leak_blacklist.create_empty_local_blacklist()
# Store the bot name and timestamp in upload metadata.
bot_name = environment.get_value('BOT_NAME')
metadata.bot_name = bot_name
metadata.timestamp = datetime.datetime.utcnow()
metadata.put()
# Adjust the test timeout, if user has provided one.
if metadata.timeout:
environment.set_value('TEST_TIMEOUT', metadata.timeout)
# Adjust the number of retries, if user has provided one.
if metadata.retries is not None:
environment.set_value('CRASH_RETRIES', metadata.retries)
# Set up testcase and get absolute testcase path.
file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type)
if not file_list:
return
# Set up build.
setup_build(testcase)
# Check if we have an application path. If not, our build failed
# to setup correctly.
if not build_manager.check_app_path():
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
'Build setup failed')
if data_handler.is_first_retry_for_task(testcase):
build_fail_wait = environment.get_value('FAIL_WAIT')
tasks.add_task('analyze',
testcase_id,
job_type,
wait_time=build_fail_wait)
else:
data_handler.close_invalid_uploaded_testcase(
testcase, metadata, 'Build setup failed')
return
# Update initial testcase information.
testcase.absolute_path = testcase_file_path
testcase.job_type = job_type
testcase.binary_flag = utils.is_binary_file(testcase_file_path)
testcase.queue = tasks.default_queue()
testcase.crash_state = ''
# Set initial testcase metadata fields (e.g. build url, etc).
data_handler.set_initial_testcase_metadata(testcase)
# Update minimized arguments and use ones provided during user upload.
if not testcase.minimized_arguments:
minimized_arguments = environment.get_value('APP_ARGS') or ''
additional_command_line_flags = testcase.get_metadata(
'uploaded_additional_args')
if additional_command_line_flags:
minimized_arguments += ' %s' % additional_command_line_flags
environment.set_value('APP_ARGS', minimized_arguments)
testcase.minimized_arguments = minimized_arguments
# Update other fields not set at upload time.
testcase.crash_revision = environment.get_value('APP_REVISION')
data_handler.set_initial_testcase_metadata(testcase)
testcase.put()
# Initialize some variables.
gestures = testcase.gestures
http_flag = testcase.http_flag
test_timeout = environment.get_value('TEST_TIMEOUT')
# Get the crash output.
result = testcase_manager.test_for_crash_with_retries(testcase,
testcase_file_path,
test_timeout,
http_flag=http_flag,
compare_crash=False)
# If we don't get a crash, try enabling http to see if we can get a crash.
# Skip engine fuzzer jobs (e.g. libFuzzer, AFL) for which http testcase paths
# are not applicable.
if (not result.is_crash() and not http_flag
and not environment.is_engine_fuzzer_job()):
result_with_http = testcase_manager.test_for_crash_with_retries(
testcase,
testcase_file_path,
test_timeout,
http_flag=True,
compare_crash=False)
if result_with_http.is_crash():
logs.log('Testcase needs http flag for crash.')
http_flag = True
result = result_with_http
# Refresh our object.
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
# Set application command line with the correct http flag.
application_command_line = (
testcase_manager.get_command_line_for_application(
testcase_file_path, needs_http=http_flag))
# Get the crash data.
crashed = result.is_crash()
crash_time = result.get_crash_time()
state = result.get_symbolized_data()
unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False)
# Get crash info object with minidump info. Also, re-generate unsymbolized
# stacktrace if needed.
crash_info, _ = (crash_uploader.get_crash_info_and_stacktrace(
application_command_line, state.crash_stacktrace, gestures))
if crash_info:
testcase.minidump_keys = crash_info.store_minidump()
if not crashed:
# Could not reproduce the crash.
log_message = ('Testcase didn\'t crash in %d seconds (with retries)' %
test_timeout)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.FINISHED,
log_message)
# In the general case, we will not attempt to symbolize if we do not detect
# a crash. For user uploads, we should symbolize anyway to provide more
# information about what might be happening.
crash_stacktrace_output = utils.get_crash_stacktrace_output(
application_command_line, state.crash_stacktrace,
unsymbolized_crash_stacktrace)
testcase.crash_stacktrace = data_handler.filter_stacktrace(
crash_stacktrace_output)
# For an unreproducible testcase, retry once on another bot to confirm
# our results and in case this bot is in a bad state which we didn't catch
# through our usual means.
if data_handler.is_first_retry_for_task(testcase):
testcase.status = 'Unreproducible, retrying'
testcase.put()
tasks.add_task('analyze', testcase_id, job_type)
return
data_handler.close_invalid_uploaded_testcase(testcase, metadata,
'Unreproducible')
# A non-reproducing testcase might still impact production branches.
# Add the impact task to get that information.
task_creation.create_impact_task_if_needed(testcase)
return
# Update testcase crash parameters.
testcase.http_flag = http_flag
testcase.crash_type = state.crash_type
testcase.crash_address = state.crash_address
testcase.crash_state = state.crash_state
crash_stacktrace_output = utils.get_crash_stacktrace_output(
application_command_line, state.crash_stacktrace,
unsymbolized_crash_stacktrace)
testcase.crash_stacktrace = data_handler.filter_stacktrace(
crash_stacktrace_output)
# Try to guess if the bug is security or not.
security_flag = crash_analyzer.is_security_issue(state.crash_stacktrace,
state.crash_type,
state.crash_address)
testcase.security_flag = security_flag
# If it is, guess the severity.
if security_flag:
testcase.security_severity = severity_analyzer.get_security_severity(
state.crash_type, state.crash_stacktrace, job_type, bool(gestures))
log_message = ('Testcase crashed in %d seconds (r%d)' %
(crash_time, testcase.crash_revision))
data_handler.update_testcase_comment(testcase,
data_types.TaskState.FINISHED,
log_message)
# See if we have to ignore this crash.
if crash_analyzer.ignore_stacktrace(state.crash_stacktrace):
data_handler.close_invalid_uploaded_testcase(testcase, metadata,
'Irrelavant')
return
# Test for reproducibility.
one_time_crasher_flag = not testcase_manager.test_for_reproducibility(
testcase.fuzzer_name, testcase.actual_fuzzer_name(),
testcase_file_path, state.crash_state, security_flag, test_timeout,
http_flag, gestures)
testcase.one_time_crasher_flag = one_time_crasher_flag
# Check to see if this is a duplicate.
data_handler.check_uploaded_testcase_duplicate(testcase, metadata)
# Set testcase and metadata status if not set already.
if testcase.status == 'Duplicate':
# For testcase uploaded by bots (with quiet flag), don't create additional
# tasks.
if metadata.quiet_flag:
data_handler.close_invalid_uploaded_testcase(
testcase, metadata, 'Duplicate')
return
else:
# New testcase.
testcase.status = 'Processed'
metadata.status = 'Confirmed'
# Reset the timestamp as well, to respect
# data_types.MIN_ELAPSED_TIME_SINCE_REPORT. Otherwise it may get filed by
# triage task prematurely without the grouper having a chance to run on this
# testcase.
testcase.timestamp = utils.utcnow()
# Add new leaks to global blacklist to avoid detecting duplicates.
# Only add if testcase has a direct leak crash and if it's reproducible.
if is_lsan_enabled:
leak_blacklist.add_crash_to_global_blacklist_if_needed(testcase)
# Update the testcase values.
testcase.put()
# Update the upload metadata.
metadata.security_flag = security_flag
metadata.put()
_add_default_issue_metadata(testcase)
# Create tasks to
# 1. Minimize testcase (minimize).
# 2. Find regression range (regression).
# 3. Find testcase impact on production branches (impact).
# 4. Check whether testcase is fixed (progression).
# 5. Get second stacktrace from another job in case of
# one-time crashes (stack).
task_creation.create_tasks(testcase)
def upload_testcases_if_needed(fuzzer_name, testcase_list, testcase_directory,
data_directory):
"""Upload test cases from the list to a cloud storage bucket."""
# Since builtin fuzzers have a coverage minimized corpus, no need to upload
# test case samples for them.
if fuzzer_name in fuzzing.ENGINES:
return
bucket_name = local_config.ProjectConfig().get(
'coverage.fuzzer-testcases.bucket')
if not bucket_name:
return
files_list = []
has_testcases_in_testcase_directory = False
has_testcases_in_data_directory = False
for testcase_path in testcase_list:
if testcase_path.startswith(testcase_directory):
files_list.append(
os.path.relpath(testcase_path, testcase_directory))
has_testcases_in_testcase_directory = True
elif testcase_path.startswith(data_directory):
files_list.append(os.path.relpath(testcase_path, data_directory))
has_testcases_in_data_directory = True
if not files_list:
return
formatted_date = str(utils.utcnow().date())
gcs_base_url = 'gs://{bucket_name}/{date}/{fuzzer_name}/'.format(
bucket_name=bucket_name, date=formatted_date, fuzzer_name=fuzzer_name)
runner = gsutil.GSUtilRunner()
batch_directory_blobs = storage.list_blobs(gcs_base_url)
total_testcases = 0
for blob in batch_directory_blobs:
if not blob.endswith(LIST_FILE_BASENAME):
continue
list_gcs_url = storage.get_cloud_storage_file_path(bucket_name, blob)
data = storage.read_data(list_gcs_url)
if not data:
logs.log_error(
'Read no data from test case list at {gcs_url}'.format(
gcs_url=list_gcs_url))
continue
total_testcases += len(data.splitlines())
# If we've already uploaded enough test cases for this fuzzer today, return.
if total_testcases >= TESTCASES_PER_DAY:
return
# Cap the number of files.
testcases_limit = min(len(files_list), TESTCASES_PER_DAY - total_testcases)
files_list = files_list[:testcases_limit]
# Upload each batch of tests to its own unique sub-bucket.
identifier = environment.get_value('BOT_NAME') + str(utils.utcnow())
gcs_base_url += utils.string_hash(identifier)
list_gcs_url = gcs_base_url + '/' + LIST_FILE_BASENAME
if not storage.write_data('\n'.join(files_list).encode('utf-8'),
list_gcs_url):
return
if has_testcases_in_testcase_directory:
# Sync everything in |testcase_directory| since it is fuzzer-generated.
runner.rsync(testcase_directory, gcs_base_url)
if has_testcases_in_data_directory:
# Sync all fuzzer generated testcase in data bundle directory.
runner.rsync(data_directory,
gcs_base_url,
exclusion_pattern=('(?!.*{fuzz_prefix})'.format(
fuzz_prefix=testcase_manager.FUZZ_PREFIX)))
# Sync all possible resource dependencies as a best effort. It matches
# |resources-| prefix that a fuzzer can use to indicate resources. Also, it
# matches resources directory that Chromium web_tests use for dependencies.
runner.rsync(data_directory,
gcs_base_url,
exclusion_pattern='(?!.*resource)')
logs.log('Synced {count} test cases to {gcs_url}.'.format(
count=len(files_list), gcs_url=gcs_base_url))
def filter_log_output(output):
"""Filters log output. Removes debug info, etc and normalize output."""
if not output:
return ''
filtered_output = ''
last_process_tuple = (None, None)
for line in output.splitlines():
if not is_line_valid(line):
continue
# To parse frames like:
# E/v8 (18890): Error installing extension 'v8/LoadTimes'.
# {log_level}/{process_name}({process_id}): {message}
m_line = re.match(r'^[VDIWEFS]/(.+)\(\s*(\d+)\)[:](.*)$', line)
if not m_line:
logs.log_error('Failed to parse logcat line: %s' % line)
continue
process_name = m_line.group(1).strip()
process_id = int(m_line.group(2))
filtered_line = m_line.group(3).rstrip()[1:]
# Process Android crash stack frames and convert into sanitizer format.
m_crash_state = re.match(r'\s*#([0-9]+)\s+pc\s+([xX0-9a-fA-F]+)\s+(.+)',
filtered_line)
if m_crash_state:
frame_no = int(m_crash_state.group(1))
frame_address = m_crash_state.group(2)
frame_binary = m_crash_state.group(3).strip()
# Ignore invalid frames, helps to prevent errors
# while symbolizing.
if '<unknown>' in frame_binary:
continue
# Normalize frame address.
if not frame_address.startswith('0x'):
frame_address = '0x%s' % frame_address
# Separate out the function argument.
frame_binary = (frame_binary.split(' '))[0]
# Normalize line into the same sanitizer tool format.
filtered_line = (' #%d %s (%s+%s)' % (frame_no, frame_address,
frame_binary, frame_address))
# Process Chrome crash stack frames and convert into sanitizer format.
# Stack frames don't have paranthesis around frame binary and address, so
# add it explicitly to allow symbolizer to catch it.
m_crash_state = re.match(
r'\s*#([0-9]+)\s+([xX0-9a-fA-F]+)\s+([^(]+\+[xX0-9a-fA-F]+)$',
filtered_line)
if m_crash_state:
frame_no = int(m_crash_state.group(1))
frame_address = m_crash_state.group(2)
frame_binary_and_address = m_crash_state.group(3).strip()
filtered_line = (' #%d %s (%s)' % (frame_no, frame_address,
frame_binary_and_address))
# Add process number if changed.
current_process_tuple = (process_name, process_id)
if current_process_tuple != last_process_tuple:
filtered_output += '--------- %s (%d):\n' % (process_name, process_id)
last_process_tuple = current_process_tuple
filtered_output += filtered_line + '\n'
return filtered_output
def execute_task(metadata_id, job_type):
"""Unpack a bundled testcase archive and create analyze jobs for each item."""
metadata = ndb.Key(data_types.BundledArchiveMetadata,
int(metadata_id)).get()
if not metadata:
logs.log_error('Invalid bundle metadata id %s.' % metadata_id)
return
bot_name = environment.get_value('BOT_NAME')
upload_metadata = data_types.TestcaseUploadMetadata.query(
data_types.TestcaseUploadMetadata.blobstore_key ==
metadata.blobstore_key).get()
if not upload_metadata:
logs.log_error('Invalid upload metadata key %s.' %
metadata.blobstore_key)
return
job = data_types.Job.query(data_types.Job.name == metadata.job_type).get()
if not job:
logs.log_error('Invalid job_type %s.' % metadata.job_type)
return
# Update the upload metadata with this bot name.
upload_metadata.bot_name = bot_name
upload_metadata.put()
# We can't use FUZZ_INPUTS directory since it is constrained
# by tmpfs limits.
testcases_directory = environment.get_value('FUZZ_INPUTS_DISK')
# Retrieve multi-testcase archive.
archive_path = os.path.join(testcases_directory, metadata.archive_filename)
if not blobs.read_blob_to_disk(metadata.blobstore_key, archive_path):
logs.log_error('Could not retrieve archive for bundle %d.' %
metadata_id)
tasks.add_task('unpack', metadata_id, job_type)
return
try:
archive.unpack(archive_path, testcases_directory)
except:
logs.log_error('Could not unpack archive for bundle %d.' % metadata_id)
tasks.add_task('unpack', metadata_id, job_type)
return
# Get additional testcase metadata (if any).
additional_metadata = None
if upload_metadata.additional_metadata_string:
additional_metadata = json.loads(
upload_metadata.additional_metadata_string)
archive_state = data_types.ArchiveStatus.NONE
bundled = True
file_list = archive.get_file_list(archive_path)
for file_path in file_list:
absolute_file_path = os.path.join(testcases_directory, file_path)
filename = os.path.basename(absolute_file_path)
# Only files are actual testcases. Skip directories.
if not os.path.isfile(absolute_file_path):
continue
try:
file_handle = open(absolute_file_path, 'rb')
blob_key = blobs.write_blob(file_handle)
file_handle.close()
except:
blob_key = None
if not blob_key:
logs.log_error('Could not write testcase %s to blobstore.' %
absolute_file_path)
continue
data_handler.create_user_uploaded_testcase(
blob_key, metadata.blobstore_key, archive_state,
metadata.archive_filename, filename, metadata.timeout, job,
metadata.job_queue, metadata.http_flag, metadata.gestures,
metadata.additional_arguments, metadata.bug_information,
metadata.crash_revision, metadata.uploader_email,
metadata.platform_id, metadata.app_launch_command,
metadata.fuzzer_name, metadata.overridden_fuzzer_name,
metadata.fuzzer_binary_name, bundled, upload_metadata.retries,
upload_metadata.bug_summary_update_flag,
upload_metadata.quiet_flag, additional_metadata)
# The upload metadata for the archive is not needed anymore since we created
# one for each testcase.
upload_metadata.key.delete()
shell.clear_testcase_directories()
def get(self):
"""Handle a get request."""
try:
grouper.group_testcases()
except:
logs.log_error('Error occurred while grouping test cases.')
return
# Free up memory after group task run.
utils.python_gc()
# Get a list of jobs excluded from bug filing.
excluded_jobs = _get_excluded_jobs()
# Get a list of all jobs. This is used to filter testcases whose jobs have
# been removed.
all_jobs = data_handler.get_all_job_type_names()
for testcase_id in data_handler.get_open_testcase_id_iterator():
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
# Already deleted.
continue
# Skip if testcase's job is removed.
if testcase.job_type not in all_jobs:
continue
# Skip if testcase's job is in exclusions list.
if testcase.job_type in excluded_jobs:
continue
# Skip if we are running progression task at this time.
if testcase.get_metadata('progression_pending'):
continue
# If the testcase has a bug filed already, no triage is needed.
if _is_bug_filed(testcase):
continue
# Check if the crash is important, i.e. it is either a reproducible crash
# or an unreproducible crash happening frequently.
if not _is_crash_important(testcase):
continue
# Require that all tasks like minimizaton, regression testing, etc have
# finished.
if not data_handler.critical_tasks_completed(testcase):
continue
# For testcases that are not part of a group, wait an additional time to
# make sure it is grouped.
# The grouper runs prior to this step in the same cron, but there is a
# window of time where new testcases can come in after the grouper starts.
# This delay needs to be longer than the maximum time the grouper can take
# to account for that.
# FIXME: In future, grouping might be dependent on regression range, so we
# would have to add an additional wait time.
if not testcase.group_id and not dates.time_has_expired(
testcase.timestamp, hours=data_types.MIN_ELAPSED_TIME_SINCE_REPORT):
continue
# If this project does not have an associated issue tracker, we cannot
# file this crash anywhere.
issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(
testcase)
if not issue_tracker:
issue_filer.notify_issue_update(testcase, 'new')
continue
# If there are similar issues to this test case already filed or recently
# closed, skip filing a duplicate bug.
if _check_and_update_similar_bug(testcase, issue_tracker):
continue
# Clean up old triage messages that would be not applicable now.
testcase.delete_metadata(TRIAGE_MESSAGE_KEY, update_testcase=False)
# File the bug first and then create filed bug metadata.
try:
issue_filer.file_issue(testcase, issue_tracker)
except Exception as e:
logs.log_error('Failed to file issue for testcase %d.' % testcase_id)
_add_triage_message(testcase,
f'Failed to file issue due to exception: {str(e)}')
continue
_create_filed_bug_metadata(testcase)
issue_filer.notify_issue_update(testcase, 'new')
logs.log('Filed new issue %s for testcase %d.' %
(testcase.bug_information, testcase_id))
def recreate_instance_with_disks(instance_name,
project,
zone,
additional_metadata=None,
wait_for_completion=False):
"""Recreate an instance and its disk."""
# Get existing instance information.
# First, try to get instance info from cache.
# TODO(ochang): Make this more general in case anything else needs to use
# this method (e.g. appengine).
instance_info = persistent_cache.get_value(GCE_INSTANCE_INFO_KEY)
if instance_info is None:
instance_info = _get_instance_info(instance_name, project, zone)
# Bail out if we don't have a valid instance information.
if (not instance_info or 'disks' not in instance_info or
not instance_info['disks']):
logs.log_error(
'Failed to get disk info from existing instance, bailing on instance '
'recreation.')
return False
# Add any additional metadata required for instance booting.
if additional_metadata:
for key, value in six.iteritems(additional_metadata):
items = instance_info.setdefault('metadata', {}).setdefault('items', [])
_add_metadata_key_value(items, key, value)
# Cache the latest instance information.
persistent_cache.set_value(
GCE_INSTANCE_INFO_KEY, instance_info, persist_across_reboots=True)
# Delete the instance.
if not _do_instance_operation(
'delete', instance_name, project, zone, wait_for_completion=True):
logs.log_error('Failed to delete instance.')
return False
# Get existing disks information, and recreate.
api = _get_api()
disks = instance_info['disks']
for disk in disks:
disk_source = disk['source']
disk_name = disk_source.split('/')[-1]
disk_info_func = api.disks().get(disk=disk_name, project=project, zone=zone)
disk_info = _execute_api_call_with_retries(disk_info_func)
if 'sourceImage' not in disk_info or 'sizeGb' not in disk_info:
logs.log_error(
'Failed to get source image and size from existing disk, bailing on '
'instance recreation.')
return False
size_gb = disk_info['sizeGb']
source_image = disk_info['sourceImage']
# Recreate the disk.
if not delete_disk(disk_name, project, zone, wait_for_completion=True):
logs.log_error('Failed to delete disk.')
return False
if not create_disk(
disk_name,
source_image,
size_gb,
project,
zone,
wait_for_completion=True):
logs.log_error('Failed to recreate disk.')
return False
# Recreate the instance with the exact same configurations, but not
# necessarily the same IPs.
try:
del instance_info['networkInterfaces'][0]['accessConfigs'][0]['natIP']
except:
# This is not a failure. When a bot is stopped, it has no ip/interface.
pass
try:
del instance_info['networkInterfaces'][0]['networkIP']
except:
# This is not a failure. When a bot is stopped, it has no ip/interface.
pass
operation = api.instances().insert(
body=instance_info, project=project, zone=zone)
return _do_operation_with_retries(
operation, project, zone, wait_for_completion=wait_for_completion)
def group_testcases():
"""Group testcases based on rules like same bug numbers, similar crash
states, etc."""
testcase_map = {}
cached_issue_map = {}
for testcase_id in data_handler.get_open_testcase_id_iterator():
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
# Already deleted.
continue
# Remove duplicates early on to avoid large groups.
if (not testcase.bug_information and not testcase.uploader_email
and _has_testcase_with_same_params(testcase, testcase_map)):
logs.log('Deleting duplicate testcase %d.' % testcase_id)
testcase.key.delete()
continue
# Wait for minimization to finish as this might change crash params such
# as type and may mark it as duplicate / closed.
if not testcase.minimized_keys:
continue
# Store needed testcase attributes into |testcase_map|.
testcase_map[testcase_id] = TestcaseAttributes(testcase_id)
testcase_attributes = testcase_map[testcase_id]
for attribute_name in FORWARDED_ATTRIBUTES:
setattr(testcase_attributes, attribute_name,
getattr(testcase, attribute_name))
# Store original issue mappings in the testcase attributes.
if testcase.bug_information:
issue_id = int(testcase.bug_information)
project_name = testcase.project_name
if (project_name in cached_issue_map
and issue_id in cached_issue_map[project_name]):
testcase_attributes.issue_id = (
cached_issue_map[project_name][issue_id])
else:
issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(
testcase)
if not issue_tracker:
logs.log_error(
'Unable to access issue tracker for issue %d.' %
issue_id)
testcase_attributes.issue_id = issue_id
continue
# Determine the original issue id traversing the list of duplicates.
try:
issue = issue_tracker.get_original_issue(issue_id)
original_issue_id = int(issue.id)
except:
# If we are unable to access the issue, then we can't determine
# the original issue id. Assume that it is the same as issue id.
logs.log_error(
'Unable to determine original issue for issue %d.' %
issue_id)
testcase_attributes.issue_id = issue_id
continue
if project_name not in cached_issue_map:
cached_issue_map[project_name] = {}
cached_issue_map[project_name][issue_id] = original_issue_id
cached_issue_map[project_name][
original_issue_id] = original_issue_id
testcase_attributes.issue_id = original_issue_id
# No longer needed. Free up some memory.
cached_issue_map.clear()
_group_testcases_with_similar_states(testcase_map)
_group_testcases_with_same_issues(testcase_map)
_shrink_large_groups_if_needed(testcase_map)
group_leader.choose(testcase_map)
# TODO(aarya): Replace with an optimized implementation using dirty flag.
# Update the group mapping in testcase object.
for testcase_id in data_handler.get_open_testcase_id_iterator():
if testcase_id not in testcase_map:
# A new testcase that was just created. Skip for now, will be grouped in
# next iteration of group task.
continue
# If we are part of a group, then calculate the number of testcases in that
# group and lowest issue id of issues associated with testcases in that
# group.
updated_group_id = testcase_map[testcase_id].group_id
updated_is_leader = testcase_map[testcase_id].is_leader
updated_group_id_count = 0
updated_group_bug_information = 0
if updated_group_id:
for other_testcase in six.itervalues(testcase_map):
if other_testcase.group_id != updated_group_id:
continue
updated_group_id_count += 1
# Update group issue id to be lowest issue id in the entire group.
if other_testcase.issue_id is None:
continue
if (not updated_group_bug_information
or updated_group_bug_information >
other_testcase.issue_id):
updated_group_bug_information = other_testcase.issue_id
# If this group id is used by only one testcase, then remove it.
if updated_group_id_count == 1:
data_handler.delete_group(updated_group_id, update_testcases=False)
updated_group_id = 0
updated_group_bug_information = 0
updated_is_leader = True
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
# Already deleted.
continue
is_changed = (
(testcase.group_id != updated_group_id) or
(testcase.group_bug_information != updated_group_bug_information)
or (testcase.is_leader != updated_is_leader))
if not testcase.get_metadata('ran_grouper'):
testcase.set_metadata('ran_grouper',
True,
update_testcase=not is_changed)
if not is_changed:
continue
testcase.group_bug_information = updated_group_bug_information
testcase.group_id = updated_group_id
testcase.is_leader = updated_is_leader
testcase.put()
logs.log('Updated testcase %d group to %d.' %
(testcase_id, updated_group_id))
def get(self):
"""Handles a GET request."""
libfuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == 'libFuzzer').get()
if not libfuzzer:
logs.log_error('Failed to get libFuzzer Fuzzer entity.')
return
afl = data_types.Fuzzer.query(data_types.Fuzzer.name == 'afl').get()
if not afl:
logs.log_error('Failed to get AFL Fuzzer entity.')
return
honggfuzz = data_types.Fuzzer.query(
data_types.Fuzzer.name == 'honggfuzz').get()
if not honggfuzz:
logs.log_error('Failed to get honggfuzz Fuzzer entity.')
return
gft = data_types.Fuzzer.query(
data_types.Fuzzer.name == 'googlefuzztest').get()
if not gft:
logs.log_error('Failed to get googlefuzztest Fuzzer entity.')
return
project_config = local_config.ProjectConfig()
segregate_projects = project_config.get('segregate_projects')
project_setup_configs = project_config.get('project_setup')
project_names = set()
job_names = set()
fuzzer_entities = {
'afl': afl.key,
'honggfuzz': honggfuzz.key,
'googlefuzztest': gft.key,
'libfuzzer': libfuzzer.key,
}
for setup_config in project_setup_configs:
bucket_config = setup_config.get('build_buckets')
if not bucket_config:
raise ProjectSetupError('Project setup buckets not specified.')
config = ProjectSetup(
BUILD_BUCKET_PATH_TEMPLATE,
REVISION_URL,
setup_config.get('build_type'),
config_suffix=setup_config.get('job_suffix', ''),
external_config=setup_config.get('external_config', ''),
segregate_projects=segregate_projects,
experimental_sanitizers=setup_config.get(
'experimental_sanitizers', []),
engine_build_buckets={
'libfuzzer': bucket_config.get('libfuzzer'),
'libfuzzer-i386': bucket_config.get('libfuzzer_i386'),
'afl': bucket_config.get('afl'),
'honggfuzz': bucket_config.get('honggfuzz'),
'googlefuzztest': bucket_config.get('googlefuzztest'),
'none': bucket_config.get('no_engine'),
'dataflow': bucket_config.get('dataflow'),
},
fuzzer_entities=fuzzer_entities,
add_info_labels=setup_config.get('add_info_labels', False),
add_revision_mappings=setup_config.get('add_revision_mappings',
False),
additional_vars=setup_config.get('additional_vars'))
projects_source = setup_config.get('source')
if projects_source == 'oss-fuzz':
projects = get_oss_fuzz_projects()
elif projects_source.startswith(storage.GS_PREFIX):
projects = get_projects_from_gcs(projects_source)
else:
raise ProjectSetupError('Invalid projects source: ' +
projects_source)
if not projects:
raise ProjectSetupError('Missing projects list.')
result = config.set_up(projects)
project_names.update(result.project_names)
job_names.update(result.job_names)
cleanup_stale_projects(list(fuzzer_entities.values()), project_names,
job_names, segregate_projects)
def main():
"""Prepare the configuration options and start requesting tasks."""
logs.configure('run_bot')
root_directory = environment.get_value('ROOT_DIR')
if not root_directory:
print('Please set ROOT_DIR environment variable to the root of the source '
'checkout before running. Exiting.')
print('For an example, check init.bash in the local directory.')
return
dates.initialize_timezone_from_environment()
environment.set_bot_environment()
monitor.initialize()
if not profiler.start_if_needed('python_profiler_bot'):
sys.exit(-1)
fuzzers_init.run()
if environment.is_trusted_host(ensure_connected=False):
from clusterfuzz._internal.bot.untrusted_runner import host
host.init()
if environment.is_untrusted_worker():
# Track revision since we won't go into the task_loop.
update_task.track_revision()
from clusterfuzz._internal.bot.untrusted_runner import \
untrusted as untrusted_worker
untrusted_worker.start_server()
assert False, 'Unreachable code'
while True:
# task_loop should be an infinite loop,
# unless we run into an exception.
error_stacktrace, clean_exit, task_payload = task_loop()
# Print the error trace to the console.
if not clean_exit:
print('Exception occurred while running "%s".' % task_payload)
print('-' * 80)
print(error_stacktrace)
print('-' * 80)
should_terminate = (
clean_exit or errors.error_in_list(error_stacktrace,
errors.BOT_ERROR_TERMINATION_LIST))
if should_terminate:
return
logs.log_error(
'Task exited with exception (payload="%s").' % task_payload,
error_stacktrace=error_stacktrace)
should_hang = errors.error_in_list(error_stacktrace,
errors.BOT_ERROR_HANG_LIST)
if should_hang:
logs.log('Start hanging forever.')
while True:
# Sleep to avoid consuming 100% of CPU.
time.sleep(60)
# See if our run timed out, if yes bail out.
if data_handler.bot_run_timed_out():
return
def setup_testcase(testcase, job_type, fuzzer_override=None):
"""Sets up the testcase and needed dependencies like fuzzer,
data bundle, etc."""
fuzzer_name = fuzzer_override or testcase.fuzzer_name
task_name = environment.get_value('TASK_NAME')
testcase_fail_wait = environment.get_value('FAIL_WAIT')
testcase_id = testcase.key.id()
# Clear testcase directories.
shell.clear_testcase_directories()
# Adjust the test timeout value if this is coming from an user uploaded
# testcase.
if testcase.uploader_email:
_set_timeout_value_from_user_upload(testcase_id)
# Update the fuzzer if necessary in order to get the updated data bundle.
if fuzzer_name:
try:
update_successful = update_fuzzer_and_data_bundles(fuzzer_name)
except errors.InvalidFuzzerError:
# Close testcase and don't recreate tasks if this fuzzer is invalid.
testcase.open = False
testcase.fixed = 'NA'
testcase.set_metadata('fuzzer_was_deleted', True)
logs.log_error('Closed testcase %d with invalid fuzzer %s.' %
(testcase_id, fuzzer_name))
error_message = 'Fuzzer %s no longer exists' % fuzzer_name
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
return None, None, None
if not update_successful:
error_message = 'Unable to setup fuzzer %s' % fuzzer_name
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
tasks.add_task(task_name,
testcase_id,
job_type,
wait_time=testcase_fail_wait)
return None, None, None
# Extract the testcase and any of its resources to the input directory.
file_list, input_directory, testcase_file_path = unpack_testcase(testcase)
if not file_list:
error_message = 'Unable to setup testcase %s' % testcase_file_path
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
tasks.add_task(task_name,
testcase_id,
job_type,
wait_time=testcase_fail_wait)
return None, None, None
# For Android/Fuchsia, we need to sync our local testcases directory with the
# one on the device.
if environment.is_android():
_copy_testcase_to_device_and_setup_environment(testcase,
testcase_file_path)
# Push testcases to worker.
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
file_host.push_testcases_to_worker()
# Copy global blacklist into local blacklist.
is_lsan_enabled = environment.get_value('LSAN')
if is_lsan_enabled:
# Get local blacklist without this testcase's entry.
leak_blacklist.copy_global_to_local_blacklist(
excluded_testcase=testcase)
prepare_environment_for_testcase(testcase, job_type, task_name)
return file_list, input_directory, testcase_file_path
def process_command(task):
"""Figures out what to do with the given task and executes the command."""
logs.log("Executing command '%s'" % task.payload())
if not task.payload().strip():
logs.log_error('Empty task received.')
return
# Parse task payload.
task_name = task.command
task_argument = task.argument
job_name = task.job
environment.set_value('TASK_NAME', task_name)
environment.set_value('TASK_ARGUMENT', task_argument)
environment.set_value('JOB_NAME', job_name)
if job_name != 'none':
job = data_types.Job.query(data_types.Job.name == job_name).get()
# Job might be removed. In that case, we don't want an exception
# raised and causing this task to be retried by another bot.
if not job:
logs.log_error("Job '%s' not found." % job_name)
return
if not job.platform:
error_string = "No platform set for job '%s'" % job_name
logs.log_error(error_string)
raise errors.BadStateError(error_string)
# A misconfiguration led to this point. Clean up the job if necessary.
job_queue_suffix = tasks.queue_suffix_for_platform(job.platform)
bot_queue_suffix = tasks.default_queue_suffix()
if job_queue_suffix != bot_queue_suffix:
# This happens rarely, store this as a hard exception.
logs.log_error(
'Wrong platform for job %s: job queue [%s], bot queue [%s].' %
(job_name, job_queue_suffix, bot_queue_suffix))
# Try to recreate the job in the correct task queue.
new_queue = (
tasks.high_end_queue() if task.high_end else tasks.regular_queue())
new_queue += job_queue_suffix
# Command override is continuously run by a bot. If we keep failing
# and recreating the task, it will just DoS the entire task queue.
# So, we don't create any new tasks in that case since it needs
# manual intervention to fix the override anyway.
if not task.is_command_override:
try:
tasks.add_task(task_name, task_argument, job_name, new_queue)
except Exception:
# This can happen on trying to publish on a non-existent topic, e.g.
# a topic for a high-end bot on another platform. In this case, just
# give up.
logs.log_error('Failed to fix platform and re-add task.')
# Add a wait interval to avoid overflowing task creation.
failure_wait_interval = environment.get_value('FAIL_WAIT')
time.sleep(failure_wait_interval)
return
if task_name != 'fuzz':
# Make sure that our platform id matches that of the testcase (for
# non-fuzz tasks).
testcase = data_handler.get_entity_by_type_and_id(data_types.Testcase,
task_argument)
if testcase:
current_platform_id = environment.get_platform_id()
testcase_platform_id = testcase.platform_id
# This indicates we are trying to run this job on the wrong platform.
# This can happen when you have different type of devices (e.g
# android) on the same platform group. In this case, we just recreate
# the task.
if (task_name != 'variant' and testcase_platform_id and
not utils.fields_match(testcase_platform_id, current_platform_id)):
logs.log(
'Testcase %d platform (%s) does not match with ours (%s), exiting'
% (testcase.key.id(), testcase_platform_id, current_platform_id))
tasks.add_task(
task_name,
task_argument,
job_name,
wait_time=utils.random_number(1, TASK_RETRY_WAIT_LIMIT))
return
# Some fuzzers contain additional environment variables that should be
# set for them. Append these for tests generated by these fuzzers and for
# the fuzz command itself.
fuzzer_name = None
if task_name == 'fuzz':
fuzzer_name = task_argument
elif testcase:
fuzzer_name = testcase.fuzzer_name
# Get job's environment string.
environment_string = job.get_environment_string()
if task_name == 'minimize':
# Let jobs specify a different job and fuzzer to minimize with.
job_environment = job.get_environment()
minimize_job_override = job_environment.get('MINIMIZE_JOB_OVERRIDE')
if minimize_job_override:
minimize_job = data_types.Job.query(
data_types.Job.name == minimize_job_override).get()
if minimize_job:
environment.set_value('JOB_NAME', minimize_job_override)
environment_string = minimize_job.get_environment_string()
environment_string += '\nORIGINAL_JOB_NAME = %s\n' % job_name
job_name = minimize_job_override
else:
logs.log_error(
'Job for minimization not found: %s.' % minimize_job_override)
# Fallback to using own job for minimization.
minimize_fuzzer_override = job_environment.get('MINIMIZE_FUZZER_OVERRIDE')
fuzzer_name = minimize_fuzzer_override or fuzzer_name
if fuzzer_name and not environment.is_engine_fuzzer_job(job_name):
fuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == fuzzer_name).get()
additional_default_variables = ''
additional_variables_for_job = ''
if (fuzzer and hasattr(fuzzer, 'additional_environment_string') and
fuzzer.additional_environment_string):
for line in fuzzer.additional_environment_string.splitlines():
# Job specific values may be defined in fuzzer additional
# environment variable name strings in the form
# job_name:VAR_NAME = VALUE.
if '=' in line and ':' in line.split('=', 1)[0]:
fuzzer_job_name, environment_definition = line.split(':', 1)
if fuzzer_job_name == job_name:
additional_variables_for_job += '\n%s' % environment_definition
continue
additional_default_variables += '\n%s' % line
environment_string += additional_default_variables
environment_string += additional_variables_for_job
# Update environment for the job.
update_environment_for_job(environment_string)
# Match the cpu architecture with the ones required in the job definition.
# If they don't match, then bail out and recreate task.
if not is_supported_cpu_arch_for_job():
logs.log(
'Unsupported cpu architecture specified in job definition, exiting.')
tasks.add_task(
task_name,
task_argument,
job_name,
wait_time=utils.random_number(1, TASK_RETRY_WAIT_LIMIT))
return
# Initial cleanup.
cleanup_task_state()
start_web_server_if_needed()
try:
run_command(task_name, task_argument, job_name)
finally:
# Final clean up.
cleanup_task_state()
def update_fuzzer_and_data_bundles(fuzzer_name):
"""Update the fuzzer with a given name if necessary."""
fuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == fuzzer_name).get()
if not fuzzer:
logs.log_error('No fuzzer exists with name %s.' % fuzzer_name)
raise errors.InvalidFuzzerError
# Set some helper environment variables.
fuzzer_directory = get_fuzzer_directory(fuzzer_name)
environment.set_value('FUZZER_DIR', fuzzer_directory)
environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content)
# If the fuzzer generates large testcases or a large number of small ones
# that don't fit on tmpfs, then use the larger disk directory.
if fuzzer.has_large_testcases:
testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK')
environment.set_value('FUZZ_INPUTS', testcase_disk_directory)
# Adjust the test timeout, if user has provided one.
if fuzzer.timeout:
environment.set_value('TEST_TIMEOUT', fuzzer.timeout)
# Increase fuzz test timeout if the fuzzer timeout is higher than its
# current value.
fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT')
if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout:
environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout)
# Adjust the max testcases if this fuzzer has specified a lower limit.
max_testcases = environment.get_value('MAX_TESTCASES')
if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases:
environment.set_value('MAX_TESTCASES', fuzzer.max_testcases)
# Check for updates to this fuzzer.
version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name)
if (not fuzzer.builtin
and revisions.needs_update(version_file, fuzzer.revision)):
logs.log('Fuzzer update was found, updating.')
# Clear the old fuzzer directory if it exists.
if not shell.remove_directory(fuzzer_directory, recreate=True):
logs.log_error('Failed to clear fuzzer directory.')
return None
# Copy the archive to local disk and unpack it.
archive_path = os.path.join(fuzzer_directory, fuzzer.filename)
if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path):
logs.log_error('Failed to copy fuzzer archive.')
return None
try:
archive.unpack(archive_path, fuzzer_directory)
except Exception:
error_message = (
'Failed to unpack fuzzer archive %s '
'(bad archive or unsupported format).') % fuzzer.filename
logs.log_error(error_message)
fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
fuzzer_name=fuzzer_name)
return None
fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path)
if not os.path.exists(fuzzer_path):
error_message = (
'Fuzzer executable %s not found. '
'Check fuzzer configuration.') % fuzzer.executable_path
logs.log_error(error_message)
fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
fuzzer_name=fuzzer_name)
return None
# Make fuzzer executable.
os.chmod(fuzzer_path, 0o750)
# Cleanup unneeded archive.
shell.remove_file(archive_path)
# Save the current revision of this fuzzer in a file for later checks.
revisions.write_revision_to_revision_file(version_file,
fuzzer.revision)
logs.log('Updated fuzzer to revision %d.' % fuzzer.revision)
_clear_old_data_bundles_if_needed()
# Setup data bundles associated with this fuzzer.
data_bundles = ndb_utils.get_all_from_query(
data_types.DataBundle.query(
data_types.DataBundle.name == fuzzer.data_bundle_name))
for data_bundle in data_bundles:
if not update_data_bundle(fuzzer, data_bundle):
return None
# Setup environment variable for launcher script path.
if fuzzer.launcher_script:
fuzzer_launcher_path = os.path.join(fuzzer_directory,
fuzzer.launcher_script)
environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path)
# For launcher script usecase, we need the entire fuzzer directory on the
# worker.
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
worker_fuzzer_directory = file_host.rebase_to_worker_root(
fuzzer_directory)
file_host.copy_directory_to_worker(fuzzer_directory,
worker_fuzzer_directory,
replace=True)
return fuzzer
def run_process(cmdline,
current_working_directory=None,
timeout=DEFAULT_TEST_TIMEOUT,
need_shell=False,
gestures=None,
env_copy=None,
testcase_run=True,
ignore_children=True):
"""Executes a process with a given command line and other parameters."""
if environment.is_trusted_host() and testcase_run:
from clusterfuzz._internal.bot.untrusted_runner import remote_process_host
return remote_process_host.run_process(cmdline,
current_working_directory,
timeout, need_shell, gestures,
env_copy, testcase_run,
ignore_children)
if gestures is None:
gestures = []
if env_copy:
os.environ.update(env_copy)
# FIXME(mbarbella): Using LAUNCHER_PATH here is error prone. It forces us to
# do certain operations before fuzzer setup (e.g. bad build check).
launcher = environment.get_value('LAUNCHER_PATH')
# This is used when running scripts on native linux OS and not on the device.
# E.g. running a fuzzer to generate testcases or launcher script.
plt = environment.platform()
runs_on_device = environment.is_android(plt) or plt == 'FUCHSIA'
if runs_on_device and (not testcase_run or launcher):
plt = 'LINUX'
is_android = environment.is_android(plt)
# Lower down testcase timeout slightly to account for time for crash analysis.
timeout -= CRASH_ANALYSIS_TIME
# LeakSanitizer hack - give time for stdout/stderr processing.
lsan = environment.get_value('LSAN', False)
if lsan:
timeout -= LSAN_ANALYSIS_TIME
# Initialize variables.
adb_output = None
process_output = ''
process_status = None
return_code = 0
process_poll_interval = environment.get_value('PROCESS_POLL_INTERVAL', 0.5)
start_time = time.time()
watch_for_process_exit = (environment.get_value('WATCH_FOR_PROCESS_EXIT')
if is_android else True)
window_list = []
# Get gesture start time from last element in gesture list.
gestures = copy.deepcopy(gestures)
if gestures and gestures[-1].startswith('Trigger'):
gesture_start_time = int(gestures[-1].split(':')[1])
gestures.pop()
else:
gesture_start_time = timeout // 2
if is_android:
# Clear the log upfront.
android.logger.clear_log()
# Run the app.
adb_output = android.adb.run_command(cmdline, timeout=timeout)
else:
cmd = shell.get_command(cmdline)
process_output = mozprocess.processhandler.StoreOutput()
process_status = ProcessStatus()
try:
process_handle = mozprocess.ProcessHandlerMixin(
cmd,
args=None,
cwd=current_working_directory,
shell=need_shell,
processOutputLine=[process_output],
onFinish=[process_status],
ignore_children=ignore_children)
start_process(process_handle)
except:
logs.log_error('Exception occurred when running command: %s.' %
cmdline)
return None, None, ''
while True:
time.sleep(process_poll_interval)
# Run the gestures at gesture_start_time or in case we didn't find windows
# in the last try.
if (gestures and time.time() - start_time >= gesture_start_time
and not window_list):
# In case, we don't find any windows, we increment the gesture start time
# so that the next check is after 1 second.
gesture_start_time += 1
if plt == 'LINUX':
linux.gestures.run_gestures(gestures, process_handle.pid,
process_status, start_time,
timeout, window_list)
elif plt == 'WINDOWS':
windows.gestures.run_gestures(gestures, process_handle.pid,
process_status, start_time,
timeout, window_list)
elif is_android:
android.gestures.run_gestures(gestures, start_time, timeout)
# TODO(mbarbella): We add a fake window here to prevent gestures on
# Android from getting executed more than once.
window_list = ['FAKE']
if time.time() - start_time >= timeout:
break
# Collect the process output.
output = (android.logger.log_output()
if is_android else b'\n'.join(process_output.output))
output = utils.decode_to_unicode(output)
if crash_analyzer.is_memory_tool_crash(output):
break
# Check if we need to bail out on process exit.
if watch_for_process_exit:
# If |watch_for_process_exit| is set, then we already completed running
# our app launch command. So, we can bail out.
if is_android:
break
# On desktop, we bail out as soon as the process finishes.
if process_status and process_status.finished:
# Wait for process shutdown and set return code.
process_handle.wait(timeout=PROCESS_CLEANUP_WAIT_TIME)
break
# Process output based on platform.
if is_android:
# Get current log output. If device is in reboot mode, logcat automatically
# waits for device to be online.
time.sleep(ANDROID_CRASH_LOGCAT_WAIT_TIME)
output = android.logger.log_output()
if android.constants.LOW_MEMORY_REGEX.search(output):
# If the device is low on memory, we should force reboot and bail out to
# prevent device from getting in a frozen state.
logs.log('Device is low on memory, rebooting.', output=output)
android.adb.hard_reset()
android.adb.wait_for_device()
elif android.adb.time_since_last_reboot() < time.time() - start_time:
# Check if a reboot has happened, if yes, append log output before reboot
# and kernel logs content to output.
log_before_last_reboot = android.logger.log_output_before_last_reboot(
)
kernel_log = android.adb.get_kernel_log_content()
output = '%s%s%s%s%s' % (
log_before_last_reboot,
utils.get_line_seperator('Device rebooted'), output,
utils.get_line_seperator('Kernel Log'), kernel_log)
# Make sure to reset SE Linux Permissive Mode. This can be done cheaply
# in ~0.15 sec and is needed especially between runs for kernel crashes.
android.adb.run_as_root()
android.settings.change_se_linux_to_permissive_mode()
return_code = 1
# Add output from adb to the front.
if adb_output:
output = '%s\n\n%s' % (adb_output, output)
# Kill the application if it is still running. We do this at the end to
# prevent this from adding noise to the logcat output.
task_name = environment.get_value('TASK_NAME')
child_process_termination_pattern = environment.get_value(
'CHILD_PROCESS_TERMINATION_PATTERN')
if task_name == 'fuzz' and child_process_termination_pattern:
# In some cases, we do not want to terminate the application after each
# run to avoid long startup times (e.g. for chrome). Terminate processes
# matching a particular pattern for light cleanup in this case.
android.adb.kill_processes_and_children_matching_name(
child_process_termination_pattern)
else:
# There is no special termination behavior. Simply stop the application.
android.app.stop()
else:
# Get the return code in case the process has finished already.
# If the process hasn't finished, return_code will be None which is what
# callers expect unless the output indicates a crash.
return_code = process_handle.poll()
# If the process is still running, then terminate it.
if not process_status.finished:
launcher_with_interpreter = shell.get_execute_command(
launcher) if launcher else None
if (launcher_with_interpreter
and cmdline.startswith(launcher_with_interpreter)):
# If this was a launcher script, we KILL all child processes created
# except for APP_NAME.
# It is expected that, if the launcher script terminated normally, it
# cleans up all the child processes it created itself.
terminate_root_and_child_processes(process_handle.pid)
else:
try:
# kill() here actually sends SIGTERM on posix.
process_handle.kill()
except:
pass
if lsan:
time.sleep(LSAN_ANALYSIS_TIME)
output = b'\n'.join(process_output.output)
output = utils.decode_to_unicode(output)
# X Server hack when max client reached.
if ('Maximum number of clients reached' in output
or 'Unable to get connection to X server' in output):
logs.log_error('Unable to connect to X server, exiting.')
os.system('sudo killall -9 Xvfb blackbox >/dev/null 2>&1')
sys.exit(0)
if testcase_run and (crash_analyzer.is_memory_tool_crash(output)
or crash_analyzer.is_check_failure_crash(output)):
return_code = 1
# If a crash is found, then we add the memory state as well.
if return_code and is_android:
ps_output = android.adb.get_ps_output()
if ps_output:
output += utils.get_line_seperator('Memory Statistics')
output += ps_output
if return_code:
logs.log_warn('Process (%s) ended with exit code (%s).' %
(repr(cmdline), str(return_code)),
output=output)
return return_code, round(time.time() - start_time, 1), output
def parse_mime_to_crash_report_info(local_minidump_mime_path):
"""Read the (local) minidump MIME file into a CrashReportInfo object."""
# Get the minidump name and path.
minidump_path_match = re.match(r'(.*)\.mime', local_minidump_mime_path)
if minidump_path_match is None:
logs.log_error('Minidump filename in unexpected format: \'%s\'.' %
local_minidump_mime_path)
return None
minidump_path = '%s.dmp' % minidump_path_match.group(1).strip()
# Reformat the minidump MIME to include the boundary.
with open(local_minidump_mime_path, 'rb') as minidump_mime_file_content:
# The boundary is the first line after the first two dashes.
boundary = minidump_mime_file_content.readline().strip()[2:]
minidump_mime_bytes = (
b'Content-Type: multipart/form-data; boundary=\"%s\"\r\n--%s\r\n' %
(boundary, boundary))
minidump_mime_bytes += minidump_mime_file_content.read()
minidump_mime_contents = email.message_from_bytes(minidump_mime_bytes)
# Parse the MIME contents, extracting the parameters needed for upload.
mime_key_values = {}
for mime_part in minidump_mime_contents.get_payload():
if isinstance(mime_part, str):
mime_part = utils.decode_to_unicode(mime_part)
logs.log_error('Unexpected str mime_part from mime path %s: %s' %
(local_minidump_mime_path, mime_part))
continue
part_descriptor = list(mime_part.values())
key_tokens = part_descriptor[0].split('; ')
key_match = re.match(r'name="(.*)".*', key_tokens[1])
# Extract from the MIME part the key-value pairs used by report uploading.
if key_match is not None:
report_key = key_match.group(1)
report_value = mime_part.get_payload(decode=True)
if report_key == MINIDUMP_FILE_KEY:
utils.write_data_to_file(report_value, minidump_path)
else:
# Take care of aliases.
if report_key in ('prod', 'buildTargetId'):
report_key = PRODUCT_KEY
elif report_key == 'ver':
report_key = VERSION_KEY
# Save the key-value pair.
mime_key_values[report_key] = report_value
# Pull out product and version explicitly since these are required
# for upload.
product, version = None, None
if PRODUCT_KEY in mime_key_values:
product = mime_key_values.pop(PRODUCT_KEY).decode('utf-8')
else:
logs.log_error(
'Could not find \'%s\' or alias in mime_key_values key.' %
PRODUCT_KEY)
if VERSION_KEY in mime_key_values:
version = mime_key_values.pop(VERSION_KEY).decode('utf-8')
else:
logs.log_error(
'Could not find \'%s\' or alias in mime_key_values key.' %
VERSION_KEY)
# If missing, return None and log keys that do exist; otherwise, construct
# CrashReportInfo and return.
if product is None or version is None:
logs.log_error('mime_key_values dict keys:\n%s' %
str(list(mime_key_values.keys())))
return None
return CrashReportInfo(minidump_path=minidump_path,
product=product,
version=version,
optional_params=mime_key_values)
def find_regression_range(testcase_id, job_type):
"""Attempt to find when the testcase regressed."""
deadline = tasks.get_task_completion_deadline()
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
if testcase.regression:
logs.log_error('Regression range is already set as %s, skip.' %
testcase.regression)
return
# This task is not applicable for custom binaries.
if build_manager.is_custom_binary():
testcase.regression = 'NA'
data_handler.update_testcase_comment(
testcase, data_types.TaskState.ERROR,
'Not applicable for custom binaries')
return
data_handler.update_testcase_comment(testcase,
data_types.TaskState.STARTED)
# Setup testcase and its dependencies.
file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type)
if not file_list:
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
'Failed to setup testcase')
tasks.add_task('regression', testcase_id, job_type)
return
build_bucket_path = build_manager.get_primary_bucket_path()
revision_list = build_manager.get_revisions_list(build_bucket_path,
testcase=testcase)
if not revision_list:
data_handler.close_testcase_with_error(
testcase_id, 'Failed to fetch revision list')
return
# Don't burden NFS server with caching these random builds.
environment.set_value('CACHE_STORE', False)
# Pick up where left off in a previous run if necessary.
min_revision = testcase.get_metadata('last_regression_min')
max_revision = testcase.get_metadata('last_regression_max')
first_run = not min_revision and not max_revision
if not min_revision:
min_revision = revisions.get_first_revision_in_list(revision_list)
if not max_revision:
max_revision = testcase.crash_revision
min_index = revisions.find_min_revision_index(revision_list, min_revision)
if min_index is None:
raise errors.BuildNotFoundError(min_revision, job_type)
max_index = revisions.find_max_revision_index(revision_list, max_revision)
if max_index is None:
raise errors.BuildNotFoundError(max_revision, job_type)
# Make sure that the revision where we noticed the crash, still crashes at
# that revision. Otherwise, our binary search algorithm won't work correctly.
max_revision = revision_list[max_index]
crashes_in_max_revision = _testcase_reproduces_in_revision(
testcase, testcase_file_path, job_type, max_revision, should_log=False)
if not crashes_in_max_revision:
testcase = data_handler.get_testcase_by_id(testcase_id)
error_message = ('Known crash revision %d did not crash' %
max_revision)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
task_creation.mark_unreproducible_if_flaky(testcase, True)
return
# If we've made it this far, the test case appears to be reproducible. Clear
# metadata from previous runs had it been marked as potentially flaky.
task_creation.mark_unreproducible_if_flaky(testcase, False)
# On the first run, check to see if we regressed near either the min or max
# revision.
if first_run and found_regression_near_extreme_revisions(
testcase, testcase_file_path, job_type, revision_list, min_index,
max_index):
return
while time.time() < deadline:
min_revision = revision_list[min_index]
max_revision = revision_list[max_index]
# If the min and max revisions are one apart (or the same, if we only have
# one build), this is as much as we can narrow the range.
if max_index - min_index <= 1:
# Verify that the regression range seems correct, and save it if so.
if not validate_regression_range(testcase, testcase_file_path,
job_type, revision_list,
min_index):
return
save_regression_range(testcase_id, min_revision, max_revision)
return
middle_index = (min_index + max_index) // 2
middle_revision = revision_list[middle_index]
try:
is_crash = _testcase_reproduces_in_revision(
testcase,
testcase_file_path,
job_type,
middle_revision,
min_revision=min_revision,
max_revision=max_revision)
except errors.BadBuildError:
# Skip this revision.
del revision_list[middle_index]
max_index -= 1
continue
if is_crash:
max_index = middle_index
else:
min_index = middle_index
_save_current_regression_range_indices(testcase_id,
revision_list[min_index],
revision_list[max_index])
# If we've broken out of the above loop, we timed out. We'll finish by
# running another regression task and picking up from this point.
testcase = data_handler.get_testcase_by_id(testcase_id)
error_message = 'Timed out, current range r%d:r%d' % (
revision_list[min_index], revision_list[max_index])
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
error_message)
tasks.add_task('regression', testcase_id, job_type)
def test_exception(self):
"""Tests exception."""
self.mock.exc_info.return_value = 'err'
logs.log_error('test', exception='exception', hello='1')
self.mock.emit.assert_called_once_with(
logging.ERROR, 'test', exc_info='err', hello='1')
def update_source_code():
"""Updates source code files with latest version from appengine."""
process_handler.cleanup_stale_processes()
shell.clear_temp_directory()
root_directory = environment.get_value('ROOT_DIR')
temp_directory = environment.get_value('BOT_TMPDIR')
temp_archive = os.path.join(temp_directory, 'clusterfuzz-source.zip')
try:
storage.copy_file_from(get_source_url(), temp_archive)
except Exception:
logs.log_error('Could not retrieve source code archive from url.')
return
try:
file_list = archive.get_file_list(temp_archive)
zip_archive = zipfile.ZipFile(temp_archive, 'r')
except Exception:
logs.log_error('Bad zip file.')
return
src_directory = os.path.join(root_directory, 'src')
output_directory = os.path.dirname(root_directory)
error_occurred = False
normalized_file_set = set()
for filepath in file_list:
filename = os.path.basename(filepath)
# This file cannot be updated on the fly since it is running as server.
if filename == 'adb':
continue
absolute_filepath = os.path.join(output_directory, filepath)
if os.path.altsep:
absolute_filepath = absolute_filepath.replace(
os.path.altsep, os.path.sep)
if os.path.realpath(absolute_filepath) != absolute_filepath:
continue
normalized_file_set.add(absolute_filepath)
try:
file_extension = os.path.splitext(filename)[1]
# Remove any .so files first before overwriting, as they can be loaded
# in the memory of existing processes. Overwriting them directly causes
# segfaults in existing processes (e.g. run.py).
if file_extension == '.so' and os.path.exists(absolute_filepath):
os.remove(absolute_filepath)
# On Windows, to update DLLs (and native .pyd extensions), we rename it
# first so that we can install the new version.
if (environment.platform() == 'WINDOWS'
and file_extension in ['.dll', '.pyd']
and os.path.exists(absolute_filepath)):
_rename_dll_for_update(absolute_filepath)
except Exception:
logs.log_error('Failed to remove or move %s before extracting new '
'version.' % absolute_filepath)
try:
extracted_path = zip_archive.extract(filepath, output_directory)
external_attr = zip_archive.getinfo(filepath).external_attr
mode = (external_attr >> 16) & 0o777
mode |= 0o440
os.chmod(extracted_path, mode)
except:
error_occurred = True
logs.log_error('Failed to extract file %s from source archive.' %
filepath)
zip_archive.close()
if error_occurred:
return
clear_pyc_files(src_directory)
clear_old_files(src_directory, normalized_file_set)
local_manifest_path = os.path.join(root_directory,
utils.LOCAL_SOURCE_MANIFEST)
source_version = utils.read_data_from_file(
local_manifest_path, eval_data=False).decode('utf-8').strip()
logs.log('Source code updated to %s.' % source_version)
def main():
"""Main build routine."""
bucket_prefix = environment.get_value('BUCKET_PREFIX')
build_dir = environment.get_value('BUILD_DIR')
wait_time = environment.get_value('WAIT_TIME')
try:
builds_metadata = build_info.get_production_builds_info_from_cd(
environment.platform())
except Exception:
logs.log_error('Errors when fetching from ChromiumDash')
# fallback to omahaproxy in the transition stage
# TODO(yuanjunh): remove the fallback logic after migration is done.
builds_metadata = build_info.get_production_builds_info(
environment.platform())
if not builds_metadata:
return
global LAST_BUILD
for build_metadata in builds_metadata:
build_type = build_metadata.build_type
revision = build_metadata.revision
version = build_metadata.version
if build_type not in ['extended_stable', 'stable', 'beta']:
# We don't need dev or canary builds atm.
continue
# Starting building the builds.
for tool in TOOLS_GN_MAPPINGS:
tool_and_build_type = '%s-%s' % (tool, build_type)
logs.log('Building %s.' % tool_and_build_type)
# Check if we already have built the same build.
if (tool_and_build_type in LAST_BUILD and
revision == LAST_BUILD[tool_and_build_type]):
logs.log('Skipping same build %s (revision %s).' % (tool_and_build_type,
revision))
continue
LAST_BUILD[tool_and_build_type] = revision
file_name_prefix = '%s-linux-%s-%s' % (tool, build_type, version)
archive_filename = '%s.zip' % file_name_prefix
archive_path_local = '%s/%s' % (build_dir, archive_filename)
bucket_name = '%s%s' % (bucket_prefix, tool.split('-')[0])
archive_path_remote = ('gs://%s/%s/%s' % (
bucket_name, TOOLS_BUCKET_DIR_MAPPINGS[tool], archive_filename))
# Run the build script with required gn arguments.
command = ''
gn_args = '%s %s' % (TOOLS_GN_MAPPINGS[tool], GN_COMMON_ARGS)
command += '%s "%s" %s %s' % (BUILD_HELPER_SCRIPT, gn_args, version,
file_name_prefix)
logs.log('Executing build script: %s.' % command)
os.system(command)
# Check if the build succeeded based on the existence of the
# local archive file.
if os.path.exists(archive_path_local):
# Build success. Now, copy it to google cloud storage and make it
# public.
os.system('gsutil cp %s %s' % (archive_path_local, archive_path_remote))
os.system('gsutil acl set public-read %s' % archive_path_remote)
logs.log('Build succeeded, created %s.' % archive_filename)
else:
LAST_BUILD[tool_and_build_type] = ''
logs.log_error('Build failed, unable to create %s.' % archive_filename)
logs.log('Completed cycle, waiting for %d secs.' % wait_time)
time.sleep(wait_time)
def configure_system_build_properties():
"""Modifies system build properties in /system/build.prop for better boot
speed and power use."""
adb.run_as_root()
# Check md5 checksum of build.prop to see if already updated,
# in which case exit. If build.prop does not exist, something
# is very wrong with the device, so bail.
old_md5 = persistent_cache.get_value(constants.BUILD_PROP_MD5_KEY)
current_md5 = adb.get_file_checksum(BUILD_PROP_PATH)
if current_md5 is None:
logs.log_error('Unable to find %s on device.' % BUILD_PROP_PATH)
return
if old_md5 == current_md5:
return
# Pull to tmp file.
bot_tmp_directory = environment.get_value('BOT_TMPDIR')
old_build_prop_path = os.path.join(bot_tmp_directory, 'old.prop')
adb.run_command(['pull', BUILD_PROP_PATH, old_build_prop_path])
if not os.path.exists(old_build_prop_path):
logs.log_error('Unable to fetch %s from device.' % BUILD_PROP_PATH)
return
# Write new build.prop.
new_build_prop_path = os.path.join(bot_tmp_directory, 'new.prop')
old_build_prop_file_content = open(old_build_prop_path, 'r')
new_build_prop_file_content = open(new_build_prop_path, 'w')
new_content_notification = '### CHANGED OR ADDED PROPERTIES ###'
for line in old_build_prop_file_content:
property_name = line.split('=')[0].strip()
if property_name in BUILD_PROPERTIES:
continue
if new_content_notification in line:
continue
new_build_prop_file_content.write(line)
new_build_prop_file_content.write(new_content_notification + '\n')
for flag, value in six.iteritems(BUILD_PROPERTIES):
new_build_prop_file_content.write('%s=%s\n' % (flag, value))
old_build_prop_file_content.close()
new_build_prop_file_content.close()
# Keep verified boot disabled for M and higher releases. This makes it easy
# to modify system's app_process to load asan libraries.
build_version = settings.get_build_version()
if is_build_at_least(build_version, 'M'):
adb.run_as_root()
adb.run_command('disable-verity')
reboot()
# Make /system writable.
adb.run_as_root()
adb.remount()
# Remove seccomp policies (on N and higher) as ASan requires extra syscalls.
if is_build_at_least(build_version, 'N'):
policy_files = adb.run_shell_command(
['find', '/system/etc/seccomp_policy/', '-type', 'f'])
for policy_file in policy_files.splitlines():
adb.run_shell_command(['rm', policy_file.strip()])
# Push new build.prop and backup to device.
logs.log('Pushing new build properties file on device.')
adb.run_command(
['push', '-p', old_build_prop_path, BUILD_PROP_BACKUP_PATH])
adb.run_command(['push', '-p', new_build_prop_path, BUILD_PROP_PATH])
adb.run_shell_command(['chmod', '644', BUILD_PROP_PATH])
# Set persistent cache key containing and md5sum.
current_md5 = adb.get_file_checksum(BUILD_PROP_PATH)
persistent_cache.set_value(constants.BUILD_PROP_MD5_KEY, current_md5)
def find_fixed_range(testcase_id, job_type):
"""Attempt to find the revision range where a testcase was fixed."""
deadline = tasks.get_task_completion_deadline()
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
if testcase.fixed:
logs.log_error('Fixed range is already set as %s, skip.' %
testcase.fixed)
return
# Setup testcase and its dependencies.
file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type)
if not file_list:
return
# Set a flag to indicate we are running progression task. This shows pending
# status on testcase report page and avoid conflicting testcase updates by
# triage cron.
testcase.set_metadata('progression_pending', True)
# Custom binaries are handled as special cases.
if build_manager.is_custom_binary():
_check_fixed_for_custom_binary(testcase, job_type, testcase_file_path)
return
build_bucket_path = build_manager.get_primary_bucket_path()
revision_list = build_manager.get_revisions_list(build_bucket_path,
testcase=testcase)
if not revision_list:
data_handler.close_testcase_with_error(
testcase_id, 'Failed to fetch revision list')
return
# Use min, max_index to mark the start and end of revision list that is used
# for bisecting the progression range. Set start to the revision where noticed
# the crash. Set end to the trunk revision. Also, use min, max from past run
# if it timed out.
min_revision = testcase.get_metadata('last_progression_min')
max_revision = testcase.get_metadata('last_progression_max')
if min_revision or max_revision:
# Clear these to avoid using them in next run. If this run fails, then we
# should try next run without them to see it succeeds. If this run succeeds,
# we should still clear them to avoid capping max revision in next run.
testcase = data_handler.get_testcase_by_id(testcase_id)
testcase.delete_metadata('last_progression_min', update_testcase=False)
testcase.delete_metadata('last_progression_max', update_testcase=False)
testcase.put()
last_tested_revision = testcase.get_metadata('last_tested_crash_revision')
known_crash_revision = last_tested_revision or testcase.crash_revision
if not min_revision:
min_revision = known_crash_revision
if not max_revision:
max_revision = revisions.get_last_revision_in_list(revision_list)
min_index = revisions.find_min_revision_index(revision_list, min_revision)
if min_index is None:
raise errors.BuildNotFoundError(min_revision, job_type)
max_index = revisions.find_max_revision_index(revision_list, max_revision)
if max_index is None:
raise errors.BuildNotFoundError(max_revision, job_type)
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.STARTED,
'r%d' % max_revision)
# Check to see if this testcase is still crashing now. If it is, then just
# bail out.
result = _testcase_reproduces_in_revision(testcase,
testcase_file_path,
job_type,
max_revision,
update_metadata=True)
if result.is_crash():
logs.log('Found crash with same signature on latest revision r%d.' %
max_revision)
app_path = environment.get_value('APP_PATH')
command = testcase_manager.get_command_line_for_application(
testcase_file_path,
app_path=app_path,
needs_http=testcase.http_flag)
symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True)
unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False)
stacktrace = utils.get_crash_stacktrace_output(
command, symbolized_crash_stacktrace,
unsymbolized_crash_stacktrace)
testcase = data_handler.get_testcase_by_id(testcase_id)
testcase.last_tested_crash_stacktrace = data_handler.filter_stacktrace(
stacktrace)
data_handler.update_progression_completion_metadata(
testcase,
max_revision,
is_crash=True,
message='still crashes on latest revision r%s' % max_revision)
# Since we've verified that the test case is still crashing, clear out any
# metadata indicating potential flake from previous runs.
task_creation.mark_unreproducible_if_flaky(testcase, False)
# For chromium project, save latest crash information for later upload
# to chromecrash/.
state = result.get_symbolized_data()
crash_uploader.save_crash_info_if_needed(testcase_id, max_revision,
job_type, state.crash_type,
state.crash_address,
state.frames)
return
if result.unexpected_crash:
testcase.set_metadata('crashes_on_unexpected_state', True)
else:
testcase.delete_metadata('crashes_on_unexpected_state')
# Don't burden NFS server with caching these random builds.
environment.set_value('CACHE_STORE', False)
# Verify that we do crash in the min revision. This is assumed to be true
# while we are doing the bisect.
result = _testcase_reproduces_in_revision(testcase, testcase_file_path,
job_type, min_revision)
if result and not result.is_crash():
testcase = data_handler.get_testcase_by_id(testcase_id)
# Retry once on another bot to confirm our result.
if data_handler.is_first_retry_for_task(testcase,
reset_after_retry=True):
tasks.add_task('progression', testcase_id, job_type)
error_message = (
'Known crash revision %d did not crash, will retry on another bot to '
'confirm result' % known_crash_revision)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
data_handler.update_progression_completion_metadata(
testcase, max_revision)
return
data_handler.clear_progression_pending(testcase)
error_message = ('Known crash revision %d did not crash' %
known_crash_revision)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
task_creation.mark_unreproducible_if_flaky(testcase, True)
return
# Start a binary search to find last non-crashing revision. At this point, we
# know that we do crash in the min_revision, and do not crash in max_revision.
while time.time() < deadline:
min_revision = revision_list[min_index]
max_revision = revision_list[max_index]
# If the min and max revisions are one apart this is as much as we can
# narrow the range.
if max_index - min_index == 1:
_save_fixed_range(testcase_id, min_revision, max_revision,
testcase_file_path)
return
# Occasionally, we get into this bad state. It seems to be related to test
# cases with flaky stacks, but the exact cause is unknown.
if max_index - min_index < 1:
testcase = data_handler.get_testcase_by_id(testcase_id)
testcase.fixed = 'NA'
testcase.open = False
message = ('Fixed testing errored out (min and max revisions '
'are both %d)' % min_revision)
data_handler.update_progression_completion_metadata(
testcase, max_revision, message=message)
# Let the bisection service know about the NA status.
bisection.request_bisection(testcase)
return
# Test the middle revision of our range.
middle_index = (min_index + max_index) // 2
middle_revision = revision_list[middle_index]
testcase = data_handler.get_testcase_by_id(testcase_id)
log_message = 'Testing r%d (current range %d:%d)' % (
middle_revision, min_revision, max_revision)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.WIP,
log_message)
try:
result = _testcase_reproduces_in_revision(testcase,
testcase_file_path,
job_type,
middle_revision)
except errors.BadBuildError:
# Skip this revision.
del revision_list[middle_index]
max_index -= 1
continue
if result.is_crash():
min_index = middle_index
else:
max_index = middle_index
_save_current_fixed_range_indices(testcase_id,
revision_list[min_index],
revision_list[max_index])
# If we've broken out of the loop, we've exceeded the deadline. Recreate the
# task to pick up where we left off.
testcase = data_handler.get_testcase_by_id(testcase_id)
error_message = ('Timed out, current range r%d:r%d' %
(revision_list[min_index], revision_list[max_index]))
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
error_message)
tasks.add_task('progression', testcase_id, job_type)
def _sync_job(self, project, info, corpus_bucket_name,
quarantine_bucket_name, logs_bucket_name,
backup_bucket_name):
"""Sync the config with ClusterFuzz."""
# Create/update ClusterFuzz jobs.
job_names = []
for template in get_jobs_for_project(project, info):
if template.engine == 'none':
# Engine-less jobs are not automatically managed.
continue
fuzzer_entity = self._fuzzer_entities.get(template.engine).get()
if not fuzzer_entity:
raise ProjectSetupError('Invalid fuzzing engine ' +
template.engine)
job_name = template.job_name(project, self._config_suffix)
job = data_types.Job.query(data_types.Job.name == job_name).get()
if not job:
job = data_types.Job()
if self._external_config:
if ('reproduction_topic' not in self._external_config or
'updates_subscription' not in self._external_config):
raise ProjectSetupError('Invalid external_config.')
job.external_reproduction_topic = self._external_config[
'reproduction_topic']
job.external_updates_subscription = self._external_config[
'updates_subscription']
else:
job.external_reproduction_topic = None
job.external_updates_subscription = None
if not info.get('disabled', False):
job_names.append(job_name)
if job_name not in fuzzer_entity.jobs and not job.is_external(
):
# Enable new job.
fuzzer_entity.jobs.append(job_name)
fuzzer_entity.put()
job.name = job_name
if self._segregate_projects:
job.platform = untrusted.platform_name(project, 'linux')
else:
# TODO(ochang): Support other platforms?
job.platform = 'LINUX'
job.templates = template.cf_job_templates
job.environment_string = JOB_TEMPLATE.format(
build_type=self._build_type,
build_bucket_path=self._get_build_bucket_path(
project, info, template.engine, template.memory_tool,
template.architecture),
engine=template.engine,
project=project)
if self._add_revision_mappings:
revision_vars_url = self._revision_url_template.format(
project=project,
bucket=self._get_build_bucket(template.engine,
template.architecture),
sanitizer=template.memory_tool)
job.environment_string += (
'REVISION_VARS_URL = {revision_vars_url}\n'.format(
revision_vars_url=revision_vars_url))
if logs_bucket_name:
job.environment_string += 'FUZZ_LOGS_BUCKET = {logs_bucket}\n'.format(
logs_bucket=logs_bucket_name)
if corpus_bucket_name:
job.environment_string += 'CORPUS_BUCKET = {corpus_bucket}\n'.format(
corpus_bucket=corpus_bucket_name)
if quarantine_bucket_name:
job.environment_string += (
'QUARANTINE_BUCKET = {quarantine_bucket}\n'.format(
quarantine_bucket=quarantine_bucket_name))
if backup_bucket_name:
job.environment_string += 'BACKUP_BUCKET = {backup_bucket}\n'.format(
backup_bucket=backup_bucket_name)
if self._add_info_labels:
job.environment_string += (
'AUTOMATIC_LABELS = Proj-{project},Engine-{engine}\n'.
format(
project=project,
engine=template.engine,
))
help_url = info.get('help_url')
if help_url:
job.environment_string += 'HELP_URL = %s\n' % help_url
if (template.experimental or
(self._experimental_sanitizers
and template.memory_tool in self._experimental_sanitizers)):
job.environment_string += 'EXPERIMENTAL = True\n'
if template.minimize_job_override:
minimize_job_override = template.minimize_job_override.job_name(
project, self._config_suffix)
job.environment_string += ('MINIMIZE_JOB_OVERRIDE = %s\n' %
minimize_job_override)
view_restrictions = info.get('view_restrictions')
if view_restrictions:
if view_restrictions in ALLOWED_VIEW_RESTRICTIONS:
job.environment_string += (
'ISSUE_VIEW_RESTRICTIONS = %s\n' % view_restrictions)
else:
logs.log_error(
'Invalid view restriction setting %s for project %s.' %
(view_restrictions, project))
selective_unpack = info.get('selective_unpack')
if selective_unpack:
job.environment_string += 'UNPACK_ALL_FUZZ_TARGETS_AND_FILES = False\n'
main_repo = info.get('main_repo')
if main_repo:
job.environment_string += f'MAIN_REPO = {main_repo}\n'
if (template.engine == 'libfuzzer'
and template.architecture == 'x86_64' and 'dataflow'
in info.get('fuzzing_engines', DEFAULT_ENGINES)):
# Dataflow binaries are built with dataflow sanitizer, but can be used
# as an auxiliary build with libFuzzer builds (e.g. with ASan or UBSan).
dataflow_build_bucket_path = self._get_build_bucket_path(
project_name=project,
info=info,
engine='dataflow',
memory_tool='dataflow',
architecture=template.architecture)
job.environment_string += (
'DATAFLOW_BUILD_BUCKET_PATH = %s\n' %
dataflow_build_bucket_path)
if self._additional_vars:
additional_vars = {}
additional_vars.update(self._additional_vars.get('all', {}))
engine_vars = self._additional_vars.get(template.engine, {})
engine_sanitizer_vars = engine_vars.get(
template.memory_tool, {})
additional_vars.update(engine_sanitizer_vars)
for key, value in sorted(six.iteritems(additional_vars)):
job.environment_string += ('{} = {}\n'.format(
key,
str(value).encode('unicode-escape').decode('utf-8')))
job.put()
return job_names
def _minimize_corpus_two_step(self, target_path, arguments,
existing_corpus_dirs, new_corpus_dir,
output_corpus_dir, reproducers_dir,
max_time):
"""Optional (but recommended): run corpus minimization.
Args:
target_path: Path to the target.
arguments: Additional arguments needed for corpus minimization.
existing_corpus_dirs: Input corpora that existed before the fuzzing run.
new_corpus_dir: Input corpus that was generated during the fuzzing run.
Must have at least one new file.
output_corpus_dir: Output directory to place minimized corpus.
reproducers_dir: The directory to put reproducers in when crashes are
found.
max_time: Maximum allowed time for the minimization.
Returns:
A Result object.
"""
if not _is_multistep_merge_supported(target_path):
# Fallback to the old single step merge. It does not support incremental
# stats and provides only `edge_coverage` and `feature_coverage` stats.
logs.log(
'Old version of libFuzzer is used. Using single step merge.')
return self.minimize_corpus(
target_path, arguments,
existing_corpus_dirs + [new_corpus_dir], output_corpus_dir,
reproducers_dir, max_time)
# The dir where merge control file is located must persist for both merge
# steps. The second step re-uses the MCF produced during the first step.
merge_control_file_dir = self._create_temp_corpus_dir('mcf_tmp_dir')
self._merge_control_file = os.path.join(merge_control_file_dir, 'MCF')
# Two step merge process to obtain accurate stats for the new corpus units.
# See https://reviews.llvm.org/D66107 for a more detailed description.
merge_stats = {}
# Step 1. Use only existing corpus and collect "initial" stats.
result_1 = self.minimize_corpus(target_path, arguments,
existing_corpus_dirs,
output_corpus_dir, reproducers_dir,
max_time)
merge_stats['initial_edge_coverage'] = result_1.stats['edge_coverage']
merge_stats['initial_feature_coverage'] = result_1.stats[
'feature_coverage']
# Clear the output dir as it does not have any new units at this point.
engine_common.recreate_directory(output_corpus_dir)
# Adjust the time limit for the time we spent on the first merge step.
max_time -= result_1.time_executed
if max_time <= 0:
logs.log_error('Merging new testcases timed out.',
fuzzer_output=result_1.logs)
raise TimeoutError('Merging new testcases timed out.')
# Step 2. Process the new corpus units as well.
result_2 = self.minimize_corpus(
target_path, arguments, existing_corpus_dirs + [new_corpus_dir],
output_corpus_dir, reproducers_dir, max_time)
merge_stats['edge_coverage'] = result_2.stats['edge_coverage']
merge_stats['feature_coverage'] = result_2.stats['feature_coverage']
# Diff the stats to obtain accurate values for the new corpus units.
merge_stats['new_edges'] = (merge_stats['edge_coverage'] -
merge_stats['initial_edge_coverage'])
merge_stats['new_features'] = (merge_stats['feature_coverage'] -
merge_stats['initial_feature_coverage'])
output = result_1.logs + '\n\n' + result_2.logs
if (merge_stats['new_edges'] < 0 or merge_stats['new_features'] < 0):
logs.log_error('Two step merge failed.',
merge_stats=merge_stats,
output=output)
merge_stats['new_edges'] = 0
merge_stats['new_features'] = 0
self._merge_control_file = None
# TODO(ochang): Get crashes found during merge.
return engine.FuzzResult(
output, result_2.command, [], merge_stats,
result_1.time_executed + result_2.time_executed)
def flash_to_latest_build_if_needed():
"""Wipes user data, resetting the device to original factory state."""
if environment.get_value('LOCAL_DEVELOPMENT'):
# Don't reimage local development devices.
return
run_timeout = environment.get_value('RUN_TIMEOUT')
if run_timeout:
# If we have a run timeout, then we are already scheduled to bail out and
# will be probably get re-imaged. E.g. using frameworks like Tradefed.
return
# Check if a flash is needed based on last recorded flash time.
last_flash_time = persistent_cache.get_value(
constants.LAST_FLASH_TIME_KEY,
constructor=datetime.datetime.utcfromtimestamp)
needs_flash = last_flash_time is None or dates.time_has_expired(
last_flash_time, seconds=FLASH_INTERVAL)
if not needs_flash:
return
is_google_device = settings.is_google_device()
if is_google_device is None:
logs.log_error('Unable to query device. Reimaging failed.')
adb.bad_state_reached()
elif not is_google_device:
# We can't reimage these, skip.
logs.log('Non-Google device found, skipping reimage.')
return
# Check if both |BUILD_BRANCH| and |BUILD_TARGET| environment variables
# are set. If not, we don't have enough data for reimaging and hence
# we bail out.
branch = environment.get_value('BUILD_BRANCH')
target = environment.get_value('BUILD_TARGET')
if not target:
# We default to userdebug configuration.
build_params = settings.get_build_parameters()
if build_params:
target = build_params.get('target') + '-userdebug'
# Cache target in environment. This is also useful for cases when
# device is bricked and we don't have this information available.
environment.set_value('BUILD_TARGET', target)
if not branch or not target:
logs.log_warn(
'BUILD_BRANCH and BUILD_TARGET are not set, skipping reimage.')
return
image_directory = environment.get_value('IMAGES_DIR')
build_info = fetch_artifact.get_latest_artifact_info(branch, target)
if not build_info:
logs.log_error('Unable to fetch information on latest build artifact for '
'branch %s and target %s.' % (branch, target))
return
if environment.is_android_cuttlefish():
download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory)
adb.recreate_cuttlefish_device()
adb.connect_to_cuttlefish_device()
else:
download_latest_build(build_info, FLASH_IMAGE_REGEXES, image_directory)
# We do one device flash at a time on one host, otherwise we run into
# failures and device being stuck in a bad state.
flash_lock_key_name = 'flash:%s' % socket.gethostname()
if not locks.acquire_lock(flash_lock_key_name, by_zone=True):
logs.log_error('Failed to acquire lock for reimaging, exiting.')
return
logs.log('Reimaging started.')
logs.log('Rebooting into bootloader mode.')
for _ in range(FLASH_RETRIES):
adb.run_as_root()
adb.run_command(['reboot-bootloader'])
time.sleep(FLASH_REBOOT_BOOTLOADER_WAIT)
adb.run_fastboot_command(['oem', 'off-mode-charge', '0'])
adb.run_fastboot_command(['-w', 'reboot-bootloader'])
for partition, partition_image_filename in FLASH_IMAGE_FILES:
partition_image_file_path = os.path.join(image_directory,
partition_image_filename)
adb.run_fastboot_command(
['flash', partition, partition_image_file_path])
if partition in ['bootloader', 'radio']:
adb.run_fastboot_command(['reboot-bootloader'])
# Disable ramdump to avoid capturing ramdumps during kernel crashes.
# This causes device lockup of several minutes during boot and we intend
# to analyze them ourselves.
adb.run_fastboot_command(['oem', 'ramdump', 'disable'])
adb.run_fastboot_command('reboot')
time.sleep(FLASH_REBOOT_WAIT)
if adb.get_device_state() == 'device':
break
logs.log_error('Reimaging failed, retrying.')
locks.release_lock(flash_lock_key_name, by_zone=True)
if adb.get_device_state() != 'device':
logs.log_error('Unable to find device. Reimaging failed.')
adb.bad_state_reached()
logs.log('Reimaging finished.')
# Reset all of our persistent keys after wipe.
persistent_cache.delete_value(constants.BUILD_PROP_MD5_KEY)
persistent_cache.delete_value(constants.LAST_TEST_ACCOUNT_CHECK_KEY)
persistent_cache.set_value(constants.LAST_FLASH_BUILD_KEY, build_info)
persistent_cache.set_value(constants.LAST_FLASH_TIME_KEY, time.time())
