def get(self):
"""Handle a get request."""
try:
grouper.group_testcases()
except:
logs.log_error('Error occurred while grouping test cases.')
return
# Free up memory after group task run.
utils.python_gc()
# Get a list of jobs excluded from bug filing.
excluded_jobs = _get_excluded_jobs()
# Get a list of all jobs. This is used to filter testcases whose jobs have
# been removed.
all_jobs = data_handler.get_all_job_type_names()
for testcase_id in data_handler.get_open_testcase_id_iterator():
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
# Already deleted.
continue
# Skip if testcase's job is removed.
if testcase.job_type not in all_jobs:
continue
# Skip if testcase's job is in exclusions list.
if testcase.job_type in excluded_jobs:
continue
# Skip if we are running progression task at this time.
if testcase.get_metadata('progression_pending'):
continue
# If the testcase has a bug filed already, no triage is needed.
if _is_bug_filed(testcase):
continue
# Check if the crash is important, i.e. it is either a reproducible crash
# or an unreproducible crash happening frequently.
if not _is_crash_important(testcase):
continue
# Require that all tasks like minimizaton, regression testing, etc have
# finished.
if not data_handler.critical_tasks_completed(testcase):
continue
# For testcases that are not part of a group, wait an additional time till
# group task completes.
# FIXME: In future, grouping might be dependent on regression range, so we
# would have to add an additional wait time.
if not testcase.group_id and not dates.time_has_expired(
testcase.timestamp,
hours=data_types.MIN_ELAPSED_TIME_SINCE_REPORT):
continue
# If this project does not have an associated issue tracker, we cannot
# file this crash anywhere.
issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(
testcase)
if not issue_tracker:
issue_filer.notify_issue_update(testcase, 'new')
continue
# If there are similar issues to this test case already filed or recently
# closed, skip filing a duplicate bug.
if _check_and_update_similar_bug(testcase, issue_tracker):
continue
# Clean up old triage messages that would be not applicable now.
testcase.delete_metadata(TRIAGE_MESSAGE_KEY, update_testcase=False)
# File the bug first and then create filed bug metadata.
try:
issue_filer.file_issue(testcase, issue_tracker)
except Exception:
logs.log_error('Failed to file issue for testcase %d.' %
testcase_id)
continue
_create_filed_bug_metadata(testcase)
issue_filer.notify_issue_update(testcase, 'new')
logs.log('Filed new issue %s for testcase %d.' %
(testcase.bug_information, testcase_id))
def get(self):
"""Handles a GET request."""
libfuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == 'libFuzzer').get()
if not libfuzzer:
logs.log_error('Failed to get libFuzzer Fuzzer entity.')
return
afl = data_types.Fuzzer.query(data_types.Fuzzer.name == 'afl').get()
if not afl:
logs.log_error('Failed to get AFL Fuzzer entity.')
return
honggfuzz = data_types.Fuzzer.query(
data_types.Fuzzer.name == 'honggfuzz').get()
if not honggfuzz:
logs.log_error('Failed to get honggfuzz Fuzzer entity.')
return
project_setup_configs = local_config.ProjectConfig().get('project_setup')
for setup_config in project_setup_configs:
bucket_config = setup_config.get('build_buckets')
if not bucket_config:
raise ProjectSetupError('Project setup buckets not specified.')
config = ProjectSetup(
BUILD_BUCKET_PATH_TEMPLATE,
REVISION_URL,
setup_config.get('build_type'),
config_suffix=setup_config.get('job_suffix', ''),
segregate_projects=setup_config.get('segregate_projects', False),
engine_build_buckets={
'libfuzzer': bucket_config.get('libfuzzer'),
'libfuzzer-i386': bucket_config.get('libfuzzer_i386'),
'afl': bucket_config.get('afl'),
'honggfuzz': bucket_config.get('honggfuzz'),
'none': bucket_config.get('no_engine'),
'dataflow': bucket_config.get('dataflow'),
},
fuzzer_entities={
'libfuzzer': libfuzzer,
'honggfuzz': honggfuzz,
'afl': afl,
},
add_info_labels=setup_config.get('add_info_labels', False),
add_revision_mappings=setup_config.get('add_revision_mappings',
False),
additional_vars=setup_config.get('additional_vars'))
projects_source = setup_config.get('source')
if projects_source == 'oss-fuzz':
projects = get_oss_fuzz_projects()
elif projects_source.startswith(storage.GS_PREFIX):
projects = get_projects_from_gcs(projects_source)
else:
raise ProjectSetupError('Invalid projects source: ' + projects_source)
if not projects:
raise ProjectSetupError('Missing projects list.')
config.set_up(projects)
def _minimize_corpus_two_step(self, target_path, arguments,
existing_corpus_dirs, new_corpus_dir,
output_corpus_dir, reproducers_dir,
max_time):
"""Optional (but recommended): run corpus minimization.
Args:
target_path: Path to the target.
arguments: Additional arguments needed for corpus minimization.
existing_corpus_dirs: Input corpora that existed before the fuzzing run.
new_corpus_dir: Input corpus that was generated during the fuzzing run.
Must have at least one new file.
output_corpus_dir: Output directory to place minimized corpus.
reproducers_dir: The directory to put reproducers in when crashes are
found.
max_time: Maximum allowed time for the minimization.
Returns:
A Result object.
"""
if not _is_multistep_merge_supported(target_path):
# Fallback to the old single step merge. It does not support incremental
# stats and provides only `edge_coverage` and `feature_coverage` stats.
logs.log(
'Old version of libFuzzer is used. Using single step merge.')
return self.minimize_corpus(
target_path, arguments,
existing_corpus_dirs + [new_corpus_dir], output_corpus_dir,
reproducers_dir, max_time)
# The dir where merge control file is located must persist for both merge
# steps. The second step re-uses the MCF produced during the first step.
merge_control_file_dir = self._create_temp_corpus_dir('mcf_tmp_dir')
self._merge_control_file = os.path.join(merge_control_file_dir, 'MCF')
# Two step merge process to obtain accurate stats for the new corpus units.
# See https://reviews.llvm.org/D66107 for a more detailed description.
merge_stats = {}
# Step 1. Use only existing corpus and collect "initial" stats.
result_1 = self.minimize_corpus(target_path, arguments,
existing_corpus_dirs,
output_corpus_dir, reproducers_dir,
max_time)
merge_stats['initial_edge_coverage'] = result_1.stats['edge_coverage']
merge_stats['initial_feature_coverage'] = result_1.stats[
'feature_coverage']
# Clear the output dir as it does not have any new units at this point.
engine_common.recreate_directory(output_corpus_dir)
# Adjust the time limit for the time we spent on the first merge step.
max_time -= result_1.time_executed
if max_time <= 0:
raise TimeoutError('Merging new testcases timed out\n' +
result_1.logs)
# Step 2. Process the new corpus units as well.
result_2 = self.minimize_corpus(
target_path, arguments, existing_corpus_dirs + [new_corpus_dir],
output_corpus_dir, reproducers_dir, max_time)
merge_stats['edge_coverage'] = result_2.stats['edge_coverage']
merge_stats['feature_coverage'] = result_2.stats['feature_coverage']
# Diff the stats to obtain accurate values for the new corpus units.
merge_stats['new_edges'] = (merge_stats['edge_coverage'] -
merge_stats['initial_edge_coverage'])
merge_stats['new_features'] = (merge_stats['feature_coverage'] -
merge_stats['initial_feature_coverage'])
output = result_1.logs + '\n\n' + result_2.logs
if (merge_stats['new_edges'] < 0 or merge_stats['new_features'] < 0):
logs.log_error('Two step merge failed.',
merge_stats=merge_stats,
output=output)
merge_stats['new_edges'] = 0
merge_stats['new_features'] = 0
self._merge_control_file = None
# TODO(ochang): Get crashes found during merge.
return engine.FuzzResult(
output, result_2.command, [], merge_stats,
result_1.time_executed + result_2.time_executed)
def update_fuzzer_and_data_bundles(fuzzer_name):
"""Update the fuzzer with a given name if necessary."""
fuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == fuzzer_name).get()
if not fuzzer:
logs.log_error('No fuzzer exists with name %s.' % fuzzer_name)
raise errors.InvalidFuzzerError
# Set some helper environment variables.
fuzzer_directory = get_fuzzer_directory(fuzzer_name)
environment.set_value('FUZZER_DIR', fuzzer_directory)
environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content)
# Adjust the test timeout, if user has provided one.
if fuzzer.timeout:
environment.set_value('TEST_TIMEOUT', fuzzer.timeout)
# Increase fuzz test timeout if the fuzzer timeout is higher than its
# current value.
fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT')
if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout:
environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout)
# Adjust the max testcases if this fuzzer has specified a lower limit.
max_testcases = environment.get_value('MAX_TESTCASES')
if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases:
environment.set_value('MAX_TESTCASES', fuzzer.max_testcases)
# Check for updates to this fuzzer.
version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name)
if (not fuzzer.builtin
and revisions.needs_update(version_file, fuzzer.revision)):
logs.log('Fuzzer update was found, updating.')
# Clear the old fuzzer directory if it exists.
if not shell.remove_directory(fuzzer_directory, recreate=True):
logs.log_error('Failed to clear fuzzer directory.')
return False
# Copy the archive to local disk and unpack it.
archive_path = os.path.join(fuzzer_directory, fuzzer.filename)
if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path):
logs.log_error('Failed to copy fuzzer archive.')
return False
try:
archive.unpack(archive_path, fuzzer_directory)
except Exception:
error_message = (
'Failed to unpack fuzzer archive %s '
'(bad archive or unsupported format).') % fuzzer.filename
logs.log_error(error_message)
fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
fuzzer_name=fuzzer_name)
return False
fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path)
if not os.path.exists(fuzzer_path):
error_message = (
'Fuzzer executable %s not found. '
'Check fuzzer configuration.') % fuzzer.executable_path
logs.log_error(error_message)
fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
fuzzer_name=fuzzer_name)
return False
# Make fuzzer executable.
os.chmod(fuzzer_path, 0750)
# Cleanup unneeded archive.
shell.remove_file(archive_path)
# Save the current revision of this fuzzer in a file for later checks.
revisions.write_revision_to_revision_file(version_file,
fuzzer.revision)
logs.log('Updated fuzzer to revision %d.' % fuzzer.revision)
# Setup data bundles associated with this fuzzer.
data_bundles = ndb_utils.get_all_from_query(
data_types.DataBundle.query(
data_types.DataBundle.name == fuzzer.data_bundle_name))
for data_bundle in data_bundles:
if not update_data_bundle(fuzzer, data_bundle):
return False
# Setup environment variable for launcher script path.
if fuzzer.launcher_script:
fuzzer_launcher_path = utils.get_launch_path_for_script(
fuzzer_directory, fuzzer.launcher_script)
environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path)
return True
def setup_testcase(testcase):
"""Sets up the testcase and needed dependencies like fuzzer,
data bundle, etc."""
fuzzer_name = testcase.fuzzer_name
job_type = testcase.job_type
task_name = environment.get_value('TASK_NAME')
testcase_fail_wait = environment.get_value('FAIL_WAIT')
testcase_id = testcase.key.id()
# Clear testcase directories.
shell.clear_testcase_directories()
# Setup memory debugging tool environment.
environment.reset_current_memory_tool_options(
redzone_size=testcase.redzone)
# Adjust the test timeout value if this is coming from an user uploaded
# testcase.
_set_timeout_value_from_user_upload(testcase_id)
if task_name == 'minimize':
# Allow minimizing with a different fuzzer set up.
minimize_fuzzer_override = environment.get_value(
'MINIMIZE_FUZZER_OVERRIDE')
fuzzer_name = minimize_fuzzer_override or fuzzer_name
# Update the fuzzer if necessary in order to get the updated data bundle.
if fuzzer_name:
try:
update_successful = update_fuzzer_and_data_bundles(fuzzer_name)
except errors.InvalidFuzzerError:
# Close testcase and don't recreate tasks if this fuzzer is invalid.
testcase.open = False
testcase.fixed = 'NA'
testcase.set_metadata('fuzzer_was_deleted', True)
logs.log_error('Closed testcase %d with invalid fuzzer %s.' %
(testcase_id, fuzzer_name))
error_message = 'Fuzzer %s no longer exists.' % fuzzer_name
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
return None, None, None
if not update_successful:
error_message = 'Unable to setup fuzzer %s.' % fuzzer_name
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
tasks.add_task(task_name,
testcase_id,
job_type,
wait_time=testcase_fail_wait)
return None, None, None
# Extract the testcase and any of its resources to the input directory.
file_list, input_directory, testcase_file_path = unpack_testcase(testcase)
if not file_list:
error_message = 'Unable to setup testcase %s.' % testcase_file_path
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
tasks.add_task(task_name,
testcase_id,
job_type,
wait_time=testcase_fail_wait)
return None, None, None
# For Android/Fuchsia, we need to sync our local testcases directory with the
# one on the device.
if environment.platform() == 'ANDROID':
_copy_testcase_to_device_and_setup_environment(testcase,
testcase_file_path)
if environment.platform() == 'FUCHSIA':
fuchsia.device.copy_testcase_to_device(testcase_file_path)
# Push testcases to worker.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.push_testcases_to_worker()
# Copy global blacklist into local blacklist.
is_lsan_enabled = environment.get_value('LSAN')
if is_lsan_enabled:
# Get local blacklist without this testcase's entry.
leak_blacklist.copy_global_to_local_blacklist(
excluded_testcase=testcase)
# Setup environment variable for windows size and location properties.
# Explicitly use empty string to indicate use of default window properties.
if hasattr(testcase, 'window_argument'):
environment.set_value('WINDOW_ARG', testcase.window_argument)
# Adjust timeout based on the stored multiplier (if available).
if hasattr(testcase, 'timeout_multiplier') and testcase.timeout_multiplier:
test_timeout = environment.get_value('TEST_TIMEOUT')
environment.set_value('TEST_TIMEOUT',
int(test_timeout * testcase.timeout_multiplier))
# Override APP_ARGS with minimized arguments (if available).
if (hasattr(testcase, 'minimized_arguments')
and testcase.minimized_arguments):
environment.set_value('APP_ARGS', testcase.minimized_arguments)
# Add FUZZ_TARGET to environment if this is a fuzz target testcase.
fuzz_target = testcase.get_metadata('fuzzer_binary_name')
if fuzz_target:
environment.set_value('FUZZ_TARGET', fuzz_target)
return file_list, input_directory, testcase_file_path
def main():
"""Prepare the configuration options and start requesting tasks."""
logs.configure('run_bot')
root_directory = environment.get_value('ROOT_DIR')
if not root_directory:
print(
'Please set ROOT_DIR environment variable to the root of the source '
'checkout before running. Exiting.')
print('For an example, check init.bash in the local directory.')
return
dates.initialize_timezone_from_environment()
environment.set_bot_environment()
monitor.initialize()
if not profiler.start_if_needed('python_profiler_bot'):
sys.exit(-1)
fuzzers_init.run()
if environment.is_trusted_host(ensure_connected=False):
from bot.untrusted_runner import host
host.init()
if environment.is_untrusted_worker():
# Track revision since we won't go into the task_loop.
update_task.track_revision()
from bot.untrusted_runner import untrusted as untrusted_worker
untrusted_worker.start_server()
assert False, 'Unreachable code'
while True:
# task_loop should be an infinite loop,
# unless we run into an exception.
error_stacktrace, clean_exit, task_payload = task_loop()
# Print the error trace to the console.
if not clean_exit:
print('Exception occurred while running "%s".' % task_payload)
print('-' * 80)
print(error_stacktrace)
print('-' * 80)
should_terminate = (clean_exit or errors.error_in_list(
error_stacktrace, errors.BOT_ERROR_TERMINATION_LIST))
if should_terminate:
return
logs.log_error('Task exited with exception (payload="%s").' %
task_payload,
error_stacktrace=error_stacktrace)
should_hang = errors.error_in_list(error_stacktrace,
errors.BOT_ERROR_HANG_LIST)
if should_hang:
logs.log('Start hanging forever.')
while True:
# Sleep to avoid consuming 100% of CPU.
time.sleep(60)
# See if our run timed out, if yes bail out.
if data_handler.bot_run_timed_out():
return
# Clear the current exception.
utils.exc_clear()
def setup_testcase(testcase, job_type, fuzzer_override=None):
"""Sets up the testcase and needed dependencies like fuzzer,
data bundle, etc."""
fuzzer_name = fuzzer_override or testcase.fuzzer_name
task_name = environment.get_value('TASK_NAME')
testcase_fail_wait = environment.get_value('FAIL_WAIT')
testcase_id = testcase.key.id()
# Clear testcase directories.
shell.clear_testcase_directories()
# Adjust the test timeout value if this is coming from an user uploaded
# testcase.
_set_timeout_value_from_user_upload(testcase_id)
# Update the fuzzer if necessary in order to get the updated data bundle.
if fuzzer_name:
try:
update_successful = update_fuzzer_and_data_bundles(fuzzer_name)
except errors.InvalidFuzzerError:
# Close testcase and don't recreate tasks if this fuzzer is invalid.
testcase.open = False
testcase.fixed = 'NA'
testcase.set_metadata('fuzzer_was_deleted', True)
logs.log_error('Closed testcase %d with invalid fuzzer %s.' %
(testcase_id, fuzzer_name))
error_message = 'Fuzzer %s no longer exists' % fuzzer_name
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
return None, None, None
if not update_successful:
error_message = 'Unable to setup fuzzer %s' % fuzzer_name
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
tasks.add_task(task_name,
testcase_id,
job_type,
wait_time=testcase_fail_wait)
return None, None, None
# Extract the testcase and any of its resources to the input directory.
file_list, input_directory, testcase_file_path = unpack_testcase(testcase)
if not file_list:
error_message = 'Unable to setup testcase %s' % testcase_file_path
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
tasks.add_task(task_name,
testcase_id,
job_type,
wait_time=testcase_fail_wait)
return None, None, None
# For Android/Fuchsia, we need to sync our local testcases directory with the
# one on the device.
if environment.platform() == 'ANDROID':
_copy_testcase_to_device_and_setup_environment(testcase,
testcase_file_path)
# Push testcases to worker.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.push_testcases_to_worker()
# Copy global blacklist into local blacklist.
is_lsan_enabled = environment.get_value('LSAN')
if is_lsan_enabled:
# Get local blacklist without this testcase's entry.
leak_blacklist.copy_global_to_local_blacklist(
excluded_testcase=testcase)
prepare_environment_for_testcase(testcase, job_type, task_name)
return file_list, input_directory, testcase_file_path
def wait_for_battery_charge_if_needed():
"""Check device battery and make sure it is charged beyond minimum level and
temperature thresholds."""
# Battery levels are not applicable on GCE.
if adb.is_gce():
return
# Make sure device is online.
adb.wait_for_device()
# Skip battery check if done recently.
last_battery_check_time = persistent_cache.get_value(
LAST_BATTERY_CHECK_TIME_KEY,
constructor=datetime.datetime.utcfromtimestamp)
if last_battery_check_time and not dates.time_has_expired(
last_battery_check_time, seconds=BATTERY_CHECK_INTERVAL):
return
# Initialize variables.
battery_level_threshold = environment.get_value('LOW_BATTERY_LEVEL_THRESHOLD',
LOW_BATTERY_LEVEL_THRESHOLD)
battery_temperature_threshold = environment.get_value(
'MAX_BATTERY_TEMPERATURE_THRESHOLD', MAX_BATTERY_TEMPERATURE_THRESHOLD)
device_restarted = False
while 1:
battery_information = get_battery_information()
if battery_information is None:
logs.log_error('Failed to get battery information, skipping check.')
return
battery_level = battery_information['level']
battery_temperature = battery_information['temperature']
logs.log('Battery information: level (%d%%), temperature (%.1f celsius).' %
(battery_level, battery_temperature))
if (battery_level >= battery_level_threshold and
battery_temperature <= battery_temperature_threshold):
persistent_cache.set_value(LAST_BATTERY_CHECK_TIME_KEY, time.time())
return
logs.log('Battery in bad battery state, putting device in sleep mode.')
if not device_restarted:
reboot()
adb.disable_wifi()
device_restarted = True
# Change thresholds to expected levels (only if they were below minimum
# thresholds).
if battery_level < battery_level_threshold:
battery_level_threshold = EXPECTED_BATTERY_LEVEL
if battery_temperature > battery_temperature_threshold:
battery_temperature_threshold = EXPECTED_BATTERY_TEMPERATURE
# Stopping shell should help with shutting off a lot of services that would
# otherwise use up the battery. However, we need to turn it back on to get
# battery status information. Also, turn off display explicitly (needed for
# Nexus 9s).
turn_off_display_if_needed()
adb.stop_shell()
time.sleep(BATTERY_CHARGE_INTERVAL)
adb.start_shell()
'*.txt',
'-x',
'*-expected.*',
'-x',
'*.git*',
'-x',
'*.svn*',
],
cwd=tests_directory)
subprocess.check_call(
['gsutil', 'cp', tests_archive_local, tests_archive_remote])
logs.log('Completed cycle, sleeping for %s seconds.' % sync_interval)
time.sleep(sync_interval)
if __name__ == '__main__':
# Make sure environment is correctly configured.
logs.configure('run_bot')
environment.set_bot_environment()
fail_wait = environment.get_value('FAIL_WAIT')
# Continue this forever.
while 1:
try:
main()
except Exception:
logs.log_error('Failed to sync tests.')
time.sleep(fail_wait)
def flash_to_latest_build_if_needed():
"""Wipes user data, resetting the device to original factory state."""
run_timeout = environment.get_value('RUN_TIMEOUT')
if run_timeout:
# If we have a run timeout, then we are already scheduled to bail out and
# will be probably get re-imaged. E.g. using frameworks like Tradefed.
return
# Check if a flash is needed based on last recorded flash time.
last_flash_time = persistent_cache.get_value(
LAST_FLASH_TIME_KEY, constructor=datetime.datetime.utcfromtimestamp)
needs_flash = last_flash_time is None or dates.time_has_expired(
last_flash_time, seconds=adb.FLASH_INTERVAL)
if not needs_flash:
return
build_info = {}
is_remote_device = environment.get_value('ANDROID_GCE')
if is_remote_device:
# To prevent thousands of devices all trying to reimage at the same time,
# reimages are done at a random time in the future.
scheduled_reimage_time = persistent_cache.get_value(
SCHEDULED_GCE_REIMAGE_TIME_KEY,
constructor=datetime.datetime.utcfromtimestamp)
if scheduled_reimage_time is None:
# No reimage scheduled yet, so we need to do so.
delay = random.randint(0, 3600)
reimage_time = int(time.time()) + delay
logs.log('Scheduling a new reimage in %d seconds.' % delay)
persistent_cache.set_value(SCHEDULED_GCE_REIMAGE_TIME_KEY, reimage_time)
return
current_time = datetime.datetime.utcnow()
if current_time < scheduled_reimage_time:
time_left = scheduled_reimage_time - current_time
# Not yet time for the reimage.
logs.log('Scheduled reimage in %d seconds.' % time_left.seconds)
return
# Recreating the virtual device will reimage this to the latest image
# available (with retry logic).
logs.log('Reimaging device to latest image.')
if not adb.recreate_virtual_device():
logs.log_error('Unable to recreate virtual device. Reimaging failed.')
adb.bad_state_reached()
else:
is_google_device = google_device()
if is_google_device is None:
logs.log_error('Unable to query device. Reimaging failed.')
adb.bad_state_reached()
if not is_google_device:
# We can't reimage these, skip.
logs.log('Non-Google device found, skipping reimage.')
return
else:
# For Google devices.
# Check if both |BUILD_BRANCH| and |BUILD_TARGET| environment variables
# are set. If not, we don't have enough data for reimaging and hence
# we bail out.
branch = environment.get_value('BUILD_BRANCH')
target = environment.get_value('BUILD_TARGET')
if not target:
# We default to userdebug configuration.
build_params = get_build_parameters()
if build_params:
target = build_params.get('target') + '-userdebug'
# Cache target in environment. This is also useful for cases when
# device is bricked and we don't have this information available.
environment.set_value('BUILD_TARGET', target)
if not branch or not target:
return
# Download the latest build artifact for this branch and target.
build_info = fetch_artifact.get_latest_artifact_info(branch, target)
if not build_info:
logs.log_error(
'Unable to fetch information on latest build artifact for '
'branch %s and target %s.' % (branch, target))
return
# Check if our local build matches the latest build. If not, we will
# download it.
build_id = build_info['bid']
target = build_info['target']
image_directory = environment.get_value('IMAGES_DIR')
last_build_info = persistent_cache.get_value(LAST_FLASH_BUILD_KEY)
if not last_build_info or last_build_info['bid'] != build_id:
# Clean up the images directory first.
shell.remove_directory(image_directory, recreate=True)
# We have a new build, download the build artifacts for it.
for image_regex in FLASH_IMAGE_REGEXES:
image_file_path = fetch_artifact.get(build_id, target, image_regex,
image_directory)
if not image_file_path:
logs.log_error(
'Failed to download image artifact %s for '
'branch %s and target %s.' % (image_file_path, branch, target))
return
if image_file_path.endswith('.zip'):
archive.unpack(image_file_path, image_directory)
# We do one device flash at a time on one host, otherwise we run into
# failures and device being stuck in a bad state.
flash_lock_key_name = 'flash:%s' % socket.gethostname()
if not locks.acquire_lock(flash_lock_key_name, by_zone=True):
logs.log_error('Failed to acquire lock for reimaging, exiting.')
return
logs.log('Reimaging started.')
logs.log('Rebooting into bootloader mode.')
for _ in xrange(FLASH_RETRIES):
adb.run_as_root()
adb.run_adb_command(['reboot-bootloader'])
time.sleep(FLASH_REBOOT_BOOTLOADER_WAIT)
adb.run_fastboot_command(['oem', 'off-mode-charge', '0'])
adb.run_fastboot_command(['-w', 'reboot-bootloader'])
for partition, partition_image_filename in FLASH_IMAGE_FILES:
partition_image_file_path = os.path.join(image_directory,
partition_image_filename)
adb.run_fastboot_command(
['flash', partition, partition_image_file_path])
if partition in ['bootloader', 'radio']:
adb.run_fastboot_command(['reboot-bootloader'])
adb.run_fastboot_command('reboot')
time.sleep(FLASH_REBOOT_WAIT)
if adb.get_device_state() == 'device':
break
logs.log_error('Reimaging failed, retrying.')
locks.release_lock(flash_lock_key_name, by_zone=True)
if adb.get_device_state() != 'device':
logs.log_error('Unable to find device. Reimaging failed.')
adb.bad_state_reached()
logs.log('Reimaging finished.')
# Reset all of our persistent keys after wipe.
persistent_cache.delete_value(BUILD_PROP_MD5_KEY)
persistent_cache.delete_value(LAST_TEST_ACCOUNT_CHECK_KEY)
persistent_cache.set_value(LAST_FLASH_BUILD_KEY, build_info)
persistent_cache.set_value(LAST_FLASH_TIME_KEY, time.time())
if is_remote_device:
persistent_cache.delete_value(SCHEDULED_GCE_REIMAGE_TIME_KEY)
def configure_build_properties_if_needed():
"""Edits /system/build.prop for better boot speed and power use."""
# Check md5 checksum of build.prop to see if already updated,
# in which case exit. If build.prop does not exist, something
# is very wrong with the device, so bail.
old_md5 = persistent_cache.get_value(BUILD_PROP_MD5_KEY)
current_md5 = adb.get_file_checksum(BUILD_PROP_PATH)
if current_md5 is None:
logs.log_error('Unable to find %s on device.' % BUILD_PROP_PATH)
return
if old_md5 == current_md5:
return
# Pull to tmp file.
bot_tmp_directory = environment.get_value('BOT_TMPDIR')
old_build_prop_path = os.path.join(bot_tmp_directory, 'old.prop')
adb.run_adb_command(['pull', BUILD_PROP_PATH, old_build_prop_path])
if not os.path.exists(old_build_prop_path):
logs.log_error('Unable to fetch %s from device.' % BUILD_PROP_PATH)
return
# Write new build.prop.
new_build_prop_path = os.path.join(bot_tmp_directory, 'new.prop')
old_build_prop_file_content = open(old_build_prop_path, 'r')
new_build_prop_file_content = open(new_build_prop_path, 'w')
new_content_notification = '### CHANGED OR ADDED PROPERTIES ###'
for line in old_build_prop_file_content:
property_name = line.split('=')[0].strip()
if property_name in BUILD_PROPERTIES:
continue
if new_content_notification in line:
continue
new_build_prop_file_content.write(line)
new_build_prop_file_content.write(new_content_notification + '\n')
for flag, value in BUILD_PROPERTIES.iteritems():
new_build_prop_file_content.write('%s=%s\n' % (flag, value))
old_build_prop_file_content.close()
new_build_prop_file_content.close()
# Keep verified boot disabled for M and higher releases. This makes it easy
# to modify system's app_process to load asan libraries.
build_version = get_build_version()
if is_build_at_least(build_version, 'M'):
adb.run_as_root()
adb.run_adb_command('disable-verity')
reboot()
# Make /system writable.
adb.run_as_root()
adb.remount()
# Remove seccomp policies (on N and higher) as ASan requires extra syscalls.
if is_build_at_least(build_version, 'N'):
policy_files = adb.run_adb_shell_command(
['find', '/system/etc/seccomp_policy/', '-type', 'f'])
for policy_file in policy_files.splitlines():
adb.run_adb_shell_command(['rm', policy_file.strip()])
# Remove Google Plus app from non-Google devices. Makes it easy to install
# older Gallery app on these devices. Otherwise, we run into duplicate
# permission errors.
if not google_device():
adb.run_adb_shell_command(['rm', '/system/app/PlusOne.apk'])
adb.run_adb_shell_command(['rm', '/system/app/PlusOne/PlusOne.apk'])
# Push new build.prop and backup to device.
logs.log('Pushing new build properties file on device.')
adb.run_adb_command(
['push', '-p', old_build_prop_path, BUILD_PROP_BACKUP_PATH])
adb.run_adb_command(['push', '-p', new_build_prop_path, BUILD_PROP_PATH])
adb.run_adb_shell_command(['chmod', '644', BUILD_PROP_PATH])
# Set persistent cache key containing and md5sum.
current_md5 = adb.get_file_checksum(BUILD_PROP_PATH)
persistent_cache.set_value(BUILD_PROP_MD5_KEY, current_md5)
def get_component_revisions_dict(revision, job_type):
"""Retrieve revision vars dict."""
if revision == 0 or revision == '0' or revision is None:
# Return empty dict for zero start revision.
return {}
config = db_config.get()
component = data_handler.get_component_name(job_type)
project_name = data_handler.get_project_name(job_type)
revision_info_url_format = db_config.get_value_for_job(
config.revision_vars_url, job_type)
if not revision_info_url_format:
return None
revisions_dict = {}
repository = data_handler.get_repository_for_component(component)
if repository and not _is_clank(revision_info_url_format):
revision_hash = _git_commit_position_to_git_hash_for_chromium(
revision, repository)
if revision_hash is None:
return None
# FIXME: While we check for this explicitly appended component in all
# applicable cases that we know of within this codebase, if the dict
# is shared with an external service (e.g. Predator) we may need to clean
# this up beforehand.
revisions_dict['/src'] = {
'name': _get_component_display_name(component, project_name),
'url': _git_url_for_chromium_repository(repository),
'rev': revision_hash,
'commit_pos': revision
}
# Use revision hash for info url later.
revision = revision_hash
revision_info_url = revision_info_url_format % revision
url_content = _get_url_content(revision_info_url)
if not url_content:
logs.log_error('Failed to get component revisions from %s.' %
revision_info_url)
return None
# Parse as per DEPS format.
if _is_deps(revision_info_url):
deps_revisions_dict = deps_to_revisions_dict(url_content)
if not deps_revisions_dict:
return None
revisions_dict.update(deps_revisions_dict)
return revisions_dict
# Parse as per Clank DEPS format.
if _is_clank(revision_info_url):
return _clank_revision_file_to_revisions_dict(url_content)
# Default case: parse content as yaml.
revisions_dict = _to_yaml_dict(url_content)
if not revisions_dict:
logs.log_error('Failed to parse component revisions from %s.' %
revision_info_url)
return None
# Parse as per source map format.
if revision_info_url.endswith(SOURCE_MAP_EXTENSION):
revisions_dict = _src_map_to_revisions_dict(revisions_dict,
project_name)
return revisions_dict
def main(argv):
"""Run libFuzzer as specified by argv."""
atexit.register(fuzzer_utils.cleanup)
# Initialize variables.
arguments = argv[1:]
testcase_file_path = arguments.pop(0)
target_name = arguments.pop(0)
fuzzer_name = data_types.fuzz_target_project_qualified_name(
utils.current_project(), target_name)
# Initialize log handler.
logs.configure(
'run_fuzzer', {
'fuzzer': fuzzer_name,
'engine': 'libFuzzer',
'job_name': environment.get_value('JOB_NAME')
})
profiler.start_if_needed('libfuzzer_launcher')
# Make sure that the fuzzer binary exists.
build_directory = environment.get_value('BUILD_DIR')
fuzzer_path = engine_common.find_fuzzer_path(build_directory, target_name)
if not fuzzer_path:
return
# Install signal handler.
signal.signal(signal.SIGTERM, engine_common.signal_term_handler)
# Set up temp dir.
engine_common.recreate_directory(fuzzer_utils.get_temp_dir())
# Setup minijail if needed.
use_minijail = environment.get_value('USE_MINIJAIL')
runner = libfuzzer.get_runner(fuzzer_path,
temp_dir=fuzzer_utils.get_temp_dir())
if use_minijail:
minijail_chroot = runner.chroot
else:
minijail_chroot = None
# Get corpus directory.
corpus_directory = environment.get_value('FUZZ_CORPUS_DIR')
# Add common arguments which are necessary to be used for every run.
arguments = expand_with_common_arguments(arguments)
# Add sanitizer options to environment that were specified in the .options
# file and options that this script requires.
set_sanitizer_options(fuzzer_path)
# If we don't have a corpus, then that means this is not a fuzzing run.
# TODO(flowerhack): Implement this to properly load past testcases.
if not corpus_directory and environment.platform() != 'FUCHSIA':
load_testcase_if_exists(runner, testcase_file_path, fuzzer_name,
use_minijail, arguments)
return
# We don't have a crash testcase, fuzz.
# Check dict argument to make sure that it's valid.
dict_argument = fuzzer_utils.extract_argument(arguments,
constants.DICT_FLAG,
remove=False)
if dict_argument and not os.path.exists(dict_argument):
logs.log_error('Invalid dict %s for %s.' %
(dict_argument, fuzzer_name))
fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG)
# If there's no dict argument, check for %target_binary_name%.dict file.
if (not fuzzer_utils.extract_argument(
arguments, constants.DICT_FLAG, remove=False)):
default_dict_path = dictionary_manager.get_default_dictionary_path(
fuzzer_path)
if os.path.exists(default_dict_path):
arguments.append(constants.DICT_FLAG + default_dict_path)
# Set up scratch directory for writing new units.
new_testcases_directory = create_corpus_directory('new')
# Strategy pool is the list of strategies that we attempt to enable, whereas
# fuzzing strategies is the list of strategies that are enabled. (e.g. if
# mutator is selected in the pool, but not available for a given target, it
# would not be added to fuzzing strategies.)
strategy_pool = strategy_selection.generate_weighted_strategy_pool(
strategy_list=strategy.LIBFUZZER_STRATEGY_LIST,
use_generator=True,
engine_name='libFuzzer')
strategy_info = pick_strategies(strategy_pool,
fuzzer_path,
corpus_directory,
arguments,
minijail_chroot=minijail_chroot)
arguments.extend(strategy_info.arguments)
# Timeout for fuzzer run.
fuzz_timeout = get_fuzz_timeout(strategy_info.is_mutations_run)
# Get list of corpus directories.
# TODO(flowerhack): Implement this to handle corpus sync'ing.
if environment.platform() == 'FUCHSIA':
corpus_directories = []
else:
corpus_directories = get_corpus_directories(
corpus_directory,
new_testcases_directory,
fuzzer_path,
strategy_info.fuzzing_strategies,
strategy_pool,
minijail_chroot=minijail_chroot,
allow_corpus_subset=not strategy_info.use_dataflow_tracing)
corpus_directories.extend(strategy_info.additional_corpus_dirs)
# Bind corpus directories in minijail.
artifact_prefix = os.path.abspath(os.path.dirname(testcase_file_path))
# Execute the fuzzer binary with original arguments.
fuzz_result = runner.fuzz(corpus_directories,
fuzz_timeout=fuzz_timeout,
artifact_prefix=artifact_prefix,
additional_args=arguments,
extra_env=strategy_info.extra_env)
if (not use_minijail
and fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE):
# Minijail returns 1 if the exit code is nonzero.
# Otherwise: we can assume that a return code of 1 means that libFuzzer
# itself ran into an error.
logs.log_error(ENGINE_ERROR_MESSAGE, engine_output=fuzz_result.output)
log_lines = fuzz_result.output.splitlines()
# Output can be large, so save some memory by removing reference to the
# original output which is no longer needed.
fuzz_result.output = None
# Check if we crashed, and get the crash testcase path.
crash_testcase_file_path = runner.get_testcase_path(log_lines)
if crash_testcase_file_path:
# Copy crash testcase contents into the main testcase path.
shutil.move(crash_testcase_file_path, testcase_file_path)
# Print the command output.
bot_name = environment.get_value('BOT_NAME', '')
command = fuzz_result.command
if use_minijail:
# Remove minijail prefix.
command = engine_common.strip_minijail_command(command, fuzzer_path)
print(
engine_common.get_log_header(command, bot_name,
fuzz_result.time_executed))
# Parse stats information based on libFuzzer output.
parsed_stats = parse_log_stats(log_lines)
# Extend parsed stats by additional performance features.
parsed_stats.update(
stats.parse_performance_features(log_lines,
strategy_info.fuzzing_strategies,
arguments))
# Set some initial stat overrides.
timeout_limit = fuzzer_utils.extract_argument(arguments,
constants.TIMEOUT_FLAG,
remove=False)
expected_duration = runner.get_max_total_time(fuzz_timeout)
actual_duration = int(fuzz_result.time_executed)
fuzzing_time_percent = 100 * actual_duration / float(expected_duration)
stat_overrides = {
'timeout_limit': int(timeout_limit),
'expected_duration': expected_duration,
'actual_duration': actual_duration,
'fuzzing_time_percent': fuzzing_time_percent,
}
# Remove fuzzing arguments before merge and dictionary analysis step.
remove_fuzzing_arguments(arguments)
# Make a decision on whether merge step is needed at all. If there are no
# new units added by libFuzzer run, then no need to do merge at all.
new_units_added = shell.get_directory_file_count(new_testcases_directory)
merge_error = None
if new_units_added:
# Merge the new units with the initial corpus.
if corpus_directory not in corpus_directories:
corpus_directories.append(corpus_directory)
# If this times out, it's possible that we will miss some units. However, if
# we're taking >10 minutes to load/merge the corpus something is going very
# wrong and we probably don't want to make things worse by adding units
# anyway.
merge_tmp_dir = None
if not use_minijail:
merge_tmp_dir = os.path.join(fuzzer_utils.get_temp_dir(),
'merge_workdir')
engine_common.recreate_directory(merge_tmp_dir)
old_corpus_len = shell.get_directory_file_count(corpus_directory)
merge_directory = create_merge_directory()
corpus_directories.insert(0, merge_directory)
if use_minijail:
bind_corpus_dirs(minijail_chroot, [merge_directory])
merge_result = runner.merge(
corpus_directories,
merge_timeout=engine_common.get_merge_timeout(
DEFAULT_MERGE_TIMEOUT),
tmp_dir=merge_tmp_dir,
additional_args=arguments)
move_mergeable_units(merge_directory, corpus_directory)
new_corpus_len = shell.get_directory_file_count(corpus_directory)
new_units_added = 0
merge_error = None
if merge_result.timed_out:
merge_error = 'Merging new testcases timed out:'
elif merge_result.return_code != 0:
merge_error = 'Merging new testcases failed:'
else:
new_units_added = new_corpus_len - old_corpus_len
stat_overrides['new_units_added'] = new_units_added
if merge_result.output:
stat_overrides.update(
stats.parse_stats_from_merge_log(
merge_result.output.splitlines()))
else:
stat_overrides['new_units_added'] = 0
logs.log('Skipped corpus merge since no new units added by fuzzing.')
# Get corpus size after merge. This removes the duplicate units that were
# created during this fuzzing session.
# TODO(flowerhack): Remove this workaround once we can handle corpus sync.
if environment.platform() != 'FUCHSIA':
stat_overrides['corpus_size'] = shell.get_directory_file_count(
corpus_directory)
# Delete all corpus directories except for the main one. These were temporary
# directories to store new testcase mutations and have already been merged to
# main corpus directory.
if corpus_directory in corpus_directories:
corpus_directories.remove(corpus_directory)
for directory in corpus_directories:
shutil.rmtree(directory, ignore_errors=True)
if use_minijail:
unbind_corpus_dirs(minijail_chroot, corpus_directories)
# Apply overridden stats to the parsed stats prior to dumping.
parsed_stats.update(stat_overrides)
# Dump stats data for further uploading to BigQuery.
engine_common.dump_big_query_data(parsed_stats, testcase_file_path,
command)
# Add custom crash state based on fuzzer name (if needed).
add_custom_crash_state_if_needed(fuzzer_name, log_lines, parsed_stats)
for line in log_lines:
print(line)
# Add fuzzing strategies used.
print(
engine_common.format_fuzzing_strategies(
strategy_info.fuzzing_strategies))
# Add merge error (if any).
if merge_error:
print(data_types.CRASH_STACKTRACE_END_MARKER)
print(merge_error)
print(
'Command:',
get_printable_command(merge_result.command, fuzzer_path,
use_minijail))
print(merge_result.output)
analyze_and_update_recommended_dictionary(runner, fuzzer_name, log_lines,
corpus_directory, arguments)
# Close minijail chroot.
if use_minijail:
minijail_chroot.close()
# Record the stats to make them easily searchable in stackdriver.
if new_units_added:
logs.log('New units added to corpus: %d.' % new_units_added,
stats=parsed_stats)
else:
logs.log('No new units found.', stats=parsed_stats)
def pick_strategies(strategy_pool,
fuzzer_path,
corpus_directory,
existing_arguments,
minijail_chroot=None):
"""Pick strategies."""
build_directory = environment.get_value('BUILD_DIR')
target_name = os.path.basename(fuzzer_path)
project_qualified_fuzzer_name = data_types.fuzz_target_project_qualified_name(
utils.current_project(), target_name)
fuzzing_strategies = []
arguments = []
additional_corpus_dirs = []
# Select a generator to attempt to use for existing testcase mutations.
candidate_generator = engine_common.select_generator(
strategy_pool, fuzzer_path)
is_mutations_run = candidate_generator != engine_common.Generator.NONE
# Depends on the presense of DFSan instrumented build.
dataflow_build_dir = environment.get_value('DATAFLOW_BUILD_DIR')
use_dataflow_tracing = (dataflow_build_dir and strategy_pool.do_strategy(
strategy.DATAFLOW_TRACING_STRATEGY))
if use_dataflow_tracing:
dataflow_binary_path = os.path.join(
dataflow_build_dir, os.path.relpath(fuzzer_path, build_directory))
if os.path.exists(dataflow_binary_path):
arguments.append(
'%s%s' %
(constants.COLLECT_DATA_FLOW_FLAG, dataflow_binary_path))
fuzzing_strategies.append(strategy.DATAFLOW_TRACING_STRATEGY.name)
else:
logs.log_error(
'Fuzz target is not found in dataflow build, skiping strategy.'
)
use_dataflow_tracing = False
# Generate new testcase mutations using radamsa, etc.
if is_mutations_run:
new_testcase_mutations_directory = create_corpus_directory('mutations')
generator_used = engine_common.generate_new_testcase_mutations(
corpus_directory, new_testcase_mutations_directory,
project_qualified_fuzzer_name, candidate_generator)
# Add the used generator strategy to our fuzzing strategies list.
if generator_used:
if candidate_generator == engine_common.Generator.RADAMSA:
fuzzing_strategies.append(
strategy.CORPUS_MUTATION_RADAMSA_STRATEGY.name)
elif candidate_generator == engine_common.Generator.ML_RNN:
fuzzing_strategies.append(
strategy.CORPUS_MUTATION_ML_RNN_STRATEGY.name)
additional_corpus_dirs.append(new_testcase_mutations_directory)
if minijail_chroot:
bind_corpus_dirs(minijail_chroot,
[new_testcase_mutations_directory])
if strategy_pool.do_strategy(strategy.RANDOM_MAX_LENGTH_STRATEGY):
max_len_argument = fuzzer_utils.extract_argument(
existing_arguments, constants.MAX_LEN_FLAG, remove=False)
if not max_len_argument:
max_length = random.SystemRandom().randint(
1, MAX_VALUE_FOR_MAX_LENGTH)
arguments.append('%s%d' % (constants.MAX_LEN_FLAG, max_length))
fuzzing_strategies.append(strategy.RANDOM_MAX_LENGTH_STRATEGY.name)
if (strategy_pool.do_strategy(strategy.RECOMMENDED_DICTIONARY_STRATEGY)
and add_recommended_dictionary(
arguments, project_qualified_fuzzer_name, fuzzer_path)):
fuzzing_strategies.append(
strategy.RECOMMENDED_DICTIONARY_STRATEGY.name)
if strategy_pool.do_strategy(strategy.VALUE_PROFILE_STRATEGY):
arguments.append(constants.VALUE_PROFILE_ARGUMENT)
fuzzing_strategies.append(strategy.VALUE_PROFILE_STRATEGY.name)
# DataFlow Tracing requires fork mode, always use it with DFT strategy.
if use_dataflow_tracing or strategy_pool.do_strategy(
strategy.FORK_STRATEGY):
max_fuzz_threads = environment.get_value('MAX_FUZZ_THREADS', 1)
num_fuzz_processes = max(
1,
multiprocessing.cpu_count() // max_fuzz_threads)
arguments.append('%s%d' % (constants.FORK_FLAG, num_fuzz_processes))
fuzzing_strategies.append(
'%s_%d' % (strategy.FORK_STRATEGY.name, num_fuzz_processes))
extra_env = {}
if (strategy_pool.do_strategy(strategy.MUTATOR_PLUGIN_STRATEGY)
and use_mutator_plugin(target_name, extra_env, minijail_chroot)):
fuzzing_strategies.append(strategy.MUTATOR_PLUGIN_STRATEGY.name)
return StrategyInfo(fuzzing_strategies, arguments, additional_corpus_dirs,
extra_env, use_dataflow_tracing, is_mutations_run)
def execute_task(metadata_id, job_type):
"""Unpack a bundled testcase archive and create analyze jobs for each item."""
metadata = ndb.Key(data_types.BundledArchiveMetadata,
int(metadata_id)).get()
if not metadata:
logs.log_error('Invalid bundle metadata id %s.' % metadata_id)
return
bot_name = environment.get_value('BOT_NAME')
upload_metadata = data_types.TestcaseUploadMetadata.query(
data_types.TestcaseUploadMetadata.blobstore_key ==
metadata.blobstore_key).get()
if not upload_metadata:
logs.log_error('Invalid upload metadata key %s.' %
metadata.blobstore_key)
return
# Update the upload metadata with this bot name.
upload_metadata.bot_name = bot_name
upload_metadata.put()
# We can't use FUZZ_INPUTS directory since it is constrained
# by tmpfs limits.
testcases_directory = environment.get_value('FUZZ_INPUTS_DISK')
# Retrieve multi-testcase archive.
archive_path = os.path.join(testcases_directory, metadata.archive_filename)
if not blobs.read_blob_to_disk(metadata.blobstore_key, archive_path):
logs.log_error('Could not retrieve archive for bundle %d.' %
metadata_id)
tasks.add_task('unpack', metadata_id, job_type)
return
try:
archive.unpack(archive_path, testcases_directory)
except:
logs.log_error('Could not unpack archive for bundle %d.' % metadata_id)
tasks.add_task('unpack', metadata_id, job_type)
return
# Get additional testcase metadata (if any).
additional_metadata = None
if upload_metadata.additional_metadata_string:
additional_metadata = json.loads(
upload_metadata.additional_metadata_string)
archive_state = data_types.ArchiveStatus.NONE
bundled = True
file_list = archive.get_file_list(archive_path)
for file_path in file_list:
absolute_file_path = os.path.join(testcases_directory, file_path)
filename = os.path.basename(absolute_file_path)
# Only files are actual testcases. Skip directories.
if not os.path.isfile(absolute_file_path):
continue
try:
file_handle = open(absolute_file_path, 'rb')
blob_key = blobs.write_blob(file_handle)
file_handle.close()
except:
blob_key = None
if not blob_key:
logs.log_error('Could not write testcase %s to blobstore.' %
absolute_file_path)
continue
data_handler.create_user_uploaded_testcase(
blob_key, metadata.blobstore_key, archive_state,
metadata.archive_filename, filename, metadata.timeout,
metadata.job_type, metadata.job_queue, metadata.http_flag,
metadata.gestures, metadata.additional_arguments,
metadata.bug_information, metadata.crash_revision,
metadata.uploader_email, metadata.platform_id,
metadata.app_launch_command, metadata.fuzzer_name,
metadata.overridden_fuzzer_name, metadata.fuzzer_binary_name,
bundled, upload_metadata.retries,
upload_metadata.bug_summary_update_flag,
upload_metadata.quiet_flag, additional_metadata)
# The upload metadata for the archive is not needed anymore since we created
# one for each testcase.
upload_metadata.key.delete()
shell.clear_testcase_directories()
def unpack_crash_testcases(crash_testcases_directory):
"""Unpacks the old crash testcases in the provided directory."""
for testcase in ndb_utils.get_all_from_model(data_types.Testcase):
testcase_id = testcase.key.id()
# 1. If we have already stored the testcase, then just skip.
if testcase_id in STORED_TESTCASES_LIST:
continue
# 2. Make sure that it is a unique crash testcase. Ignore duplicates,
# uploaded repros.
if testcase.status != 'Processed':
continue
# 3. Check if the testcase is fixed. If not, skip.
if testcase.open:
continue
# 4. Check if the testcase has a minimized repro. If not, skip.
if not testcase.minimized_keys or testcase.minimized_keys == 'NA':
continue
# 5. Only use testcases that have bugs associated with them.
if not testcase.bug_information:
continue
# 6. Existing IPC testcases are un-interesting and unused in furthur
# mutations. Due to size bloat, ignoring these for now.
if testcase.absolute_path.endswith(testcase_manager.IPCDUMP_EXTENSION):
continue
# 7. Ignore testcases that are archives (e.g. Langfuzz fuzzer tests).
if archive.get_archive_type(testcase.absolute_path):
continue
# 8. Skip in-process fuzzer testcases, since these are only applicable to
# fuzz targets and don't run with blackbox binaries.
if testcase.fuzzer_name and testcase.fuzzer_name in ['afl', 'libFuzzer']:
continue
# Un-pack testcase.
try:
_, input_directory, _ = setup.unpack_testcase(testcase)
except Exception:
logs.log_error('Failed to unpack testcase %d.' % testcase.key.id())
continue
# Move this to our crash testcases directory.
crash_testcase_directory = os.path.join(crash_testcases_directory,
str(testcase_id))
shell.move(input_directory, crash_testcase_directory)
# Re-create input directory for unpacking testcase in next iteration.
shell.create_directory(input_directory)
STORED_TESTCASES_LIST.append(testcase_id)
# Remove testcase directories that exceed the max size limit.
for directory_name in os.listdir(crash_testcases_directory):
directory_path = os.path.join(crash_testcases_directory, directory_name)
if not os.path.isdir(directory_path):
continue
if shell.get_directory_size(directory_path) <= MAX_TESTCASE_DIRECTORY_SIZE:
continue
shell.remove_directory(directory_path)
# Rename all fuzzed testcase files as regular files.
for root, _, files in os.walk(crash_testcases_directory):
for filename in files:
if not filename.startswith(testcase_manager.FUZZ_PREFIX):
continue
file_path = os.path.join(root, filename)
stripped_file_name = os.path.basename(file_path)[len(
testcase_manager.FUZZ_PREFIX):]
stripped_file_path = os.path.join(
os.path.dirname(file_path), stripped_file_name)
try:
os.rename(file_path, stripped_file_path)
except:
raise Exception('Failed to rename testcase %s.' % file_path)
# Remove empty files and dirs to avoid the case where a fuzzer randomly
# chooses an empty dir/file and generates zero testcases.
shell.remove_empty_files(crash_testcases_directory)
shell.remove_empty_directories(crash_testcases_directory)
def _do_run_testcase_and_return_result_in_queue(crash_queue,
thread_index,
file_path,
gestures,
env_copy,
upload_output=False):
"""Run a single testcase and return crash results in the crash queue."""
try:
# Run testcase and check whether a crash occurred or not.
return_code, crash_time, output = run_testcase(thread_index, file_path,
gestures, env_copy)
# Pull testcase directory to host to get any stats files.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.pull_testcases_from_worker()
# Analyze the crash.
crash_output = _get_crash_output(output)
crash_result = CrashResult(return_code, crash_time, crash_output)
# To provide consistency between stats and logs, we use timestamp taken
# from stats when uploading logs and testcase.
if upload_output:
log_time = _get_testcase_time(file_path)
if crash_result.is_crash():
# Initialize resource list with the testcase path.
resource_list = [file_path]
resource_list += get_resource_paths(crash_output)
# Store the crash stack file in the crash stacktrace directory
# with filename as the hash of the testcase path.
crash_stacks_directory = environment.get_value(
'CRASH_STACKTRACES_DIR')
stack_file_path = os.path.join(crash_stacks_directory,
utils.string_hash(file_path))
utils.write_data_to_file(crash_output, stack_file_path)
# Put crash/no-crash results in the crash queue.
crash_queue.put(
Crash(file_path=file_path,
crash_time=crash_time,
return_code=return_code,
resource_list=resource_list,
gestures=gestures,
stack_file_path=stack_file_path))
# Don't upload uninteresting testcases (no crash) or if there is no log to
# correlate it with (not upload_output).
if upload_output:
upload_testcase(file_path, log_time)
if upload_output:
# Include full output for uploaded logs (crash output, merge output, etc).
crash_result_full = CrashResult(return_code, crash_time, output)
log = prepare_log_for_upload(crash_result_full.get_stacktrace(),
return_code)
upload_log(log, log_time)
except Exception:
logs.log_error('Exception occurred while running '
'run_testcase_and_return_result_in_queue.')
def download_system_symbols_if_needed(symbols_directory, is_kernel=False):
"""Download system libraries from |SYMBOLS_URL| and cache locally."""
# For local testing, we do not have access to the cloud storage bucket with
# the symbols. In this case, just bail out.
if environment.get_value('LOCAL_DEVELOPMENT'):
return
# When running reproduce tool locally, we do not have access to the cloud
# storage bucket with the symbols. In this case, just bail out.
if environment.get_value('REPRODUCE_TOOL'):
return
# We have archived symbols for google builds only.
if not settings.is_google_device():
return
# For Android kernel we want to get the repro.prop
# Note: kasan and non-kasan kernel should have the same repo.prop for a given
# build_id.
if is_kernel:
build_id = settings.get_kernel_build_id()
target = settings.get_kernel_name()
if not build_id or not target:
logs.log_error('Could not get kernel parameters, exiting.')
return
artifact_file_name = 'repo.prop'
symbols_archive_filename = get_symbols_archive_filename(
build_id, target)
output_filename_override = symbols_archive_filename
# We create our own build_params for cache
build_params = {
'build_id': build_id,
'target': target,
'type': 'kernel'
}
else:
# Get the build fingerprint parameters.
build_params = settings.get_build_parameters()
if not build_params:
logs.log_error('Unable to determine build parameters.')
return
build_id = build_params.get('build_id')
target = build_params.get('target')
build_type = build_params.get('type')
if not build_id or not target or not build_type:
logs.log_error('Null build parameters found, exiting.')
return
symbols_archive_filename = '%s-symbols-%s.zip' % (target, build_id)
artifact_file_name = symbols_archive_filename
output_filename_override = None
# Check if we already have the symbols in cache.
build_params_check_path = os.path.join(symbols_directory,
'.cached_build_params')
cached_build_params = utils.read_data_from_file(build_params_check_path,
eval_data=True)
if cached_build_params and cached_build_params == build_params:
# No work to do, same system symbols already in cache.
return
symbols_archive_path = os.path.join(symbols_directory,
symbols_archive_filename)
# Delete existing symbols directory first.
shell.remove_directory(symbols_directory, recreate=True)
# Fetch symbol file from cloud storage cache (if available).
found_in_cache = storage.get_file_from_cache_if_exists(
symbols_archive_path, update_modification_time_on_access=False)
if not found_in_cache:
tool_suffix = environment.get_value('SANITIZER_TOOL_NAME')
if is_kernel:
# Some kernels are just 'kernel', some are kernel_target
if tool_suffix:
targets_with_type_and_san = [
'kernel_%s' % tool_suffix,
'kernel_%s_%s' % (tool_suffix, target)
]
else:
targets_with_type_and_san = ['kernel', 'kernel_%s' % target]
else:
# Include type and sanitizer information in the target.
target_with_type_and_san = '%s-%s' % (target, build_type)
if tool_suffix and not tool_suffix in target_with_type_and_san:
target_with_type_and_san += '_%s' % tool_suffix
targets_with_type_and_san = [target_with_type_and_san]
for target_with_type_and_san in targets_with_type_and_san:
# Fetch the artifact now.
fetch_artifact.get(build_id, target_with_type_and_san,
artifact_file_name, symbols_directory,
output_filename_override)
if os.path.exists(symbols_archive_path):
break
if not os.path.exists(symbols_archive_path):
logs.log_error('Unable to locate symbols archive %s.' %
symbols_archive_path)
return
# Store the artifact for later use or for use by other bots.
storage.store_file_in_cache(symbols_archive_path)
# repo.prop is not a zip archive.
if not is_kernel:
archive.unpack(symbols_archive_path, symbols_directory, trusted=True)
shell.remove_file(symbols_archive_path)
utils.write_data_to_file(build_params, build_params_check_path)
def find_fixed_range(testcase_id, job_type):
"""Attempt to find the revision range where a testcase was fixed."""
deadline = tasks.get_task_completion_deadline()
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
if testcase.fixed:
logs.log_error('Fixed range is already set as %s, skip.' %
testcase.fixed)
return
# Setup testcase and its dependencies.
file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type)
if not file_list:
return
# Set a flag to indicate we are running progression task. This shows pending
# status on testcase report page and avoid conflicting testcase updates by
# triage cron.
testcase.set_metadata('progression_pending', True)
# Custom binaries are handled as special cases.
if build_manager.is_custom_binary():
_check_fixed_for_custom_binary(testcase, job_type, testcase_file_path)
return
build_bucket_path = build_manager.get_primary_bucket_path()
revision_list = build_manager.get_revisions_list(build_bucket_path,
testcase=testcase)
if not revision_list:
data_handler.close_testcase_with_error(
testcase_id, 'Failed to fetch revision list')
return
# Use min, max_index to mark the start and end of revision list that is used
# for bisecting the progression range. Set start to the revision where noticed
# the crash. Set end to the trunk revision. Also, use min, max from past run
# if it timed out.
min_revision = testcase.get_metadata('last_progression_min')
max_revision = testcase.get_metadata('last_progression_max')
if min_revision or max_revision:
# Clear these to avoid using them in next run. If this run fails, then we
# should try next run without them to see it succeeds. If this run succeeds,
# we should still clear them to avoid capping max revision in next run.
testcase = data_handler.get_testcase_by_id(testcase_id)
testcase.delete_metadata('last_progression_min', update_testcase=False)
testcase.delete_metadata('last_progression_max', update_testcase=False)
testcase.put()
last_tested_revision = testcase.get_metadata('last_tested_crash_revision')
known_crash_revision = last_tested_revision or testcase.crash_revision
if not min_revision:
min_revision = known_crash_revision
if not max_revision:
max_revision = revisions.get_last_revision_in_list(revision_list)
min_index = revisions.find_min_revision_index(revision_list, min_revision)
if min_index is None:
raise errors.BuildNotFoundError(min_revision, job_type)
max_index = revisions.find_max_revision_index(revision_list, max_revision)
if max_index is None:
raise errors.BuildNotFoundError(max_revision, job_type)
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.STARTED,
'r%d' % max_revision)
# Check to see if this testcase is still crashing now. If it is, then just
# bail out.
result = _testcase_reproduces_in_revision(testcase,
testcase_file_path,
job_type,
max_revision,
update_metadata=True)
if result.is_crash():
logs.log('Found crash with same signature on latest revision r%d.' %
max_revision)
app_path = environment.get_value('APP_PATH')
command = testcase_manager.get_command_line_for_application(
testcase_file_path,
app_path=app_path,
needs_http=testcase.http_flag)
symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True)
unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False)
stacktrace = utils.get_crash_stacktrace_output(
command, symbolized_crash_stacktrace,
unsymbolized_crash_stacktrace)
testcase = data_handler.get_testcase_by_id(testcase_id)
testcase.last_tested_crash_stacktrace = data_handler.filter_stacktrace(
stacktrace)
data_handler.update_progression_completion_metadata(
testcase,
max_revision,
is_crash=True,
message='still crashes on latest revision r%s' % max_revision)
# Since we've verified that the test case is still crashing, clear out any
# metadata indicating potential flake from previous runs.
task_creation.mark_unreproducible_if_flaky(testcase, False)
# For chromium project, save latest crash information for later upload
# to chromecrash/.
state = result.get_symbolized_data()
crash_uploader.save_crash_info_if_needed(testcase_id, max_revision,
job_type, state.crash_type,
state.crash_address,
state.frames)
return
# Don't burden NFS server with caching these random builds.
environment.set_value('CACHE_STORE', False)
# Verify that we do crash in the min revision. This is assumed to be true
# while we are doing the bisect.
result = _testcase_reproduces_in_revision(testcase, testcase_file_path,
job_type, min_revision)
if result and not result.is_crash():
testcase = data_handler.get_testcase_by_id(testcase_id)
# Retry once on another bot to confirm our result.
if data_handler.is_first_retry_for_task(testcase,
reset_after_retry=True):
tasks.add_task('progression', testcase_id, job_type)
error_message = (
'Known crash revision %d did not crash, will retry on another bot to '
'confirm result' % known_crash_revision)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
data_handler.update_progression_completion_metadata(
testcase, max_revision)
return
data_handler.clear_progression_pending(testcase)
error_message = ('Known crash revision %d did not crash' %
known_crash_revision)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
error_message)
task_creation.mark_unreproducible_if_flaky(testcase, True)
return
# Start a binary search to find last non-crashing revision. At this point, we
# know that we do crash in the min_revision, and do not crash in max_revision.
while time.time() < deadline:
min_revision = revision_list[min_index]
max_revision = revision_list[max_index]
# If the min and max revisions are one apart this is as much as we can
# narrow the range.
if max_index - min_index == 1:
_save_fixed_range(testcase_id, min_revision, max_revision,
testcase_file_path)
return
# Occasionally, we get into this bad state. It seems to be related to test
# cases with flaky stacks, but the exact cause is unknown.
if max_index - min_index < 1:
testcase = data_handler.get_testcase_by_id(testcase_id)
testcase.fixed = 'NA'
testcase.open = False
message = ('Fixed testing errored out (min and max revisions '
'are both %d)' % min_revision)
data_handler.update_progression_completion_metadata(
testcase, max_revision, message=message)
# Let the bisection service know about the NA status.
bisection.request_bisection(testcase)
return
# Test the middle revision of our range.
middle_index = (min_index + max_index) // 2
middle_revision = revision_list[middle_index]
testcase = data_handler.get_testcase_by_id(testcase_id)
log_message = 'Testing r%d (current range %d:%d)' % (
middle_revision, min_revision, max_revision)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.WIP,
log_message)
try:
result = _testcase_reproduces_in_revision(testcase,
testcase_file_path,
job_type,
middle_revision)
except errors.BadBuildError:
# Skip this revision.
del revision_list[middle_index]
max_index -= 1
continue
if result.is_crash():
min_index = middle_index
else:
max_index = middle_index
_save_current_fixed_range_indices(testcase_id,
revision_list[min_index],
revision_list[max_index])
# If we've broken out of the loop, we've exceeded the deadline. Recreate the
# task to pick up where we left off.
testcase = data_handler.get_testcase_by_id(testcase_id)
error_message = ('Timed out, current range r%d:r%d' %
(revision_list[min_index], revision_list[max_index]))
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
error_message)
tasks.add_task('progression', testcase_id, job_type)
def execute(input_directory, output_directory, fuzzer_name,
generation_timeout):
"""Execute ML RNN generator to produce new inputs.
This method should be called inside launcher, to generate a number of
new inputs based on ML RNN model.
It will fetch ML model from GCS bucket specified in environment
variable `CORPUS_BUCKET`. The script to run the model resides
in folder `tools/fuzzers/ml/rnn`.
Args:
input_directory: Seed corpus path. The directory should not be empty.
output_directory: The directory to place generated inputs.
fuzzer_name: Name of the fuzzer, e.g libpng_read_fuzzer. It indicates the
subdirectory in gcs bucket to store models.
generation_timeout: Time in seconds for the generator to run. Normally it
takes <1s to generate an input, assuming the input length is <4KB.
"""
if environment.platform() != 'LINUX':
logs.log('Unsupported platform for ML RNN generation, skipping.')
return
# Validate corpus folder.
file_count = shell.get_directory_file_count(input_directory)
if not file_count:
logs.log('Corpus is empty. Skip generation.')
return
# Number of existing new inputs. They are possibly generated by other
# generators.
old_corpus_units = shell.get_directory_file_count(output_directory)
old_corpus_bytes = shell.get_directory_size(output_directory)
# Get model path.
model_path = prepare_model_directory(fuzzer_name)
if not model_path:
return
result = run(input_directory, output_directory, model_path,
generation_timeout)
# Generation process exited abnormally but not caused by timeout, meaning
# error occurred during execution.
if result.return_code and not result.timed_out:
if result.return_code == constants.ExitCode.CORPUS_TOO_SMALL:
logs.log_warn(
'ML RNN generation for fuzzer %s aborted due to small corpus.'
% fuzzer_name)
else:
logs.log_error(
'ML RNN generation for fuzzer %s failed with ExitCode = %d.' %
(fuzzer_name, result.return_code),
output=result.output)
return
# Timeout is not error, if we have new units generated.
if result.timed_out:
logs.log_warn('ML RNN generation for fuzzer %s timed out.' %
fuzzer_name)
new_corpus_units = (shell.get_directory_file_count(output_directory) -
old_corpus_units)
new_corpus_bytes = (shell.get_directory_size(output_directory) -
old_corpus_bytes)
if new_corpus_units:
logs.log(
'Added %d new inputs (%d bytes) using ML RNN generator for %s.' %
(new_corpus_units, new_corpus_bytes, fuzzer_name))
else:
logs.log_error('ML RNN generator did not produce any inputs for %s' %
fuzzer_name,
output=result.output)
def update_data_bundle(fuzzer, data_bundle):
"""Updates a data bundle to the latest version."""
# This module can't be in the global imports due to appengine issues
# with multiprocessing and psutil imports.
from google_cloud_utils import gsutil
# If we are using a data bundle on NFS, it is expected that our testcases
# will usually be large enough that we would fill up our tmpfs directory
# pretty quickly. So, change it to use an on-disk directory.
if not data_bundle.is_local:
testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK')
environment.set_value('FUZZ_INPUTS', testcase_disk_directory)
data_bundle_directory = get_data_bundle_directory(fuzzer.name)
if not data_bundle_directory:
logs.log_error('Failed to setup data bundle %s.' % data_bundle.name)
return False
if not shell.create_directory_if_needed(data_bundle_directory,
create_intermediates=True):
logs.log_error('Failed to create data bundle %s directory.' %
data_bundle.name)
return False
# Check if data bundle is up to date. If yes, skip the update.
if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
logs.log('Data bundle was recently synced, skip.')
return True
# Fetch lock for this data bundle.
if not _fetch_lock_for_data_bundle_update(data_bundle):
logs.log_error('Failed to lock data bundle %s.' % data_bundle.name)
return False
# Re-check if another bot did the sync already. If yes, skip.
if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
logs.log('Another bot finished the sync, skip.')
_release_lock_for_data_bundle_update(data_bundle)
return True
time_before_sync_start = time.time()
# No need to sync anything if this is a search index data bundle. In that
# case, the fuzzer will generate testcases from a gcs bucket periodically.
if not _is_search_index_data_bundle(data_bundle.name):
bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from bot.untrusted_runner import corpus_manager
from bot.untrusted_runner import file_host
worker_data_bundle_directory = file_host.rebase_to_worker_root(
data_bundle_directory)
file_host.create_directory(worker_data_bundle_directory,
create_intermediates=True)
result = corpus_manager.RemoteGSUtilRunner().rsync(
bucket_url, worker_data_bundle_directory, delete=False)
else:
result = gsutil.GSUtilRunner().rsync(bucket_url,
data_bundle_directory,
delete=False)
if result.return_code != 0:
logs.log_error('Failed to sync data bundle %s: %s.' %
(data_bundle.name, result.output))
_release_lock_for_data_bundle_update(data_bundle)
return False
# Update the testcase list file.
tests.create_testcase_list_file(data_bundle_directory)
# Write last synced time in the sync file.
sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory)
utils.write_data_to_file(time_before_sync_start, sync_file_path)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from bot.untrusted_runner import file_host
worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path)
file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path)
# Release acquired lock.
_release_lock_for_data_bundle_update(data_bundle)
return True
def get(self):
"""Handle a get request."""
try:
grouper.group_testcases()
except:
logs.log_error('Error occurred while grouping test cases.')
return
# Free up memory after group task run.
utils.python_gc()
# Get list of jobs excluded from bug filing.
excluded_jobs = get_excluded_jobs()
for testcase_id in data_handler.get_open_testcase_id_iterator():
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
# Already deleted.
continue
# Skip if testcase's job type is in exclusions list.
if testcase.job_type in excluded_jobs:
continue
# If the testcase has a bug filed already, no triage is needed.
if is_bug_filed(testcase):
continue
# Check if the crash is important, i.e. it is either a reproducible crash
# or an unreproducible crash happening frequently.
if not is_crash_important(testcase):
continue
# Require that all tasks like minimizaton, regression testing, etc have
# finished.
if not data_handler.critical_tasks_completed(testcase):
continue
# For testcases that are not part of a group, wait an additional time till
# group task completes.
# FIXME: In future, grouping might be dependent on regression range, so we
# would have to add an additional wait time.
if not testcase.group_id and not dates.time_has_expired(
testcase.timestamp,
hours=data_types.MIN_ELAPSED_TIME_SINCE_REPORT):
continue
# If this project does not have an associated issue tracker, we cannot
# file this crash anywhere.
issue_tracker_manager = issue_tracker_utils.get_issue_tracker_manager(
testcase, use_cache=True)
if not issue_tracker_manager:
continue
# If there are similar issues to this test case already filed or recently
# closed, skip filing a duplicate bug.
if is_similar_bug_open_or_recently_closed(testcase,
issue_tracker_manager):
continue
# File the bug first and then create filed bug metadata.
issue_filer.file_issue(testcase, issue_tracker_manager)
create_filed_bug_metadata(testcase)
logs.log('Filed new issue %s for testcase %d.' %
(testcase.bug_information, testcase_id))
def archive_testcase_and_dependencies_in_gcs(resource_list, testcase_path):
"""Archive testcase and its dependencies, and store in blobstore."""
if not os.path.exists(testcase_path):
logs.log_error('Unable to find testcase %s.' % testcase_path)
return None, None, None, None
absolute_filename = testcase_path
archived = False
zip_filename = None
zip_path = None
if not resource_list:
resource_list = []
# Add resource dependencies based on testcase path. These include
# stuff like extensions directory, dependency files, etc.
resource_list.extend(tests.get_resource_dependencies(testcase_path))
# Filter out duplicates, directories, and files that do not exist.
resource_list = utils.filter_file_list(resource_list)
logs.log('Testcase and related files :\n%s' % str(resource_list))
if len(resource_list) <= 1:
# If this does not have any resources, just save the testcase.
try:
file_handle = open(testcase_path, 'rb')
except IOError:
logs.log_error('Unable to open testcase %s.' % testcase_path)
return None, None, None, None
else:
# If there are resources, create an archive.
# Find the common root directory for all of the resources.
# Assumption: resource_list[0] is the testcase path.
base_directory_list = resource_list[0].split(os.path.sep)
for list_index in xrange(1, len(resource_list)):
current_directory_list = resource_list[list_index].split(
os.path.sep)
length = min(len(base_directory_list), len(current_directory_list))
for directory_index in xrange(length):
if (current_directory_list[directory_index] !=
base_directory_list[directory_index]):
base_directory_list = base_directory_list[
0:directory_index]
break
base_directory = os.path.sep.join(base_directory_list)
logs.log('Subresource common base directory: %s' % base_directory)
if base_directory:
# Common parent directory, archive sub-paths only.
base_len = len(base_directory) + len(os.path.sep)
else:
# No common parent directory, archive all paths as it-is.
base_len = 0
# Prepare the filename for the archive.
zip_filename, _ = os.path.splitext(os.path.basename(testcase_path))
zip_filename += _TESTCASE_ARCHIVE_EXTENSION
# Create the archive.
zip_path = os.path.join(environment.get_value('INPUT_DIR'),
zip_filename)
zip_file = zipfile.ZipFile(zip_path, 'w')
for file_name in resource_list:
if os.path.exists(file_name):
relative_filename = file_name[base_len:]
zip_file.write(file_name, relative_filename,
zipfile.ZIP_DEFLATED)
zip_file.close()
try:
file_handle = open(zip_path, 'rb')
except IOError:
logs.log_error('Unable to open testcase archive %s.' % zip_path)
return None, None, None, None
archived = True
absolute_filename = testcase_path[base_len:]
fuzzed_key = blobs.write_blob(file_handle)
file_handle.close()
# Don't need the archive after writing testcase to blobstore.
if zip_path:
shell.remove_file(zip_path)
return fuzzed_key, archived, absolute_filename, zip_filename
def update_issue_owner_and_ccs_from_predator_results(policy,
testcase,
issue,
only_allow_ccs=False):
"""Assign the issue to an appropriate owner if possible."""
if not issue or not issue.is_open:
return
# If the issue already has an owner, we don't need to update the bug.
if issue.assignee:
return
# If we've assigned an owner or cc once before, it likely means we were
# incorrect. Don't try again for this particular issue.
if (issue_tracker_utils.was_label_added(
issue, data_types.CHROMIUM_ISSUE_PREDATOR_AUTO_OWNER_LABEL)
or issue_tracker_utils.was_label_added(
issue, data_types.CHROMIUM_ISSUE_PREDATOR_AUTO_CC_LABEL)):
return
# If there are more than 3 suspected CLs, we can't be confident in the
# results. Just skip any sort of notification to CL authors in this case.
suspected_cls = _get_predator_result_item(testcase, 'suspected_cls')
if not suspected_cls or len(suspected_cls) > 3:
return
# Validate that the suspected CLs have all of the information we need before
# continuing. This allows us to assume that they are well-formed later,
# avoiding any potential exceptions that would interrupt this task.
for suspected_cl in suspected_cls:
url = suspected_cl.get('url')
description = suspected_cl.get('description')
author = suspected_cl.get('author')
if not url or not description or not author:
logs.log_error(
'Suspected CL for testcase %d is missing required information.'
% testcase.key.id())
return
if len(suspected_cls) == 1 and not only_allow_ccs:
suspected_cl = suspected_cls[0]
# If this owner has already been assigned before but has since been removed,
# don't assign it to them again.
for action in issue.actions:
if action.assignee == suspected_cls[0]['author']:
return
# We have high confidence for the single-CL case, so we assign the owner.
issue.labels.add(data_types.CHROMIUM_ISSUE_PREDATOR_AUTO_OWNER_LABEL)
issue.assignee = suspected_cl['author']
issue.status = policy.status('assigned')
issue_comment = (
'Automatically assigning owner based on suspected regression '
'changelist %s (%s).\n\n'
'If this is incorrect, please let us know why and apply the %s '
'label. If you aren\'t the correct owner for this issue, please '
'unassign yourself as soon as possible so it can be re-triaged.' %
(suspected_cl['url'], suspected_cl['description'],
data_types.CHROMIUM_ISSUE_PREDATOR_WRONG_CL_LABEL))
else:
if testcase.get_metadata('has_issue_ccs_from_predator_results'):
return
issue_comment = (
'Automatically adding ccs based on suspected regression changelists:'
'\n\n')
ccs_added = False
for suspected_cl in suspected_cls:
# Update the comment with the suspected CL, regardless of whether or not
# we're ccing the author. This might, for example, catch the attention of
# someone who has already been cced.
author = suspected_cl['author']
issue_comment += '%s by %s - %s\n\n' % (
suspected_cl['description'], author, suspected_cl['url'])
if author in issue.ccs:
continue
# If an author has previously been manually removed from the cc list,
# we assume they were incorrectly added. Don't try to add them again.
author_was_removed = False
for action in issue.actions:
if author in action.ccs.removed:
author_was_removed = True
break
if author_was_removed:
continue
issue.ccs.add(author)
ccs_added = True
if not ccs_added:
# Everyone we'd expect to see has already been cced on the issue. No need
# to spam it with another comment. Also, set the metadata to avoid doing
# this again.
testcase.set_metadata('has_issue_ccs_from_owners_file', True)
return
issue.labels.add(data_types.CHROMIUM_ISSUE_PREDATOR_AUTO_CC_LABEL)
issue_comment += ((
'If this is incorrect, please let us know why and apply the '
'{label_text}.').format(label_text=issue.issue_tracker.label_text(
data_types.CHROMIUM_ISSUE_PREDATOR_WRONG_CL_LABEL)))
try:
issue.save(new_comment=issue_comment, notify=True)
except HttpError:
# If we see such an error when we aren't setting an owner, it's unexpected.
if only_allow_ccs or not issue.assignee:
logs.log_error('Unable to update issue for test case %d.' %
testcase.key.id())
return
# Retry without setting the owner. They may not be a chromium project
# member, in which case we can try falling back to cc.
issue = issue_tracker_utils.get_issue_for_testcase(testcase)
update_issue_owner_and_ccs_from_predator_results(policy,
testcase,
issue,
only_allow_ccs=True)
def _sync_job(self, project, info, corpus_bucket_name, quarantine_bucket_name,
logs_bucket_name, backup_bucket_name):
"""Sync the config with ClusterFuzz."""
# Create/update ClusterFuzz jobs.
for template in get_jobs_for_project(project, info):
if template.engine == 'none':
# Engine-less jobs are not automatically managed.
continue
fuzzer_entity = self._fuzzer_entities.get(template.engine)
if not fuzzer_entity:
raise ProjectSetupError('Invalid fuzzing engine ' + template.engine)
job_name = template.job_name(project, self._config_suffix)
job = data_types.Job.query(data_types.Job.name == job_name).get()
if not job:
job = data_types.Job()
if job_name not in fuzzer_entity.jobs and not info.get('disabled', False):
# Enable new job.
fuzzer_entity.jobs.append(job_name)
job.name = job_name
if self._segregate_projects:
job.platform = untrusted.platform_name(project, 'linux')
else:
# TODO(ochang): Support other platforms?
job.platform = 'LINUX'
job.templates = template.cf_job_templates
job.environment_string = JOB_TEMPLATE.format(
build_type=self._build_type,
build_bucket_path=self._get_build_bucket_path(
project, info, template.engine, template.memory_tool,
template.architecture),
engine=template.engine,
project=project)
if self._add_revision_mappings:
revision_vars_url = self._revision_url_template.format(
project=project,
bucket=self._get_build_bucket(template.engine,
template.architecture),
sanitizer=template.memory_tool)
job.environment_string += (
'REVISION_VARS_URL = {revision_vars_url}\n'.format(
revision_vars_url=revision_vars_url))
if logs_bucket_name:
job.environment_string += 'FUZZ_LOGS_BUCKET = {logs_bucket}\n'.format(
logs_bucket=logs_bucket_name)
if corpus_bucket_name:
job.environment_string += 'CORPUS_BUCKET = {corpus_bucket}\n'.format(
corpus_bucket=corpus_bucket_name)
if quarantine_bucket_name:
job.environment_string += (
'QUARANTINE_BUCKET = {quarantine_bucket}\n'.format(
quarantine_bucket=quarantine_bucket_name))
if backup_bucket_name:
job.environment_string += 'BACKUP_BUCKET = {backup_bucket}\n'.format(
backup_bucket=backup_bucket_name)
if self._add_info_labels:
job.environment_string += (
'AUTOMATIC_LABELS = Proj-{project},Engine-{engine}\n'.format(
project=project,
engine=template.engine,
))
help_url = info.get('help_url')
if help_url:
job.environment_string += 'HELP_URL = %s\n' % help_url
if template.experimental:
job.environment_string += 'EXPERIMENTAL = True\n'
if template.minimize_job_override:
minimize_job_override = template.minimize_job_override.job_name(
project, self._config_suffix)
job.environment_string += (
'MINIMIZE_JOB_OVERRIDE = %s\n' % minimize_job_override)
view_restrictions = info.get('view_restrictions')
if view_restrictions:
if view_restrictions in ALLOWED_VIEW_RESTRICTIONS:
job.environment_string += (
'ISSUE_VIEW_RESTRICTIONS = %s\n' % view_restrictions)
else:
logs.log_error('Invalid view restriction setting %s for project %s.' %
(view_restrictions, project))
selective_unpack = info.get('selective_unpack')
if selective_unpack:
job.environment_string += 'UNPACK_ALL_FUZZ_TARGETS_AND_FILES = False\n'
if (template.engine == 'libfuzzer' and
template.architecture == 'x86_64' and
'dataflow' in info.get('fuzzing_engines', DEFAULT_ENGINES)):
# Dataflow binaries are built with dataflow sanitizer, but can be used
# as an auxiliary build with libFuzzer builds (e.g. with ASan or UBSan).
dataflow_build_bucket_path = self._get_build_bucket_path(
project_name=project,
info=info,
engine='dataflow',
memory_tool='dataflow',
architecture=template.architecture)
job.environment_string += (
'DATAFLOW_BUILD_BUCKET_PATH = %s\n' % dataflow_build_bucket_path)
if self._additional_vars:
additional_vars = {}
additional_vars.update(self._additional_vars.get('all', {}))
engine_vars = self._additional_vars.get(template.engine, {})
engine_sanitizer_vars = engine_vars.get(template.memory_tool, {})
additional_vars.update(engine_sanitizer_vars)
for key, value in sorted(six.iteritems(additional_vars)):
job.environment_string += ('{} = {}\n'.format(
key,
str(value).encode('unicode-escape').decode('utf-8')))
job.put()
def fuzz(self, target_path, options, reproducers_dir, max_time):
"""Run a fuzz session.
Args:
target_path: Path to the target.
options: The FuzzOptions object returned by prepare().
reproducers_dir: The directory to put reproducers in when crashes
are found.
max_time: Maximum allowed time for the fuzzing to run.
Returns:
A FuzzResult object.
"""
profiler.start_if_needed('libfuzzer_fuzz')
runner = libfuzzer.get_runner(target_path)
launcher.set_sanitizer_options(target_path)
# Directory to place new units.
new_corpus_dir = self._create_temp_corpus_dir('new')
corpus_directories = [new_corpus_dir] + options.fuzz_corpus_dirs
fuzz_timeout = launcher.get_fuzz_timeout(options.is_mutations_run,
total_timeout=max_time)
fuzz_result = runner.fuzz(corpus_directories,
fuzz_timeout=fuzz_timeout,
additional_args=options.arguments,
artifact_prefix=reproducers_dir,
extra_env=options.extra_env)
if (not environment.get_value('USE_MINIJAIL') and
fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE):
# Minijail returns 1 if the exit code is nonzero.
# Otherwise: we can assume that a return code of 1 means that libFuzzer
# itself ran into an error.
logs.log_error(ENGINE_ERROR_MESSAGE,
engine_output=fuzz_result.output)
log_lines = utils.decode_to_unicode(fuzz_result.output).splitlines()
# Output can be large, so save some memory by removing reference to the
# original output which is no longer needed.
fuzz_result.output = None
# Check if we crashed, and get the crash testcase path.
crash_testcase_file_path = runner.get_testcase_path(log_lines)
# Parse stats information based on libFuzzer output.
parsed_stats = launcher.parse_log_stats(log_lines)
# Extend parsed stats by additional performance features.
parsed_stats.update(
stats.parse_performance_features(log_lines,
options.strategies,
options.arguments,
include_strategies=False))
# Set some initial stat overrides.
timeout_limit = fuzzer_utils.extract_argument(options.arguments,
constants.TIMEOUT_FLAG,
remove=False)
expected_duration = runner.get_max_total_time(fuzz_timeout)
actual_duration = int(fuzz_result.time_executed)
fuzzing_time_percent = 100 * actual_duration / float(expected_duration)
parsed_stats.update({
'timeout_limit': int(timeout_limit),
'expected_duration': expected_duration,
'actual_duration': actual_duration,
'fuzzing_time_percent': fuzzing_time_percent,
})
# Remove fuzzing arguments before merge and dictionary analysis step.
arguments = options.arguments[:]
launcher.remove_fuzzing_arguments(arguments)
self._merge_new_units(target_path, options.corpus_dir, new_corpus_dir,
options.fuzz_corpus_dirs, arguments,
parsed_stats)
fuzz_logs = '\n'.join(log_lines)
crashes = []
if crash_testcase_file_path:
# Write the new testcase.
# Copy crash testcase contents into the main testcase path.
crashes.append(
engine.Crash(crash_testcase_file_path, fuzz_logs, arguments,
actual_duration))
project_qualified_fuzzer_name = (
data_types.fuzz_target_project_qualified_name(
utils.current_project(), os.path.basename(target_path)))
launcher.analyze_and_update_recommended_dictionary(
runner, project_qualified_fuzzer_name, log_lines,
options.corpus_dir, arguments)
return engine.FuzzResult(fuzz_logs, fuzz_result.command, crashes,
parsed_stats, fuzz_result.time_executed)
def fuzz(self, target_path, options, reproducers_dir, max_time):
"""Run a fuzz session.
Args:
target_path: Path to the target.
options: The FuzzOptions object returned by prepare().
reproducers_dir: The directory to put reproducers in when crashes
are found.
max_time: Maximum allowed time for the fuzzing to run.
Returns:
A FuzzResult object.
"""
profiler.start_if_needed('libfuzzer_fuzz')
runner = libfuzzer.get_runner(target_path)
libfuzzer.set_sanitizer_options(target_path, fuzz_options=options)
# Directory to place new units.
new_corpus_dir = self._create_temp_corpus_dir('new')
corpus_directories = [new_corpus_dir] + options.fuzz_corpus_dirs
fuzz_timeout = libfuzzer.get_fuzz_timeout(options.is_mutations_run,
total_timeout=max_time)
fuzz_result = runner.fuzz(corpus_directories,
fuzz_timeout=fuzz_timeout,
additional_args=options.arguments,
artifact_prefix=reproducers_dir,
extra_env=options.extra_env)
project_qualified_fuzzer_name = _project_qualified_fuzzer_name(
target_path)
dict_error_match = DICT_PARSING_FAILED_REGEX.search(fuzz_result.output)
if dict_error_match:
logs.log_error(
'Dictionary parsing failed (target={target}, line={line}).'.
format(target=project_qualified_fuzzer_name,
line=dict_error_match.group(1)),
engine_output=fuzz_result.output)
elif (not environment.get_value('USE_MINIJAIL') and
fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE):
# Minijail returns 1 if the exit code is nonzero.
# Otherwise: we can assume that a return code of 1 means that libFuzzer
# itself ran into an error.
logs.log_error(ENGINE_ERROR_MESSAGE + ' (target={target}).'.format(
target=project_qualified_fuzzer_name),
engine_output=fuzz_result.output)
log_lines = fuzz_result.output.splitlines()
# Output can be large, so save some memory by removing reference to the
# original output which is no longer needed.
fuzz_result.output = None
# Check if we crashed, and get the crash testcase path.
crash_testcase_file_path = runner.get_testcase_path(log_lines)
# If we exited with a non-zero return code with no crash file in output from
# libFuzzer, this is most likely a startup crash. Use an empty testcase to
# to store it as a crash.
if not crash_testcase_file_path and fuzz_result.return_code:
crash_testcase_file_path = self._create_empty_testcase_file(
reproducers_dir)
# Parse stats information based on libFuzzer output.
parsed_stats = libfuzzer.parse_log_stats(log_lines)
# Extend parsed stats by additional performance features.
parsed_stats.update(
stats.parse_performance_features(log_lines, options.strategies,
options.arguments))
# Set some initial stat overrides.
timeout_limit = fuzzer_utils.extract_argument(options.arguments,
constants.TIMEOUT_FLAG,
remove=False)
expected_duration = runner.get_max_total_time(fuzz_timeout)
actual_duration = int(fuzz_result.time_executed)
fuzzing_time_percent = 100 * actual_duration / float(expected_duration)
parsed_stats.update({
'timeout_limit': int(timeout_limit),
'expected_duration': expected_duration,
'actual_duration': actual_duration,
'fuzzing_time_percent': fuzzing_time_percent,
})
# Remove fuzzing arguments before merge and dictionary analysis step.
merge_arguments = options.arguments[:]
libfuzzer.remove_fuzzing_arguments(merge_arguments, is_merge=True)
self._merge_new_units(target_path, options.corpus_dir, new_corpus_dir,
options.fuzz_corpus_dirs, merge_arguments,
parsed_stats)
fuzz_logs = '\n'.join(log_lines)
crashes = []
if crash_testcase_file_path:
reproduce_arguments = options.arguments[:]
libfuzzer.remove_fuzzing_arguments(reproduce_arguments)
# Use higher timeout for reproduction.
libfuzzer.fix_timeout_argument_for_reproduction(
reproduce_arguments)
# Write the new testcase.
# Copy crash testcase contents into the main testcase path.
crashes.append(
engine.Crash(crash_testcase_file_path, fuzz_logs,
reproduce_arguments, actual_duration))
libfuzzer.analyze_and_update_recommended_dictionary(
runner, project_qualified_fuzzer_name, log_lines,
options.corpus_dir, merge_arguments)
return engine.FuzzResult(fuzz_logs, fuzz_result.command, crashes,
parsed_stats, fuzz_result.time_executed)
def run_process(cmdline,
current_working_directory=None,
timeout=DEFAULT_TEST_TIMEOUT,
need_shell=False,
gestures=None,
env_copy=None,
testcase_run=True,
ignore_children=True):
"""Executes a process with a given command line and other parameters."""
# FIXME(mbarbella): Using LAUNCHER_PATH here is error prone. It forces us to
# do certain operations before fuzzer setup (e.g. bad build check).
launcher = environment.get_value('LAUNCHER_PATH')
if environment.is_trusted_host() and testcase_run and not launcher:
from bot.untrusted_runner import remote_process_host
return remote_process_host.run_process(cmdline,
current_working_directory,
timeout, need_shell, gestures,
env_copy, testcase_run,
ignore_children)
if gestures is None:
gestures = []
if env_copy:
os.environ.update(env_copy)
# This is used when running scripts on native linux OS and not on the device.
# E.g. running a fuzzer to generate testcases or launcher script.
plt = environment.platform()
if plt in ['ANDROID', 'FUCHSIA'] and (not testcase_run or launcher):
plt = 'LINUX'
elif plt == 'IOS' and (not testcase_run or launcher):
plt = 'MAC'
# Lower down testcase timeout slightly to account for time for crash analysis.
timeout -= CRASH_ANALYSIS_TIME
# LeakSanitizer hack - give time for stdout/stderr processing.
lsan = environment.get_value('LSAN', False)
if lsan:
timeout -= LSAN_ANALYSIS_TIME
# Initialize variables.
adb_output = None
process_output = ''
process_status = None
return_code = 0
process_poll_interval = environment.get_value('PROCESS_POLL_INTERVAL', 0.5)
start_time = time.time()
watch_for_process_exit = (environment.get_value('WATCH_FOR_PROCESS_EXIT')
if plt == 'ANDROID' else True)
window_list = []
# Get gesture start time from last element in gesture list.
gestures = copy.deepcopy(gestures)
if gestures and gestures[-1].startswith('Trigger'):
gesture_start_time = int(gestures[-1].split(':')[1])
gestures.pop()
else:
gesture_start_time = timeout / 2
logs.log('Process (%s) started.' % str(cmdline), level=logging.DEBUG)
if plt == 'ANDROID':
# Clear the log upfront.
android.logger.clear_log()
# Run the app.
adb_output = android.adb.run_adb_command(cmdline, timeout=timeout)
else:
cmd, args = shell.get_command_and_arguments(cmdline)
process_output = mozprocess.processhandler.StoreOutput()
process_status = ProcessStatus()
try:
process_handle = mozprocess.ProcessHandlerMixin(
cmd,
args,
cwd=current_working_directory,
shell=need_shell,
processOutputLine=[process_output],
onFinish=[process_status],
ignore_children=ignore_children)
start_process(process_handle)
except:
logs.log_error('Exception occurred when running command: %s.' %
cmdline)
return None, None, ''
while True:
time.sleep(process_poll_interval)
# Run the gestures at gesture_start_time or in case we didn't find windows
# in the last try.
if (gestures and time.time() - start_time >= gesture_start_time
and not window_list):
# In case, we don't find any windows, we increment the gesture start time
# so that the next check is after 1 second.
gesture_start_time += 1
if plt == 'LINUX':
linux.gestures.run_gestures(gestures, process_handle.pid,
process_status, start_time,
timeout, window_list)
elif plt == 'WINDOWS':
windows.gestures.run_gestures(gestures, process_handle.pid,
process_status, start_time,
timeout, window_list)
elif plt == 'ANDROID':
android.gestures.run_gestures(gestures, start_time, timeout)
# TODO(mbarbella): We add a fake window here to prevent gestures on
# Android from getting executed more than once.
window_list = ['FAKE']
if time.time() - start_time >= timeout:
break
# Collect the process output.
output = (android.logger.log_output()
if plt == 'ANDROID' else '\n'.join(process_output.output))
if crash_analyzer.is_memory_tool_crash(output):
break
# Check if we need to bail out on process exit.
if watch_for_process_exit:
# If |watch_for_process_exit| is set, then we already completed running
# our app launch command. So, we can bail out.
if plt == 'ANDROID':
break
# On desktop, we bail out as soon as the process finishes.
if process_status and process_status.finished:
# Wait for process shutdown and set return code.
process_handle.wait(timeout=PROCESS_CLEANUP_WAIT_TIME)
break
# Process output based on platform.
if plt == 'ANDROID':
# Get current log output. If device is in reboot mode, logcat automatically
# waits for device to be online.
time.sleep(ANDROID_CRASH_LOGCAT_WAIT_TIME)
output = android.logger.log_output()
if android.device.LOW_MEMORY_REGEX.search(output):
# If the device is low on memory, we should force reboot and bail out to
# prevent device from getting in a frozen state.
logs.log('Device is low on memory, rebooting.', output=output)
android.adb.hard_reset()
android.adb.wait_for_device()
android.device.setup_memory_monitor_script_if_needed()
elif android.adb.time_since_last_reboot() < time.time() - start_time:
# Check if a reboot has happened, if yes, append log output before reboot
# and kernel logs content to output.
log_before_last_reboot = android.logger.log_output_before_last_reboot(
)
kernel_log = android.device.get_kernel_log_content()
output = '%s%s%s%s%s' % (
log_before_last_reboot,
utils.get_line_seperator('Device rebooted'), output,
utils.get_line_seperator('Kernel Log'), kernel_log)
# Make sure to reset SE Linux Permissive Mode. This can be done cheaply
# in ~0.15 sec and is needed especially between runs for kernel crashes.
android.adb.run_as_root()
android.adb.change_se_linux_to_permissive_mode()
return_code = 1
# Add output from adb to the front.
if adb_output:
output = '%s\n\n%s' % (adb_output, output)
# Kill the application if it is still running. We do this at the end to
# prevent this from adding noise to the logcat output.
task_name = environment.get_value('TASK_NAME')
child_process_termination_pattern = environment.get_value(
'CHILD_PROCESS_TERMINATION_PATTERN')
if task_name == 'fuzz' and child_process_termination_pattern:
# In some cases, we do not want to terminate the application after each
# run to avoid long startup times (e.g. for chrome). Terminate processes
# matching a particular pattern for light cleanup in this case.
android.adb.kill_processes_and_children_matching_name(
child_process_termination_pattern)
else:
# There is no special termination behavior. Simply stop the application.
android.adb.stop_application()
else:
# Get the return code in case the process has finished already.
# If the process hasn't finished, return_code will be None which is what
# callers expect unless the output indicates a crash.
return_code = process_handle.poll()
# If the process is still running, then terminate it.
if not process_status.finished:
if launcher and cmdline.startswith(launcher):
# If this was a launcher script, we KILL all child processes created
# except for APP_NAME.
# It is expected that, if the launcher script terminated normally, it
# cleans up all the child processes it created itself.
terminate_root_and_child_processes(process_handle.pid)
else:
try:
# kill() here actually sends SIGTERM on posix.
process_handle.kill()
except:
pass
if lsan:
time.sleep(LSAN_ANALYSIS_TIME)
output = '\n'.join(process_output.output)
# X Server hack when max client reached.
if ('Maximum number of clients reached' in output
or 'Unable to get connection to X server' in output):
logs.log_error('Unable to connect to X server, exiting.')
os.system('sudo killall -9 Xvfb blackbox >/dev/null 2>&1')
sys.exit(0)
if testcase_run and (crash_analyzer.is_memory_tool_crash(output)
or crash_analyzer.is_check_failure_crash(output)):
return_code = 1
# If a crash is found, then we add the memory state as well.
if return_code and plt == 'ANDROID':
ps_output = android.adb.get_ps_output()
if ps_output:
output += utils.get_line_seperator('Memory Statistics')
output += ps_output
logs.log('Process (%s) ended, exit code (%s), output (%s).' %
(str(cmdline), str(return_code), str(output)),
level=logging.DEBUG)
return return_code, round(time.time() - start_time, 1), output
def prepare(self, corpus_dir, target_path, _):
"""Prepare for a fuzzing session, by generating options. Returns a
FuzzOptions object.
Args:
corpus_dir: The main corpus directory.
target_path: Path to the target.
build_dir: Path to the build directory.
Returns:
A FuzzOptions object.
"""
arguments = fuzzer.get_arguments(target_path)
grammar = fuzzer.get_grammar(target_path)
strategy_pool = strategy_selection.generate_weighted_strategy_pool(
strategy_list=strategy.LIBFUZZER_STRATEGY_LIST,
use_generator=True,
engine_name=self.name)
strategy_info = libfuzzer.pick_strategies(strategy_pool, target_path,
corpus_dir, arguments,
grammar)
arguments.extend(strategy_info.arguments)
# Check for seed corpus and add it into corpus directory.
engine_common.unpack_seed_corpus_if_needed(target_path, corpus_dir)
# Pick a few testcases from our corpus to use as the initial corpus.
subset_size = engine_common.random_choice(
engine_common.CORPUS_SUBSET_NUM_TESTCASES)
if (not strategy_info.use_dataflow_tracing
and strategy_pool.do_strategy(strategy.CORPUS_SUBSET_STRATEGY)
and shell.get_directory_file_count(corpus_dir) > subset_size):
# Copy |subset_size| testcases into 'subset' directory.
corpus_subset_dir = self._create_temp_corpus_dir('subset')
libfuzzer.copy_from_corpus(corpus_subset_dir, corpus_dir,
subset_size)
strategy_info.fuzzing_strategies.append(
strategy.CORPUS_SUBSET_STRATEGY.name + '_' + str(subset_size))
strategy_info.additional_corpus_dirs.append(corpus_subset_dir)
else:
strategy_info.additional_corpus_dirs.append(corpus_dir)
# Check dict argument to make sure that it's valid.
dict_path = fuzzer_utils.extract_argument(arguments,
constants.DICT_FLAG,
remove=False)
if dict_path and not os.path.exists(dict_path):
logs.log_error('Invalid dict %s for %s.' %
(dict_path, target_path))
fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG)
# If there's no dict argument, check for %target_binary_name%.dict file.
dict_path = fuzzer_utils.extract_argument(arguments,
constants.DICT_FLAG,
remove=False)
if not dict_path:
dict_path = dictionary_manager.get_default_dictionary_path(
target_path)
if os.path.exists(dict_path):
arguments.append(constants.DICT_FLAG + dict_path)
# If we have a dictionary, correct any items that are not formatted properly
# (e.g. quote items that are missing them).
dictionary_manager.correct_if_needed(dict_path)
strategies = stats.process_strategies(strategy_info.fuzzing_strategies,
name_modifier=lambda x: x)
return LibFuzzerOptions(corpus_dir, arguments, strategies,
strategy_info.additional_corpus_dirs,
strategy_info.extra_env,
strategy_info.use_dataflow_tracing,
strategy_info.is_mutations_run)
def main(argv):
"""Run libFuzzer as specified by argv."""
atexit.register(fuzzer_utils.cleanup)
# Initialize variables.
arguments = argv[1:]
testcase_file_path = arguments.pop(0)
target_name = arguments.pop(0)
fuzzer_name = data_types.fuzz_target_project_qualified_name(
utils.current_project(), target_name)
# Initialize log handler.
logs.configure(
'run_fuzzer', {
'fuzzer': fuzzer_name,
'engine': 'libFuzzer',
'job_name': environment.get_value('JOB_NAME')
})
profiler.start_if_needed('libfuzzer_launcher')
# Make sure that the fuzzer binary exists.
build_directory = environment.get_value('BUILD_DIR')
fuzzer_path = engine_common.find_fuzzer_path(build_directory, target_name)
if not fuzzer_path:
# This is an expected case when doing regression testing with old builds
# that do not have that fuzz target. It can also happen when a host sends a
# message to an untrusted worker that just restarted and lost information on
# build directory.
logs.log_warn('Could not find fuzz target %s.' % target_name)
return
# Install signal handler.
signal.signal(signal.SIGTERM, engine_common.signal_term_handler)
# Set up temp dir.
engine_common.recreate_directory(fuzzer_utils.get_temp_dir())
# Setup minijail if needed.
use_minijail = environment.get_value('USE_MINIJAIL')
runner = libfuzzer.get_runner(fuzzer_path,
temp_dir=fuzzer_utils.get_temp_dir())
if use_minijail:
minijail_chroot = runner.chroot
else:
minijail_chroot = None
# Get corpus directory.
corpus_directory = environment.get_value('FUZZ_CORPUS_DIR')
# Add common arguments which are necessary to be used for every run.
arguments = expand_with_common_arguments(arguments)
# Add sanitizer options to environment that were specified in the .options
# file and options that this script requires.
set_sanitizer_options(fuzzer_path)
# Minimize test argument.
minimize_to = fuzzer_utils.extract_argument(arguments,
MINIMIZE_TO_ARGUMENT)
minimize_timeout = fuzzer_utils.extract_argument(
arguments, MINIMIZE_TIMEOUT_ARGUMENT)
if minimize_to and minimize_timeout:
minimize_testcase(runner, testcase_file_path, minimize_to,
int(minimize_timeout), arguments, use_minijail)
return
# Cleanse argument.
cleanse_to = fuzzer_utils.extract_argument(arguments, CLEANSE_TO_ARGUMENT)
cleanse_timeout = fuzzer_utils.extract_argument(arguments,
CLEANSE_TIMEOUT_ARGUMENT)
if cleanse_to and cleanse_timeout:
cleanse_testcase(runner, testcase_file_path, cleanse_to,
int(cleanse_timeout), arguments, use_minijail)
return
# If we don't have a corpus, then that means this is not a fuzzing run.
if not corpus_directory:
load_testcase_if_exists(runner, testcase_file_path, fuzzer_name,
use_minijail, arguments)
return
# We don't have a crash testcase, fuzz.
# Check dict argument to make sure that it's valid.
dict_argument = fuzzer_utils.extract_argument(arguments,
constants.DICT_FLAG,
remove=False)
if dict_argument and not os.path.exists(dict_argument):
logs.log_error('Invalid dict %s for %s.' %
(dict_argument, fuzzer_name))
fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG)
# If there's no dict argument, check for %target_binary_name%.dict file.
if (not fuzzer_utils.extract_argument(
arguments, constants.DICT_FLAG, remove=False)):
default_dict_path = dictionary_manager.get_default_dictionary_path(
fuzzer_path)
if os.path.exists(default_dict_path):
arguments.append(constants.DICT_FLAG + default_dict_path)
fuzzing_strategies = []
# Select a generator to use for existing testcase mutations.
generator = _select_generator()
is_mutations_run = generator != Generator.NONE
# Timeout for fuzzer run.
fuzz_timeout = get_fuzz_timeout(is_mutations_run)
# Get list of corpus directories.
corpus_directories = get_corpus_directories(corpus_directory, fuzzer_path,
fuzzing_strategies,
minijail_chroot)
# Bind corpus directories in minijail.
if use_minijail:
artifact_prefix = constants.ARTIFACT_PREFIX_FLAG + '/'
else:
artifact_prefix = '%s%s/' % (constants.ARTIFACT_PREFIX_FLAG,
os.path.abspath(
os.path.dirname(testcase_file_path)))
# Generate new testcase mutations using radamsa, etc.
if is_mutations_run:
new_testcase_mutations_directory = generate_new_testcase_mutations(
corpus_directory, fuzzer_name, generator, fuzzing_strategies)
corpus_directories.append(new_testcase_mutations_directory)
if use_minijail:
bind_corpus_dirs(minijail_chroot,
[new_testcase_mutations_directory])
max_len_argument = fuzzer_utils.extract_argument(arguments,
constants.MAX_LEN_FLAG,
remove=False)
if not max_len_argument and do_random_max_length():
max_length = random.SystemRandom().randint(1, MAX_VALUE_FOR_MAX_LENGTH)
arguments.append('%s%d' % (constants.MAX_LEN_FLAG, max_length))
fuzzing_strategies.append(strategy.RANDOM_MAX_LENGTH_STRATEGY)
if do_recommended_dictionary():
if add_recommended_dictionary(arguments, fuzzer_name, fuzzer_path):
fuzzing_strategies.append(strategy.RECOMMENDED_DICTIONARY_STRATEGY)
if do_value_profile():
arguments.append(constants.VALUE_PROFILE_ARGUMENT)
fuzzing_strategies.append(strategy.VALUE_PROFILE_STRATEGY)
if do_fork():
max_fuzz_threads = environment.get_value('MAX_FUZZ_THREADS', 1)
num_fuzz_processes = max(
1,
multiprocessing.cpu_count() / max_fuzz_threads)
arguments.append('%s%d' % (constants.FORK_FLAG, num_fuzz_processes))
fuzzing_strategies.append('%s_%d' %
(strategy.FORK_STRATEGY, num_fuzz_processes))
extra_env = {}
if do_mutator_plugin():
if use_mutator_plugin(target_name, extra_env, minijail_chroot):
fuzzing_strategies.append(strategy.MUTATOR_PLUGIN_STRATEGY)
# Execute the fuzzer binary with original arguments.
fuzz_result = runner.fuzz(corpus_directories,
fuzz_timeout=fuzz_timeout,
additional_args=arguments + [artifact_prefix],
extra_env=extra_env)
if (not use_minijail
and fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE):
# Minijail returns 1 if the exit code is nonzero.
# Otherwise: we can assume that a return code of 1 means that libFuzzer
# itself ran into an error.
logs.log_error(ENGINE_ERROR_MESSAGE, engine_output=fuzz_result.output)
log_lines = fuzz_result.output.splitlines()
# Output can be large, so save some memory by removing reference to the
# original output which is no longer needed.
fuzz_result.output = None
# Check if we crashed, and get the crash testcase path.
crash_testcase_file_path = None
for line in log_lines:
match = re.match(CRASH_TESTCASE_REGEX, line)
if match:
crash_testcase_file_path = match.group(1)
break
if crash_testcase_file_path:
# Write the new testcase.
if use_minijail:
# Convert chroot relative path to host path. Remove the leading '/' before
# joining.
crash_testcase_file_path = os.path.join(
minijail_chroot.directory, crash_testcase_file_path[1:])
# Copy crash testcase contents into the main testcase path.
shutil.move(crash_testcase_file_path, testcase_file_path)
# Print the command output.
log_header_format = ('Command: %s\n' 'Bot: %s\n' 'Time ran: %f\n')
bot_name = environment.get_value('BOT_NAME', '')
command = fuzz_result.command
if use_minijail:
# Remove minijail prefix.
command = engine_common.strip_minijail_command(command, fuzzer_path)
print(log_header_format % (engine_common.get_command_quoted(command),
bot_name, fuzz_result.time_executed))
# Parse stats information based on libFuzzer output.
parsed_stats = parse_log_stats(log_lines)
# Extend parsed stats by additional performance features.
parsed_stats.update(
stats.parse_performance_features(log_lines, fuzzing_strategies,
arguments))
# Set some initial stat overrides.
timeout_limit = fuzzer_utils.extract_argument(arguments,
constants.TIMEOUT_FLAG,
remove=False)
expected_duration = runner.get_max_total_time(fuzz_timeout)
actual_duration = int(fuzz_result.time_executed)
fuzzing_time_percent = 100 * actual_duration / float(expected_duration)
stat_overrides = {
'timeout_limit': int(timeout_limit),
'expected_duration': expected_duration,
'actual_duration': actual_duration,
'fuzzing_time_percent': fuzzing_time_percent,
}
# Remove fuzzing arguments before merge and dictionary analysis step.
remove_fuzzing_arguments(arguments)
# Make a decision on whether merge step is needed at all. If there are no
# new units added by libFuzzer run, then no need to do merge at all.
new_units_added = parsed_stats.get('new_units_added', 0)
merge_error = None
if new_units_added:
# Merge the new units with the initial corpus.
if corpus_directory not in corpus_directories:
corpus_directories.append(corpus_directory)
# If this times out, it's possible that we will miss some units. However, if
# we're taking >10 minutes to load/merge the corpus something is going very
# wrong and we probably don't want to make things worse by adding units
# anyway.
merge_tmp_dir = None
if not use_minijail:
merge_tmp_dir = os.path.join(fuzzer_utils.get_temp_dir(),
'merge_workdir')
engine_common.recreate_directory(merge_tmp_dir)
old_corpus_len = shell.get_directory_file_count(corpus_directory)
merge_directory = create_merge_directory()
corpus_directories.insert(0, merge_directory)
if use_minijail:
bind_corpus_dirs(minijail_chroot, [merge_directory])
merge_result = runner.merge(
corpus_directories,
merge_timeout=engine_common.get_merge_timeout(
DEFAULT_MERGE_TIMEOUT),
tmp_dir=merge_tmp_dir,
additional_args=arguments)
move_mergeable_units(merge_directory, corpus_directory)
new_corpus_len = shell.get_directory_file_count(corpus_directory)
new_units_added = 0
merge_error = None
if merge_result.timed_out:
merge_error = 'Merging new testcases timed out:'
elif merge_result.return_code != 0:
merge_error = 'Merging new testcases failed:'
else:
new_units_added = new_corpus_len - old_corpus_len
stat_overrides['new_units_added'] = new_units_added
if merge_result.output:
stat_overrides.update(
stats.parse_stats_from_merge_log(
merge_result.output.splitlines()))
else:
stat_overrides['new_units_added'] = 0
logs.log('Skipped corpus merge since no new units added by fuzzing.')
# Get corpus size after merge. This removes the duplicate units that were
# created during this fuzzing session.
stat_overrides['corpus_size'] = shell.get_directory_file_count(
corpus_directory)
# Delete all corpus directories except for the main one. These were temporary
# directories to store new testcase mutations and have already been merged to
# main corpus directory.
if corpus_directory in corpus_directories:
corpus_directories.remove(corpus_directory)
for directory in corpus_directories:
shutil.rmtree(directory, ignore_errors=True)
if use_minijail:
unbind_corpus_dirs(minijail_chroot, corpus_directories)
# Apply overridden stats to the parsed stats prior to dumping.
parsed_stats.update(stat_overrides)
# Dump stats data for further uploading to BigQuery.
engine_common.dump_big_query_data(parsed_stats, testcase_file_path,
LIBFUZZER_PREFIX, fuzzer_name, command)
# Add custom crash state based on fuzzer name (if needed).
add_custom_crash_state_if_needed(fuzzer_name, log_lines, parsed_stats)
for line in log_lines:
print(line)
# Add fuzzing strategies used.
engine_common.print_fuzzing_strategies(fuzzing_strategies)
# Add merge error (if any).
if merge_error:
print(data_types.CRASH_STACKTRACE_END_MARKER)
print(merge_error)
print(
'Command:',
get_printable_command(merge_result.command, fuzzer_path,
use_minijail))
print(merge_result.output)
analyze_and_update_recommended_dictionary(runner, fuzzer_name, log_lines,
corpus_directory, arguments)
# Close minijail chroot.
if use_minijail:
minijail_chroot.close()
# Record the stats to make them easily searchable in stackdriver.
if new_units_added:
logs.log('New units added to corpus: %d.' % new_units_added,
stats=parsed_stats)
else:
logs.log('No new units found.', stats=parsed_stats)
