def get(self): """Handle a get request.""" try: grouper.group_testcases() except: logs.log_error('Error occurred while grouping test cases.') return # Free up memory after group task run. utils.python_gc() # Get a list of jobs excluded from bug filing. excluded_jobs = _get_excluded_jobs() # Get a list of all jobs. This is used to filter testcases whose jobs have # been removed. all_jobs = data_handler.get_all_job_type_names() for testcase_id in data_handler.get_open_testcase_id_iterator(): try: testcase = data_handler.get_testcase_by_id(testcase_id) except errors.InvalidTestcaseError: # Already deleted. continue # Skip if testcase's job is removed. if testcase.job_type not in all_jobs: continue # Skip if testcase's job is in exclusions list. if testcase.job_type in excluded_jobs: continue # Skip if we are running progression task at this time. if testcase.get_metadata('progression_pending'): continue # If the testcase has a bug filed already, no triage is needed. if _is_bug_filed(testcase): continue # Check if the crash is important, i.e. it is either a reproducible crash # or an unreproducible crash happening frequently. if not _is_crash_important(testcase): continue # Require that all tasks like minimizaton, regression testing, etc have # finished. if not data_handler.critical_tasks_completed(testcase): continue # For testcases that are not part of a group, wait an additional time till # group task completes. # FIXME: In future, grouping might be dependent on regression range, so we # would have to add an additional wait time. if not testcase.group_id and not dates.time_has_expired( testcase.timestamp, hours=data_types.MIN_ELAPSED_TIME_SINCE_REPORT): continue # If this project does not have an associated issue tracker, we cannot # file this crash anywhere. issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase( testcase) if not issue_tracker: issue_filer.notify_issue_update(testcase, 'new') continue # If there are similar issues to this test case already filed or recently # closed, skip filing a duplicate bug. if _check_and_update_similar_bug(testcase, issue_tracker): continue # Clean up old triage messages that would be not applicable now. testcase.delete_metadata(TRIAGE_MESSAGE_KEY, update_testcase=False) # File the bug first and then create filed bug metadata. try: issue_filer.file_issue(testcase, issue_tracker) except Exception: logs.log_error('Failed to file issue for testcase %d.' % testcase_id) continue _create_filed_bug_metadata(testcase) issue_filer.notify_issue_update(testcase, 'new') logs.log('Filed new issue %s for testcase %d.' % (testcase.bug_information, testcase_id))
def get(self): """Handles a GET request.""" libfuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == 'libFuzzer').get() if not libfuzzer: logs.log_error('Failed to get libFuzzer Fuzzer entity.') return afl = data_types.Fuzzer.query(data_types.Fuzzer.name == 'afl').get() if not afl: logs.log_error('Failed to get AFL Fuzzer entity.') return honggfuzz = data_types.Fuzzer.query( data_types.Fuzzer.name == 'honggfuzz').get() if not honggfuzz: logs.log_error('Failed to get honggfuzz Fuzzer entity.') return project_setup_configs = local_config.ProjectConfig().get('project_setup') for setup_config in project_setup_configs: bucket_config = setup_config.get('build_buckets') if not bucket_config: raise ProjectSetupError('Project setup buckets not specified.') config = ProjectSetup( BUILD_BUCKET_PATH_TEMPLATE, REVISION_URL, setup_config.get('build_type'), config_suffix=setup_config.get('job_suffix', ''), segregate_projects=setup_config.get('segregate_projects', False), engine_build_buckets={ 'libfuzzer': bucket_config.get('libfuzzer'), 'libfuzzer-i386': bucket_config.get('libfuzzer_i386'), 'afl': bucket_config.get('afl'), 'honggfuzz': bucket_config.get('honggfuzz'), 'none': bucket_config.get('no_engine'), 'dataflow': bucket_config.get('dataflow'), }, fuzzer_entities={ 'libfuzzer': libfuzzer, 'honggfuzz': honggfuzz, 'afl': afl, }, add_info_labels=setup_config.get('add_info_labels', False), add_revision_mappings=setup_config.get('add_revision_mappings', False), additional_vars=setup_config.get('additional_vars')) projects_source = setup_config.get('source') if projects_source == 'oss-fuzz': projects = get_oss_fuzz_projects() elif projects_source.startswith(storage.GS_PREFIX): projects = get_projects_from_gcs(projects_source) else: raise ProjectSetupError('Invalid projects source: ' + projects_source) if not projects: raise ProjectSetupError('Missing projects list.') config.set_up(projects)
def _minimize_corpus_two_step(self, target_path, arguments, existing_corpus_dirs, new_corpus_dir, output_corpus_dir, reproducers_dir, max_time): """Optional (but recommended): run corpus minimization. Args: target_path: Path to the target. arguments: Additional arguments needed for corpus minimization. existing_corpus_dirs: Input corpora that existed before the fuzzing run. new_corpus_dir: Input corpus that was generated during the fuzzing run. Must have at least one new file. output_corpus_dir: Output directory to place minimized corpus. reproducers_dir: The directory to put reproducers in when crashes are found. max_time: Maximum allowed time for the minimization. Returns: A Result object. """ if not _is_multistep_merge_supported(target_path): # Fallback to the old single step merge. It does not support incremental # stats and provides only `edge_coverage` and `feature_coverage` stats. logs.log( 'Old version of libFuzzer is used. Using single step merge.') return self.minimize_corpus( target_path, arguments, existing_corpus_dirs + [new_corpus_dir], output_corpus_dir, reproducers_dir, max_time) # The dir where merge control file is located must persist for both merge # steps. The second step re-uses the MCF produced during the first step. merge_control_file_dir = self._create_temp_corpus_dir('mcf_tmp_dir') self._merge_control_file = os.path.join(merge_control_file_dir, 'MCF') # Two step merge process to obtain accurate stats for the new corpus units. # See https://reviews.llvm.org/D66107 for a more detailed description. merge_stats = {} # Step 1. Use only existing corpus and collect "initial" stats. result_1 = self.minimize_corpus(target_path, arguments, existing_corpus_dirs, output_corpus_dir, reproducers_dir, max_time) merge_stats['initial_edge_coverage'] = result_1.stats['edge_coverage'] merge_stats['initial_feature_coverage'] = result_1.stats[ 'feature_coverage'] # Clear the output dir as it does not have any new units at this point. engine_common.recreate_directory(output_corpus_dir) # Adjust the time limit for the time we spent on the first merge step. max_time -= result_1.time_executed if max_time <= 0: raise TimeoutError('Merging new testcases timed out\n' + result_1.logs) # Step 2. Process the new corpus units as well. result_2 = self.minimize_corpus( target_path, arguments, existing_corpus_dirs + [new_corpus_dir], output_corpus_dir, reproducers_dir, max_time) merge_stats['edge_coverage'] = result_2.stats['edge_coverage'] merge_stats['feature_coverage'] = result_2.stats['feature_coverage'] # Diff the stats to obtain accurate values for the new corpus units. merge_stats['new_edges'] = (merge_stats['edge_coverage'] - merge_stats['initial_edge_coverage']) merge_stats['new_features'] = (merge_stats['feature_coverage'] - merge_stats['initial_feature_coverage']) output = result_1.logs + '\n\n' + result_2.logs if (merge_stats['new_edges'] < 0 or merge_stats['new_features'] < 0): logs.log_error('Two step merge failed.', merge_stats=merge_stats, output=output) merge_stats['new_edges'] = 0 merge_stats['new_features'] = 0 self._merge_control_file = None # TODO(ochang): Get crashes found during merge. return engine.FuzzResult( output, result_2.command, [], merge_stats, result_1.time_executed + result_2.time_executed)
def update_fuzzer_and_data_bundles(fuzzer_name): """Update the fuzzer with a given name if necessary.""" fuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == fuzzer_name).get() if not fuzzer: logs.log_error('No fuzzer exists with name %s.' % fuzzer_name) raise errors.InvalidFuzzerError # Set some helper environment variables. fuzzer_directory = get_fuzzer_directory(fuzzer_name) environment.set_value('FUZZER_DIR', fuzzer_directory) environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content) # Adjust the test timeout, if user has provided one. if fuzzer.timeout: environment.set_value('TEST_TIMEOUT', fuzzer.timeout) # Increase fuzz test timeout if the fuzzer timeout is higher than its # current value. fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT') if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout: environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout) # Adjust the max testcases if this fuzzer has specified a lower limit. max_testcases = environment.get_value('MAX_TESTCASES') if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases: environment.set_value('MAX_TESTCASES', fuzzer.max_testcases) # Check for updates to this fuzzer. version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name) if (not fuzzer.builtin and revisions.needs_update(version_file, fuzzer.revision)): logs.log('Fuzzer update was found, updating.') # Clear the old fuzzer directory if it exists. if not shell.remove_directory(fuzzer_directory, recreate=True): logs.log_error('Failed to clear fuzzer directory.') return False # Copy the archive to local disk and unpack it. archive_path = os.path.join(fuzzer_directory, fuzzer.filename) if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path): logs.log_error('Failed to copy fuzzer archive.') return False try: archive.unpack(archive_path, fuzzer_directory) except Exception: error_message = ( 'Failed to unpack fuzzer archive %s ' '(bad archive or unsupported format).') % fuzzer.filename logs.log_error(error_message) fuzzer_logs.upload_script_log('Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return False fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path) if not os.path.exists(fuzzer_path): error_message = ( 'Fuzzer executable %s not found. ' 'Check fuzzer configuration.') % fuzzer.executable_path logs.log_error(error_message) fuzzer_logs.upload_script_log('Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return False # Make fuzzer executable. os.chmod(fuzzer_path, 0750) # Cleanup unneeded archive. shell.remove_file(archive_path) # Save the current revision of this fuzzer in a file for later checks. revisions.write_revision_to_revision_file(version_file, fuzzer.revision) logs.log('Updated fuzzer to revision %d.' % fuzzer.revision) # Setup data bundles associated with this fuzzer. data_bundles = ndb_utils.get_all_from_query( data_types.DataBundle.query( data_types.DataBundle.name == fuzzer.data_bundle_name)) for data_bundle in data_bundles: if not update_data_bundle(fuzzer, data_bundle): return False # Setup environment variable for launcher script path. if fuzzer.launcher_script: fuzzer_launcher_path = utils.get_launch_path_for_script( fuzzer_directory, fuzzer.launcher_script) environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path) return True
def setup_testcase(testcase): """Sets up the testcase and needed dependencies like fuzzer, data bundle, etc.""" fuzzer_name = testcase.fuzzer_name job_type = testcase.job_type task_name = environment.get_value('TASK_NAME') testcase_fail_wait = environment.get_value('FAIL_WAIT') testcase_id = testcase.key.id() # Clear testcase directories. shell.clear_testcase_directories() # Setup memory debugging tool environment. environment.reset_current_memory_tool_options( redzone_size=testcase.redzone) # Adjust the test timeout value if this is coming from an user uploaded # testcase. _set_timeout_value_from_user_upload(testcase_id) if task_name == 'minimize': # Allow minimizing with a different fuzzer set up. minimize_fuzzer_override = environment.get_value( 'MINIMIZE_FUZZER_OVERRIDE') fuzzer_name = minimize_fuzzer_override or fuzzer_name # Update the fuzzer if necessary in order to get the updated data bundle. if fuzzer_name: try: update_successful = update_fuzzer_and_data_bundles(fuzzer_name) except errors.InvalidFuzzerError: # Close testcase and don't recreate tasks if this fuzzer is invalid. testcase.open = False testcase.fixed = 'NA' testcase.set_metadata('fuzzer_was_deleted', True) logs.log_error('Closed testcase %d with invalid fuzzer %s.' % (testcase_id, fuzzer_name)) error_message = 'Fuzzer %s no longer exists.' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) return None, None, None if not update_successful: error_message = 'Unable to setup fuzzer %s.' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task(task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # Extract the testcase and any of its resources to the input directory. file_list, input_directory, testcase_file_path = unpack_testcase(testcase) if not file_list: error_message = 'Unable to setup testcase %s.' % testcase_file_path data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task(task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # For Android/Fuchsia, we need to sync our local testcases directory with the # one on the device. if environment.platform() == 'ANDROID': _copy_testcase_to_device_and_setup_environment(testcase, testcase_file_path) if environment.platform() == 'FUCHSIA': fuchsia.device.copy_testcase_to_device(testcase_file_path) # Push testcases to worker. if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.push_testcases_to_worker() # Copy global blacklist into local blacklist. is_lsan_enabled = environment.get_value('LSAN') if is_lsan_enabled: # Get local blacklist without this testcase's entry. leak_blacklist.copy_global_to_local_blacklist( excluded_testcase=testcase) # Setup environment variable for windows size and location properties. # Explicitly use empty string to indicate use of default window properties. if hasattr(testcase, 'window_argument'): environment.set_value('WINDOW_ARG', testcase.window_argument) # Adjust timeout based on the stored multiplier (if available). if hasattr(testcase, 'timeout_multiplier') and testcase.timeout_multiplier: test_timeout = environment.get_value('TEST_TIMEOUT') environment.set_value('TEST_TIMEOUT', int(test_timeout * testcase.timeout_multiplier)) # Override APP_ARGS with minimized arguments (if available). if (hasattr(testcase, 'minimized_arguments') and testcase.minimized_arguments): environment.set_value('APP_ARGS', testcase.minimized_arguments) # Add FUZZ_TARGET to environment if this is a fuzz target testcase. fuzz_target = testcase.get_metadata('fuzzer_binary_name') if fuzz_target: environment.set_value('FUZZ_TARGET', fuzz_target) return file_list, input_directory, testcase_file_path
def main(): """Prepare the configuration options and start requesting tasks.""" logs.configure('run_bot') root_directory = environment.get_value('ROOT_DIR') if not root_directory: print( 'Please set ROOT_DIR environment variable to the root of the source ' 'checkout before running. Exiting.') print('For an example, check init.bash in the local directory.') return dates.initialize_timezone_from_environment() environment.set_bot_environment() monitor.initialize() if not profiler.start_if_needed('python_profiler_bot'): sys.exit(-1) fuzzers_init.run() if environment.is_trusted_host(ensure_connected=False): from bot.untrusted_runner import host host.init() if environment.is_untrusted_worker(): # Track revision since we won't go into the task_loop. update_task.track_revision() from bot.untrusted_runner import untrusted as untrusted_worker untrusted_worker.start_server() assert False, 'Unreachable code' while True: # task_loop should be an infinite loop, # unless we run into an exception. error_stacktrace, clean_exit, task_payload = task_loop() # Print the error trace to the console. if not clean_exit: print('Exception occurred while running "%s".' % task_payload) print('-' * 80) print(error_stacktrace) print('-' * 80) should_terminate = (clean_exit or errors.error_in_list( error_stacktrace, errors.BOT_ERROR_TERMINATION_LIST)) if should_terminate: return logs.log_error('Task exited with exception (payload="%s").' % task_payload, error_stacktrace=error_stacktrace) should_hang = errors.error_in_list(error_stacktrace, errors.BOT_ERROR_HANG_LIST) if should_hang: logs.log('Start hanging forever.') while True: # Sleep to avoid consuming 100% of CPU. time.sleep(60) # See if our run timed out, if yes bail out. if data_handler.bot_run_timed_out(): return # Clear the current exception. utils.exc_clear()
def setup_testcase(testcase, job_type, fuzzer_override=None): """Sets up the testcase and needed dependencies like fuzzer, data bundle, etc.""" fuzzer_name = fuzzer_override or testcase.fuzzer_name task_name = environment.get_value('TASK_NAME') testcase_fail_wait = environment.get_value('FAIL_WAIT') testcase_id = testcase.key.id() # Clear testcase directories. shell.clear_testcase_directories() # Adjust the test timeout value if this is coming from an user uploaded # testcase. _set_timeout_value_from_user_upload(testcase_id) # Update the fuzzer if necessary in order to get the updated data bundle. if fuzzer_name: try: update_successful = update_fuzzer_and_data_bundles(fuzzer_name) except errors.InvalidFuzzerError: # Close testcase and don't recreate tasks if this fuzzer is invalid. testcase.open = False testcase.fixed = 'NA' testcase.set_metadata('fuzzer_was_deleted', True) logs.log_error('Closed testcase %d with invalid fuzzer %s.' % (testcase_id, fuzzer_name)) error_message = 'Fuzzer %s no longer exists' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) return None, None, None if not update_successful: error_message = 'Unable to setup fuzzer %s' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task(task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # Extract the testcase and any of its resources to the input directory. file_list, input_directory, testcase_file_path = unpack_testcase(testcase) if not file_list: error_message = 'Unable to setup testcase %s' % testcase_file_path data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task(task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # For Android/Fuchsia, we need to sync our local testcases directory with the # one on the device. if environment.platform() == 'ANDROID': _copy_testcase_to_device_and_setup_environment(testcase, testcase_file_path) # Push testcases to worker. if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.push_testcases_to_worker() # Copy global blacklist into local blacklist. is_lsan_enabled = environment.get_value('LSAN') if is_lsan_enabled: # Get local blacklist without this testcase's entry. leak_blacklist.copy_global_to_local_blacklist( excluded_testcase=testcase) prepare_environment_for_testcase(testcase, job_type, task_name) return file_list, input_directory, testcase_file_path
def wait_for_battery_charge_if_needed(): """Check device battery and make sure it is charged beyond minimum level and temperature thresholds.""" # Battery levels are not applicable on GCE. if adb.is_gce(): return # Make sure device is online. adb.wait_for_device() # Skip battery check if done recently. last_battery_check_time = persistent_cache.get_value( LAST_BATTERY_CHECK_TIME_KEY, constructor=datetime.datetime.utcfromtimestamp) if last_battery_check_time and not dates.time_has_expired( last_battery_check_time, seconds=BATTERY_CHECK_INTERVAL): return # Initialize variables. battery_level_threshold = environment.get_value('LOW_BATTERY_LEVEL_THRESHOLD', LOW_BATTERY_LEVEL_THRESHOLD) battery_temperature_threshold = environment.get_value( 'MAX_BATTERY_TEMPERATURE_THRESHOLD', MAX_BATTERY_TEMPERATURE_THRESHOLD) device_restarted = False while 1: battery_information = get_battery_information() if battery_information is None: logs.log_error('Failed to get battery information, skipping check.') return battery_level = battery_information['level'] battery_temperature = battery_information['temperature'] logs.log('Battery information: level (%d%%), temperature (%.1f celsius).' % (battery_level, battery_temperature)) if (battery_level >= battery_level_threshold and battery_temperature <= battery_temperature_threshold): persistent_cache.set_value(LAST_BATTERY_CHECK_TIME_KEY, time.time()) return logs.log('Battery in bad battery state, putting device in sleep mode.') if not device_restarted: reboot() adb.disable_wifi() device_restarted = True # Change thresholds to expected levels (only if they were below minimum # thresholds). if battery_level < battery_level_threshold: battery_level_threshold = EXPECTED_BATTERY_LEVEL if battery_temperature > battery_temperature_threshold: battery_temperature_threshold = EXPECTED_BATTERY_TEMPERATURE # Stopping shell should help with shutting off a lot of services that would # otherwise use up the battery. However, we need to turn it back on to get # battery status information. Also, turn off display explicitly (needed for # Nexus 9s). turn_off_display_if_needed() adb.stop_shell() time.sleep(BATTERY_CHARGE_INTERVAL) adb.start_shell()
'*.txt', '-x', '*-expected.*', '-x', '*.git*', '-x', '*.svn*', ], cwd=tests_directory) subprocess.check_call( ['gsutil', 'cp', tests_archive_local, tests_archive_remote]) logs.log('Completed cycle, sleeping for %s seconds.' % sync_interval) time.sleep(sync_interval) if __name__ == '__main__': # Make sure environment is correctly configured. logs.configure('run_bot') environment.set_bot_environment() fail_wait = environment.get_value('FAIL_WAIT') # Continue this forever. while 1: try: main() except Exception: logs.log_error('Failed to sync tests.') time.sleep(fail_wait)
def flash_to_latest_build_if_needed(): """Wipes user data, resetting the device to original factory state.""" run_timeout = environment.get_value('RUN_TIMEOUT') if run_timeout: # If we have a run timeout, then we are already scheduled to bail out and # will be probably get re-imaged. E.g. using frameworks like Tradefed. return # Check if a flash is needed based on last recorded flash time. last_flash_time = persistent_cache.get_value( LAST_FLASH_TIME_KEY, constructor=datetime.datetime.utcfromtimestamp) needs_flash = last_flash_time is None or dates.time_has_expired( last_flash_time, seconds=adb.FLASH_INTERVAL) if not needs_flash: return build_info = {} is_remote_device = environment.get_value('ANDROID_GCE') if is_remote_device: # To prevent thousands of devices all trying to reimage at the same time, # reimages are done at a random time in the future. scheduled_reimage_time = persistent_cache.get_value( SCHEDULED_GCE_REIMAGE_TIME_KEY, constructor=datetime.datetime.utcfromtimestamp) if scheduled_reimage_time is None: # No reimage scheduled yet, so we need to do so. delay = random.randint(0, 3600) reimage_time = int(time.time()) + delay logs.log('Scheduling a new reimage in %d seconds.' % delay) persistent_cache.set_value(SCHEDULED_GCE_REIMAGE_TIME_KEY, reimage_time) return current_time = datetime.datetime.utcnow() if current_time < scheduled_reimage_time: time_left = scheduled_reimage_time - current_time # Not yet time for the reimage. logs.log('Scheduled reimage in %d seconds.' % time_left.seconds) return # Recreating the virtual device will reimage this to the latest image # available (with retry logic). logs.log('Reimaging device to latest image.') if not adb.recreate_virtual_device(): logs.log_error('Unable to recreate virtual device. Reimaging failed.') adb.bad_state_reached() else: is_google_device = google_device() if is_google_device is None: logs.log_error('Unable to query device. Reimaging failed.') adb.bad_state_reached() if not is_google_device: # We can't reimage these, skip. logs.log('Non-Google device found, skipping reimage.') return else: # For Google devices. # Check if both |BUILD_BRANCH| and |BUILD_TARGET| environment variables # are set. If not, we don't have enough data for reimaging and hence # we bail out. branch = environment.get_value('BUILD_BRANCH') target = environment.get_value('BUILD_TARGET') if not target: # We default to userdebug configuration. build_params = get_build_parameters() if build_params: target = build_params.get('target') + '-userdebug' # Cache target in environment. This is also useful for cases when # device is bricked and we don't have this information available. environment.set_value('BUILD_TARGET', target) if not branch or not target: return # Download the latest build artifact for this branch and target. build_info = fetch_artifact.get_latest_artifact_info(branch, target) if not build_info: logs.log_error( 'Unable to fetch information on latest build artifact for ' 'branch %s and target %s.' % (branch, target)) return # Check if our local build matches the latest build. If not, we will # download it. build_id = build_info['bid'] target = build_info['target'] image_directory = environment.get_value('IMAGES_DIR') last_build_info = persistent_cache.get_value(LAST_FLASH_BUILD_KEY) if not last_build_info or last_build_info['bid'] != build_id: # Clean up the images directory first. shell.remove_directory(image_directory, recreate=True) # We have a new build, download the build artifacts for it. for image_regex in FLASH_IMAGE_REGEXES: image_file_path = fetch_artifact.get(build_id, target, image_regex, image_directory) if not image_file_path: logs.log_error( 'Failed to download image artifact %s for ' 'branch %s and target %s.' % (image_file_path, branch, target)) return if image_file_path.endswith('.zip'): archive.unpack(image_file_path, image_directory) # We do one device flash at a time on one host, otherwise we run into # failures and device being stuck in a bad state. flash_lock_key_name = 'flash:%s' % socket.gethostname() if not locks.acquire_lock(flash_lock_key_name, by_zone=True): logs.log_error('Failed to acquire lock for reimaging, exiting.') return logs.log('Reimaging started.') logs.log('Rebooting into bootloader mode.') for _ in xrange(FLASH_RETRIES): adb.run_as_root() adb.run_adb_command(['reboot-bootloader']) time.sleep(FLASH_REBOOT_BOOTLOADER_WAIT) adb.run_fastboot_command(['oem', 'off-mode-charge', '0']) adb.run_fastboot_command(['-w', 'reboot-bootloader']) for partition, partition_image_filename in FLASH_IMAGE_FILES: partition_image_file_path = os.path.join(image_directory, partition_image_filename) adb.run_fastboot_command( ['flash', partition, partition_image_file_path]) if partition in ['bootloader', 'radio']: adb.run_fastboot_command(['reboot-bootloader']) adb.run_fastboot_command('reboot') time.sleep(FLASH_REBOOT_WAIT) if adb.get_device_state() == 'device': break logs.log_error('Reimaging failed, retrying.') locks.release_lock(flash_lock_key_name, by_zone=True) if adb.get_device_state() != 'device': logs.log_error('Unable to find device. Reimaging failed.') adb.bad_state_reached() logs.log('Reimaging finished.') # Reset all of our persistent keys after wipe. persistent_cache.delete_value(BUILD_PROP_MD5_KEY) persistent_cache.delete_value(LAST_TEST_ACCOUNT_CHECK_KEY) persistent_cache.set_value(LAST_FLASH_BUILD_KEY, build_info) persistent_cache.set_value(LAST_FLASH_TIME_KEY, time.time()) if is_remote_device: persistent_cache.delete_value(SCHEDULED_GCE_REIMAGE_TIME_KEY)
def configure_build_properties_if_needed(): """Edits /system/build.prop for better boot speed and power use.""" # Check md5 checksum of build.prop to see if already updated, # in which case exit. If build.prop does not exist, something # is very wrong with the device, so bail. old_md5 = persistent_cache.get_value(BUILD_PROP_MD5_KEY) current_md5 = adb.get_file_checksum(BUILD_PROP_PATH) if current_md5 is None: logs.log_error('Unable to find %s on device.' % BUILD_PROP_PATH) return if old_md5 == current_md5: return # Pull to tmp file. bot_tmp_directory = environment.get_value('BOT_TMPDIR') old_build_prop_path = os.path.join(bot_tmp_directory, 'old.prop') adb.run_adb_command(['pull', BUILD_PROP_PATH, old_build_prop_path]) if not os.path.exists(old_build_prop_path): logs.log_error('Unable to fetch %s from device.' % BUILD_PROP_PATH) return # Write new build.prop. new_build_prop_path = os.path.join(bot_tmp_directory, 'new.prop') old_build_prop_file_content = open(old_build_prop_path, 'r') new_build_prop_file_content = open(new_build_prop_path, 'w') new_content_notification = '### CHANGED OR ADDED PROPERTIES ###' for line in old_build_prop_file_content: property_name = line.split('=')[0].strip() if property_name in BUILD_PROPERTIES: continue if new_content_notification in line: continue new_build_prop_file_content.write(line) new_build_prop_file_content.write(new_content_notification + '\n') for flag, value in BUILD_PROPERTIES.iteritems(): new_build_prop_file_content.write('%s=%s\n' % (flag, value)) old_build_prop_file_content.close() new_build_prop_file_content.close() # Keep verified boot disabled for M and higher releases. This makes it easy # to modify system's app_process to load asan libraries. build_version = get_build_version() if is_build_at_least(build_version, 'M'): adb.run_as_root() adb.run_adb_command('disable-verity') reboot() # Make /system writable. adb.run_as_root() adb.remount() # Remove seccomp policies (on N and higher) as ASan requires extra syscalls. if is_build_at_least(build_version, 'N'): policy_files = adb.run_adb_shell_command( ['find', '/system/etc/seccomp_policy/', '-type', 'f']) for policy_file in policy_files.splitlines(): adb.run_adb_shell_command(['rm', policy_file.strip()]) # Remove Google Plus app from non-Google devices. Makes it easy to install # older Gallery app on these devices. Otherwise, we run into duplicate # permission errors. if not google_device(): adb.run_adb_shell_command(['rm', '/system/app/PlusOne.apk']) adb.run_adb_shell_command(['rm', '/system/app/PlusOne/PlusOne.apk']) # Push new build.prop and backup to device. logs.log('Pushing new build properties file on device.') adb.run_adb_command( ['push', '-p', old_build_prop_path, BUILD_PROP_BACKUP_PATH]) adb.run_adb_command(['push', '-p', new_build_prop_path, BUILD_PROP_PATH]) adb.run_adb_shell_command(['chmod', '644', BUILD_PROP_PATH]) # Set persistent cache key containing and md5sum. current_md5 = adb.get_file_checksum(BUILD_PROP_PATH) persistent_cache.set_value(BUILD_PROP_MD5_KEY, current_md5)
def get_component_revisions_dict(revision, job_type): """Retrieve revision vars dict.""" if revision == 0 or revision == '0' or revision is None: # Return empty dict for zero start revision. return {} config = db_config.get() component = data_handler.get_component_name(job_type) project_name = data_handler.get_project_name(job_type) revision_info_url_format = db_config.get_value_for_job( config.revision_vars_url, job_type) if not revision_info_url_format: return None revisions_dict = {} repository = data_handler.get_repository_for_component(component) if repository and not _is_clank(revision_info_url_format): revision_hash = _git_commit_position_to_git_hash_for_chromium( revision, repository) if revision_hash is None: return None # FIXME: While we check for this explicitly appended component in all # applicable cases that we know of within this codebase, if the dict # is shared with an external service (e.g. Predator) we may need to clean # this up beforehand. revisions_dict['/src'] = { 'name': _get_component_display_name(component, project_name), 'url': _git_url_for_chromium_repository(repository), 'rev': revision_hash, 'commit_pos': revision } # Use revision hash for info url later. revision = revision_hash revision_info_url = revision_info_url_format % revision url_content = _get_url_content(revision_info_url) if not url_content: logs.log_error('Failed to get component revisions from %s.' % revision_info_url) return None # Parse as per DEPS format. if _is_deps(revision_info_url): deps_revisions_dict = deps_to_revisions_dict(url_content) if not deps_revisions_dict: return None revisions_dict.update(deps_revisions_dict) return revisions_dict # Parse as per Clank DEPS format. if _is_clank(revision_info_url): return _clank_revision_file_to_revisions_dict(url_content) # Default case: parse content as yaml. revisions_dict = _to_yaml_dict(url_content) if not revisions_dict: logs.log_error('Failed to parse component revisions from %s.' % revision_info_url) return None # Parse as per source map format. if revision_info_url.endswith(SOURCE_MAP_EXTENSION): revisions_dict = _src_map_to_revisions_dict(revisions_dict, project_name) return revisions_dict
def main(argv): """Run libFuzzer as specified by argv.""" atexit.register(fuzzer_utils.cleanup) # Initialize variables. arguments = argv[1:] testcase_file_path = arguments.pop(0) target_name = arguments.pop(0) fuzzer_name = data_types.fuzz_target_project_qualified_name( utils.current_project(), target_name) # Initialize log handler. logs.configure( 'run_fuzzer', { 'fuzzer': fuzzer_name, 'engine': 'libFuzzer', 'job_name': environment.get_value('JOB_NAME') }) profiler.start_if_needed('libfuzzer_launcher') # Make sure that the fuzzer binary exists. build_directory = environment.get_value('BUILD_DIR') fuzzer_path = engine_common.find_fuzzer_path(build_directory, target_name) if not fuzzer_path: return # Install signal handler. signal.signal(signal.SIGTERM, engine_common.signal_term_handler) # Set up temp dir. engine_common.recreate_directory(fuzzer_utils.get_temp_dir()) # Setup minijail if needed. use_minijail = environment.get_value('USE_MINIJAIL') runner = libfuzzer.get_runner(fuzzer_path, temp_dir=fuzzer_utils.get_temp_dir()) if use_minijail: minijail_chroot = runner.chroot else: minijail_chroot = None # Get corpus directory. corpus_directory = environment.get_value('FUZZ_CORPUS_DIR') # Add common arguments which are necessary to be used for every run. arguments = expand_with_common_arguments(arguments) # Add sanitizer options to environment that were specified in the .options # file and options that this script requires. set_sanitizer_options(fuzzer_path) # If we don't have a corpus, then that means this is not a fuzzing run. # TODO(flowerhack): Implement this to properly load past testcases. if not corpus_directory and environment.platform() != 'FUCHSIA': load_testcase_if_exists(runner, testcase_file_path, fuzzer_name, use_minijail, arguments) return # We don't have a crash testcase, fuzz. # Check dict argument to make sure that it's valid. dict_argument = fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG, remove=False) if dict_argument and not os.path.exists(dict_argument): logs.log_error('Invalid dict %s for %s.' % (dict_argument, fuzzer_name)) fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG) # If there's no dict argument, check for %target_binary_name%.dict file. if (not fuzzer_utils.extract_argument( arguments, constants.DICT_FLAG, remove=False)): default_dict_path = dictionary_manager.get_default_dictionary_path( fuzzer_path) if os.path.exists(default_dict_path): arguments.append(constants.DICT_FLAG + default_dict_path) # Set up scratch directory for writing new units. new_testcases_directory = create_corpus_directory('new') # Strategy pool is the list of strategies that we attempt to enable, whereas # fuzzing strategies is the list of strategies that are enabled. (e.g. if # mutator is selected in the pool, but not available for a given target, it # would not be added to fuzzing strategies.) strategy_pool = strategy_selection.generate_weighted_strategy_pool( strategy_list=strategy.LIBFUZZER_STRATEGY_LIST, use_generator=True, engine_name='libFuzzer') strategy_info = pick_strategies(strategy_pool, fuzzer_path, corpus_directory, arguments, minijail_chroot=minijail_chroot) arguments.extend(strategy_info.arguments) # Timeout for fuzzer run. fuzz_timeout = get_fuzz_timeout(strategy_info.is_mutations_run) # Get list of corpus directories. # TODO(flowerhack): Implement this to handle corpus sync'ing. if environment.platform() == 'FUCHSIA': corpus_directories = [] else: corpus_directories = get_corpus_directories( corpus_directory, new_testcases_directory, fuzzer_path, strategy_info.fuzzing_strategies, strategy_pool, minijail_chroot=minijail_chroot, allow_corpus_subset=not strategy_info.use_dataflow_tracing) corpus_directories.extend(strategy_info.additional_corpus_dirs) # Bind corpus directories in minijail. artifact_prefix = os.path.abspath(os.path.dirname(testcase_file_path)) # Execute the fuzzer binary with original arguments. fuzz_result = runner.fuzz(corpus_directories, fuzz_timeout=fuzz_timeout, artifact_prefix=artifact_prefix, additional_args=arguments, extra_env=strategy_info.extra_env) if (not use_minijail and fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE): # Minijail returns 1 if the exit code is nonzero. # Otherwise: we can assume that a return code of 1 means that libFuzzer # itself ran into an error. logs.log_error(ENGINE_ERROR_MESSAGE, engine_output=fuzz_result.output) log_lines = fuzz_result.output.splitlines() # Output can be large, so save some memory by removing reference to the # original output which is no longer needed. fuzz_result.output = None # Check if we crashed, and get the crash testcase path. crash_testcase_file_path = runner.get_testcase_path(log_lines) if crash_testcase_file_path: # Copy crash testcase contents into the main testcase path. shutil.move(crash_testcase_file_path, testcase_file_path) # Print the command output. bot_name = environment.get_value('BOT_NAME', '') command = fuzz_result.command if use_minijail: # Remove minijail prefix. command = engine_common.strip_minijail_command(command, fuzzer_path) print( engine_common.get_log_header(command, bot_name, fuzz_result.time_executed)) # Parse stats information based on libFuzzer output. parsed_stats = parse_log_stats(log_lines) # Extend parsed stats by additional performance features. parsed_stats.update( stats.parse_performance_features(log_lines, strategy_info.fuzzing_strategies, arguments)) # Set some initial stat overrides. timeout_limit = fuzzer_utils.extract_argument(arguments, constants.TIMEOUT_FLAG, remove=False) expected_duration = runner.get_max_total_time(fuzz_timeout) actual_duration = int(fuzz_result.time_executed) fuzzing_time_percent = 100 * actual_duration / float(expected_duration) stat_overrides = { 'timeout_limit': int(timeout_limit), 'expected_duration': expected_duration, 'actual_duration': actual_duration, 'fuzzing_time_percent': fuzzing_time_percent, } # Remove fuzzing arguments before merge and dictionary analysis step. remove_fuzzing_arguments(arguments) # Make a decision on whether merge step is needed at all. If there are no # new units added by libFuzzer run, then no need to do merge at all. new_units_added = shell.get_directory_file_count(new_testcases_directory) merge_error = None if new_units_added: # Merge the new units with the initial corpus. if corpus_directory not in corpus_directories: corpus_directories.append(corpus_directory) # If this times out, it's possible that we will miss some units. However, if # we're taking >10 minutes to load/merge the corpus something is going very # wrong and we probably don't want to make things worse by adding units # anyway. merge_tmp_dir = None if not use_minijail: merge_tmp_dir = os.path.join(fuzzer_utils.get_temp_dir(), 'merge_workdir') engine_common.recreate_directory(merge_tmp_dir) old_corpus_len = shell.get_directory_file_count(corpus_directory) merge_directory = create_merge_directory() corpus_directories.insert(0, merge_directory) if use_minijail: bind_corpus_dirs(minijail_chroot, [merge_directory]) merge_result = runner.merge( corpus_directories, merge_timeout=engine_common.get_merge_timeout( DEFAULT_MERGE_TIMEOUT), tmp_dir=merge_tmp_dir, additional_args=arguments) move_mergeable_units(merge_directory, corpus_directory) new_corpus_len = shell.get_directory_file_count(corpus_directory) new_units_added = 0 merge_error = None if merge_result.timed_out: merge_error = 'Merging new testcases timed out:' elif merge_result.return_code != 0: merge_error = 'Merging new testcases failed:' else: new_units_added = new_corpus_len - old_corpus_len stat_overrides['new_units_added'] = new_units_added if merge_result.output: stat_overrides.update( stats.parse_stats_from_merge_log( merge_result.output.splitlines())) else: stat_overrides['new_units_added'] = 0 logs.log('Skipped corpus merge since no new units added by fuzzing.') # Get corpus size after merge. This removes the duplicate units that were # created during this fuzzing session. # TODO(flowerhack): Remove this workaround once we can handle corpus sync. if environment.platform() != 'FUCHSIA': stat_overrides['corpus_size'] = shell.get_directory_file_count( corpus_directory) # Delete all corpus directories except for the main one. These were temporary # directories to store new testcase mutations and have already been merged to # main corpus directory. if corpus_directory in corpus_directories: corpus_directories.remove(corpus_directory) for directory in corpus_directories: shutil.rmtree(directory, ignore_errors=True) if use_minijail: unbind_corpus_dirs(minijail_chroot, corpus_directories) # Apply overridden stats to the parsed stats prior to dumping. parsed_stats.update(stat_overrides) # Dump stats data for further uploading to BigQuery. engine_common.dump_big_query_data(parsed_stats, testcase_file_path, command) # Add custom crash state based on fuzzer name (if needed). add_custom_crash_state_if_needed(fuzzer_name, log_lines, parsed_stats) for line in log_lines: print(line) # Add fuzzing strategies used. print( engine_common.format_fuzzing_strategies( strategy_info.fuzzing_strategies)) # Add merge error (if any). if merge_error: print(data_types.CRASH_STACKTRACE_END_MARKER) print(merge_error) print( 'Command:', get_printable_command(merge_result.command, fuzzer_path, use_minijail)) print(merge_result.output) analyze_and_update_recommended_dictionary(runner, fuzzer_name, log_lines, corpus_directory, arguments) # Close minijail chroot. if use_minijail: minijail_chroot.close() # Record the stats to make them easily searchable in stackdriver. if new_units_added: logs.log('New units added to corpus: %d.' % new_units_added, stats=parsed_stats) else: logs.log('No new units found.', stats=parsed_stats)
def pick_strategies(strategy_pool, fuzzer_path, corpus_directory, existing_arguments, minijail_chroot=None): """Pick strategies.""" build_directory = environment.get_value('BUILD_DIR') target_name = os.path.basename(fuzzer_path) project_qualified_fuzzer_name = data_types.fuzz_target_project_qualified_name( utils.current_project(), target_name) fuzzing_strategies = [] arguments = [] additional_corpus_dirs = [] # Select a generator to attempt to use for existing testcase mutations. candidate_generator = engine_common.select_generator( strategy_pool, fuzzer_path) is_mutations_run = candidate_generator != engine_common.Generator.NONE # Depends on the presense of DFSan instrumented build. dataflow_build_dir = environment.get_value('DATAFLOW_BUILD_DIR') use_dataflow_tracing = (dataflow_build_dir and strategy_pool.do_strategy( strategy.DATAFLOW_TRACING_STRATEGY)) if use_dataflow_tracing: dataflow_binary_path = os.path.join( dataflow_build_dir, os.path.relpath(fuzzer_path, build_directory)) if os.path.exists(dataflow_binary_path): arguments.append( '%s%s' % (constants.COLLECT_DATA_FLOW_FLAG, dataflow_binary_path)) fuzzing_strategies.append(strategy.DATAFLOW_TRACING_STRATEGY.name) else: logs.log_error( 'Fuzz target is not found in dataflow build, skiping strategy.' ) use_dataflow_tracing = False # Generate new testcase mutations using radamsa, etc. if is_mutations_run: new_testcase_mutations_directory = create_corpus_directory('mutations') generator_used = engine_common.generate_new_testcase_mutations( corpus_directory, new_testcase_mutations_directory, project_qualified_fuzzer_name, candidate_generator) # Add the used generator strategy to our fuzzing strategies list. if generator_used: if candidate_generator == engine_common.Generator.RADAMSA: fuzzing_strategies.append( strategy.CORPUS_MUTATION_RADAMSA_STRATEGY.name) elif candidate_generator == engine_common.Generator.ML_RNN: fuzzing_strategies.append( strategy.CORPUS_MUTATION_ML_RNN_STRATEGY.name) additional_corpus_dirs.append(new_testcase_mutations_directory) if minijail_chroot: bind_corpus_dirs(minijail_chroot, [new_testcase_mutations_directory]) if strategy_pool.do_strategy(strategy.RANDOM_MAX_LENGTH_STRATEGY): max_len_argument = fuzzer_utils.extract_argument( existing_arguments, constants.MAX_LEN_FLAG, remove=False) if not max_len_argument: max_length = random.SystemRandom().randint( 1, MAX_VALUE_FOR_MAX_LENGTH) arguments.append('%s%d' % (constants.MAX_LEN_FLAG, max_length)) fuzzing_strategies.append(strategy.RANDOM_MAX_LENGTH_STRATEGY.name) if (strategy_pool.do_strategy(strategy.RECOMMENDED_DICTIONARY_STRATEGY) and add_recommended_dictionary( arguments, project_qualified_fuzzer_name, fuzzer_path)): fuzzing_strategies.append( strategy.RECOMMENDED_DICTIONARY_STRATEGY.name) if strategy_pool.do_strategy(strategy.VALUE_PROFILE_STRATEGY): arguments.append(constants.VALUE_PROFILE_ARGUMENT) fuzzing_strategies.append(strategy.VALUE_PROFILE_STRATEGY.name) # DataFlow Tracing requires fork mode, always use it with DFT strategy. if use_dataflow_tracing or strategy_pool.do_strategy( strategy.FORK_STRATEGY): max_fuzz_threads = environment.get_value('MAX_FUZZ_THREADS', 1) num_fuzz_processes = max( 1, multiprocessing.cpu_count() // max_fuzz_threads) arguments.append('%s%d' % (constants.FORK_FLAG, num_fuzz_processes)) fuzzing_strategies.append( '%s_%d' % (strategy.FORK_STRATEGY.name, num_fuzz_processes)) extra_env = {} if (strategy_pool.do_strategy(strategy.MUTATOR_PLUGIN_STRATEGY) and use_mutator_plugin(target_name, extra_env, minijail_chroot)): fuzzing_strategies.append(strategy.MUTATOR_PLUGIN_STRATEGY.name) return StrategyInfo(fuzzing_strategies, arguments, additional_corpus_dirs, extra_env, use_dataflow_tracing, is_mutations_run)
def execute_task(metadata_id, job_type): """Unpack a bundled testcase archive and create analyze jobs for each item.""" metadata = ndb.Key(data_types.BundledArchiveMetadata, int(metadata_id)).get() if not metadata: logs.log_error('Invalid bundle metadata id %s.' % metadata_id) return bot_name = environment.get_value('BOT_NAME') upload_metadata = data_types.TestcaseUploadMetadata.query( data_types.TestcaseUploadMetadata.blobstore_key == metadata.blobstore_key).get() if not upload_metadata: logs.log_error('Invalid upload metadata key %s.' % metadata.blobstore_key) return # Update the upload metadata with this bot name. upload_metadata.bot_name = bot_name upload_metadata.put() # We can't use FUZZ_INPUTS directory since it is constrained # by tmpfs limits. testcases_directory = environment.get_value('FUZZ_INPUTS_DISK') # Retrieve multi-testcase archive. archive_path = os.path.join(testcases_directory, metadata.archive_filename) if not blobs.read_blob_to_disk(metadata.blobstore_key, archive_path): logs.log_error('Could not retrieve archive for bundle %d.' % metadata_id) tasks.add_task('unpack', metadata_id, job_type) return try: archive.unpack(archive_path, testcases_directory) except: logs.log_error('Could not unpack archive for bundle %d.' % metadata_id) tasks.add_task('unpack', metadata_id, job_type) return # Get additional testcase metadata (if any). additional_metadata = None if upload_metadata.additional_metadata_string: additional_metadata = json.loads( upload_metadata.additional_metadata_string) archive_state = data_types.ArchiveStatus.NONE bundled = True file_list = archive.get_file_list(archive_path) for file_path in file_list: absolute_file_path = os.path.join(testcases_directory, file_path) filename = os.path.basename(absolute_file_path) # Only files are actual testcases. Skip directories. if not os.path.isfile(absolute_file_path): continue try: file_handle = open(absolute_file_path, 'rb') blob_key = blobs.write_blob(file_handle) file_handle.close() except: blob_key = None if not blob_key: logs.log_error('Could not write testcase %s to blobstore.' % absolute_file_path) continue data_handler.create_user_uploaded_testcase( blob_key, metadata.blobstore_key, archive_state, metadata.archive_filename, filename, metadata.timeout, metadata.job_type, metadata.job_queue, metadata.http_flag, metadata.gestures, metadata.additional_arguments, metadata.bug_information, metadata.crash_revision, metadata.uploader_email, metadata.platform_id, metadata.app_launch_command, metadata.fuzzer_name, metadata.overridden_fuzzer_name, metadata.fuzzer_binary_name, bundled, upload_metadata.retries, upload_metadata.bug_summary_update_flag, upload_metadata.quiet_flag, additional_metadata) # The upload metadata for the archive is not needed anymore since we created # one for each testcase. upload_metadata.key.delete() shell.clear_testcase_directories()
def unpack_crash_testcases(crash_testcases_directory): """Unpacks the old crash testcases in the provided directory.""" for testcase in ndb_utils.get_all_from_model(data_types.Testcase): testcase_id = testcase.key.id() # 1. If we have already stored the testcase, then just skip. if testcase_id in STORED_TESTCASES_LIST: continue # 2. Make sure that it is a unique crash testcase. Ignore duplicates, # uploaded repros. if testcase.status != 'Processed': continue # 3. Check if the testcase is fixed. If not, skip. if testcase.open: continue # 4. Check if the testcase has a minimized repro. If not, skip. if not testcase.minimized_keys or testcase.minimized_keys == 'NA': continue # 5. Only use testcases that have bugs associated with them. if not testcase.bug_information: continue # 6. Existing IPC testcases are un-interesting and unused in furthur # mutations. Due to size bloat, ignoring these for now. if testcase.absolute_path.endswith(testcase_manager.IPCDUMP_EXTENSION): continue # 7. Ignore testcases that are archives (e.g. Langfuzz fuzzer tests). if archive.get_archive_type(testcase.absolute_path): continue # 8. Skip in-process fuzzer testcases, since these are only applicable to # fuzz targets and don't run with blackbox binaries. if testcase.fuzzer_name and testcase.fuzzer_name in ['afl', 'libFuzzer']: continue # Un-pack testcase. try: _, input_directory, _ = setup.unpack_testcase(testcase) except Exception: logs.log_error('Failed to unpack testcase %d.' % testcase.key.id()) continue # Move this to our crash testcases directory. crash_testcase_directory = os.path.join(crash_testcases_directory, str(testcase_id)) shell.move(input_directory, crash_testcase_directory) # Re-create input directory for unpacking testcase in next iteration. shell.create_directory(input_directory) STORED_TESTCASES_LIST.append(testcase_id) # Remove testcase directories that exceed the max size limit. for directory_name in os.listdir(crash_testcases_directory): directory_path = os.path.join(crash_testcases_directory, directory_name) if not os.path.isdir(directory_path): continue if shell.get_directory_size(directory_path) <= MAX_TESTCASE_DIRECTORY_SIZE: continue shell.remove_directory(directory_path) # Rename all fuzzed testcase files as regular files. for root, _, files in os.walk(crash_testcases_directory): for filename in files: if not filename.startswith(testcase_manager.FUZZ_PREFIX): continue file_path = os.path.join(root, filename) stripped_file_name = os.path.basename(file_path)[len( testcase_manager.FUZZ_PREFIX):] stripped_file_path = os.path.join( os.path.dirname(file_path), stripped_file_name) try: os.rename(file_path, stripped_file_path) except: raise Exception('Failed to rename testcase %s.' % file_path) # Remove empty files and dirs to avoid the case where a fuzzer randomly # chooses an empty dir/file and generates zero testcases. shell.remove_empty_files(crash_testcases_directory) shell.remove_empty_directories(crash_testcases_directory)
def _do_run_testcase_and_return_result_in_queue(crash_queue, thread_index, file_path, gestures, env_copy, upload_output=False): """Run a single testcase and return crash results in the crash queue.""" try: # Run testcase and check whether a crash occurred or not. return_code, crash_time, output = run_testcase(thread_index, file_path, gestures, env_copy) # Pull testcase directory to host to get any stats files. if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.pull_testcases_from_worker() # Analyze the crash. crash_output = _get_crash_output(output) crash_result = CrashResult(return_code, crash_time, crash_output) # To provide consistency between stats and logs, we use timestamp taken # from stats when uploading logs and testcase. if upload_output: log_time = _get_testcase_time(file_path) if crash_result.is_crash(): # Initialize resource list with the testcase path. resource_list = [file_path] resource_list += get_resource_paths(crash_output) # Store the crash stack file in the crash stacktrace directory # with filename as the hash of the testcase path. crash_stacks_directory = environment.get_value( 'CRASH_STACKTRACES_DIR') stack_file_path = os.path.join(crash_stacks_directory, utils.string_hash(file_path)) utils.write_data_to_file(crash_output, stack_file_path) # Put crash/no-crash results in the crash queue. crash_queue.put( Crash(file_path=file_path, crash_time=crash_time, return_code=return_code, resource_list=resource_list, gestures=gestures, stack_file_path=stack_file_path)) # Don't upload uninteresting testcases (no crash) or if there is no log to # correlate it with (not upload_output). if upload_output: upload_testcase(file_path, log_time) if upload_output: # Include full output for uploaded logs (crash output, merge output, etc). crash_result_full = CrashResult(return_code, crash_time, output) log = prepare_log_for_upload(crash_result_full.get_stacktrace(), return_code) upload_log(log, log_time) except Exception: logs.log_error('Exception occurred while running ' 'run_testcase_and_return_result_in_queue.')
def download_system_symbols_if_needed(symbols_directory, is_kernel=False): """Download system libraries from |SYMBOLS_URL| and cache locally.""" # For local testing, we do not have access to the cloud storage bucket with # the symbols. In this case, just bail out. if environment.get_value('LOCAL_DEVELOPMENT'): return # When running reproduce tool locally, we do not have access to the cloud # storage bucket with the symbols. In this case, just bail out. if environment.get_value('REPRODUCE_TOOL'): return # We have archived symbols for google builds only. if not settings.is_google_device(): return # For Android kernel we want to get the repro.prop # Note: kasan and non-kasan kernel should have the same repo.prop for a given # build_id. if is_kernel: build_id = settings.get_kernel_build_id() target = settings.get_kernel_name() if not build_id or not target: logs.log_error('Could not get kernel parameters, exiting.') return artifact_file_name = 'repo.prop' symbols_archive_filename = get_symbols_archive_filename( build_id, target) output_filename_override = symbols_archive_filename # We create our own build_params for cache build_params = { 'build_id': build_id, 'target': target, 'type': 'kernel' } else: # Get the build fingerprint parameters. build_params = settings.get_build_parameters() if not build_params: logs.log_error('Unable to determine build parameters.') return build_id = build_params.get('build_id') target = build_params.get('target') build_type = build_params.get('type') if not build_id or not target or not build_type: logs.log_error('Null build parameters found, exiting.') return symbols_archive_filename = '%s-symbols-%s.zip' % (target, build_id) artifact_file_name = symbols_archive_filename output_filename_override = None # Check if we already have the symbols in cache. build_params_check_path = os.path.join(symbols_directory, '.cached_build_params') cached_build_params = utils.read_data_from_file(build_params_check_path, eval_data=True) if cached_build_params and cached_build_params == build_params: # No work to do, same system symbols already in cache. return symbols_archive_path = os.path.join(symbols_directory, symbols_archive_filename) # Delete existing symbols directory first. shell.remove_directory(symbols_directory, recreate=True) # Fetch symbol file from cloud storage cache (if available). found_in_cache = storage.get_file_from_cache_if_exists( symbols_archive_path, update_modification_time_on_access=False) if not found_in_cache: tool_suffix = environment.get_value('SANITIZER_TOOL_NAME') if is_kernel: # Some kernels are just 'kernel', some are kernel_target if tool_suffix: targets_with_type_and_san = [ 'kernel_%s' % tool_suffix, 'kernel_%s_%s' % (tool_suffix, target) ] else: targets_with_type_and_san = ['kernel', 'kernel_%s' % target] else: # Include type and sanitizer information in the target. target_with_type_and_san = '%s-%s' % (target, build_type) if tool_suffix and not tool_suffix in target_with_type_and_san: target_with_type_and_san += '_%s' % tool_suffix targets_with_type_and_san = [target_with_type_and_san] for target_with_type_and_san in targets_with_type_and_san: # Fetch the artifact now. fetch_artifact.get(build_id, target_with_type_and_san, artifact_file_name, symbols_directory, output_filename_override) if os.path.exists(symbols_archive_path): break if not os.path.exists(symbols_archive_path): logs.log_error('Unable to locate symbols archive %s.' % symbols_archive_path) return # Store the artifact for later use or for use by other bots. storage.store_file_in_cache(symbols_archive_path) # repo.prop is not a zip archive. if not is_kernel: archive.unpack(symbols_archive_path, symbols_directory, trusted=True) shell.remove_file(symbols_archive_path) utils.write_data_to_file(build_params, build_params_check_path)
def find_fixed_range(testcase_id, job_type): """Attempt to find the revision range where a testcase was fixed.""" deadline = tasks.get_task_completion_deadline() testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return if testcase.fixed: logs.log_error('Fixed range is already set as %s, skip.' % testcase.fixed) return # Setup testcase and its dependencies. file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type) if not file_list: return # Set a flag to indicate we are running progression task. This shows pending # status on testcase report page and avoid conflicting testcase updates by # triage cron. testcase.set_metadata('progression_pending', True) # Custom binaries are handled as special cases. if build_manager.is_custom_binary(): _check_fixed_for_custom_binary(testcase, job_type, testcase_file_path) return build_bucket_path = build_manager.get_primary_bucket_path() revision_list = build_manager.get_revisions_list(build_bucket_path, testcase=testcase) if not revision_list: data_handler.close_testcase_with_error( testcase_id, 'Failed to fetch revision list') return # Use min, max_index to mark the start and end of revision list that is used # for bisecting the progression range. Set start to the revision where noticed # the crash. Set end to the trunk revision. Also, use min, max from past run # if it timed out. min_revision = testcase.get_metadata('last_progression_min') max_revision = testcase.get_metadata('last_progression_max') if min_revision or max_revision: # Clear these to avoid using them in next run. If this run fails, then we # should try next run without them to see it succeeds. If this run succeeds, # we should still clear them to avoid capping max revision in next run. testcase = data_handler.get_testcase_by_id(testcase_id) testcase.delete_metadata('last_progression_min', update_testcase=False) testcase.delete_metadata('last_progression_max', update_testcase=False) testcase.put() last_tested_revision = testcase.get_metadata('last_tested_crash_revision') known_crash_revision = last_tested_revision or testcase.crash_revision if not min_revision: min_revision = known_crash_revision if not max_revision: max_revision = revisions.get_last_revision_in_list(revision_list) min_index = revisions.find_min_revision_index(revision_list, min_revision) if min_index is None: raise errors.BuildNotFoundError(min_revision, job_type) max_index = revisions.find_max_revision_index(revision_list, max_revision) if max_index is None: raise errors.BuildNotFoundError(max_revision, job_type) testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED, 'r%d' % max_revision) # Check to see if this testcase is still crashing now. If it is, then just # bail out. result = _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, max_revision, update_metadata=True) if result.is_crash(): logs.log('Found crash with same signature on latest revision r%d.' % max_revision) app_path = environment.get_value('APP_PATH') command = testcase_manager.get_command_line_for_application( testcase_file_path, app_path=app_path, needs_http=testcase.http_flag) symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True) unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False) stacktrace = utils.get_crash_stacktrace_output( command, symbolized_crash_stacktrace, unsymbolized_crash_stacktrace) testcase = data_handler.get_testcase_by_id(testcase_id) testcase.last_tested_crash_stacktrace = data_handler.filter_stacktrace( stacktrace) data_handler.update_progression_completion_metadata( testcase, max_revision, is_crash=True, message='still crashes on latest revision r%s' % max_revision) # Since we've verified that the test case is still crashing, clear out any # metadata indicating potential flake from previous runs. task_creation.mark_unreproducible_if_flaky(testcase, False) # For chromium project, save latest crash information for later upload # to chromecrash/. state = result.get_symbolized_data() crash_uploader.save_crash_info_if_needed(testcase_id, max_revision, job_type, state.crash_type, state.crash_address, state.frames) return # Don't burden NFS server with caching these random builds. environment.set_value('CACHE_STORE', False) # Verify that we do crash in the min revision. This is assumed to be true # while we are doing the bisect. result = _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, min_revision) if result and not result.is_crash(): testcase = data_handler.get_testcase_by_id(testcase_id) # Retry once on another bot to confirm our result. if data_handler.is_first_retry_for_task(testcase, reset_after_retry=True): tasks.add_task('progression', testcase_id, job_type) error_message = ( 'Known crash revision %d did not crash, will retry on another bot to ' 'confirm result' % known_crash_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) data_handler.update_progression_completion_metadata( testcase, max_revision) return data_handler.clear_progression_pending(testcase) error_message = ('Known crash revision %d did not crash' % known_crash_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) task_creation.mark_unreproducible_if_flaky(testcase, True) return # Start a binary search to find last non-crashing revision. At this point, we # know that we do crash in the min_revision, and do not crash in max_revision. while time.time() < deadline: min_revision = revision_list[min_index] max_revision = revision_list[max_index] # If the min and max revisions are one apart this is as much as we can # narrow the range. if max_index - min_index == 1: _save_fixed_range(testcase_id, min_revision, max_revision, testcase_file_path) return # Occasionally, we get into this bad state. It seems to be related to test # cases with flaky stacks, but the exact cause is unknown. if max_index - min_index < 1: testcase = data_handler.get_testcase_by_id(testcase_id) testcase.fixed = 'NA' testcase.open = False message = ('Fixed testing errored out (min and max revisions ' 'are both %d)' % min_revision) data_handler.update_progression_completion_metadata( testcase, max_revision, message=message) # Let the bisection service know about the NA status. bisection.request_bisection(testcase) return # Test the middle revision of our range. middle_index = (min_index + max_index) // 2 middle_revision = revision_list[middle_index] testcase = data_handler.get_testcase_by_id(testcase_id) log_message = 'Testing r%d (current range %d:%d)' % ( middle_revision, min_revision, max_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.WIP, log_message) try: result = _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, middle_revision) except errors.BadBuildError: # Skip this revision. del revision_list[middle_index] max_index -= 1 continue if result.is_crash(): min_index = middle_index else: max_index = middle_index _save_current_fixed_range_indices(testcase_id, revision_list[min_index], revision_list[max_index]) # If we've broken out of the loop, we've exceeded the deadline. Recreate the # task to pick up where we left off. testcase = data_handler.get_testcase_by_id(testcase_id) error_message = ('Timed out, current range r%d:r%d' % (revision_list[min_index], revision_list[max_index])) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task('progression', testcase_id, job_type)
def execute(input_directory, output_directory, fuzzer_name, generation_timeout): """Execute ML RNN generator to produce new inputs. This method should be called inside launcher, to generate a number of new inputs based on ML RNN model. It will fetch ML model from GCS bucket specified in environment variable `CORPUS_BUCKET`. The script to run the model resides in folder `tools/fuzzers/ml/rnn`. Args: input_directory: Seed corpus path. The directory should not be empty. output_directory: The directory to place generated inputs. fuzzer_name: Name of the fuzzer, e.g libpng_read_fuzzer. It indicates the subdirectory in gcs bucket to store models. generation_timeout: Time in seconds for the generator to run. Normally it takes <1s to generate an input, assuming the input length is <4KB. """ if environment.platform() != 'LINUX': logs.log('Unsupported platform for ML RNN generation, skipping.') return # Validate corpus folder. file_count = shell.get_directory_file_count(input_directory) if not file_count: logs.log('Corpus is empty. Skip generation.') return # Number of existing new inputs. They are possibly generated by other # generators. old_corpus_units = shell.get_directory_file_count(output_directory) old_corpus_bytes = shell.get_directory_size(output_directory) # Get model path. model_path = prepare_model_directory(fuzzer_name) if not model_path: return result = run(input_directory, output_directory, model_path, generation_timeout) # Generation process exited abnormally but not caused by timeout, meaning # error occurred during execution. if result.return_code and not result.timed_out: if result.return_code == constants.ExitCode.CORPUS_TOO_SMALL: logs.log_warn( 'ML RNN generation for fuzzer %s aborted due to small corpus.' % fuzzer_name) else: logs.log_error( 'ML RNN generation for fuzzer %s failed with ExitCode = %d.' % (fuzzer_name, result.return_code), output=result.output) return # Timeout is not error, if we have new units generated. if result.timed_out: logs.log_warn('ML RNN generation for fuzzer %s timed out.' % fuzzer_name) new_corpus_units = (shell.get_directory_file_count(output_directory) - old_corpus_units) new_corpus_bytes = (shell.get_directory_size(output_directory) - old_corpus_bytes) if new_corpus_units: logs.log( 'Added %d new inputs (%d bytes) using ML RNN generator for %s.' % (new_corpus_units, new_corpus_bytes, fuzzer_name)) else: logs.log_error('ML RNN generator did not produce any inputs for %s' % fuzzer_name, output=result.output)
def update_data_bundle(fuzzer, data_bundle): """Updates a data bundle to the latest version.""" # This module can't be in the global imports due to appengine issues # with multiprocessing and psutil imports. from google_cloud_utils import gsutil # If we are using a data bundle on NFS, it is expected that our testcases # will usually be large enough that we would fill up our tmpfs directory # pretty quickly. So, change it to use an on-disk directory. if not data_bundle.is_local: testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK') environment.set_value('FUZZ_INPUTS', testcase_disk_directory) data_bundle_directory = get_data_bundle_directory(fuzzer.name) if not data_bundle_directory: logs.log_error('Failed to setup data bundle %s.' % data_bundle.name) return False if not shell.create_directory_if_needed(data_bundle_directory, create_intermediates=True): logs.log_error('Failed to create data bundle %s directory.' % data_bundle.name) return False # Check if data bundle is up to date. If yes, skip the update. if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): logs.log('Data bundle was recently synced, skip.') return True # Fetch lock for this data bundle. if not _fetch_lock_for_data_bundle_update(data_bundle): logs.log_error('Failed to lock data bundle %s.' % data_bundle.name) return False # Re-check if another bot did the sync already. If yes, skip. if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): logs.log('Another bot finished the sync, skip.') _release_lock_for_data_bundle_update(data_bundle) return True time_before_sync_start = time.time() # No need to sync anything if this is a search index data bundle. In that # case, the fuzzer will generate testcases from a gcs bucket periodically. if not _is_search_index_data_bundle(data_bundle.name): bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name) if environment.is_trusted_host() and data_bundle.sync_to_worker: from bot.untrusted_runner import corpus_manager from bot.untrusted_runner import file_host worker_data_bundle_directory = file_host.rebase_to_worker_root( data_bundle_directory) file_host.create_directory(worker_data_bundle_directory, create_intermediates=True) result = corpus_manager.RemoteGSUtilRunner().rsync( bucket_url, worker_data_bundle_directory, delete=False) else: result = gsutil.GSUtilRunner().rsync(bucket_url, data_bundle_directory, delete=False) if result.return_code != 0: logs.log_error('Failed to sync data bundle %s: %s.' % (data_bundle.name, result.output)) _release_lock_for_data_bundle_update(data_bundle) return False # Update the testcase list file. tests.create_testcase_list_file(data_bundle_directory) # Write last synced time in the sync file. sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory) utils.write_data_to_file(time_before_sync_start, sync_file_path) if environment.is_trusted_host() and data_bundle.sync_to_worker: from bot.untrusted_runner import file_host worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path) file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path) # Release acquired lock. _release_lock_for_data_bundle_update(data_bundle) return True
def get(self): """Handle a get request.""" try: grouper.group_testcases() except: logs.log_error('Error occurred while grouping test cases.') return # Free up memory after group task run. utils.python_gc() # Get list of jobs excluded from bug filing. excluded_jobs = get_excluded_jobs() for testcase_id in data_handler.get_open_testcase_id_iterator(): try: testcase = data_handler.get_testcase_by_id(testcase_id) except errors.InvalidTestcaseError: # Already deleted. continue # Skip if testcase's job type is in exclusions list. if testcase.job_type in excluded_jobs: continue # If the testcase has a bug filed already, no triage is needed. if is_bug_filed(testcase): continue # Check if the crash is important, i.e. it is either a reproducible crash # or an unreproducible crash happening frequently. if not is_crash_important(testcase): continue # Require that all tasks like minimizaton, regression testing, etc have # finished. if not data_handler.critical_tasks_completed(testcase): continue # For testcases that are not part of a group, wait an additional time till # group task completes. # FIXME: In future, grouping might be dependent on regression range, so we # would have to add an additional wait time. if not testcase.group_id and not dates.time_has_expired( testcase.timestamp, hours=data_types.MIN_ELAPSED_TIME_SINCE_REPORT): continue # If this project does not have an associated issue tracker, we cannot # file this crash anywhere. issue_tracker_manager = issue_tracker_utils.get_issue_tracker_manager( testcase, use_cache=True) if not issue_tracker_manager: continue # If there are similar issues to this test case already filed or recently # closed, skip filing a duplicate bug. if is_similar_bug_open_or_recently_closed(testcase, issue_tracker_manager): continue # File the bug first and then create filed bug metadata. issue_filer.file_issue(testcase, issue_tracker_manager) create_filed_bug_metadata(testcase) logs.log('Filed new issue %s for testcase %d.' % (testcase.bug_information, testcase_id))
def archive_testcase_and_dependencies_in_gcs(resource_list, testcase_path): """Archive testcase and its dependencies, and store in blobstore.""" if not os.path.exists(testcase_path): logs.log_error('Unable to find testcase %s.' % testcase_path) return None, None, None, None absolute_filename = testcase_path archived = False zip_filename = None zip_path = None if not resource_list: resource_list = [] # Add resource dependencies based on testcase path. These include # stuff like extensions directory, dependency files, etc. resource_list.extend(tests.get_resource_dependencies(testcase_path)) # Filter out duplicates, directories, and files that do not exist. resource_list = utils.filter_file_list(resource_list) logs.log('Testcase and related files :\n%s' % str(resource_list)) if len(resource_list) <= 1: # If this does not have any resources, just save the testcase. try: file_handle = open(testcase_path, 'rb') except IOError: logs.log_error('Unable to open testcase %s.' % testcase_path) return None, None, None, None else: # If there are resources, create an archive. # Find the common root directory for all of the resources. # Assumption: resource_list[0] is the testcase path. base_directory_list = resource_list[0].split(os.path.sep) for list_index in xrange(1, len(resource_list)): current_directory_list = resource_list[list_index].split( os.path.sep) length = min(len(base_directory_list), len(current_directory_list)) for directory_index in xrange(length): if (current_directory_list[directory_index] != base_directory_list[directory_index]): base_directory_list = base_directory_list[ 0:directory_index] break base_directory = os.path.sep.join(base_directory_list) logs.log('Subresource common base directory: %s' % base_directory) if base_directory: # Common parent directory, archive sub-paths only. base_len = len(base_directory) + len(os.path.sep) else: # No common parent directory, archive all paths as it-is. base_len = 0 # Prepare the filename for the archive. zip_filename, _ = os.path.splitext(os.path.basename(testcase_path)) zip_filename += _TESTCASE_ARCHIVE_EXTENSION # Create the archive. zip_path = os.path.join(environment.get_value('INPUT_DIR'), zip_filename) zip_file = zipfile.ZipFile(zip_path, 'w') for file_name in resource_list: if os.path.exists(file_name): relative_filename = file_name[base_len:] zip_file.write(file_name, relative_filename, zipfile.ZIP_DEFLATED) zip_file.close() try: file_handle = open(zip_path, 'rb') except IOError: logs.log_error('Unable to open testcase archive %s.' % zip_path) return None, None, None, None archived = True absolute_filename = testcase_path[base_len:] fuzzed_key = blobs.write_blob(file_handle) file_handle.close() # Don't need the archive after writing testcase to blobstore. if zip_path: shell.remove_file(zip_path) return fuzzed_key, archived, absolute_filename, zip_filename
def update_issue_owner_and_ccs_from_predator_results(policy, testcase, issue, only_allow_ccs=False): """Assign the issue to an appropriate owner if possible.""" if not issue or not issue.is_open: return # If the issue already has an owner, we don't need to update the bug. if issue.assignee: return # If we've assigned an owner or cc once before, it likely means we were # incorrect. Don't try again for this particular issue. if (issue_tracker_utils.was_label_added( issue, data_types.CHROMIUM_ISSUE_PREDATOR_AUTO_OWNER_LABEL) or issue_tracker_utils.was_label_added( issue, data_types.CHROMIUM_ISSUE_PREDATOR_AUTO_CC_LABEL)): return # If there are more than 3 suspected CLs, we can't be confident in the # results. Just skip any sort of notification to CL authors in this case. suspected_cls = _get_predator_result_item(testcase, 'suspected_cls') if not suspected_cls or len(suspected_cls) > 3: return # Validate that the suspected CLs have all of the information we need before # continuing. This allows us to assume that they are well-formed later, # avoiding any potential exceptions that would interrupt this task. for suspected_cl in suspected_cls: url = suspected_cl.get('url') description = suspected_cl.get('description') author = suspected_cl.get('author') if not url or not description or not author: logs.log_error( 'Suspected CL for testcase %d is missing required information.' % testcase.key.id()) return if len(suspected_cls) == 1 and not only_allow_ccs: suspected_cl = suspected_cls[0] # If this owner has already been assigned before but has since been removed, # don't assign it to them again. for action in issue.actions: if action.assignee == suspected_cls[0]['author']: return # We have high confidence for the single-CL case, so we assign the owner. issue.labels.add(data_types.CHROMIUM_ISSUE_PREDATOR_AUTO_OWNER_LABEL) issue.assignee = suspected_cl['author'] issue.status = policy.status('assigned') issue_comment = ( 'Automatically assigning owner based on suspected regression ' 'changelist %s (%s).\n\n' 'If this is incorrect, please let us know why and apply the %s ' 'label. If you aren\'t the correct owner for this issue, please ' 'unassign yourself as soon as possible so it can be re-triaged.' % (suspected_cl['url'], suspected_cl['description'], data_types.CHROMIUM_ISSUE_PREDATOR_WRONG_CL_LABEL)) else: if testcase.get_metadata('has_issue_ccs_from_predator_results'): return issue_comment = ( 'Automatically adding ccs based on suspected regression changelists:' '\n\n') ccs_added = False for suspected_cl in suspected_cls: # Update the comment with the suspected CL, regardless of whether or not # we're ccing the author. This might, for example, catch the attention of # someone who has already been cced. author = suspected_cl['author'] issue_comment += '%s by %s - %s\n\n' % ( suspected_cl['description'], author, suspected_cl['url']) if author in issue.ccs: continue # If an author has previously been manually removed from the cc list, # we assume they were incorrectly added. Don't try to add them again. author_was_removed = False for action in issue.actions: if author in action.ccs.removed: author_was_removed = True break if author_was_removed: continue issue.ccs.add(author) ccs_added = True if not ccs_added: # Everyone we'd expect to see has already been cced on the issue. No need # to spam it with another comment. Also, set the metadata to avoid doing # this again. testcase.set_metadata('has_issue_ccs_from_owners_file', True) return issue.labels.add(data_types.CHROMIUM_ISSUE_PREDATOR_AUTO_CC_LABEL) issue_comment += (( 'If this is incorrect, please let us know why and apply the ' '{label_text}.').format(label_text=issue.issue_tracker.label_text( data_types.CHROMIUM_ISSUE_PREDATOR_WRONG_CL_LABEL))) try: issue.save(new_comment=issue_comment, notify=True) except HttpError: # If we see such an error when we aren't setting an owner, it's unexpected. if only_allow_ccs or not issue.assignee: logs.log_error('Unable to update issue for test case %d.' % testcase.key.id()) return # Retry without setting the owner. They may not be a chromium project # member, in which case we can try falling back to cc. issue = issue_tracker_utils.get_issue_for_testcase(testcase) update_issue_owner_and_ccs_from_predator_results(policy, testcase, issue, only_allow_ccs=True)
def _sync_job(self, project, info, corpus_bucket_name, quarantine_bucket_name, logs_bucket_name, backup_bucket_name): """Sync the config with ClusterFuzz.""" # Create/update ClusterFuzz jobs. for template in get_jobs_for_project(project, info): if template.engine == 'none': # Engine-less jobs are not automatically managed. continue fuzzer_entity = self._fuzzer_entities.get(template.engine) if not fuzzer_entity: raise ProjectSetupError('Invalid fuzzing engine ' + template.engine) job_name = template.job_name(project, self._config_suffix) job = data_types.Job.query(data_types.Job.name == job_name).get() if not job: job = data_types.Job() if job_name not in fuzzer_entity.jobs and not info.get('disabled', False): # Enable new job. fuzzer_entity.jobs.append(job_name) job.name = job_name if self._segregate_projects: job.platform = untrusted.platform_name(project, 'linux') else: # TODO(ochang): Support other platforms? job.platform = 'LINUX' job.templates = template.cf_job_templates job.environment_string = JOB_TEMPLATE.format( build_type=self._build_type, build_bucket_path=self._get_build_bucket_path( project, info, template.engine, template.memory_tool, template.architecture), engine=template.engine, project=project) if self._add_revision_mappings: revision_vars_url = self._revision_url_template.format( project=project, bucket=self._get_build_bucket(template.engine, template.architecture), sanitizer=template.memory_tool) job.environment_string += ( 'REVISION_VARS_URL = {revision_vars_url}\n'.format( revision_vars_url=revision_vars_url)) if logs_bucket_name: job.environment_string += 'FUZZ_LOGS_BUCKET = {logs_bucket}\n'.format( logs_bucket=logs_bucket_name) if corpus_bucket_name: job.environment_string += 'CORPUS_BUCKET = {corpus_bucket}\n'.format( corpus_bucket=corpus_bucket_name) if quarantine_bucket_name: job.environment_string += ( 'QUARANTINE_BUCKET = {quarantine_bucket}\n'.format( quarantine_bucket=quarantine_bucket_name)) if backup_bucket_name: job.environment_string += 'BACKUP_BUCKET = {backup_bucket}\n'.format( backup_bucket=backup_bucket_name) if self._add_info_labels: job.environment_string += ( 'AUTOMATIC_LABELS = Proj-{project},Engine-{engine}\n'.format( project=project, engine=template.engine, )) help_url = info.get('help_url') if help_url: job.environment_string += 'HELP_URL = %s\n' % help_url if template.experimental: job.environment_string += 'EXPERIMENTAL = True\n' if template.minimize_job_override: minimize_job_override = template.minimize_job_override.job_name( project, self._config_suffix) job.environment_string += ( 'MINIMIZE_JOB_OVERRIDE = %s\n' % minimize_job_override) view_restrictions = info.get('view_restrictions') if view_restrictions: if view_restrictions in ALLOWED_VIEW_RESTRICTIONS: job.environment_string += ( 'ISSUE_VIEW_RESTRICTIONS = %s\n' % view_restrictions) else: logs.log_error('Invalid view restriction setting %s for project %s.' % (view_restrictions, project)) selective_unpack = info.get('selective_unpack') if selective_unpack: job.environment_string += 'UNPACK_ALL_FUZZ_TARGETS_AND_FILES = False\n' if (template.engine == 'libfuzzer' and template.architecture == 'x86_64' and 'dataflow' in info.get('fuzzing_engines', DEFAULT_ENGINES)): # Dataflow binaries are built with dataflow sanitizer, but can be used # as an auxiliary build with libFuzzer builds (e.g. with ASan or UBSan). dataflow_build_bucket_path = self._get_build_bucket_path( project_name=project, info=info, engine='dataflow', memory_tool='dataflow', architecture=template.architecture) job.environment_string += ( 'DATAFLOW_BUILD_BUCKET_PATH = %s\n' % dataflow_build_bucket_path) if self._additional_vars: additional_vars = {} additional_vars.update(self._additional_vars.get('all', {})) engine_vars = self._additional_vars.get(template.engine, {}) engine_sanitizer_vars = engine_vars.get(template.memory_tool, {}) additional_vars.update(engine_sanitizer_vars) for key, value in sorted(six.iteritems(additional_vars)): job.environment_string += ('{} = {}\n'.format( key, str(value).encode('unicode-escape').decode('utf-8'))) job.put()
def fuzz(self, target_path, options, reproducers_dir, max_time): """Run a fuzz session. Args: target_path: Path to the target. options: The FuzzOptions object returned by prepare(). reproducers_dir: The directory to put reproducers in when crashes are found. max_time: Maximum allowed time for the fuzzing to run. Returns: A FuzzResult object. """ profiler.start_if_needed('libfuzzer_fuzz') runner = libfuzzer.get_runner(target_path) launcher.set_sanitizer_options(target_path) # Directory to place new units. new_corpus_dir = self._create_temp_corpus_dir('new') corpus_directories = [new_corpus_dir] + options.fuzz_corpus_dirs fuzz_timeout = launcher.get_fuzz_timeout(options.is_mutations_run, total_timeout=max_time) fuzz_result = runner.fuzz(corpus_directories, fuzz_timeout=fuzz_timeout, additional_args=options.arguments, artifact_prefix=reproducers_dir, extra_env=options.extra_env) if (not environment.get_value('USE_MINIJAIL') and fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE): # Minijail returns 1 if the exit code is nonzero. # Otherwise: we can assume that a return code of 1 means that libFuzzer # itself ran into an error. logs.log_error(ENGINE_ERROR_MESSAGE, engine_output=fuzz_result.output) log_lines = utils.decode_to_unicode(fuzz_result.output).splitlines() # Output can be large, so save some memory by removing reference to the # original output which is no longer needed. fuzz_result.output = None # Check if we crashed, and get the crash testcase path. crash_testcase_file_path = runner.get_testcase_path(log_lines) # Parse stats information based on libFuzzer output. parsed_stats = launcher.parse_log_stats(log_lines) # Extend parsed stats by additional performance features. parsed_stats.update( stats.parse_performance_features(log_lines, options.strategies, options.arguments, include_strategies=False)) # Set some initial stat overrides. timeout_limit = fuzzer_utils.extract_argument(options.arguments, constants.TIMEOUT_FLAG, remove=False) expected_duration = runner.get_max_total_time(fuzz_timeout) actual_duration = int(fuzz_result.time_executed) fuzzing_time_percent = 100 * actual_duration / float(expected_duration) parsed_stats.update({ 'timeout_limit': int(timeout_limit), 'expected_duration': expected_duration, 'actual_duration': actual_duration, 'fuzzing_time_percent': fuzzing_time_percent, }) # Remove fuzzing arguments before merge and dictionary analysis step. arguments = options.arguments[:] launcher.remove_fuzzing_arguments(arguments) self._merge_new_units(target_path, options.corpus_dir, new_corpus_dir, options.fuzz_corpus_dirs, arguments, parsed_stats) fuzz_logs = '\n'.join(log_lines) crashes = [] if crash_testcase_file_path: # Write the new testcase. # Copy crash testcase contents into the main testcase path. crashes.append( engine.Crash(crash_testcase_file_path, fuzz_logs, arguments, actual_duration)) project_qualified_fuzzer_name = ( data_types.fuzz_target_project_qualified_name( utils.current_project(), os.path.basename(target_path))) launcher.analyze_and_update_recommended_dictionary( runner, project_qualified_fuzzer_name, log_lines, options.corpus_dir, arguments) return engine.FuzzResult(fuzz_logs, fuzz_result.command, crashes, parsed_stats, fuzz_result.time_executed)
def fuzz(self, target_path, options, reproducers_dir, max_time): """Run a fuzz session. Args: target_path: Path to the target. options: The FuzzOptions object returned by prepare(). reproducers_dir: The directory to put reproducers in when crashes are found. max_time: Maximum allowed time for the fuzzing to run. Returns: A FuzzResult object. """ profiler.start_if_needed('libfuzzer_fuzz') runner = libfuzzer.get_runner(target_path) libfuzzer.set_sanitizer_options(target_path, fuzz_options=options) # Directory to place new units. new_corpus_dir = self._create_temp_corpus_dir('new') corpus_directories = [new_corpus_dir] + options.fuzz_corpus_dirs fuzz_timeout = libfuzzer.get_fuzz_timeout(options.is_mutations_run, total_timeout=max_time) fuzz_result = runner.fuzz(corpus_directories, fuzz_timeout=fuzz_timeout, additional_args=options.arguments, artifact_prefix=reproducers_dir, extra_env=options.extra_env) project_qualified_fuzzer_name = _project_qualified_fuzzer_name( target_path) dict_error_match = DICT_PARSING_FAILED_REGEX.search(fuzz_result.output) if dict_error_match: logs.log_error( 'Dictionary parsing failed (target={target}, line={line}).'. format(target=project_qualified_fuzzer_name, line=dict_error_match.group(1)), engine_output=fuzz_result.output) elif (not environment.get_value('USE_MINIJAIL') and fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE): # Minijail returns 1 if the exit code is nonzero. # Otherwise: we can assume that a return code of 1 means that libFuzzer # itself ran into an error. logs.log_error(ENGINE_ERROR_MESSAGE + ' (target={target}).'.format( target=project_qualified_fuzzer_name), engine_output=fuzz_result.output) log_lines = fuzz_result.output.splitlines() # Output can be large, so save some memory by removing reference to the # original output which is no longer needed. fuzz_result.output = None # Check if we crashed, and get the crash testcase path. crash_testcase_file_path = runner.get_testcase_path(log_lines) # If we exited with a non-zero return code with no crash file in output from # libFuzzer, this is most likely a startup crash. Use an empty testcase to # to store it as a crash. if not crash_testcase_file_path and fuzz_result.return_code: crash_testcase_file_path = self._create_empty_testcase_file( reproducers_dir) # Parse stats information based on libFuzzer output. parsed_stats = libfuzzer.parse_log_stats(log_lines) # Extend parsed stats by additional performance features. parsed_stats.update( stats.parse_performance_features(log_lines, options.strategies, options.arguments)) # Set some initial stat overrides. timeout_limit = fuzzer_utils.extract_argument(options.arguments, constants.TIMEOUT_FLAG, remove=False) expected_duration = runner.get_max_total_time(fuzz_timeout) actual_duration = int(fuzz_result.time_executed) fuzzing_time_percent = 100 * actual_duration / float(expected_duration) parsed_stats.update({ 'timeout_limit': int(timeout_limit), 'expected_duration': expected_duration, 'actual_duration': actual_duration, 'fuzzing_time_percent': fuzzing_time_percent, }) # Remove fuzzing arguments before merge and dictionary analysis step. merge_arguments = options.arguments[:] libfuzzer.remove_fuzzing_arguments(merge_arguments, is_merge=True) self._merge_new_units(target_path, options.corpus_dir, new_corpus_dir, options.fuzz_corpus_dirs, merge_arguments, parsed_stats) fuzz_logs = '\n'.join(log_lines) crashes = [] if crash_testcase_file_path: reproduce_arguments = options.arguments[:] libfuzzer.remove_fuzzing_arguments(reproduce_arguments) # Use higher timeout for reproduction. libfuzzer.fix_timeout_argument_for_reproduction( reproduce_arguments) # Write the new testcase. # Copy crash testcase contents into the main testcase path. crashes.append( engine.Crash(crash_testcase_file_path, fuzz_logs, reproduce_arguments, actual_duration)) libfuzzer.analyze_and_update_recommended_dictionary( runner, project_qualified_fuzzer_name, log_lines, options.corpus_dir, merge_arguments) return engine.FuzzResult(fuzz_logs, fuzz_result.command, crashes, parsed_stats, fuzz_result.time_executed)
def run_process(cmdline, current_working_directory=None, timeout=DEFAULT_TEST_TIMEOUT, need_shell=False, gestures=None, env_copy=None, testcase_run=True, ignore_children=True): """Executes a process with a given command line and other parameters.""" # FIXME(mbarbella): Using LAUNCHER_PATH here is error prone. It forces us to # do certain operations before fuzzer setup (e.g. bad build check). launcher = environment.get_value('LAUNCHER_PATH') if environment.is_trusted_host() and testcase_run and not launcher: from bot.untrusted_runner import remote_process_host return remote_process_host.run_process(cmdline, current_working_directory, timeout, need_shell, gestures, env_copy, testcase_run, ignore_children) if gestures is None: gestures = [] if env_copy: os.environ.update(env_copy) # This is used when running scripts on native linux OS and not on the device. # E.g. running a fuzzer to generate testcases or launcher script. plt = environment.platform() if plt in ['ANDROID', 'FUCHSIA'] and (not testcase_run or launcher): plt = 'LINUX' elif plt == 'IOS' and (not testcase_run or launcher): plt = 'MAC' # Lower down testcase timeout slightly to account for time for crash analysis. timeout -= CRASH_ANALYSIS_TIME # LeakSanitizer hack - give time for stdout/stderr processing. lsan = environment.get_value('LSAN', False) if lsan: timeout -= LSAN_ANALYSIS_TIME # Initialize variables. adb_output = None process_output = '' process_status = None return_code = 0 process_poll_interval = environment.get_value('PROCESS_POLL_INTERVAL', 0.5) start_time = time.time() watch_for_process_exit = (environment.get_value('WATCH_FOR_PROCESS_EXIT') if plt == 'ANDROID' else True) window_list = [] # Get gesture start time from last element in gesture list. gestures = copy.deepcopy(gestures) if gestures and gestures[-1].startswith('Trigger'): gesture_start_time = int(gestures[-1].split(':')[1]) gestures.pop() else: gesture_start_time = timeout / 2 logs.log('Process (%s) started.' % str(cmdline), level=logging.DEBUG) if plt == 'ANDROID': # Clear the log upfront. android.logger.clear_log() # Run the app. adb_output = android.adb.run_adb_command(cmdline, timeout=timeout) else: cmd, args = shell.get_command_and_arguments(cmdline) process_output = mozprocess.processhandler.StoreOutput() process_status = ProcessStatus() try: process_handle = mozprocess.ProcessHandlerMixin( cmd, args, cwd=current_working_directory, shell=need_shell, processOutputLine=[process_output], onFinish=[process_status], ignore_children=ignore_children) start_process(process_handle) except: logs.log_error('Exception occurred when running command: %s.' % cmdline) return None, None, '' while True: time.sleep(process_poll_interval) # Run the gestures at gesture_start_time or in case we didn't find windows # in the last try. if (gestures and time.time() - start_time >= gesture_start_time and not window_list): # In case, we don't find any windows, we increment the gesture start time # so that the next check is after 1 second. gesture_start_time += 1 if plt == 'LINUX': linux.gestures.run_gestures(gestures, process_handle.pid, process_status, start_time, timeout, window_list) elif plt == 'WINDOWS': windows.gestures.run_gestures(gestures, process_handle.pid, process_status, start_time, timeout, window_list) elif plt == 'ANDROID': android.gestures.run_gestures(gestures, start_time, timeout) # TODO(mbarbella): We add a fake window here to prevent gestures on # Android from getting executed more than once. window_list = ['FAKE'] if time.time() - start_time >= timeout: break # Collect the process output. output = (android.logger.log_output() if plt == 'ANDROID' else '\n'.join(process_output.output)) if crash_analyzer.is_memory_tool_crash(output): break # Check if we need to bail out on process exit. if watch_for_process_exit: # If |watch_for_process_exit| is set, then we already completed running # our app launch command. So, we can bail out. if plt == 'ANDROID': break # On desktop, we bail out as soon as the process finishes. if process_status and process_status.finished: # Wait for process shutdown and set return code. process_handle.wait(timeout=PROCESS_CLEANUP_WAIT_TIME) break # Process output based on platform. if plt == 'ANDROID': # Get current log output. If device is in reboot mode, logcat automatically # waits for device to be online. time.sleep(ANDROID_CRASH_LOGCAT_WAIT_TIME) output = android.logger.log_output() if android.device.LOW_MEMORY_REGEX.search(output): # If the device is low on memory, we should force reboot and bail out to # prevent device from getting in a frozen state. logs.log('Device is low on memory, rebooting.', output=output) android.adb.hard_reset() android.adb.wait_for_device() android.device.setup_memory_monitor_script_if_needed() elif android.adb.time_since_last_reboot() < time.time() - start_time: # Check if a reboot has happened, if yes, append log output before reboot # and kernel logs content to output. log_before_last_reboot = android.logger.log_output_before_last_reboot( ) kernel_log = android.device.get_kernel_log_content() output = '%s%s%s%s%s' % ( log_before_last_reboot, utils.get_line_seperator('Device rebooted'), output, utils.get_line_seperator('Kernel Log'), kernel_log) # Make sure to reset SE Linux Permissive Mode. This can be done cheaply # in ~0.15 sec and is needed especially between runs for kernel crashes. android.adb.run_as_root() android.adb.change_se_linux_to_permissive_mode() return_code = 1 # Add output from adb to the front. if adb_output: output = '%s\n\n%s' % (adb_output, output) # Kill the application if it is still running. We do this at the end to # prevent this from adding noise to the logcat output. task_name = environment.get_value('TASK_NAME') child_process_termination_pattern = environment.get_value( 'CHILD_PROCESS_TERMINATION_PATTERN') if task_name == 'fuzz' and child_process_termination_pattern: # In some cases, we do not want to terminate the application after each # run to avoid long startup times (e.g. for chrome). Terminate processes # matching a particular pattern for light cleanup in this case. android.adb.kill_processes_and_children_matching_name( child_process_termination_pattern) else: # There is no special termination behavior. Simply stop the application. android.adb.stop_application() else: # Get the return code in case the process has finished already. # If the process hasn't finished, return_code will be None which is what # callers expect unless the output indicates a crash. return_code = process_handle.poll() # If the process is still running, then terminate it. if not process_status.finished: if launcher and cmdline.startswith(launcher): # If this was a launcher script, we KILL all child processes created # except for APP_NAME. # It is expected that, if the launcher script terminated normally, it # cleans up all the child processes it created itself. terminate_root_and_child_processes(process_handle.pid) else: try: # kill() here actually sends SIGTERM on posix. process_handle.kill() except: pass if lsan: time.sleep(LSAN_ANALYSIS_TIME) output = '\n'.join(process_output.output) # X Server hack when max client reached. if ('Maximum number of clients reached' in output or 'Unable to get connection to X server' in output): logs.log_error('Unable to connect to X server, exiting.') os.system('sudo killall -9 Xvfb blackbox >/dev/null 2>&1') sys.exit(0) if testcase_run and (crash_analyzer.is_memory_tool_crash(output) or crash_analyzer.is_check_failure_crash(output)): return_code = 1 # If a crash is found, then we add the memory state as well. if return_code and plt == 'ANDROID': ps_output = android.adb.get_ps_output() if ps_output: output += utils.get_line_seperator('Memory Statistics') output += ps_output logs.log('Process (%s) ended, exit code (%s), output (%s).' % (str(cmdline), str(return_code), str(output)), level=logging.DEBUG) return return_code, round(time.time() - start_time, 1), output
def prepare(self, corpus_dir, target_path, _): """Prepare for a fuzzing session, by generating options. Returns a FuzzOptions object. Args: corpus_dir: The main corpus directory. target_path: Path to the target. build_dir: Path to the build directory. Returns: A FuzzOptions object. """ arguments = fuzzer.get_arguments(target_path) grammar = fuzzer.get_grammar(target_path) strategy_pool = strategy_selection.generate_weighted_strategy_pool( strategy_list=strategy.LIBFUZZER_STRATEGY_LIST, use_generator=True, engine_name=self.name) strategy_info = libfuzzer.pick_strategies(strategy_pool, target_path, corpus_dir, arguments, grammar) arguments.extend(strategy_info.arguments) # Check for seed corpus and add it into corpus directory. engine_common.unpack_seed_corpus_if_needed(target_path, corpus_dir) # Pick a few testcases from our corpus to use as the initial corpus. subset_size = engine_common.random_choice( engine_common.CORPUS_SUBSET_NUM_TESTCASES) if (not strategy_info.use_dataflow_tracing and strategy_pool.do_strategy(strategy.CORPUS_SUBSET_STRATEGY) and shell.get_directory_file_count(corpus_dir) > subset_size): # Copy |subset_size| testcases into 'subset' directory. corpus_subset_dir = self._create_temp_corpus_dir('subset') libfuzzer.copy_from_corpus(corpus_subset_dir, corpus_dir, subset_size) strategy_info.fuzzing_strategies.append( strategy.CORPUS_SUBSET_STRATEGY.name + '_' + str(subset_size)) strategy_info.additional_corpus_dirs.append(corpus_subset_dir) else: strategy_info.additional_corpus_dirs.append(corpus_dir) # Check dict argument to make sure that it's valid. dict_path = fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG, remove=False) if dict_path and not os.path.exists(dict_path): logs.log_error('Invalid dict %s for %s.' % (dict_path, target_path)) fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG) # If there's no dict argument, check for %target_binary_name%.dict file. dict_path = fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG, remove=False) if not dict_path: dict_path = dictionary_manager.get_default_dictionary_path( target_path) if os.path.exists(dict_path): arguments.append(constants.DICT_FLAG + dict_path) # If we have a dictionary, correct any items that are not formatted properly # (e.g. quote items that are missing them). dictionary_manager.correct_if_needed(dict_path) strategies = stats.process_strategies(strategy_info.fuzzing_strategies, name_modifier=lambda x: x) return LibFuzzerOptions(corpus_dir, arguments, strategies, strategy_info.additional_corpus_dirs, strategy_info.extra_env, strategy_info.use_dataflow_tracing, strategy_info.is_mutations_run)
def main(argv): """Run libFuzzer as specified by argv.""" atexit.register(fuzzer_utils.cleanup) # Initialize variables. arguments = argv[1:] testcase_file_path = arguments.pop(0) target_name = arguments.pop(0) fuzzer_name = data_types.fuzz_target_project_qualified_name( utils.current_project(), target_name) # Initialize log handler. logs.configure( 'run_fuzzer', { 'fuzzer': fuzzer_name, 'engine': 'libFuzzer', 'job_name': environment.get_value('JOB_NAME') }) profiler.start_if_needed('libfuzzer_launcher') # Make sure that the fuzzer binary exists. build_directory = environment.get_value('BUILD_DIR') fuzzer_path = engine_common.find_fuzzer_path(build_directory, target_name) if not fuzzer_path: # This is an expected case when doing regression testing with old builds # that do not have that fuzz target. It can also happen when a host sends a # message to an untrusted worker that just restarted and lost information on # build directory. logs.log_warn('Could not find fuzz target %s.' % target_name) return # Install signal handler. signal.signal(signal.SIGTERM, engine_common.signal_term_handler) # Set up temp dir. engine_common.recreate_directory(fuzzer_utils.get_temp_dir()) # Setup minijail if needed. use_minijail = environment.get_value('USE_MINIJAIL') runner = libfuzzer.get_runner(fuzzer_path, temp_dir=fuzzer_utils.get_temp_dir()) if use_minijail: minijail_chroot = runner.chroot else: minijail_chroot = None # Get corpus directory. corpus_directory = environment.get_value('FUZZ_CORPUS_DIR') # Add common arguments which are necessary to be used for every run. arguments = expand_with_common_arguments(arguments) # Add sanitizer options to environment that were specified in the .options # file and options that this script requires. set_sanitizer_options(fuzzer_path) # Minimize test argument. minimize_to = fuzzer_utils.extract_argument(arguments, MINIMIZE_TO_ARGUMENT) minimize_timeout = fuzzer_utils.extract_argument( arguments, MINIMIZE_TIMEOUT_ARGUMENT) if minimize_to and minimize_timeout: minimize_testcase(runner, testcase_file_path, minimize_to, int(minimize_timeout), arguments, use_minijail) return # Cleanse argument. cleanse_to = fuzzer_utils.extract_argument(arguments, CLEANSE_TO_ARGUMENT) cleanse_timeout = fuzzer_utils.extract_argument(arguments, CLEANSE_TIMEOUT_ARGUMENT) if cleanse_to and cleanse_timeout: cleanse_testcase(runner, testcase_file_path, cleanse_to, int(cleanse_timeout), arguments, use_minijail) return # If we don't have a corpus, then that means this is not a fuzzing run. if not corpus_directory: load_testcase_if_exists(runner, testcase_file_path, fuzzer_name, use_minijail, arguments) return # We don't have a crash testcase, fuzz. # Check dict argument to make sure that it's valid. dict_argument = fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG, remove=False) if dict_argument and not os.path.exists(dict_argument): logs.log_error('Invalid dict %s for %s.' % (dict_argument, fuzzer_name)) fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG) # If there's no dict argument, check for %target_binary_name%.dict file. if (not fuzzer_utils.extract_argument( arguments, constants.DICT_FLAG, remove=False)): default_dict_path = dictionary_manager.get_default_dictionary_path( fuzzer_path) if os.path.exists(default_dict_path): arguments.append(constants.DICT_FLAG + default_dict_path) fuzzing_strategies = [] # Select a generator to use for existing testcase mutations. generator = _select_generator() is_mutations_run = generator != Generator.NONE # Timeout for fuzzer run. fuzz_timeout = get_fuzz_timeout(is_mutations_run) # Get list of corpus directories. corpus_directories = get_corpus_directories(corpus_directory, fuzzer_path, fuzzing_strategies, minijail_chroot) # Bind corpus directories in minijail. if use_minijail: artifact_prefix = constants.ARTIFACT_PREFIX_FLAG + '/' else: artifact_prefix = '%s%s/' % (constants.ARTIFACT_PREFIX_FLAG, os.path.abspath( os.path.dirname(testcase_file_path))) # Generate new testcase mutations using radamsa, etc. if is_mutations_run: new_testcase_mutations_directory = generate_new_testcase_mutations( corpus_directory, fuzzer_name, generator, fuzzing_strategies) corpus_directories.append(new_testcase_mutations_directory) if use_minijail: bind_corpus_dirs(minijail_chroot, [new_testcase_mutations_directory]) max_len_argument = fuzzer_utils.extract_argument(arguments, constants.MAX_LEN_FLAG, remove=False) if not max_len_argument and do_random_max_length(): max_length = random.SystemRandom().randint(1, MAX_VALUE_FOR_MAX_LENGTH) arguments.append('%s%d' % (constants.MAX_LEN_FLAG, max_length)) fuzzing_strategies.append(strategy.RANDOM_MAX_LENGTH_STRATEGY) if do_recommended_dictionary(): if add_recommended_dictionary(arguments, fuzzer_name, fuzzer_path): fuzzing_strategies.append(strategy.RECOMMENDED_DICTIONARY_STRATEGY) if do_value_profile(): arguments.append(constants.VALUE_PROFILE_ARGUMENT) fuzzing_strategies.append(strategy.VALUE_PROFILE_STRATEGY) if do_fork(): max_fuzz_threads = environment.get_value('MAX_FUZZ_THREADS', 1) num_fuzz_processes = max( 1, multiprocessing.cpu_count() / max_fuzz_threads) arguments.append('%s%d' % (constants.FORK_FLAG, num_fuzz_processes)) fuzzing_strategies.append('%s_%d' % (strategy.FORK_STRATEGY, num_fuzz_processes)) extra_env = {} if do_mutator_plugin(): if use_mutator_plugin(target_name, extra_env, minijail_chroot): fuzzing_strategies.append(strategy.MUTATOR_PLUGIN_STRATEGY) # Execute the fuzzer binary with original arguments. fuzz_result = runner.fuzz(corpus_directories, fuzz_timeout=fuzz_timeout, additional_args=arguments + [artifact_prefix], extra_env=extra_env) if (not use_minijail and fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE): # Minijail returns 1 if the exit code is nonzero. # Otherwise: we can assume that a return code of 1 means that libFuzzer # itself ran into an error. logs.log_error(ENGINE_ERROR_MESSAGE, engine_output=fuzz_result.output) log_lines = fuzz_result.output.splitlines() # Output can be large, so save some memory by removing reference to the # original output which is no longer needed. fuzz_result.output = None # Check if we crashed, and get the crash testcase path. crash_testcase_file_path = None for line in log_lines: match = re.match(CRASH_TESTCASE_REGEX, line) if match: crash_testcase_file_path = match.group(1) break if crash_testcase_file_path: # Write the new testcase. if use_minijail: # Convert chroot relative path to host path. Remove the leading '/' before # joining. crash_testcase_file_path = os.path.join( minijail_chroot.directory, crash_testcase_file_path[1:]) # Copy crash testcase contents into the main testcase path. shutil.move(crash_testcase_file_path, testcase_file_path) # Print the command output. log_header_format = ('Command: %s\n' 'Bot: %s\n' 'Time ran: %f\n') bot_name = environment.get_value('BOT_NAME', '') command = fuzz_result.command if use_minijail: # Remove minijail prefix. command = engine_common.strip_minijail_command(command, fuzzer_path) print(log_header_format % (engine_common.get_command_quoted(command), bot_name, fuzz_result.time_executed)) # Parse stats information based on libFuzzer output. parsed_stats = parse_log_stats(log_lines) # Extend parsed stats by additional performance features. parsed_stats.update( stats.parse_performance_features(log_lines, fuzzing_strategies, arguments)) # Set some initial stat overrides. timeout_limit = fuzzer_utils.extract_argument(arguments, constants.TIMEOUT_FLAG, remove=False) expected_duration = runner.get_max_total_time(fuzz_timeout) actual_duration = int(fuzz_result.time_executed) fuzzing_time_percent = 100 * actual_duration / float(expected_duration) stat_overrides = { 'timeout_limit': int(timeout_limit), 'expected_duration': expected_duration, 'actual_duration': actual_duration, 'fuzzing_time_percent': fuzzing_time_percent, } # Remove fuzzing arguments before merge and dictionary analysis step. remove_fuzzing_arguments(arguments) # Make a decision on whether merge step is needed at all. If there are no # new units added by libFuzzer run, then no need to do merge at all. new_units_added = parsed_stats.get('new_units_added', 0) merge_error = None if new_units_added: # Merge the new units with the initial corpus. if corpus_directory not in corpus_directories: corpus_directories.append(corpus_directory) # If this times out, it's possible that we will miss some units. However, if # we're taking >10 minutes to load/merge the corpus something is going very # wrong and we probably don't want to make things worse by adding units # anyway. merge_tmp_dir = None if not use_minijail: merge_tmp_dir = os.path.join(fuzzer_utils.get_temp_dir(), 'merge_workdir') engine_common.recreate_directory(merge_tmp_dir) old_corpus_len = shell.get_directory_file_count(corpus_directory) merge_directory = create_merge_directory() corpus_directories.insert(0, merge_directory) if use_minijail: bind_corpus_dirs(minijail_chroot, [merge_directory]) merge_result = runner.merge( corpus_directories, merge_timeout=engine_common.get_merge_timeout( DEFAULT_MERGE_TIMEOUT), tmp_dir=merge_tmp_dir, additional_args=arguments) move_mergeable_units(merge_directory, corpus_directory) new_corpus_len = shell.get_directory_file_count(corpus_directory) new_units_added = 0 merge_error = None if merge_result.timed_out: merge_error = 'Merging new testcases timed out:' elif merge_result.return_code != 0: merge_error = 'Merging new testcases failed:' else: new_units_added = new_corpus_len - old_corpus_len stat_overrides['new_units_added'] = new_units_added if merge_result.output: stat_overrides.update( stats.parse_stats_from_merge_log( merge_result.output.splitlines())) else: stat_overrides['new_units_added'] = 0 logs.log('Skipped corpus merge since no new units added by fuzzing.') # Get corpus size after merge. This removes the duplicate units that were # created during this fuzzing session. stat_overrides['corpus_size'] = shell.get_directory_file_count( corpus_directory) # Delete all corpus directories except for the main one. These were temporary # directories to store new testcase mutations and have already been merged to # main corpus directory. if corpus_directory in corpus_directories: corpus_directories.remove(corpus_directory) for directory in corpus_directories: shutil.rmtree(directory, ignore_errors=True) if use_minijail: unbind_corpus_dirs(minijail_chroot, corpus_directories) # Apply overridden stats to the parsed stats prior to dumping. parsed_stats.update(stat_overrides) # Dump stats data for further uploading to BigQuery. engine_common.dump_big_query_data(parsed_stats, testcase_file_path, LIBFUZZER_PREFIX, fuzzer_name, command) # Add custom crash state based on fuzzer name (if needed). add_custom_crash_state_if_needed(fuzzer_name, log_lines, parsed_stats) for line in log_lines: print(line) # Add fuzzing strategies used. engine_common.print_fuzzing_strategies(fuzzing_strategies) # Add merge error (if any). if merge_error: print(data_types.CRASH_STACKTRACE_END_MARKER) print(merge_error) print( 'Command:', get_printable_command(merge_result.command, fuzzer_path, use_minijail)) print(merge_result.output) analyze_and_update_recommended_dictionary(runner, fuzzer_name, log_lines, corpus_directory, arguments) # Close minijail chroot. if use_minijail: minijail_chroot.close() # Record the stats to make them easily searchable in stackdriver. if new_units_added: logs.log('New units added to corpus: %d.' % new_units_added, stats=parsed_stats) else: logs.log('No new units found.', stats=parsed_stats)