def prepare(self, corpus_dir, target_path, build_dir): """Prepare for a fuzzing session, by generating options. Returns a FuzzOptions object. Args: corpus_dir: The main corpus directory. target_path: Path to the target. build_dir: Path to the build directory. Returns: A FuzzOptions object. """ del build_dir arguments = fuzzer.get_arguments(target_path) grammar = fuzzer.get_grammar(target_path) if self.do_strategies: strategy_pool = strategy_selection.generate_weighted_strategy_pool( strategy_list=strategy.LIBFUZZER_STRATEGY_LIST, use_generator=True, engine_name=self.name) else: strategy_pool = strategy_selection.StrategyPool() strategy_info = libfuzzer.pick_strategies(strategy_pool, target_path, corpus_dir, arguments, grammar) arguments.extend(strategy_info.arguments) # Check for seed corpus and add it into corpus directory. engine_common.unpack_seed_corpus_if_needed(target_path, corpus_dir) # Pick a few testcases from our corpus to use as the initial corpus. subset_size = engine_common.random_choice( engine_common.CORPUS_SUBSET_NUM_TESTCASES) if (not strategy_info.use_dataflow_tracing and strategy_pool.do_strategy(strategy.CORPUS_SUBSET_STRATEGY) and shell.get_directory_file_count(corpus_dir) > subset_size): # Copy |subset_size| testcases into 'subset' directory. corpus_subset_dir = self._create_temp_corpus_dir('subset') libfuzzer.copy_from_corpus(corpus_subset_dir, corpus_dir, subset_size) strategy_info.fuzzing_strategies.append( strategy.CORPUS_SUBSET_STRATEGY.name + '_' + str(subset_size)) strategy_info.additional_corpus_dirs.append(corpus_subset_dir) else: strategy_info.additional_corpus_dirs.append(corpus_dir) # Check dict argument to make sure that it's valid. dict_path = fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG, remove=False) if dict_path and not os.path.exists(dict_path): logs.log_error('Invalid dict %s for %s.' % (dict_path, target_path)) fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG) # If there's no dict argument, check for %target_binary_name%.dict file. dict_path = fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG, remove=False) if not dict_path: dict_path = dictionary_manager.get_default_dictionary_path( target_path) if os.path.exists(dict_path): arguments.append(constants.DICT_FLAG + dict_path) # If we have a dictionary, correct any items that are not formatted properly # (e.g. quote items that are missing them). dictionary_manager.correct_if_needed(dict_path) strategies = stats.process_strategies(strategy_info.fuzzing_strategies, name_modifier=lambda x: x) return LibFuzzerOptions(corpus_dir, arguments, strategies, strategy_info.additional_corpus_dirs, strategy_info.extra_env, strategy_info.use_dataflow_tracing, strategy_info.is_mutations_run)
def _do_run_testcase_and_return_result_in_queue(crash_queue, thread_index, file_path, gestures, env_copy, upload_output=False): """Run a single testcase and return crash results in the crash queue.""" try: # Run testcase and check whether a crash occurred or not. return_code, crash_time, output = run_testcase(thread_index, file_path, gestures, env_copy) # Pull testcase directory to host to get any stats files. if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host file_host.pull_testcases_from_worker() # Analyze the crash. crash_output = _get_crash_output(output) crash_result = CrashResult(return_code, crash_time, crash_output) # To provide consistency between stats and logs, we use timestamp taken # from stats when uploading logs and testcase. if upload_output: log_time = _get_testcase_time(file_path) if crash_result.is_crash(): # Initialize resource list with the testcase path. resource_list = [file_path] resource_list += get_resource_paths(crash_output) # Store the crash stack file in the crash stacktrace directory # with filename as the hash of the testcase path. crash_stacks_directory = environment.get_value( 'CRASH_STACKTRACES_DIR') stack_file_path = os.path.join(crash_stacks_directory, utils.string_hash(file_path)) utils.write_data_to_file(crash_output, stack_file_path) # Put crash/no-crash results in the crash queue. crash_queue.put( Crash(file_path=file_path, crash_time=crash_time, return_code=return_code, resource_list=resource_list, gestures=gestures, stack_file_path=stack_file_path)) # Don't upload uninteresting testcases (no crash) or if there is no log to # correlate it with (not upload_output). if upload_output: upload_testcase(file_path, log_time) if upload_output: # Include full output for uploaded logs (crash output, merge output, etc). crash_result_full = CrashResult(return_code, crash_time, output) log = prepare_log_for_upload(crash_result_full.get_stacktrace(), return_code) upload_log(log, log_time) except Exception: logs.log_error('Exception occurred while running ' 'run_testcase_and_return_result_in_queue.')
def fuzz(self, target_path, options, reproducers_dir, max_time): """Run a fuzz session. Args: target_path: Path to the target. options: The FuzzOptions object returned by prepare(). reproducers_dir: The directory to put reproducers in when crashes are found. max_time: Maximum allowed time for the fuzzing to run. Returns: A FuzzResult object. """ profiler.start_if_needed('libfuzzer_fuzz') runner = libfuzzer.get_runner(target_path) libfuzzer.set_sanitizer_options(target_path, fuzz_options=options) # Directory to place new units. if options.merge_back_new_testcases: new_corpus_dir = self._create_temp_corpus_dir('new') corpus_directories = [new_corpus_dir] + options.fuzz_corpus_dirs else: corpus_directories = options.fuzz_corpus_dirs fuzz_result = runner.fuzz(corpus_directories, fuzz_timeout=max_time, additional_args=options.arguments, artifact_prefix=reproducers_dir, extra_env=options.extra_env) project_qualified_fuzzer_name = ( engine_common.get_project_qualified_fuzzer_name(target_path)) dict_error_match = DICT_PARSING_FAILED_REGEX.search(fuzz_result.output) if dict_error_match: logs.log_error( 'Dictionary parsing failed ' f'(target={project_qualified_fuzzer_name}, ' f'line={dict_error_match.group(1)}).', engine_output=fuzz_result.output) elif (not environment.get_value('USE_MINIJAIL') and fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE): # Minijail returns 1 if the exit code is nonzero. # Otherwise: we can assume that a return code of 1 means that libFuzzer # itself ran into an error. logs.log_error(ENGINE_ERROR_MESSAGE + f' (target={project_qualified_fuzzer_name}).', engine_output=fuzz_result.output) log_lines = fuzz_result.output.splitlines() # Output can be large, so save some memory by removing reference to the # original output which is no longer needed. fuzz_result.output = None # Check if we crashed, and get the crash testcase path. crash_testcase_file_path = runner.get_testcase_path(log_lines) # If we exited with a non-zero return code with no crash file in output from # libFuzzer, this is most likely a startup crash. Use an empty testcase to # to store it as a crash. if (not crash_testcase_file_path and fuzz_result.return_code not in constants.NONCRASH_RETURN_CODES): crash_testcase_file_path = self._create_empty_testcase_file( reproducers_dir) # Parse stats information based on libFuzzer output. parsed_stats = libfuzzer.parse_log_stats(log_lines) # Extend parsed stats by additional performance features. parsed_stats.update( stats.parse_performance_features(log_lines, options.strategies, options.arguments)) # Set some initial stat overrides. timeout_limit = fuzzer_utils.extract_argument(options.arguments, constants.TIMEOUT_FLAG, remove=False) actual_duration = int(fuzz_result.time_executed) fuzzing_time_percent = 100 * actual_duration / float(max_time) parsed_stats.update({ 'timeout_limit': int(timeout_limit), 'expected_duration': int(max_time), 'actual_duration': actual_duration, 'fuzzing_time_percent': fuzzing_time_percent, }) # Remove fuzzing arguments before merge and dictionary analysis step. non_fuzz_arguments = options.arguments.copy() libfuzzer.remove_fuzzing_arguments(non_fuzz_arguments, is_merge=True) if options.merge_back_new_testcases: self._merge_new_units(target_path, options.corpus_dir, new_corpus_dir, options.fuzz_corpus_dirs, non_fuzz_arguments, parsed_stats) fuzz_logs = '\n'.join(log_lines) crashes = [] if crash_testcase_file_path: reproduce_arguments = options.arguments[:] libfuzzer.remove_fuzzing_arguments(reproduce_arguments) # Use higher timeout for reproduction. libfuzzer.fix_timeout_argument_for_reproduction( reproduce_arguments) # Write the new testcase. # Copy crash testcase contents into the main testcase path. crashes.append( engine.Crash(crash_testcase_file_path, fuzz_logs, reproduce_arguments, actual_duration)) if options.analyze_dictionary: libfuzzer.analyze_and_update_recommended_dictionary( runner, project_qualified_fuzzer_name, log_lines, options.corpus_dir, non_fuzz_arguments) return engine.FuzzResult(fuzz_logs, fuzz_result.command, crashes, parsed_stats, fuzz_result.time_executed)
def get_crash_info(output): """Parse crash output to get (local) minidump path and any other information useful for crash uploading, and store in a CrashReportInfo object.""" crash_stacks_directory = environment.get_value('CRASH_STACKTRACES_DIR') output_lines = output.splitlines() num_lines = len(output_lines) is_android = environment.is_android() for i, line in enumerate(output_lines): if is_android: # If we are on Android, the dump extraction is more complicated. # The location placed in the crash-stacktrace is of the dump itself but # in fact only the MIME of the dump exists, and will have a different # extension. We need to pull the MIME and process it. match = re.match(CRASH_DUMP_PATH_MARKER, line) if not match: continue minidump_mime_filename_base = None for j in range(i + 1, num_lines): line = output_lines[j] match = re.match(r'(.*)\.dmp', line) if match: minidump_mime_filename_base = os.path.basename( match.group(1).strip()) break if not minidump_mime_filename_base: logs.log_error( 'Minidump marker was found, but no path in stacktrace.') return None # Look for MIME. If none found, bail. # We might not have copied over the crash dumps yet (copying is buffered), # so we want to search both the original directory and the one to which # the minidumps should later be copied. device_directories_to_search = [ constants.CRASH_DUMPS_DIR, os.path.dirname(line.strip()) ] device_minidump_search_paths = [] device_minidump_mime_path = None for device_directory in device_directories_to_search: device_minidump_mime_potential_paths = adb.run_shell_command( ['ls', '"%s"' % device_directory], root=True).splitlines() device_minidump_search_paths += device_minidump_mime_potential_paths for potential_path in device_minidump_mime_potential_paths: # Check that we actually found a file, and the right one (not logcat). if 'No such file or directory' in potential_path: continue if minidump_mime_filename_base not in potential_path: continue if '.up' in potential_path or '.dmp' in potential_path: device_minidump_mime_path = os.path.join( device_directory, potential_path) break # Break if we found a path. if device_minidump_mime_path is not None: break # If we still didn't find a minidump path, bail. if device_minidump_mime_path is None: logs.log_error('Could not get MIME path from ls:\n%s' % str(device_minidump_search_paths)) return None # Pull out MIME and parse to minidump file and MIME parameters. minidump_mime_filename = '%s.mime' % minidump_mime_filename_base local_minidump_mime_path = os.path.join(crash_stacks_directory, minidump_mime_filename) adb.run_command([ 'pull', '"%s"' % device_minidump_mime_path, local_minidump_mime_path ]) if not os.path.exists(local_minidump_mime_path): logs.log_error( 'Could not pull MIME from %s to %s.' % (device_minidump_mime_path, local_minidump_mime_path)) return None crash_info = parse_mime_to_crash_report_info( local_minidump_mime_path) if crash_info is None: return None crash_info.unsymbolized_stacktrace = output return crash_info # Other platforms are not currently supported. logs.log_error('Unable to fetch crash information for this platform.') return None # Could not find dump location, bail out. This could also happen when we don't # have a minidump location in stack at all, e.g. when testcase does not crash # during minimization. return None
# Check if the build succeeded based on the existence of the # local archive file. if os.path.exists(archive_path_local): # Build success. Now, copy it to google cloud storage and make it # public. os.system('gsutil cp %s %s' % (archive_path_local, archive_path_remote)) os.system('gsutil acl set public-read %s' % archive_path_remote) logs.log('Build succeeded, created %s.' % archive_filename) else: LAST_BUILD[tool_and_build_type] = '' logs.log_error('Build failed, unable to create %s.' % archive_filename) logs.log('Completed cycle, waiting for %d secs.' % wait_time) time.sleep(wait_time) if __name__ == '__main__': # Make sure environment is correctly configured. logs.configure('run_bot') environment.set_bot_environment() fail_wait = environment.get_value('FAIL_WAIT') # Continue this forever. while True: try: main() except Exception: logs.log_error('Failed to create build.') time.sleep(fail_wait)
def update_data_bundle(fuzzer, data_bundle): """Updates a data bundle to the latest version.""" # This module can't be in the global imports due to appengine issues # with multiprocessing and psutil imports. from clusterfuzz._internal.google_cloud_utils import gsutil # If we are using a data bundle on NFS, it is expected that our testcases # will usually be large enough that we would fill up our tmpfs directory # pretty quickly. So, change it to use an on-disk directory. if not data_bundle.is_local: testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK') environment.set_value('FUZZ_INPUTS', testcase_disk_directory) data_bundle_directory = get_data_bundle_directory(fuzzer.name) if not data_bundle_directory: logs.log_error('Failed to setup data bundle %s.' % data_bundle.name) return False if not shell.create_directory(data_bundle_directory, create_intermediates=True): logs.log_error('Failed to create data bundle %s directory.' % data_bundle.name) return False # Check if data bundle is up to date. If yes, skip the update. if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): logs.log('Data bundle was recently synced, skip.') return True # Fetch lock for this data bundle. if not _fetch_lock_for_data_bundle_update(data_bundle): logs.log_error('Failed to lock data bundle %s.' % data_bundle.name) return False # Re-check if another bot did the sync already. If yes, skip. if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): logs.log('Another bot finished the sync, skip.') _release_lock_for_data_bundle_update(data_bundle) return True time_before_sync_start = time.time() # No need to sync anything if this is a search index data bundle. In that # case, the fuzzer will generate testcases from a gcs bucket periodically. if not _is_search_index_data_bundle(data_bundle.name): bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name) if environment.is_trusted_host() and data_bundle.sync_to_worker: from clusterfuzz._internal.bot.untrusted_runner import corpus_manager from clusterfuzz._internal.bot.untrusted_runner import file_host worker_data_bundle_directory = file_host.rebase_to_worker_root( data_bundle_directory) file_host.create_directory(worker_data_bundle_directory, create_intermediates=True) result = corpus_manager.RemoteGSUtilRunner().rsync( bucket_url, worker_data_bundle_directory, delete=False) else: result = gsutil.GSUtilRunner().rsync(bucket_url, data_bundle_directory, delete=False) if result.return_code != 0: logs.log_error('Failed to sync data bundle %s: %s.' % (data_bundle.name, result.output)) _release_lock_for_data_bundle_update(data_bundle) return False # Update the testcase list file. testcase_manager.create_testcase_list_file(data_bundle_directory) # Write last synced time in the sync file. sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory) utils.write_data_to_file(time_before_sync_start, sync_file_path) if environment.is_trusted_host() and data_bundle.sync_to_worker: from clusterfuzz._internal.bot.untrusted_runner import file_host worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path) file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path) # Release acquired lock. _release_lock_for_data_bundle_update(data_bundle) return True
def archive_testcase_and_dependencies_in_gcs(resource_list, testcase_path): """Archive testcase and its dependencies, and store in blobstore.""" if not os.path.exists(testcase_path): logs.log_error('Unable to find testcase %s.' % testcase_path) return None, None, None, None absolute_filename = testcase_path archived = False zip_filename = None zip_path = None if not resource_list: resource_list = [] # Add resource dependencies based on testcase path. These include # stuff like extensions directory, dependency files, etc. resource_list.extend( testcase_manager.get_resource_dependencies(testcase_path)) # Filter out duplicates, directories, and files that do not exist. resource_list = utils.filter_file_list(resource_list) logs.log('Testcase and related files :\n%s' % str(resource_list)) if len(resource_list) <= 1: # If this does not have any resources, just save the testcase. # TODO(flowerhack): Update this when we teach CF how to download testcases. try: file_handle = open(testcase_path, 'rb') except IOError: logs.log_error('Unable to open testcase %s.' % testcase_path) return None, None, None, None else: # If there are resources, create an archive. # Find the common root directory for all of the resources. # Assumption: resource_list[0] is the testcase path. base_directory_list = resource_list[0].split(os.path.sep) for list_index in range(1, len(resource_list)): current_directory_list = resource_list[list_index].split( os.path.sep) length = min(len(base_directory_list), len(current_directory_list)) for directory_index in range(length): if (current_directory_list[directory_index] != base_directory_list[directory_index]): base_directory_list = base_directory_list[ 0:directory_index] break base_directory = os.path.sep.join(base_directory_list) logs.log('Subresource common base directory: %s' % base_directory) if base_directory: # Common parent directory, archive sub-paths only. base_len = len(base_directory) + len(os.path.sep) else: # No common parent directory, archive all paths as it-is. base_len = 0 # Prepare the filename for the archive. zip_filename, _ = os.path.splitext(os.path.basename(testcase_path)) zip_filename += _TESTCASE_ARCHIVE_EXTENSION # Create the archive. zip_path = os.path.join(environment.get_value('INPUT_DIR'), zip_filename) zip_file = zipfile.ZipFile(zip_path, 'w') for file_name in resource_list: if os.path.exists(file_name): relative_filename = file_name[base_len:] zip_file.write(file_name, relative_filename, zipfile.ZIP_DEFLATED) zip_file.close() try: file_handle = open(zip_path, 'rb') except IOError: logs.log_error('Unable to open testcase archive %s.' % zip_path) return None, None, None, None archived = True absolute_filename = testcase_path[base_len:] fuzzed_key = blobs.write_blob(file_handle) file_handle.close() # Don't need the archive after writing testcase to blobstore. if zip_path: shell.remove_file(zip_path) return fuzzed_key, archived, absolute_filename, zip_filename
def execute_task(testcase_id, job_type): """Run analyze task.""" # Reset redzones. environment.reset_current_memory_tool_options(redzone_size=128) # Unset window location size and position properties so as to use default. environment.set_value('WINDOW_ARG', '') # Locate the testcase associated with the id. testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) metadata = data_types.TestcaseUploadMetadata.query( data_types.TestcaseUploadMetadata.testcase_id == int( testcase_id)).get() if not metadata: logs.log_error('Testcase %s has no associated upload metadata.' % testcase_id) testcase.key.delete() return is_lsan_enabled = environment.get_value('LSAN') if is_lsan_enabled: # Creates empty local blacklist so all leaks will be visible to uploader. leak_blacklist.create_empty_local_blacklist() # Store the bot name and timestamp in upload metadata. bot_name = environment.get_value('BOT_NAME') metadata.bot_name = bot_name metadata.timestamp = datetime.datetime.utcnow() metadata.put() # Adjust the test timeout, if user has provided one. if metadata.timeout: environment.set_value('TEST_TIMEOUT', metadata.timeout) # Adjust the number of retries, if user has provided one. if metadata.retries is not None: environment.set_value('CRASH_RETRIES', metadata.retries) # Set up testcase and get absolute testcase path. file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type) if not file_list: return # Set up build. setup_build(testcase) # Check if we have an application path. If not, our build failed # to setup correctly. if not build_manager.check_app_path(): data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, 'Build setup failed') if data_handler.is_first_retry_for_task(testcase): build_fail_wait = environment.get_value('FAIL_WAIT') tasks.add_task('analyze', testcase_id, job_type, wait_time=build_fail_wait) else: data_handler.close_invalid_uploaded_testcase( testcase, metadata, 'Build setup failed') return # Update initial testcase information. testcase.absolute_path = testcase_file_path testcase.job_type = job_type testcase.binary_flag = utils.is_binary_file(testcase_file_path) testcase.queue = tasks.default_queue() testcase.crash_state = '' # Set initial testcase metadata fields (e.g. build url, etc). data_handler.set_initial_testcase_metadata(testcase) # Update minimized arguments and use ones provided during user upload. if not testcase.minimized_arguments: minimized_arguments = environment.get_value('APP_ARGS') or '' additional_command_line_flags = testcase.get_metadata( 'uploaded_additional_args') if additional_command_line_flags: minimized_arguments += ' %s' % additional_command_line_flags environment.set_value('APP_ARGS', minimized_arguments) testcase.minimized_arguments = minimized_arguments # Update other fields not set at upload time. testcase.crash_revision = environment.get_value('APP_REVISION') data_handler.set_initial_testcase_metadata(testcase) testcase.put() # Initialize some variables. gestures = testcase.gestures http_flag = testcase.http_flag test_timeout = environment.get_value('TEST_TIMEOUT') # Get the crash output. result = testcase_manager.test_for_crash_with_retries(testcase, testcase_file_path, test_timeout, http_flag=http_flag, compare_crash=False) # If we don't get a crash, try enabling http to see if we can get a crash. # Skip engine fuzzer jobs (e.g. libFuzzer, AFL) for which http testcase paths # are not applicable. if (not result.is_crash() and not http_flag and not environment.is_engine_fuzzer_job()): result_with_http = testcase_manager.test_for_crash_with_retries( testcase, testcase_file_path, test_timeout, http_flag=True, compare_crash=False) if result_with_http.is_crash(): logs.log('Testcase needs http flag for crash.') http_flag = True result = result_with_http # Refresh our object. testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return # Set application command line with the correct http flag. application_command_line = ( testcase_manager.get_command_line_for_application( testcase_file_path, needs_http=http_flag)) # Get the crash data. crashed = result.is_crash() crash_time = result.get_crash_time() state = result.get_symbolized_data() unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False) # Get crash info object with minidump info. Also, re-generate unsymbolized # stacktrace if needed. crash_info, _ = (crash_uploader.get_crash_info_and_stacktrace( application_command_line, state.crash_stacktrace, gestures)) if crash_info: testcase.minidump_keys = crash_info.store_minidump() if not crashed: # Could not reproduce the crash. log_message = ('Testcase didn\'t crash in %d seconds (with retries)' % test_timeout) data_handler.update_testcase_comment(testcase, data_types.TaskState.FINISHED, log_message) # In the general case, we will not attempt to symbolize if we do not detect # a crash. For user uploads, we should symbolize anyway to provide more # information about what might be happening. crash_stacktrace_output = utils.get_crash_stacktrace_output( application_command_line, state.crash_stacktrace, unsymbolized_crash_stacktrace) testcase.crash_stacktrace = data_handler.filter_stacktrace( crash_stacktrace_output) # For an unreproducible testcase, retry once on another bot to confirm # our results and in case this bot is in a bad state which we didn't catch # through our usual means. if data_handler.is_first_retry_for_task(testcase): testcase.status = 'Unreproducible, retrying' testcase.put() tasks.add_task('analyze', testcase_id, job_type) return data_handler.close_invalid_uploaded_testcase(testcase, metadata, 'Unreproducible') # A non-reproducing testcase might still impact production branches. # Add the impact task to get that information. task_creation.create_impact_task_if_needed(testcase) return # Update testcase crash parameters. testcase.http_flag = http_flag testcase.crash_type = state.crash_type testcase.crash_address = state.crash_address testcase.crash_state = state.crash_state crash_stacktrace_output = utils.get_crash_stacktrace_output( application_command_line, state.crash_stacktrace, unsymbolized_crash_stacktrace) testcase.crash_stacktrace = data_handler.filter_stacktrace( crash_stacktrace_output) # Try to guess if the bug is security or not. security_flag = crash_analyzer.is_security_issue(state.crash_stacktrace, state.crash_type, state.crash_address) testcase.security_flag = security_flag # If it is, guess the severity. if security_flag: testcase.security_severity = severity_analyzer.get_security_severity( state.crash_type, state.crash_stacktrace, job_type, bool(gestures)) log_message = ('Testcase crashed in %d seconds (r%d)' % (crash_time, testcase.crash_revision)) data_handler.update_testcase_comment(testcase, data_types.TaskState.FINISHED, log_message) # See if we have to ignore this crash. if crash_analyzer.ignore_stacktrace(state.crash_stacktrace): data_handler.close_invalid_uploaded_testcase(testcase, metadata, 'Irrelavant') return # Test for reproducibility. one_time_crasher_flag = not testcase_manager.test_for_reproducibility( testcase.fuzzer_name, testcase.actual_fuzzer_name(), testcase_file_path, state.crash_state, security_flag, test_timeout, http_flag, gestures) testcase.one_time_crasher_flag = one_time_crasher_flag # Check to see if this is a duplicate. data_handler.check_uploaded_testcase_duplicate(testcase, metadata) # Set testcase and metadata status if not set already. if testcase.status == 'Duplicate': # For testcase uploaded by bots (with quiet flag), don't create additional # tasks. if metadata.quiet_flag: data_handler.close_invalid_uploaded_testcase( testcase, metadata, 'Duplicate') return else: # New testcase. testcase.status = 'Processed' metadata.status = 'Confirmed' # Reset the timestamp as well, to respect # data_types.MIN_ELAPSED_TIME_SINCE_REPORT. Otherwise it may get filed by # triage task prematurely without the grouper having a chance to run on this # testcase. testcase.timestamp = utils.utcnow() # Add new leaks to global blacklist to avoid detecting duplicates. # Only add if testcase has a direct leak crash and if it's reproducible. if is_lsan_enabled: leak_blacklist.add_crash_to_global_blacklist_if_needed(testcase) # Update the testcase values. testcase.put() # Update the upload metadata. metadata.security_flag = security_flag metadata.put() _add_default_issue_metadata(testcase) # Create tasks to # 1. Minimize testcase (minimize). # 2. Find regression range (regression). # 3. Find testcase impact on production branches (impact). # 4. Check whether testcase is fixed (progression). # 5. Get second stacktrace from another job in case of # one-time crashes (stack). task_creation.create_tasks(testcase)
def upload_testcases_if_needed(fuzzer_name, testcase_list, testcase_directory, data_directory): """Upload test cases from the list to a cloud storage bucket.""" # Since builtin fuzzers have a coverage minimized corpus, no need to upload # test case samples for them. if fuzzer_name in fuzzing.ENGINES: return bucket_name = local_config.ProjectConfig().get( 'coverage.fuzzer-testcases.bucket') if not bucket_name: return files_list = [] has_testcases_in_testcase_directory = False has_testcases_in_data_directory = False for testcase_path in testcase_list: if testcase_path.startswith(testcase_directory): files_list.append( os.path.relpath(testcase_path, testcase_directory)) has_testcases_in_testcase_directory = True elif testcase_path.startswith(data_directory): files_list.append(os.path.relpath(testcase_path, data_directory)) has_testcases_in_data_directory = True if not files_list: return formatted_date = str(utils.utcnow().date()) gcs_base_url = 'gs://{bucket_name}/{date}/{fuzzer_name}/'.format( bucket_name=bucket_name, date=formatted_date, fuzzer_name=fuzzer_name) runner = gsutil.GSUtilRunner() batch_directory_blobs = storage.list_blobs(gcs_base_url) total_testcases = 0 for blob in batch_directory_blobs: if not blob.endswith(LIST_FILE_BASENAME): continue list_gcs_url = storage.get_cloud_storage_file_path(bucket_name, blob) data = storage.read_data(list_gcs_url) if not data: logs.log_error( 'Read no data from test case list at {gcs_url}'.format( gcs_url=list_gcs_url)) continue total_testcases += len(data.splitlines()) # If we've already uploaded enough test cases for this fuzzer today, return. if total_testcases >= TESTCASES_PER_DAY: return # Cap the number of files. testcases_limit = min(len(files_list), TESTCASES_PER_DAY - total_testcases) files_list = files_list[:testcases_limit] # Upload each batch of tests to its own unique sub-bucket. identifier = environment.get_value('BOT_NAME') + str(utils.utcnow()) gcs_base_url += utils.string_hash(identifier) list_gcs_url = gcs_base_url + '/' + LIST_FILE_BASENAME if not storage.write_data('\n'.join(files_list).encode('utf-8'), list_gcs_url): return if has_testcases_in_testcase_directory: # Sync everything in |testcase_directory| since it is fuzzer-generated. runner.rsync(testcase_directory, gcs_base_url) if has_testcases_in_data_directory: # Sync all fuzzer generated testcase in data bundle directory. runner.rsync(data_directory, gcs_base_url, exclusion_pattern=('(?!.*{fuzz_prefix})'.format( fuzz_prefix=testcase_manager.FUZZ_PREFIX))) # Sync all possible resource dependencies as a best effort. It matches # |resources-| prefix that a fuzzer can use to indicate resources. Also, it # matches resources directory that Chromium web_tests use for dependencies. runner.rsync(data_directory, gcs_base_url, exclusion_pattern='(?!.*resource)') logs.log('Synced {count} test cases to {gcs_url}.'.format( count=len(files_list), gcs_url=gcs_base_url))
def filter_log_output(output): """Filters log output. Removes debug info, etc and normalize output.""" if not output: return '' filtered_output = '' last_process_tuple = (None, None) for line in output.splitlines(): if not is_line_valid(line): continue # To parse frames like: # E/v8 (18890): Error installing extension 'v8/LoadTimes'. # {log_level}/{process_name}({process_id}): {message} m_line = re.match(r'^[VDIWEFS]/(.+)\(\s*(\d+)\)[:](.*)$', line) if not m_line: logs.log_error('Failed to parse logcat line: %s' % line) continue process_name = m_line.group(1).strip() process_id = int(m_line.group(2)) filtered_line = m_line.group(3).rstrip()[1:] # Process Android crash stack frames and convert into sanitizer format. m_crash_state = re.match(r'\s*#([0-9]+)\s+pc\s+([xX0-9a-fA-F]+)\s+(.+)', filtered_line) if m_crash_state: frame_no = int(m_crash_state.group(1)) frame_address = m_crash_state.group(2) frame_binary = m_crash_state.group(3).strip() # Ignore invalid frames, helps to prevent errors # while symbolizing. if '<unknown>' in frame_binary: continue # Normalize frame address. if not frame_address.startswith('0x'): frame_address = '0x%s' % frame_address # Separate out the function argument. frame_binary = (frame_binary.split(' '))[0] # Normalize line into the same sanitizer tool format. filtered_line = (' #%d %s (%s+%s)' % (frame_no, frame_address, frame_binary, frame_address)) # Process Chrome crash stack frames and convert into sanitizer format. # Stack frames don't have paranthesis around frame binary and address, so # add it explicitly to allow symbolizer to catch it. m_crash_state = re.match( r'\s*#([0-9]+)\s+([xX0-9a-fA-F]+)\s+([^(]+\+[xX0-9a-fA-F]+)$', filtered_line) if m_crash_state: frame_no = int(m_crash_state.group(1)) frame_address = m_crash_state.group(2) frame_binary_and_address = m_crash_state.group(3).strip() filtered_line = (' #%d %s (%s)' % (frame_no, frame_address, frame_binary_and_address)) # Add process number if changed. current_process_tuple = (process_name, process_id) if current_process_tuple != last_process_tuple: filtered_output += '--------- %s (%d):\n' % (process_name, process_id) last_process_tuple = current_process_tuple filtered_output += filtered_line + '\n' return filtered_output
def execute_task(metadata_id, job_type): """Unpack a bundled testcase archive and create analyze jobs for each item.""" metadata = ndb.Key(data_types.BundledArchiveMetadata, int(metadata_id)).get() if not metadata: logs.log_error('Invalid bundle metadata id %s.' % metadata_id) return bot_name = environment.get_value('BOT_NAME') upload_metadata = data_types.TestcaseUploadMetadata.query( data_types.TestcaseUploadMetadata.blobstore_key == metadata.blobstore_key).get() if not upload_metadata: logs.log_error('Invalid upload metadata key %s.' % metadata.blobstore_key) return job = data_types.Job.query(data_types.Job.name == metadata.job_type).get() if not job: logs.log_error('Invalid job_type %s.' % metadata.job_type) return # Update the upload metadata with this bot name. upload_metadata.bot_name = bot_name upload_metadata.put() # We can't use FUZZ_INPUTS directory since it is constrained # by tmpfs limits. testcases_directory = environment.get_value('FUZZ_INPUTS_DISK') # Retrieve multi-testcase archive. archive_path = os.path.join(testcases_directory, metadata.archive_filename) if not blobs.read_blob_to_disk(metadata.blobstore_key, archive_path): logs.log_error('Could not retrieve archive for bundle %d.' % metadata_id) tasks.add_task('unpack', metadata_id, job_type) return try: archive.unpack(archive_path, testcases_directory) except: logs.log_error('Could not unpack archive for bundle %d.' % metadata_id) tasks.add_task('unpack', metadata_id, job_type) return # Get additional testcase metadata (if any). additional_metadata = None if upload_metadata.additional_metadata_string: additional_metadata = json.loads( upload_metadata.additional_metadata_string) archive_state = data_types.ArchiveStatus.NONE bundled = True file_list = archive.get_file_list(archive_path) for file_path in file_list: absolute_file_path = os.path.join(testcases_directory, file_path) filename = os.path.basename(absolute_file_path) # Only files are actual testcases. Skip directories. if not os.path.isfile(absolute_file_path): continue try: file_handle = open(absolute_file_path, 'rb') blob_key = blobs.write_blob(file_handle) file_handle.close() except: blob_key = None if not blob_key: logs.log_error('Could not write testcase %s to blobstore.' % absolute_file_path) continue data_handler.create_user_uploaded_testcase( blob_key, metadata.blobstore_key, archive_state, metadata.archive_filename, filename, metadata.timeout, job, metadata.job_queue, metadata.http_flag, metadata.gestures, metadata.additional_arguments, metadata.bug_information, metadata.crash_revision, metadata.uploader_email, metadata.platform_id, metadata.app_launch_command, metadata.fuzzer_name, metadata.overridden_fuzzer_name, metadata.fuzzer_binary_name, bundled, upload_metadata.retries, upload_metadata.bug_summary_update_flag, upload_metadata.quiet_flag, additional_metadata) # The upload metadata for the archive is not needed anymore since we created # one for each testcase. upload_metadata.key.delete() shell.clear_testcase_directories()
def get(self): """Handle a get request.""" try: grouper.group_testcases() except: logs.log_error('Error occurred while grouping test cases.') return # Free up memory after group task run. utils.python_gc() # Get a list of jobs excluded from bug filing. excluded_jobs = _get_excluded_jobs() # Get a list of all jobs. This is used to filter testcases whose jobs have # been removed. all_jobs = data_handler.get_all_job_type_names() for testcase_id in data_handler.get_open_testcase_id_iterator(): try: testcase = data_handler.get_testcase_by_id(testcase_id) except errors.InvalidTestcaseError: # Already deleted. continue # Skip if testcase's job is removed. if testcase.job_type not in all_jobs: continue # Skip if testcase's job is in exclusions list. if testcase.job_type in excluded_jobs: continue # Skip if we are running progression task at this time. if testcase.get_metadata('progression_pending'): continue # If the testcase has a bug filed already, no triage is needed. if _is_bug_filed(testcase): continue # Check if the crash is important, i.e. it is either a reproducible crash # or an unreproducible crash happening frequently. if not _is_crash_important(testcase): continue # Require that all tasks like minimizaton, regression testing, etc have # finished. if not data_handler.critical_tasks_completed(testcase): continue # For testcases that are not part of a group, wait an additional time to # make sure it is grouped. # The grouper runs prior to this step in the same cron, but there is a # window of time where new testcases can come in after the grouper starts. # This delay needs to be longer than the maximum time the grouper can take # to account for that. # FIXME: In future, grouping might be dependent on regression range, so we # would have to add an additional wait time. if not testcase.group_id and not dates.time_has_expired( testcase.timestamp, hours=data_types.MIN_ELAPSED_TIME_SINCE_REPORT): continue # If this project does not have an associated issue tracker, we cannot # file this crash anywhere. issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase( testcase) if not issue_tracker: issue_filer.notify_issue_update(testcase, 'new') continue # If there are similar issues to this test case already filed or recently # closed, skip filing a duplicate bug. if _check_and_update_similar_bug(testcase, issue_tracker): continue # Clean up old triage messages that would be not applicable now. testcase.delete_metadata(TRIAGE_MESSAGE_KEY, update_testcase=False) # File the bug first and then create filed bug metadata. try: issue_filer.file_issue(testcase, issue_tracker) except Exception as e: logs.log_error('Failed to file issue for testcase %d.' % testcase_id) _add_triage_message(testcase, f'Failed to file issue due to exception: {str(e)}') continue _create_filed_bug_metadata(testcase) issue_filer.notify_issue_update(testcase, 'new') logs.log('Filed new issue %s for testcase %d.' % (testcase.bug_information, testcase_id))
def recreate_instance_with_disks(instance_name, project, zone, additional_metadata=None, wait_for_completion=False): """Recreate an instance and its disk.""" # Get existing instance information. # First, try to get instance info from cache. # TODO(ochang): Make this more general in case anything else needs to use # this method (e.g. appengine). instance_info = persistent_cache.get_value(GCE_INSTANCE_INFO_KEY) if instance_info is None: instance_info = _get_instance_info(instance_name, project, zone) # Bail out if we don't have a valid instance information. if (not instance_info or 'disks' not in instance_info or not instance_info['disks']): logs.log_error( 'Failed to get disk info from existing instance, bailing on instance ' 'recreation.') return False # Add any additional metadata required for instance booting. if additional_metadata: for key, value in six.iteritems(additional_metadata): items = instance_info.setdefault('metadata', {}).setdefault('items', []) _add_metadata_key_value(items, key, value) # Cache the latest instance information. persistent_cache.set_value( GCE_INSTANCE_INFO_KEY, instance_info, persist_across_reboots=True) # Delete the instance. if not _do_instance_operation( 'delete', instance_name, project, zone, wait_for_completion=True): logs.log_error('Failed to delete instance.') return False # Get existing disks information, and recreate. api = _get_api() disks = instance_info['disks'] for disk in disks: disk_source = disk['source'] disk_name = disk_source.split('/')[-1] disk_info_func = api.disks().get(disk=disk_name, project=project, zone=zone) disk_info = _execute_api_call_with_retries(disk_info_func) if 'sourceImage' not in disk_info or 'sizeGb' not in disk_info: logs.log_error( 'Failed to get source image and size from existing disk, bailing on ' 'instance recreation.') return False size_gb = disk_info['sizeGb'] source_image = disk_info['sourceImage'] # Recreate the disk. if not delete_disk(disk_name, project, zone, wait_for_completion=True): logs.log_error('Failed to delete disk.') return False if not create_disk( disk_name, source_image, size_gb, project, zone, wait_for_completion=True): logs.log_error('Failed to recreate disk.') return False # Recreate the instance with the exact same configurations, but not # necessarily the same IPs. try: del instance_info['networkInterfaces'][0]['accessConfigs'][0]['natIP'] except: # This is not a failure. When a bot is stopped, it has no ip/interface. pass try: del instance_info['networkInterfaces'][0]['networkIP'] except: # This is not a failure. When a bot is stopped, it has no ip/interface. pass operation = api.instances().insert( body=instance_info, project=project, zone=zone) return _do_operation_with_retries( operation, project, zone, wait_for_completion=wait_for_completion)
def group_testcases(): """Group testcases based on rules like same bug numbers, similar crash states, etc.""" testcase_map = {} cached_issue_map = {} for testcase_id in data_handler.get_open_testcase_id_iterator(): try: testcase = data_handler.get_testcase_by_id(testcase_id) except errors.InvalidTestcaseError: # Already deleted. continue # Remove duplicates early on to avoid large groups. if (not testcase.bug_information and not testcase.uploader_email and _has_testcase_with_same_params(testcase, testcase_map)): logs.log('Deleting duplicate testcase %d.' % testcase_id) testcase.key.delete() continue # Wait for minimization to finish as this might change crash params such # as type and may mark it as duplicate / closed. if not testcase.minimized_keys: continue # Store needed testcase attributes into |testcase_map|. testcase_map[testcase_id] = TestcaseAttributes(testcase_id) testcase_attributes = testcase_map[testcase_id] for attribute_name in FORWARDED_ATTRIBUTES: setattr(testcase_attributes, attribute_name, getattr(testcase, attribute_name)) # Store original issue mappings in the testcase attributes. if testcase.bug_information: issue_id = int(testcase.bug_information) project_name = testcase.project_name if (project_name in cached_issue_map and issue_id in cached_issue_map[project_name]): testcase_attributes.issue_id = ( cached_issue_map[project_name][issue_id]) else: issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase( testcase) if not issue_tracker: logs.log_error( 'Unable to access issue tracker for issue %d.' % issue_id) testcase_attributes.issue_id = issue_id continue # Determine the original issue id traversing the list of duplicates. try: issue = issue_tracker.get_original_issue(issue_id) original_issue_id = int(issue.id) except: # If we are unable to access the issue, then we can't determine # the original issue id. Assume that it is the same as issue id. logs.log_error( 'Unable to determine original issue for issue %d.' % issue_id) testcase_attributes.issue_id = issue_id continue if project_name not in cached_issue_map: cached_issue_map[project_name] = {} cached_issue_map[project_name][issue_id] = original_issue_id cached_issue_map[project_name][ original_issue_id] = original_issue_id testcase_attributes.issue_id = original_issue_id # No longer needed. Free up some memory. cached_issue_map.clear() _group_testcases_with_similar_states(testcase_map) _group_testcases_with_same_issues(testcase_map) _shrink_large_groups_if_needed(testcase_map) group_leader.choose(testcase_map) # TODO(aarya): Replace with an optimized implementation using dirty flag. # Update the group mapping in testcase object. for testcase_id in data_handler.get_open_testcase_id_iterator(): if testcase_id not in testcase_map: # A new testcase that was just created. Skip for now, will be grouped in # next iteration of group task. continue # If we are part of a group, then calculate the number of testcases in that # group and lowest issue id of issues associated with testcases in that # group. updated_group_id = testcase_map[testcase_id].group_id updated_is_leader = testcase_map[testcase_id].is_leader updated_group_id_count = 0 updated_group_bug_information = 0 if updated_group_id: for other_testcase in six.itervalues(testcase_map): if other_testcase.group_id != updated_group_id: continue updated_group_id_count += 1 # Update group issue id to be lowest issue id in the entire group. if other_testcase.issue_id is None: continue if (not updated_group_bug_information or updated_group_bug_information > other_testcase.issue_id): updated_group_bug_information = other_testcase.issue_id # If this group id is used by only one testcase, then remove it. if updated_group_id_count == 1: data_handler.delete_group(updated_group_id, update_testcases=False) updated_group_id = 0 updated_group_bug_information = 0 updated_is_leader = True try: testcase = data_handler.get_testcase_by_id(testcase_id) except errors.InvalidTestcaseError: # Already deleted. continue is_changed = ( (testcase.group_id != updated_group_id) or (testcase.group_bug_information != updated_group_bug_information) or (testcase.is_leader != updated_is_leader)) if not testcase.get_metadata('ran_grouper'): testcase.set_metadata('ran_grouper', True, update_testcase=not is_changed) if not is_changed: continue testcase.group_bug_information = updated_group_bug_information testcase.group_id = updated_group_id testcase.is_leader = updated_is_leader testcase.put() logs.log('Updated testcase %d group to %d.' % (testcase_id, updated_group_id))
def get(self): """Handles a GET request.""" libfuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == 'libFuzzer').get() if not libfuzzer: logs.log_error('Failed to get libFuzzer Fuzzer entity.') return afl = data_types.Fuzzer.query(data_types.Fuzzer.name == 'afl').get() if not afl: logs.log_error('Failed to get AFL Fuzzer entity.') return honggfuzz = data_types.Fuzzer.query( data_types.Fuzzer.name == 'honggfuzz').get() if not honggfuzz: logs.log_error('Failed to get honggfuzz Fuzzer entity.') return gft = data_types.Fuzzer.query( data_types.Fuzzer.name == 'googlefuzztest').get() if not gft: logs.log_error('Failed to get googlefuzztest Fuzzer entity.') return project_config = local_config.ProjectConfig() segregate_projects = project_config.get('segregate_projects') project_setup_configs = project_config.get('project_setup') project_names = set() job_names = set() fuzzer_entities = { 'afl': afl.key, 'honggfuzz': honggfuzz.key, 'googlefuzztest': gft.key, 'libfuzzer': libfuzzer.key, } for setup_config in project_setup_configs: bucket_config = setup_config.get('build_buckets') if not bucket_config: raise ProjectSetupError('Project setup buckets not specified.') config = ProjectSetup( BUILD_BUCKET_PATH_TEMPLATE, REVISION_URL, setup_config.get('build_type'), config_suffix=setup_config.get('job_suffix', ''), external_config=setup_config.get('external_config', ''), segregate_projects=segregate_projects, experimental_sanitizers=setup_config.get( 'experimental_sanitizers', []), engine_build_buckets={ 'libfuzzer': bucket_config.get('libfuzzer'), 'libfuzzer-i386': bucket_config.get('libfuzzer_i386'), 'afl': bucket_config.get('afl'), 'honggfuzz': bucket_config.get('honggfuzz'), 'googlefuzztest': bucket_config.get('googlefuzztest'), 'none': bucket_config.get('no_engine'), 'dataflow': bucket_config.get('dataflow'), }, fuzzer_entities=fuzzer_entities, add_info_labels=setup_config.get('add_info_labels', False), add_revision_mappings=setup_config.get('add_revision_mappings', False), additional_vars=setup_config.get('additional_vars')) projects_source = setup_config.get('source') if projects_source == 'oss-fuzz': projects = get_oss_fuzz_projects() elif projects_source.startswith(storage.GS_PREFIX): projects = get_projects_from_gcs(projects_source) else: raise ProjectSetupError('Invalid projects source: ' + projects_source) if not projects: raise ProjectSetupError('Missing projects list.') result = config.set_up(projects) project_names.update(result.project_names) job_names.update(result.job_names) cleanup_stale_projects(list(fuzzer_entities.values()), project_names, job_names, segregate_projects)
def main(): """Prepare the configuration options and start requesting tasks.""" logs.configure('run_bot') root_directory = environment.get_value('ROOT_DIR') if not root_directory: print('Please set ROOT_DIR environment variable to the root of the source ' 'checkout before running. Exiting.') print('For an example, check init.bash in the local directory.') return dates.initialize_timezone_from_environment() environment.set_bot_environment() monitor.initialize() if not profiler.start_if_needed('python_profiler_bot'): sys.exit(-1) fuzzers_init.run() if environment.is_trusted_host(ensure_connected=False): from clusterfuzz._internal.bot.untrusted_runner import host host.init() if environment.is_untrusted_worker(): # Track revision since we won't go into the task_loop. update_task.track_revision() from clusterfuzz._internal.bot.untrusted_runner import \ untrusted as untrusted_worker untrusted_worker.start_server() assert False, 'Unreachable code' while True: # task_loop should be an infinite loop, # unless we run into an exception. error_stacktrace, clean_exit, task_payload = task_loop() # Print the error trace to the console. if not clean_exit: print('Exception occurred while running "%s".' % task_payload) print('-' * 80) print(error_stacktrace) print('-' * 80) should_terminate = ( clean_exit or errors.error_in_list(error_stacktrace, errors.BOT_ERROR_TERMINATION_LIST)) if should_terminate: return logs.log_error( 'Task exited with exception (payload="%s").' % task_payload, error_stacktrace=error_stacktrace) should_hang = errors.error_in_list(error_stacktrace, errors.BOT_ERROR_HANG_LIST) if should_hang: logs.log('Start hanging forever.') while True: # Sleep to avoid consuming 100% of CPU. time.sleep(60) # See if our run timed out, if yes bail out. if data_handler.bot_run_timed_out(): return
def setup_testcase(testcase, job_type, fuzzer_override=None): """Sets up the testcase and needed dependencies like fuzzer, data bundle, etc.""" fuzzer_name = fuzzer_override or testcase.fuzzer_name task_name = environment.get_value('TASK_NAME') testcase_fail_wait = environment.get_value('FAIL_WAIT') testcase_id = testcase.key.id() # Clear testcase directories. shell.clear_testcase_directories() # Adjust the test timeout value if this is coming from an user uploaded # testcase. if testcase.uploader_email: _set_timeout_value_from_user_upload(testcase_id) # Update the fuzzer if necessary in order to get the updated data bundle. if fuzzer_name: try: update_successful = update_fuzzer_and_data_bundles(fuzzer_name) except errors.InvalidFuzzerError: # Close testcase and don't recreate tasks if this fuzzer is invalid. testcase.open = False testcase.fixed = 'NA' testcase.set_metadata('fuzzer_was_deleted', True) logs.log_error('Closed testcase %d with invalid fuzzer %s.' % (testcase_id, fuzzer_name)) error_message = 'Fuzzer %s no longer exists' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) return None, None, None if not update_successful: error_message = 'Unable to setup fuzzer %s' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task(task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # Extract the testcase and any of its resources to the input directory. file_list, input_directory, testcase_file_path = unpack_testcase(testcase) if not file_list: error_message = 'Unable to setup testcase %s' % testcase_file_path data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task(task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # For Android/Fuchsia, we need to sync our local testcases directory with the # one on the device. if environment.is_android(): _copy_testcase_to_device_and_setup_environment(testcase, testcase_file_path) # Push testcases to worker. if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host file_host.push_testcases_to_worker() # Copy global blacklist into local blacklist. is_lsan_enabled = environment.get_value('LSAN') if is_lsan_enabled: # Get local blacklist without this testcase's entry. leak_blacklist.copy_global_to_local_blacklist( excluded_testcase=testcase) prepare_environment_for_testcase(testcase, job_type, task_name) return file_list, input_directory, testcase_file_path
def process_command(task): """Figures out what to do with the given task and executes the command.""" logs.log("Executing command '%s'" % task.payload()) if not task.payload().strip(): logs.log_error('Empty task received.') return # Parse task payload. task_name = task.command task_argument = task.argument job_name = task.job environment.set_value('TASK_NAME', task_name) environment.set_value('TASK_ARGUMENT', task_argument) environment.set_value('JOB_NAME', job_name) if job_name != 'none': job = data_types.Job.query(data_types.Job.name == job_name).get() # Job might be removed. In that case, we don't want an exception # raised and causing this task to be retried by another bot. if not job: logs.log_error("Job '%s' not found." % job_name) return if not job.platform: error_string = "No platform set for job '%s'" % job_name logs.log_error(error_string) raise errors.BadStateError(error_string) # A misconfiguration led to this point. Clean up the job if necessary. job_queue_suffix = tasks.queue_suffix_for_platform(job.platform) bot_queue_suffix = tasks.default_queue_suffix() if job_queue_suffix != bot_queue_suffix: # This happens rarely, store this as a hard exception. logs.log_error( 'Wrong platform for job %s: job queue [%s], bot queue [%s].' % (job_name, job_queue_suffix, bot_queue_suffix)) # Try to recreate the job in the correct task queue. new_queue = ( tasks.high_end_queue() if task.high_end else tasks.regular_queue()) new_queue += job_queue_suffix # Command override is continuously run by a bot. If we keep failing # and recreating the task, it will just DoS the entire task queue. # So, we don't create any new tasks in that case since it needs # manual intervention to fix the override anyway. if not task.is_command_override: try: tasks.add_task(task_name, task_argument, job_name, new_queue) except Exception: # This can happen on trying to publish on a non-existent topic, e.g. # a topic for a high-end bot on another platform. In this case, just # give up. logs.log_error('Failed to fix platform and re-add task.') # Add a wait interval to avoid overflowing task creation. failure_wait_interval = environment.get_value('FAIL_WAIT') time.sleep(failure_wait_interval) return if task_name != 'fuzz': # Make sure that our platform id matches that of the testcase (for # non-fuzz tasks). testcase = data_handler.get_entity_by_type_and_id(data_types.Testcase, task_argument) if testcase: current_platform_id = environment.get_platform_id() testcase_platform_id = testcase.platform_id # This indicates we are trying to run this job on the wrong platform. # This can happen when you have different type of devices (e.g # android) on the same platform group. In this case, we just recreate # the task. if (task_name != 'variant' and testcase_platform_id and not utils.fields_match(testcase_platform_id, current_platform_id)): logs.log( 'Testcase %d platform (%s) does not match with ours (%s), exiting' % (testcase.key.id(), testcase_platform_id, current_platform_id)) tasks.add_task( task_name, task_argument, job_name, wait_time=utils.random_number(1, TASK_RETRY_WAIT_LIMIT)) return # Some fuzzers contain additional environment variables that should be # set for them. Append these for tests generated by these fuzzers and for # the fuzz command itself. fuzzer_name = None if task_name == 'fuzz': fuzzer_name = task_argument elif testcase: fuzzer_name = testcase.fuzzer_name # Get job's environment string. environment_string = job.get_environment_string() if task_name == 'minimize': # Let jobs specify a different job and fuzzer to minimize with. job_environment = job.get_environment() minimize_job_override = job_environment.get('MINIMIZE_JOB_OVERRIDE') if minimize_job_override: minimize_job = data_types.Job.query( data_types.Job.name == minimize_job_override).get() if minimize_job: environment.set_value('JOB_NAME', minimize_job_override) environment_string = minimize_job.get_environment_string() environment_string += '\nORIGINAL_JOB_NAME = %s\n' % job_name job_name = minimize_job_override else: logs.log_error( 'Job for minimization not found: %s.' % minimize_job_override) # Fallback to using own job for minimization. minimize_fuzzer_override = job_environment.get('MINIMIZE_FUZZER_OVERRIDE') fuzzer_name = minimize_fuzzer_override or fuzzer_name if fuzzer_name and not environment.is_engine_fuzzer_job(job_name): fuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == fuzzer_name).get() additional_default_variables = '' additional_variables_for_job = '' if (fuzzer and hasattr(fuzzer, 'additional_environment_string') and fuzzer.additional_environment_string): for line in fuzzer.additional_environment_string.splitlines(): # Job specific values may be defined in fuzzer additional # environment variable name strings in the form # job_name:VAR_NAME = VALUE. if '=' in line and ':' in line.split('=', 1)[0]: fuzzer_job_name, environment_definition = line.split(':', 1) if fuzzer_job_name == job_name: additional_variables_for_job += '\n%s' % environment_definition continue additional_default_variables += '\n%s' % line environment_string += additional_default_variables environment_string += additional_variables_for_job # Update environment for the job. update_environment_for_job(environment_string) # Match the cpu architecture with the ones required in the job definition. # If they don't match, then bail out and recreate task. if not is_supported_cpu_arch_for_job(): logs.log( 'Unsupported cpu architecture specified in job definition, exiting.') tasks.add_task( task_name, task_argument, job_name, wait_time=utils.random_number(1, TASK_RETRY_WAIT_LIMIT)) return # Initial cleanup. cleanup_task_state() start_web_server_if_needed() try: run_command(task_name, task_argument, job_name) finally: # Final clean up. cleanup_task_state()
def update_fuzzer_and_data_bundles(fuzzer_name): """Update the fuzzer with a given name if necessary.""" fuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == fuzzer_name).get() if not fuzzer: logs.log_error('No fuzzer exists with name %s.' % fuzzer_name) raise errors.InvalidFuzzerError # Set some helper environment variables. fuzzer_directory = get_fuzzer_directory(fuzzer_name) environment.set_value('FUZZER_DIR', fuzzer_directory) environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content) # If the fuzzer generates large testcases or a large number of small ones # that don't fit on tmpfs, then use the larger disk directory. if fuzzer.has_large_testcases: testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK') environment.set_value('FUZZ_INPUTS', testcase_disk_directory) # Adjust the test timeout, if user has provided one. if fuzzer.timeout: environment.set_value('TEST_TIMEOUT', fuzzer.timeout) # Increase fuzz test timeout if the fuzzer timeout is higher than its # current value. fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT') if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout: environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout) # Adjust the max testcases if this fuzzer has specified a lower limit. max_testcases = environment.get_value('MAX_TESTCASES') if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases: environment.set_value('MAX_TESTCASES', fuzzer.max_testcases) # Check for updates to this fuzzer. version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name) if (not fuzzer.builtin and revisions.needs_update(version_file, fuzzer.revision)): logs.log('Fuzzer update was found, updating.') # Clear the old fuzzer directory if it exists. if not shell.remove_directory(fuzzer_directory, recreate=True): logs.log_error('Failed to clear fuzzer directory.') return None # Copy the archive to local disk and unpack it. archive_path = os.path.join(fuzzer_directory, fuzzer.filename) if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path): logs.log_error('Failed to copy fuzzer archive.') return None try: archive.unpack(archive_path, fuzzer_directory) except Exception: error_message = ( 'Failed to unpack fuzzer archive %s ' '(bad archive or unsupported format).') % fuzzer.filename logs.log_error(error_message) fuzzer_logs.upload_script_log('Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return None fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path) if not os.path.exists(fuzzer_path): error_message = ( 'Fuzzer executable %s not found. ' 'Check fuzzer configuration.') % fuzzer.executable_path logs.log_error(error_message) fuzzer_logs.upload_script_log('Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return None # Make fuzzer executable. os.chmod(fuzzer_path, 0o750) # Cleanup unneeded archive. shell.remove_file(archive_path) # Save the current revision of this fuzzer in a file for later checks. revisions.write_revision_to_revision_file(version_file, fuzzer.revision) logs.log('Updated fuzzer to revision %d.' % fuzzer.revision) _clear_old_data_bundles_if_needed() # Setup data bundles associated with this fuzzer. data_bundles = ndb_utils.get_all_from_query( data_types.DataBundle.query( data_types.DataBundle.name == fuzzer.data_bundle_name)) for data_bundle in data_bundles: if not update_data_bundle(fuzzer, data_bundle): return None # Setup environment variable for launcher script path. if fuzzer.launcher_script: fuzzer_launcher_path = os.path.join(fuzzer_directory, fuzzer.launcher_script) environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path) # For launcher script usecase, we need the entire fuzzer directory on the # worker. if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host worker_fuzzer_directory = file_host.rebase_to_worker_root( fuzzer_directory) file_host.copy_directory_to_worker(fuzzer_directory, worker_fuzzer_directory, replace=True) return fuzzer
def run_process(cmdline, current_working_directory=None, timeout=DEFAULT_TEST_TIMEOUT, need_shell=False, gestures=None, env_copy=None, testcase_run=True, ignore_children=True): """Executes a process with a given command line and other parameters.""" if environment.is_trusted_host() and testcase_run: from clusterfuzz._internal.bot.untrusted_runner import remote_process_host return remote_process_host.run_process(cmdline, current_working_directory, timeout, need_shell, gestures, env_copy, testcase_run, ignore_children) if gestures is None: gestures = [] if env_copy: os.environ.update(env_copy) # FIXME(mbarbella): Using LAUNCHER_PATH here is error prone. It forces us to # do certain operations before fuzzer setup (e.g. bad build check). launcher = environment.get_value('LAUNCHER_PATH') # This is used when running scripts on native linux OS and not on the device. # E.g. running a fuzzer to generate testcases or launcher script. plt = environment.platform() runs_on_device = environment.is_android(plt) or plt == 'FUCHSIA' if runs_on_device and (not testcase_run or launcher): plt = 'LINUX' is_android = environment.is_android(plt) # Lower down testcase timeout slightly to account for time for crash analysis. timeout -= CRASH_ANALYSIS_TIME # LeakSanitizer hack - give time for stdout/stderr processing. lsan = environment.get_value('LSAN', False) if lsan: timeout -= LSAN_ANALYSIS_TIME # Initialize variables. adb_output = None process_output = '' process_status = None return_code = 0 process_poll_interval = environment.get_value('PROCESS_POLL_INTERVAL', 0.5) start_time = time.time() watch_for_process_exit = (environment.get_value('WATCH_FOR_PROCESS_EXIT') if is_android else True) window_list = [] # Get gesture start time from last element in gesture list. gestures = copy.deepcopy(gestures) if gestures and gestures[-1].startswith('Trigger'): gesture_start_time = int(gestures[-1].split(':')[1]) gestures.pop() else: gesture_start_time = timeout // 2 if is_android: # Clear the log upfront. android.logger.clear_log() # Run the app. adb_output = android.adb.run_command(cmdline, timeout=timeout) else: cmd = shell.get_command(cmdline) process_output = mozprocess.processhandler.StoreOutput() process_status = ProcessStatus() try: process_handle = mozprocess.ProcessHandlerMixin( cmd, args=None, cwd=current_working_directory, shell=need_shell, processOutputLine=[process_output], onFinish=[process_status], ignore_children=ignore_children) start_process(process_handle) except: logs.log_error('Exception occurred when running command: %s.' % cmdline) return None, None, '' while True: time.sleep(process_poll_interval) # Run the gestures at gesture_start_time or in case we didn't find windows # in the last try. if (gestures and time.time() - start_time >= gesture_start_time and not window_list): # In case, we don't find any windows, we increment the gesture start time # so that the next check is after 1 second. gesture_start_time += 1 if plt == 'LINUX': linux.gestures.run_gestures(gestures, process_handle.pid, process_status, start_time, timeout, window_list) elif plt == 'WINDOWS': windows.gestures.run_gestures(gestures, process_handle.pid, process_status, start_time, timeout, window_list) elif is_android: android.gestures.run_gestures(gestures, start_time, timeout) # TODO(mbarbella): We add a fake window here to prevent gestures on # Android from getting executed more than once. window_list = ['FAKE'] if time.time() - start_time >= timeout: break # Collect the process output. output = (android.logger.log_output() if is_android else b'\n'.join(process_output.output)) output = utils.decode_to_unicode(output) if crash_analyzer.is_memory_tool_crash(output): break # Check if we need to bail out on process exit. if watch_for_process_exit: # If |watch_for_process_exit| is set, then we already completed running # our app launch command. So, we can bail out. if is_android: break # On desktop, we bail out as soon as the process finishes. if process_status and process_status.finished: # Wait for process shutdown and set return code. process_handle.wait(timeout=PROCESS_CLEANUP_WAIT_TIME) break # Process output based on platform. if is_android: # Get current log output. If device is in reboot mode, logcat automatically # waits for device to be online. time.sleep(ANDROID_CRASH_LOGCAT_WAIT_TIME) output = android.logger.log_output() if android.constants.LOW_MEMORY_REGEX.search(output): # If the device is low on memory, we should force reboot and bail out to # prevent device from getting in a frozen state. logs.log('Device is low on memory, rebooting.', output=output) android.adb.hard_reset() android.adb.wait_for_device() elif android.adb.time_since_last_reboot() < time.time() - start_time: # Check if a reboot has happened, if yes, append log output before reboot # and kernel logs content to output. log_before_last_reboot = android.logger.log_output_before_last_reboot( ) kernel_log = android.adb.get_kernel_log_content() output = '%s%s%s%s%s' % ( log_before_last_reboot, utils.get_line_seperator('Device rebooted'), output, utils.get_line_seperator('Kernel Log'), kernel_log) # Make sure to reset SE Linux Permissive Mode. This can be done cheaply # in ~0.15 sec and is needed especially between runs for kernel crashes. android.adb.run_as_root() android.settings.change_se_linux_to_permissive_mode() return_code = 1 # Add output from adb to the front. if adb_output: output = '%s\n\n%s' % (adb_output, output) # Kill the application if it is still running. We do this at the end to # prevent this from adding noise to the logcat output. task_name = environment.get_value('TASK_NAME') child_process_termination_pattern = environment.get_value( 'CHILD_PROCESS_TERMINATION_PATTERN') if task_name == 'fuzz' and child_process_termination_pattern: # In some cases, we do not want to terminate the application after each # run to avoid long startup times (e.g. for chrome). Terminate processes # matching a particular pattern for light cleanup in this case. android.adb.kill_processes_and_children_matching_name( child_process_termination_pattern) else: # There is no special termination behavior. Simply stop the application. android.app.stop() else: # Get the return code in case the process has finished already. # If the process hasn't finished, return_code will be None which is what # callers expect unless the output indicates a crash. return_code = process_handle.poll() # If the process is still running, then terminate it. if not process_status.finished: launcher_with_interpreter = shell.get_execute_command( launcher) if launcher else None if (launcher_with_interpreter and cmdline.startswith(launcher_with_interpreter)): # If this was a launcher script, we KILL all child processes created # except for APP_NAME. # It is expected that, if the launcher script terminated normally, it # cleans up all the child processes it created itself. terminate_root_and_child_processes(process_handle.pid) else: try: # kill() here actually sends SIGTERM on posix. process_handle.kill() except: pass if lsan: time.sleep(LSAN_ANALYSIS_TIME) output = b'\n'.join(process_output.output) output = utils.decode_to_unicode(output) # X Server hack when max client reached. if ('Maximum number of clients reached' in output or 'Unable to get connection to X server' in output): logs.log_error('Unable to connect to X server, exiting.') os.system('sudo killall -9 Xvfb blackbox >/dev/null 2>&1') sys.exit(0) if testcase_run and (crash_analyzer.is_memory_tool_crash(output) or crash_analyzer.is_check_failure_crash(output)): return_code = 1 # If a crash is found, then we add the memory state as well. if return_code and is_android: ps_output = android.adb.get_ps_output() if ps_output: output += utils.get_line_seperator('Memory Statistics') output += ps_output if return_code: logs.log_warn('Process (%s) ended with exit code (%s).' % (repr(cmdline), str(return_code)), output=output) return return_code, round(time.time() - start_time, 1), output
def parse_mime_to_crash_report_info(local_minidump_mime_path): """Read the (local) minidump MIME file into a CrashReportInfo object.""" # Get the minidump name and path. minidump_path_match = re.match(r'(.*)\.mime', local_minidump_mime_path) if minidump_path_match is None: logs.log_error('Minidump filename in unexpected format: \'%s\'.' % local_minidump_mime_path) return None minidump_path = '%s.dmp' % minidump_path_match.group(1).strip() # Reformat the minidump MIME to include the boundary. with open(local_minidump_mime_path, 'rb') as minidump_mime_file_content: # The boundary is the first line after the first two dashes. boundary = minidump_mime_file_content.readline().strip()[2:] minidump_mime_bytes = ( b'Content-Type: multipart/form-data; boundary=\"%s\"\r\n--%s\r\n' % (boundary, boundary)) minidump_mime_bytes += minidump_mime_file_content.read() minidump_mime_contents = email.message_from_bytes(minidump_mime_bytes) # Parse the MIME contents, extracting the parameters needed for upload. mime_key_values = {} for mime_part in minidump_mime_contents.get_payload(): if isinstance(mime_part, str): mime_part = utils.decode_to_unicode(mime_part) logs.log_error('Unexpected str mime_part from mime path %s: %s' % (local_minidump_mime_path, mime_part)) continue part_descriptor = list(mime_part.values()) key_tokens = part_descriptor[0].split('; ') key_match = re.match(r'name="(.*)".*', key_tokens[1]) # Extract from the MIME part the key-value pairs used by report uploading. if key_match is not None: report_key = key_match.group(1) report_value = mime_part.get_payload(decode=True) if report_key == MINIDUMP_FILE_KEY: utils.write_data_to_file(report_value, minidump_path) else: # Take care of aliases. if report_key in ('prod', 'buildTargetId'): report_key = PRODUCT_KEY elif report_key == 'ver': report_key = VERSION_KEY # Save the key-value pair. mime_key_values[report_key] = report_value # Pull out product and version explicitly since these are required # for upload. product, version = None, None if PRODUCT_KEY in mime_key_values: product = mime_key_values.pop(PRODUCT_KEY).decode('utf-8') else: logs.log_error( 'Could not find \'%s\' or alias in mime_key_values key.' % PRODUCT_KEY) if VERSION_KEY in mime_key_values: version = mime_key_values.pop(VERSION_KEY).decode('utf-8') else: logs.log_error( 'Could not find \'%s\' or alias in mime_key_values key.' % VERSION_KEY) # If missing, return None and log keys that do exist; otherwise, construct # CrashReportInfo and return. if product is None or version is None: logs.log_error('mime_key_values dict keys:\n%s' % str(list(mime_key_values.keys()))) return None return CrashReportInfo(minidump_path=minidump_path, product=product, version=version, optional_params=mime_key_values)
def find_regression_range(testcase_id, job_type): """Attempt to find when the testcase regressed.""" deadline = tasks.get_task_completion_deadline() testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return if testcase.regression: logs.log_error('Regression range is already set as %s, skip.' % testcase.regression) return # This task is not applicable for custom binaries. if build_manager.is_custom_binary(): testcase.regression = 'NA' data_handler.update_testcase_comment( testcase, data_types.TaskState.ERROR, 'Not applicable for custom binaries') return data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) # Setup testcase and its dependencies. file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type) if not file_list: testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, 'Failed to setup testcase') tasks.add_task('regression', testcase_id, job_type) return build_bucket_path = build_manager.get_primary_bucket_path() revision_list = build_manager.get_revisions_list(build_bucket_path, testcase=testcase) if not revision_list: data_handler.close_testcase_with_error( testcase_id, 'Failed to fetch revision list') return # Don't burden NFS server with caching these random builds. environment.set_value('CACHE_STORE', False) # Pick up where left off in a previous run if necessary. min_revision = testcase.get_metadata('last_regression_min') max_revision = testcase.get_metadata('last_regression_max') first_run = not min_revision and not max_revision if not min_revision: min_revision = revisions.get_first_revision_in_list(revision_list) if not max_revision: max_revision = testcase.crash_revision min_index = revisions.find_min_revision_index(revision_list, min_revision) if min_index is None: raise errors.BuildNotFoundError(min_revision, job_type) max_index = revisions.find_max_revision_index(revision_list, max_revision) if max_index is None: raise errors.BuildNotFoundError(max_revision, job_type) # Make sure that the revision where we noticed the crash, still crashes at # that revision. Otherwise, our binary search algorithm won't work correctly. max_revision = revision_list[max_index] crashes_in_max_revision = _testcase_reproduces_in_revision( testcase, testcase_file_path, job_type, max_revision, should_log=False) if not crashes_in_max_revision: testcase = data_handler.get_testcase_by_id(testcase_id) error_message = ('Known crash revision %d did not crash' % max_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) task_creation.mark_unreproducible_if_flaky(testcase, True) return # If we've made it this far, the test case appears to be reproducible. Clear # metadata from previous runs had it been marked as potentially flaky. task_creation.mark_unreproducible_if_flaky(testcase, False) # On the first run, check to see if we regressed near either the min or max # revision. if first_run and found_regression_near_extreme_revisions( testcase, testcase_file_path, job_type, revision_list, min_index, max_index): return while time.time() < deadline: min_revision = revision_list[min_index] max_revision = revision_list[max_index] # If the min and max revisions are one apart (or the same, if we only have # one build), this is as much as we can narrow the range. if max_index - min_index <= 1: # Verify that the regression range seems correct, and save it if so. if not validate_regression_range(testcase, testcase_file_path, job_type, revision_list, min_index): return save_regression_range(testcase_id, min_revision, max_revision) return middle_index = (min_index + max_index) // 2 middle_revision = revision_list[middle_index] try: is_crash = _testcase_reproduces_in_revision( testcase, testcase_file_path, job_type, middle_revision, min_revision=min_revision, max_revision=max_revision) except errors.BadBuildError: # Skip this revision. del revision_list[middle_index] max_index -= 1 continue if is_crash: max_index = middle_index else: min_index = middle_index _save_current_regression_range_indices(testcase_id, revision_list[min_index], revision_list[max_index]) # If we've broken out of the above loop, we timed out. We'll finish by # running another regression task and picking up from this point. testcase = data_handler.get_testcase_by_id(testcase_id) error_message = 'Timed out, current range r%d:r%d' % ( revision_list[min_index], revision_list[max_index]) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task('regression', testcase_id, job_type)
def test_exception(self): """Tests exception.""" self.mock.exc_info.return_value = 'err' logs.log_error('test', exception='exception', hello='1') self.mock.emit.assert_called_once_with( logging.ERROR, 'test', exc_info='err', hello='1')
def update_source_code(): """Updates source code files with latest version from appengine.""" process_handler.cleanup_stale_processes() shell.clear_temp_directory() root_directory = environment.get_value('ROOT_DIR') temp_directory = environment.get_value('BOT_TMPDIR') temp_archive = os.path.join(temp_directory, 'clusterfuzz-source.zip') try: storage.copy_file_from(get_source_url(), temp_archive) except Exception: logs.log_error('Could not retrieve source code archive from url.') return try: file_list = archive.get_file_list(temp_archive) zip_archive = zipfile.ZipFile(temp_archive, 'r') except Exception: logs.log_error('Bad zip file.') return src_directory = os.path.join(root_directory, 'src') output_directory = os.path.dirname(root_directory) error_occurred = False normalized_file_set = set() for filepath in file_list: filename = os.path.basename(filepath) # This file cannot be updated on the fly since it is running as server. if filename == 'adb': continue absolute_filepath = os.path.join(output_directory, filepath) if os.path.altsep: absolute_filepath = absolute_filepath.replace( os.path.altsep, os.path.sep) if os.path.realpath(absolute_filepath) != absolute_filepath: continue normalized_file_set.add(absolute_filepath) try: file_extension = os.path.splitext(filename)[1] # Remove any .so files first before overwriting, as they can be loaded # in the memory of existing processes. Overwriting them directly causes # segfaults in existing processes (e.g. run.py). if file_extension == '.so' and os.path.exists(absolute_filepath): os.remove(absolute_filepath) # On Windows, to update DLLs (and native .pyd extensions), we rename it # first so that we can install the new version. if (environment.platform() == 'WINDOWS' and file_extension in ['.dll', '.pyd'] and os.path.exists(absolute_filepath)): _rename_dll_for_update(absolute_filepath) except Exception: logs.log_error('Failed to remove or move %s before extracting new ' 'version.' % absolute_filepath) try: extracted_path = zip_archive.extract(filepath, output_directory) external_attr = zip_archive.getinfo(filepath).external_attr mode = (external_attr >> 16) & 0o777 mode |= 0o440 os.chmod(extracted_path, mode) except: error_occurred = True logs.log_error('Failed to extract file %s from source archive.' % filepath) zip_archive.close() if error_occurred: return clear_pyc_files(src_directory) clear_old_files(src_directory, normalized_file_set) local_manifest_path = os.path.join(root_directory, utils.LOCAL_SOURCE_MANIFEST) source_version = utils.read_data_from_file( local_manifest_path, eval_data=False).decode('utf-8').strip() logs.log('Source code updated to %s.' % source_version)
def main(): """Main build routine.""" bucket_prefix = environment.get_value('BUCKET_PREFIX') build_dir = environment.get_value('BUILD_DIR') wait_time = environment.get_value('WAIT_TIME') try: builds_metadata = build_info.get_production_builds_info_from_cd( environment.platform()) except Exception: logs.log_error('Errors when fetching from ChromiumDash') # fallback to omahaproxy in the transition stage # TODO(yuanjunh): remove the fallback logic after migration is done. builds_metadata = build_info.get_production_builds_info( environment.platform()) if not builds_metadata: return global LAST_BUILD for build_metadata in builds_metadata: build_type = build_metadata.build_type revision = build_metadata.revision version = build_metadata.version if build_type not in ['extended_stable', 'stable', 'beta']: # We don't need dev or canary builds atm. continue # Starting building the builds. for tool in TOOLS_GN_MAPPINGS: tool_and_build_type = '%s-%s' % (tool, build_type) logs.log('Building %s.' % tool_and_build_type) # Check if we already have built the same build. if (tool_and_build_type in LAST_BUILD and revision == LAST_BUILD[tool_and_build_type]): logs.log('Skipping same build %s (revision %s).' % (tool_and_build_type, revision)) continue LAST_BUILD[tool_and_build_type] = revision file_name_prefix = '%s-linux-%s-%s' % (tool, build_type, version) archive_filename = '%s.zip' % file_name_prefix archive_path_local = '%s/%s' % (build_dir, archive_filename) bucket_name = '%s%s' % (bucket_prefix, tool.split('-')[0]) archive_path_remote = ('gs://%s/%s/%s' % ( bucket_name, TOOLS_BUCKET_DIR_MAPPINGS[tool], archive_filename)) # Run the build script with required gn arguments. command = '' gn_args = '%s %s' % (TOOLS_GN_MAPPINGS[tool], GN_COMMON_ARGS) command += '%s "%s" %s %s' % (BUILD_HELPER_SCRIPT, gn_args, version, file_name_prefix) logs.log('Executing build script: %s.' % command) os.system(command) # Check if the build succeeded based on the existence of the # local archive file. if os.path.exists(archive_path_local): # Build success. Now, copy it to google cloud storage and make it # public. os.system('gsutil cp %s %s' % (archive_path_local, archive_path_remote)) os.system('gsutil acl set public-read %s' % archive_path_remote) logs.log('Build succeeded, created %s.' % archive_filename) else: LAST_BUILD[tool_and_build_type] = '' logs.log_error('Build failed, unable to create %s.' % archive_filename) logs.log('Completed cycle, waiting for %d secs.' % wait_time) time.sleep(wait_time)
def configure_system_build_properties(): """Modifies system build properties in /system/build.prop for better boot speed and power use.""" adb.run_as_root() # Check md5 checksum of build.prop to see if already updated, # in which case exit. If build.prop does not exist, something # is very wrong with the device, so bail. old_md5 = persistent_cache.get_value(constants.BUILD_PROP_MD5_KEY) current_md5 = adb.get_file_checksum(BUILD_PROP_PATH) if current_md5 is None: logs.log_error('Unable to find %s on device.' % BUILD_PROP_PATH) return if old_md5 == current_md5: return # Pull to tmp file. bot_tmp_directory = environment.get_value('BOT_TMPDIR') old_build_prop_path = os.path.join(bot_tmp_directory, 'old.prop') adb.run_command(['pull', BUILD_PROP_PATH, old_build_prop_path]) if not os.path.exists(old_build_prop_path): logs.log_error('Unable to fetch %s from device.' % BUILD_PROP_PATH) return # Write new build.prop. new_build_prop_path = os.path.join(bot_tmp_directory, 'new.prop') old_build_prop_file_content = open(old_build_prop_path, 'r') new_build_prop_file_content = open(new_build_prop_path, 'w') new_content_notification = '### CHANGED OR ADDED PROPERTIES ###' for line in old_build_prop_file_content: property_name = line.split('=')[0].strip() if property_name in BUILD_PROPERTIES: continue if new_content_notification in line: continue new_build_prop_file_content.write(line) new_build_prop_file_content.write(new_content_notification + '\n') for flag, value in six.iteritems(BUILD_PROPERTIES): new_build_prop_file_content.write('%s=%s\n' % (flag, value)) old_build_prop_file_content.close() new_build_prop_file_content.close() # Keep verified boot disabled for M and higher releases. This makes it easy # to modify system's app_process to load asan libraries. build_version = settings.get_build_version() if is_build_at_least(build_version, 'M'): adb.run_as_root() adb.run_command('disable-verity') reboot() # Make /system writable. adb.run_as_root() adb.remount() # Remove seccomp policies (on N and higher) as ASan requires extra syscalls. if is_build_at_least(build_version, 'N'): policy_files = adb.run_shell_command( ['find', '/system/etc/seccomp_policy/', '-type', 'f']) for policy_file in policy_files.splitlines(): adb.run_shell_command(['rm', policy_file.strip()]) # Push new build.prop and backup to device. logs.log('Pushing new build properties file on device.') adb.run_command( ['push', '-p', old_build_prop_path, BUILD_PROP_BACKUP_PATH]) adb.run_command(['push', '-p', new_build_prop_path, BUILD_PROP_PATH]) adb.run_shell_command(['chmod', '644', BUILD_PROP_PATH]) # Set persistent cache key containing and md5sum. current_md5 = adb.get_file_checksum(BUILD_PROP_PATH) persistent_cache.set_value(constants.BUILD_PROP_MD5_KEY, current_md5)
def find_fixed_range(testcase_id, job_type): """Attempt to find the revision range where a testcase was fixed.""" deadline = tasks.get_task_completion_deadline() testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return if testcase.fixed: logs.log_error('Fixed range is already set as %s, skip.' % testcase.fixed) return # Setup testcase and its dependencies. file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type) if not file_list: return # Set a flag to indicate we are running progression task. This shows pending # status on testcase report page and avoid conflicting testcase updates by # triage cron. testcase.set_metadata('progression_pending', True) # Custom binaries are handled as special cases. if build_manager.is_custom_binary(): _check_fixed_for_custom_binary(testcase, job_type, testcase_file_path) return build_bucket_path = build_manager.get_primary_bucket_path() revision_list = build_manager.get_revisions_list(build_bucket_path, testcase=testcase) if not revision_list: data_handler.close_testcase_with_error( testcase_id, 'Failed to fetch revision list') return # Use min, max_index to mark the start and end of revision list that is used # for bisecting the progression range. Set start to the revision where noticed # the crash. Set end to the trunk revision. Also, use min, max from past run # if it timed out. min_revision = testcase.get_metadata('last_progression_min') max_revision = testcase.get_metadata('last_progression_max') if min_revision or max_revision: # Clear these to avoid using them in next run. If this run fails, then we # should try next run without them to see it succeeds. If this run succeeds, # we should still clear them to avoid capping max revision in next run. testcase = data_handler.get_testcase_by_id(testcase_id) testcase.delete_metadata('last_progression_min', update_testcase=False) testcase.delete_metadata('last_progression_max', update_testcase=False) testcase.put() last_tested_revision = testcase.get_metadata('last_tested_crash_revision') known_crash_revision = last_tested_revision or testcase.crash_revision if not min_revision: min_revision = known_crash_revision if not max_revision: max_revision = revisions.get_last_revision_in_list(revision_list) min_index = revisions.find_min_revision_index(revision_list, min_revision) if min_index is None: raise errors.BuildNotFoundError(min_revision, job_type) max_index = revisions.find_max_revision_index(revision_list, max_revision) if max_index is None: raise errors.BuildNotFoundError(max_revision, job_type) testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED, 'r%d' % max_revision) # Check to see if this testcase is still crashing now. If it is, then just # bail out. result = _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, max_revision, update_metadata=True) if result.is_crash(): logs.log('Found crash with same signature on latest revision r%d.' % max_revision) app_path = environment.get_value('APP_PATH') command = testcase_manager.get_command_line_for_application( testcase_file_path, app_path=app_path, needs_http=testcase.http_flag) symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True) unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False) stacktrace = utils.get_crash_stacktrace_output( command, symbolized_crash_stacktrace, unsymbolized_crash_stacktrace) testcase = data_handler.get_testcase_by_id(testcase_id) testcase.last_tested_crash_stacktrace = data_handler.filter_stacktrace( stacktrace) data_handler.update_progression_completion_metadata( testcase, max_revision, is_crash=True, message='still crashes on latest revision r%s' % max_revision) # Since we've verified that the test case is still crashing, clear out any # metadata indicating potential flake from previous runs. task_creation.mark_unreproducible_if_flaky(testcase, False) # For chromium project, save latest crash information for later upload # to chromecrash/. state = result.get_symbolized_data() crash_uploader.save_crash_info_if_needed(testcase_id, max_revision, job_type, state.crash_type, state.crash_address, state.frames) return if result.unexpected_crash: testcase.set_metadata('crashes_on_unexpected_state', True) else: testcase.delete_metadata('crashes_on_unexpected_state') # Don't burden NFS server with caching these random builds. environment.set_value('CACHE_STORE', False) # Verify that we do crash in the min revision. This is assumed to be true # while we are doing the bisect. result = _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, min_revision) if result and not result.is_crash(): testcase = data_handler.get_testcase_by_id(testcase_id) # Retry once on another bot to confirm our result. if data_handler.is_first_retry_for_task(testcase, reset_after_retry=True): tasks.add_task('progression', testcase_id, job_type) error_message = ( 'Known crash revision %d did not crash, will retry on another bot to ' 'confirm result' % known_crash_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) data_handler.update_progression_completion_metadata( testcase, max_revision) return data_handler.clear_progression_pending(testcase) error_message = ('Known crash revision %d did not crash' % known_crash_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) task_creation.mark_unreproducible_if_flaky(testcase, True) return # Start a binary search to find last non-crashing revision. At this point, we # know that we do crash in the min_revision, and do not crash in max_revision. while time.time() < deadline: min_revision = revision_list[min_index] max_revision = revision_list[max_index] # If the min and max revisions are one apart this is as much as we can # narrow the range. if max_index - min_index == 1: _save_fixed_range(testcase_id, min_revision, max_revision, testcase_file_path) return # Occasionally, we get into this bad state. It seems to be related to test # cases with flaky stacks, but the exact cause is unknown. if max_index - min_index < 1: testcase = data_handler.get_testcase_by_id(testcase_id) testcase.fixed = 'NA' testcase.open = False message = ('Fixed testing errored out (min and max revisions ' 'are both %d)' % min_revision) data_handler.update_progression_completion_metadata( testcase, max_revision, message=message) # Let the bisection service know about the NA status. bisection.request_bisection(testcase) return # Test the middle revision of our range. middle_index = (min_index + max_index) // 2 middle_revision = revision_list[middle_index] testcase = data_handler.get_testcase_by_id(testcase_id) log_message = 'Testing r%d (current range %d:%d)' % ( middle_revision, min_revision, max_revision) data_handler.update_testcase_comment(testcase, data_types.TaskState.WIP, log_message) try: result = _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, middle_revision) except errors.BadBuildError: # Skip this revision. del revision_list[middle_index] max_index -= 1 continue if result.is_crash(): min_index = middle_index else: max_index = middle_index _save_current_fixed_range_indices(testcase_id, revision_list[min_index], revision_list[max_index]) # If we've broken out of the loop, we've exceeded the deadline. Recreate the # task to pick up where we left off. testcase = data_handler.get_testcase_by_id(testcase_id) error_message = ('Timed out, current range r%d:r%d' % (revision_list[min_index], revision_list[max_index])) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task('progression', testcase_id, job_type)
def _sync_job(self, project, info, corpus_bucket_name, quarantine_bucket_name, logs_bucket_name, backup_bucket_name): """Sync the config with ClusterFuzz.""" # Create/update ClusterFuzz jobs. job_names = [] for template in get_jobs_for_project(project, info): if template.engine == 'none': # Engine-less jobs are not automatically managed. continue fuzzer_entity = self._fuzzer_entities.get(template.engine).get() if not fuzzer_entity: raise ProjectSetupError('Invalid fuzzing engine ' + template.engine) job_name = template.job_name(project, self._config_suffix) job = data_types.Job.query(data_types.Job.name == job_name).get() if not job: job = data_types.Job() if self._external_config: if ('reproduction_topic' not in self._external_config or 'updates_subscription' not in self._external_config): raise ProjectSetupError('Invalid external_config.') job.external_reproduction_topic = self._external_config[ 'reproduction_topic'] job.external_updates_subscription = self._external_config[ 'updates_subscription'] else: job.external_reproduction_topic = None job.external_updates_subscription = None if not info.get('disabled', False): job_names.append(job_name) if job_name not in fuzzer_entity.jobs and not job.is_external( ): # Enable new job. fuzzer_entity.jobs.append(job_name) fuzzer_entity.put() job.name = job_name if self._segregate_projects: job.platform = untrusted.platform_name(project, 'linux') else: # TODO(ochang): Support other platforms? job.platform = 'LINUX' job.templates = template.cf_job_templates job.environment_string = JOB_TEMPLATE.format( build_type=self._build_type, build_bucket_path=self._get_build_bucket_path( project, info, template.engine, template.memory_tool, template.architecture), engine=template.engine, project=project) if self._add_revision_mappings: revision_vars_url = self._revision_url_template.format( project=project, bucket=self._get_build_bucket(template.engine, template.architecture), sanitizer=template.memory_tool) job.environment_string += ( 'REVISION_VARS_URL = {revision_vars_url}\n'.format( revision_vars_url=revision_vars_url)) if logs_bucket_name: job.environment_string += 'FUZZ_LOGS_BUCKET = {logs_bucket}\n'.format( logs_bucket=logs_bucket_name) if corpus_bucket_name: job.environment_string += 'CORPUS_BUCKET = {corpus_bucket}\n'.format( corpus_bucket=corpus_bucket_name) if quarantine_bucket_name: job.environment_string += ( 'QUARANTINE_BUCKET = {quarantine_bucket}\n'.format( quarantine_bucket=quarantine_bucket_name)) if backup_bucket_name: job.environment_string += 'BACKUP_BUCKET = {backup_bucket}\n'.format( backup_bucket=backup_bucket_name) if self._add_info_labels: job.environment_string += ( 'AUTOMATIC_LABELS = Proj-{project},Engine-{engine}\n'. format( project=project, engine=template.engine, )) help_url = info.get('help_url') if help_url: job.environment_string += 'HELP_URL = %s\n' % help_url if (template.experimental or (self._experimental_sanitizers and template.memory_tool in self._experimental_sanitizers)): job.environment_string += 'EXPERIMENTAL = True\n' if template.minimize_job_override: minimize_job_override = template.minimize_job_override.job_name( project, self._config_suffix) job.environment_string += ('MINIMIZE_JOB_OVERRIDE = %s\n' % minimize_job_override) view_restrictions = info.get('view_restrictions') if view_restrictions: if view_restrictions in ALLOWED_VIEW_RESTRICTIONS: job.environment_string += ( 'ISSUE_VIEW_RESTRICTIONS = %s\n' % view_restrictions) else: logs.log_error( 'Invalid view restriction setting %s for project %s.' % (view_restrictions, project)) selective_unpack = info.get('selective_unpack') if selective_unpack: job.environment_string += 'UNPACK_ALL_FUZZ_TARGETS_AND_FILES = False\n' main_repo = info.get('main_repo') if main_repo: job.environment_string += f'MAIN_REPO = {main_repo}\n' if (template.engine == 'libfuzzer' and template.architecture == 'x86_64' and 'dataflow' in info.get('fuzzing_engines', DEFAULT_ENGINES)): # Dataflow binaries are built with dataflow sanitizer, but can be used # as an auxiliary build with libFuzzer builds (e.g. with ASan or UBSan). dataflow_build_bucket_path = self._get_build_bucket_path( project_name=project, info=info, engine='dataflow', memory_tool='dataflow', architecture=template.architecture) job.environment_string += ( 'DATAFLOW_BUILD_BUCKET_PATH = %s\n' % dataflow_build_bucket_path) if self._additional_vars: additional_vars = {} additional_vars.update(self._additional_vars.get('all', {})) engine_vars = self._additional_vars.get(template.engine, {}) engine_sanitizer_vars = engine_vars.get( template.memory_tool, {}) additional_vars.update(engine_sanitizer_vars) for key, value in sorted(six.iteritems(additional_vars)): job.environment_string += ('{} = {}\n'.format( key, str(value).encode('unicode-escape').decode('utf-8'))) job.put() return job_names
def _minimize_corpus_two_step(self, target_path, arguments, existing_corpus_dirs, new_corpus_dir, output_corpus_dir, reproducers_dir, max_time): """Optional (but recommended): run corpus minimization. Args: target_path: Path to the target. arguments: Additional arguments needed for corpus minimization. existing_corpus_dirs: Input corpora that existed before the fuzzing run. new_corpus_dir: Input corpus that was generated during the fuzzing run. Must have at least one new file. output_corpus_dir: Output directory to place minimized corpus. reproducers_dir: The directory to put reproducers in when crashes are found. max_time: Maximum allowed time for the minimization. Returns: A Result object. """ if not _is_multistep_merge_supported(target_path): # Fallback to the old single step merge. It does not support incremental # stats and provides only `edge_coverage` and `feature_coverage` stats. logs.log( 'Old version of libFuzzer is used. Using single step merge.') return self.minimize_corpus( target_path, arguments, existing_corpus_dirs + [new_corpus_dir], output_corpus_dir, reproducers_dir, max_time) # The dir where merge control file is located must persist for both merge # steps. The second step re-uses the MCF produced during the first step. merge_control_file_dir = self._create_temp_corpus_dir('mcf_tmp_dir') self._merge_control_file = os.path.join(merge_control_file_dir, 'MCF') # Two step merge process to obtain accurate stats for the new corpus units. # See https://reviews.llvm.org/D66107 for a more detailed description. merge_stats = {} # Step 1. Use only existing corpus and collect "initial" stats. result_1 = self.minimize_corpus(target_path, arguments, existing_corpus_dirs, output_corpus_dir, reproducers_dir, max_time) merge_stats['initial_edge_coverage'] = result_1.stats['edge_coverage'] merge_stats['initial_feature_coverage'] = result_1.stats[ 'feature_coverage'] # Clear the output dir as it does not have any new units at this point. engine_common.recreate_directory(output_corpus_dir) # Adjust the time limit for the time we spent on the first merge step. max_time -= result_1.time_executed if max_time <= 0: logs.log_error('Merging new testcases timed out.', fuzzer_output=result_1.logs) raise TimeoutError('Merging new testcases timed out.') # Step 2. Process the new corpus units as well. result_2 = self.minimize_corpus( target_path, arguments, existing_corpus_dirs + [new_corpus_dir], output_corpus_dir, reproducers_dir, max_time) merge_stats['edge_coverage'] = result_2.stats['edge_coverage'] merge_stats['feature_coverage'] = result_2.stats['feature_coverage'] # Diff the stats to obtain accurate values for the new corpus units. merge_stats['new_edges'] = (merge_stats['edge_coverage'] - merge_stats['initial_edge_coverage']) merge_stats['new_features'] = (merge_stats['feature_coverage'] - merge_stats['initial_feature_coverage']) output = result_1.logs + '\n\n' + result_2.logs if (merge_stats['new_edges'] < 0 or merge_stats['new_features'] < 0): logs.log_error('Two step merge failed.', merge_stats=merge_stats, output=output) merge_stats['new_edges'] = 0 merge_stats['new_features'] = 0 self._merge_control_file = None # TODO(ochang): Get crashes found during merge. return engine.FuzzResult( output, result_2.command, [], merge_stats, result_1.time_executed + result_2.time_executed)
def flash_to_latest_build_if_needed(): """Wipes user data, resetting the device to original factory state.""" if environment.get_value('LOCAL_DEVELOPMENT'): # Don't reimage local development devices. return run_timeout = environment.get_value('RUN_TIMEOUT') if run_timeout: # If we have a run timeout, then we are already scheduled to bail out and # will be probably get re-imaged. E.g. using frameworks like Tradefed. return # Check if a flash is needed based on last recorded flash time. last_flash_time = persistent_cache.get_value( constants.LAST_FLASH_TIME_KEY, constructor=datetime.datetime.utcfromtimestamp) needs_flash = last_flash_time is None or dates.time_has_expired( last_flash_time, seconds=FLASH_INTERVAL) if not needs_flash: return is_google_device = settings.is_google_device() if is_google_device is None: logs.log_error('Unable to query device. Reimaging failed.') adb.bad_state_reached() elif not is_google_device: # We can't reimage these, skip. logs.log('Non-Google device found, skipping reimage.') return # Check if both |BUILD_BRANCH| and |BUILD_TARGET| environment variables # are set. If not, we don't have enough data for reimaging and hence # we bail out. branch = environment.get_value('BUILD_BRANCH') target = environment.get_value('BUILD_TARGET') if not target: # We default to userdebug configuration. build_params = settings.get_build_parameters() if build_params: target = build_params.get('target') + '-userdebug' # Cache target in environment. This is also useful for cases when # device is bricked and we don't have this information available. environment.set_value('BUILD_TARGET', target) if not branch or not target: logs.log_warn( 'BUILD_BRANCH and BUILD_TARGET are not set, skipping reimage.') return image_directory = environment.get_value('IMAGES_DIR') build_info = fetch_artifact.get_latest_artifact_info(branch, target) if not build_info: logs.log_error('Unable to fetch information on latest build artifact for ' 'branch %s and target %s.' % (branch, target)) return if environment.is_android_cuttlefish(): download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory) adb.recreate_cuttlefish_device() adb.connect_to_cuttlefish_device() else: download_latest_build(build_info, FLASH_IMAGE_REGEXES, image_directory) # We do one device flash at a time on one host, otherwise we run into # failures and device being stuck in a bad state. flash_lock_key_name = 'flash:%s' % socket.gethostname() if not locks.acquire_lock(flash_lock_key_name, by_zone=True): logs.log_error('Failed to acquire lock for reimaging, exiting.') return logs.log('Reimaging started.') logs.log('Rebooting into bootloader mode.') for _ in range(FLASH_RETRIES): adb.run_as_root() adb.run_command(['reboot-bootloader']) time.sleep(FLASH_REBOOT_BOOTLOADER_WAIT) adb.run_fastboot_command(['oem', 'off-mode-charge', '0']) adb.run_fastboot_command(['-w', 'reboot-bootloader']) for partition, partition_image_filename in FLASH_IMAGE_FILES: partition_image_file_path = os.path.join(image_directory, partition_image_filename) adb.run_fastboot_command( ['flash', partition, partition_image_file_path]) if partition in ['bootloader', 'radio']: adb.run_fastboot_command(['reboot-bootloader']) # Disable ramdump to avoid capturing ramdumps during kernel crashes. # This causes device lockup of several minutes during boot and we intend # to analyze them ourselves. adb.run_fastboot_command(['oem', 'ramdump', 'disable']) adb.run_fastboot_command('reboot') time.sleep(FLASH_REBOOT_WAIT) if adb.get_device_state() == 'device': break logs.log_error('Reimaging failed, retrying.') locks.release_lock(flash_lock_key_name, by_zone=True) if adb.get_device_state() != 'device': logs.log_error('Unable to find device. Reimaging failed.') adb.bad_state_reached() logs.log('Reimaging finished.') # Reset all of our persistent keys after wipe. persistent_cache.delete_value(constants.BUILD_PROP_MD5_KEY) persistent_cache.delete_value(constants.LAST_TEST_ACCOUNT_CHECK_KEY) persistent_cache.set_value(constants.LAST_FLASH_BUILD_KEY, build_info) persistent_cache.set_value(constants.LAST_FLASH_TIME_KEY, time.time())