def setUp(self): if environment.platform() != 'LINUX': self.skipTest( 'Minijail tests are only applicable for linux platform.') super(CorpusPruningTestMinijail, self).setUp() os.environ['USE_MINIJAIL'] = 'True'
def _setup_x(): """Start Xvfb and blackbox before running the test application.""" if environment.platform() != 'LINUX': return [] if environment.is_engine_fuzzer_job(): # For engine fuzzer jobs like AFL, libFuzzer, Xvfb is not needed as the # those fuzz targets do not needed a UI. return [] environment.set_value('DISPLAY', DISPLAY) print('Creating virtual display...') xvfb_runner = new_process.ProcessRunner('/usr/bin/Xvfb') xvfb_process = xvfb_runner.run(additional_args=[ DISPLAY, '-screen', '0', '1280x1024x24', '-ac', '-nolisten', 'tcp' ]) time.sleep(PROCESS_START_WAIT_SECONDS) blackbox_runner = new_process.ProcessRunner('/usr/bin/blackbox') blackbox_process = blackbox_runner.run() time.sleep(PROCESS_START_WAIT_SECONDS) # Return all handles we create so they can be terminated properly at exit. return [xvfb_process, blackbox_process]
def get_build_to_revision_mappings(platform=None): """Gets the build information.""" if not platform: platform = environment.platform() result = {} build_info_json = _fetch_releases_from_chromiumdash(platform) for info in build_info_json: build_type = info['channel'].lower() if build_type == 'extended': build_type = 'extended_stable' version = info['version'] revision = str(info['chromium_main_branch_position']) result[build_type] = {'revision': revision, 'version': version} # Hack: pretend Windows extended stable info to be Linux extended stable info. # Because Linux doesn't have extended stable channel. if platform.lower() == 'linux': es_info = _fetch_releases_from_chromiumdash('WINDOWS', channel='Extended')[0] result['extended_stable'] = { 'revision': str(es_info['chromium_main_branch_position']), 'version': es_info['version'] } return result
def _fake_get_libfuzzer_testcase(*_): """Fake test case output intended to run "echo -n".""" testcase_map = { 'crash_type': 'type', 'crash_state': 'state', 'security_flag': False, 'gestures': [], 'flaky_stack': False, 'job_type': 'test_job', 'redzone': 32, 'disable_ubsan': False, 'additional_metadata': '{}', 'fuzzer_name': 'libFuzzer', 'job_definition': 'APP_NAME = launcher.py\n', 'overridden_fuzzer_name': 'libFuzzer_test_fuzzer', 'platform': environment.platform().lower(), 'minimized_arguments': '', 'window_argument': '', 'timeout_multiplier': 1.0, 'serialized_fuzz_target': { 'binary': 'test_fuzzer', 'engine': 'libFuzzer', 'project': 'test_project', }, 'one_time_crasher_flag': False, } return reproduce.SerializedTestcase(testcase_map)
def save_crash_info_if_needed(testcase_id, crash_revision, job_type, crash_type, crash_address, crash_frames): """Saves crash report for chromium project, skip otherwise.""" if data_handler.get_project_name(job_type) != 'chromium': return None serialized_crash_stack_frames = get_symbolized_stack_bytes( crash_type, crash_address, crash_frames) if not serialized_crash_stack_frames: return None crash_info = CrashReportInfo( serialized_crash_stack_frames=serialized_crash_stack_frames) # Get product and version (required). platform = environment.platform() crash_info.product = PRODUCT_MAP[platform] crash_info.version = revisions.get_real_revision(crash_revision, job_type, display=True) # Update crash_info object with bot information and testcase id. crash_info.bot_id = environment.get_value('BOT_NAME') crash_info.testcase_id = int(testcase_id) # Store CrashInfo metadata. crash_report_metadata = crash_info.to_report_metadata() crash_report_metadata.job_type = job_type crash_report_metadata.crash_revision = crash_revision crash_report_metadata.put() logs.log('Created crash report entry for testcase %s.' % testcase_id) return crash_info
def default_queue_suffix(): """Get the queue suffix for the current platform.""" queue_override = environment.get_value('QUEUE_OVERRIDE') if queue_override: return queue_suffix_for_platform(queue_override) return queue_suffix_for_platform(environment.platform())
def _get_testcase_file_and_path(testcase): """Figure out the relative path and input directory for this testcase.""" testcase_absolute_path = testcase.absolute_path # This hack is needed so that we can run a testcase generated on windows, on # linux. os.path.isabs return false on paths like c:\a\b\c. testcase_path_is_absolute = (testcase_absolute_path[1:3] == ':\\' or os.path.isabs(testcase_absolute_path)) # Fix os.sep in testcase path if we are running this on non-windows platform. # It is unusual to have '\\' on linux paths, so substitution should be safe. if environment.platform() != 'WINDOWS' and '\\' in testcase_absolute_path: testcase_absolute_path = testcase_absolute_path.replace('\\', os.sep) # Default directory for testcases. input_directory = environment.get_value('FUZZ_INPUTS') if not testcase_path_is_absolute: testcase_path = os.path.join(input_directory, testcase_absolute_path) return input_directory, testcase_path # Check if the testcase is on a nfs data bundle. If yes, then just # return it without doing root directory path fix. nfs_root = environment.get_value('NFS_ROOT') if nfs_root and testcase_absolute_path.startswith(nfs_root): return input_directory, testcase_absolute_path # Root directory can be different on bots. Fix the path to account for this. root_directory = environment.get_value('ROOT_DIR') search_string = '%s%s%s' % (os.sep, _BOT_DIR, os.sep) search_index = testcase_absolute_path.find(search_string) relative_path = testcase_absolute_path[search_index + len(search_string):] testcase_path = os.path.join(root_directory, _BOT_DIR, relative_path) return input_directory, testcase_path
def _get_fuzzer_binary_name_and_path(self): """Returns the fuzzer binary name and its path.""" # Fuchsia doesn't use file paths to call fuzzers, just the name of the # fuzzer, so we set both from FUZZ_TARGET here. if environment.platform() == 'FUCHSIA': fuzzer_binary_name = fuzzer_path = environment.get_value( 'FUZZ_TARGET') return fuzzer_binary_name, fuzzer_path build_directory = environment.get_value('BUILD_DIR') if not build_directory: raise BuiltinFuzzerException( 'BUILD_DIR environment variable is not set.') fuzzers = fuzzers_utils.get_fuzz_targets(build_directory) if not fuzzers: raise BuiltinFuzzerException( 'No fuzzer binaries found in |BUILD_DIR| directory.') fuzzer_binary_name = environment.get_value('FUZZ_TARGET') if fuzzer_binary_name: fuzzer_path = _get_fuzzer_path(fuzzers, fuzzer_binary_name) else: fuzzer_path = random.SystemRandom().choice(fuzzers) fuzzer_binary_name = os.path.basename(fuzzer_path) return fuzzer_binary_name, fuzzer_path
def initialize_timezone_from_environment(): """Initializes timezone for date functions based on environment.""" plt = environment.platform() if plt == 'WINDOWS': return # Only available on Unix platforms. time.tzset()
def wait_process(process, timeout, input_data=None, terminate_before_kill=False, terminate_wait_time=None): """Waits until either the process exits or times out. Args: process: A subprocess.Popen object. timeout: Maximum number of seconds to wait for before sending a signal. input_data: Input to be sent to the process. terminate_before_kill: A bool indicating that SIGTERM should be sent to the process first before SIGKILL (to let the SIGTERM handler run). terminate_wait_time: Maximum number of seconds to wait for the SIGTERM handler. Returns: A ProcessResult. """ result = ProcessResult() is_windows = environment.platform() == 'WINDOWS' # On Windows, terminate() just calls Win32 API function TerminateProcess() # which is equivalent to process kill. So, skip terminate_before_kill. if terminate_before_kill and not is_windows: first_timeout_function = process.terminate # Use a second timer to send the process kill. second_timer = threading.Timer(timeout + terminate_wait_time, _end_process, [process.kill, result]) else: first_timeout_function = process.kill second_timer = None first_timer = threading.Timer(timeout, _end_process, [first_timeout_function, result]) output = None start_time = time.time() try: first_timer.start() if second_timer: second_timer.start() output = process.communicate(input_data)[0] finally: first_timer.cancel() if second_timer: second_timer.cancel() result.return_code = process.poll() result.output = output result.time_executed = time.time() - start_time return result
def select_generator(strategy_pool, fuzzer_path): """Pick a generator to generate new testcases before fuzzing or return Generator.NONE if no generator selected.""" if environment.is_lib() or environment.platform() == 'FUCHSIA': # Unsupported. return Generator.NONE # We can't use radamsa binary on Windows. Disable ML for now until we know it # works on Win. # These generators don't produce testcases that LPM fuzzers can use. if (environment.platform() == 'WINDOWS' or is_lpm_fuzz_target(fuzzer_path)): return Generator.NONE if strategy_pool.do_strategy(strategy.CORPUS_MUTATION_ML_RNN_STRATEGY): return Generator.ML_RNN if strategy_pool.do_strategy(strategy.CORPUS_MUTATION_RADAMSA_STRATEGY): return Generator.RADAMSA return Generator.NONE
def android_device_required(func): """Skip Android-specific tests if we cannot run them.""" reason = None if not environment.get_value('ANDROID_SERIAL'): reason = 'Android device tests require that ANDROID_SERIAL is set.' elif not environment.get_value('INTEGRATION'): reason = 'Integration tests are not enabled.' elif environment.platform() != 'LINUX': reason = 'Android device tests can only run on a Linux host.' return unittest.skipIf(reason is not None, reason)(func)
def get_radamsa_path(): """Return path to radamsa binary for current platform.""" bin_directory_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'bin') platform = environment.platform() if platform == 'LINUX': return os.path.join(bin_directory_path, 'linux', 'radamsa') if platform == 'MAC': return os.path.join(bin_directory_path, 'mac', 'radamsa') return None
def get_gestures(gesture_count): """Return a list of random gestures.""" plt = environment.platform() if environment.is_android(plt): return android.gestures.get_random_gestures(gesture_count) if plt == 'LINUX': return linux.gestures.get_random_gestures(gesture_count) if plt == 'WINDOWS': return windows.gestures.get_random_gestures(gesture_count) return []
def clear_testcase_directories(): """Clears the testcase directories.""" remove_directory(environment.get_value('FUZZ_INPUTS'), recreate=True) remove_directory(environment.get_value('FUZZ_INPUTS_DISK'), recreate=True) if environment.is_android() and environment.get_value('ANDROID_SERIAL'): from clusterfuzz._internal.platforms import android android.device.clear_testcase_directory() if environment.platform() == 'FUCHSIA': from clusterfuzz._internal.platforms import fuchsia fuchsia.device.clear_testcase_directory() if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host file_host.clear_testcase_directories()
def cleanup_defunct_processes(): """Cleans up defunct processes.""" # Defunct processes happen only on unix platforms. if environment.platform() != 'WINDOWS': while True: try: # Matches any defunct child process. p, _ = os.waitpid(-1, os.WNOHANG) if not p: break logs.log('Clearing defunct process %s.' % str(p)) except: break
def start_process(process_handle): """Start the process using process handle and override list2cmdline for Windows.""" is_win = environment.platform() == 'WINDOWS' if is_win: # Override list2cmdline on Windows to return first index of list as string. # This is to workaround a mozprocess bug since it passes command as list # and not as string. subprocess.list2cmdline_orig = subprocess.list2cmdline subprocess.list2cmdline = lambda s: s[0] try: process_handle.run() finally: if is_win: subprocess.list2cmdline = subprocess.list2cmdline_orig
def symbolize_stacktrace(unsymbolized_crash_stacktrace, enable_inline_frames=True): """Symbolize a crash stacktrace.""" if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import symbolize_host return symbolize_host.symbolize_stacktrace(unsymbolized_crash_stacktrace, enable_inline_frames) platform = environment.platform() if platform == 'WINDOWS': # Windows Clang ASAN provides symbolized stacktraces anyway. return unsymbolized_crash_stacktrace if platform == 'FUCHSIA': # Fuchsia Clang ASAN provides symbolized stacktraces anyway. return unsymbolized_crash_stacktrace # FIXME: Support symbolization on ChromeOS device. if platform == 'CHROMEOS': return unsymbolized_crash_stacktrace # Initialize variables. global llvm_symbolizer_path global pipes global stack_inlining global symbolizers pipes = [] stack_inlining = str(enable_inline_frames).lower() symbolizers = {} # Make sure we have a llvm symbolizer for this platform. llvm_symbolizer_path = environment.get_llvm_symbolizer_path() if not llvm_symbolizer_path: return unsymbolized_crash_stacktrace # Disable buffering for stdout. disable_buffering() loop = SymbolizationLoop( binary_path_filter=filter_binary_path, dsym_hint_producer=chrome_dsym_hints) symbolized_crash_stacktrace = loop.process_stacktrace( unsymbolized_crash_stacktrace) return symbolized_crash_stacktrace
def convert_dependency_url_to_local_path(url): """Convert a dependency URL to a corresponding local path.""" # Bot-specific import. from clusterfuzz._internal.bot.webserver import http_server logs.log('Process dependency: %s.' % url) file_match = FILE_URL_REGEX.search(url) http_match = HTTP_URL_REGEX.search(url) platform = environment.platform() local_path = None if file_match: file_path = file_match.group(1) logs.log('Detected file dependency: %s.' % file_path) if platform == 'WINDOWS': local_path = file_path else: local_path = '/' + file_path # Convert remote to local path for android. if environment.is_android(): remote_testcases_directory = android.constants.DEVICE_TESTCASES_DIR local_testcases_directory = environment.get_value( 'FUZZ_INPUTS') local_path = local_path.replace(remote_testcases_directory, local_testcases_directory) elif http_match: relative_http_path = os.path.sep + http_match.group(2) logs.log('Detected http dependency: %s.' % relative_http_path) local_path = http_server.get_absolute_testcase_file(relative_http_path) if not local_path: # This needs to be a warning since in many cases, it is actually a # non-existent path. For others, we need to add the directory aliases in # file http_server.py. logs.log_warn('Unable to find server resource %s, skipping.' % relative_http_path) if local_path: local_path = utils.normalize_path(local_path) return local_path
def setUp(self): """Setup for minijail test.""" if environment.platform() != 'LINUX': self.skipTest( 'Minijail tests are only applicable for linux platform.') self.setUpPyfakefs() for subdir in ['dev', 'lib', 'lib32', 'lib64', 'proc']: self.fs.create_dir(os.path.join('/', subdir)) self.fs.create_dir(os.path.join('/', 'usr', 'lib')) self.fs.create_dir(os.path.join('/', 'usr', 'lib32')) test_helpers.patch(self, [ 'clusterfuzz._internal.system.minijail._get_minijail_path', 'clusterfuzz._internal.system.minijail.MinijailChroot._mknod', ]) test_helpers.patch_environ(self) self.mock._get_minijail_path.return_value = '/sbin/minijail' # pylint: disable=protected-access
def get_command(self, additional_args=None): """Overridden get_command.""" if environment.platform() != 'LINUX': raise RuntimeError('UnshareProcessRunner only supported on Linux') unshare_path = environment.get_default_tool_path('unshare') if not unshare_path: raise RuntimeError('unshare not found') command = [ unshare_path, '-c', # Map current user to same user in user namespace. '-n', # Enter network namespace. ] command.append(self._executable_path) command.extend(self._default_args) if additional_args: command.extend(additional_args) return command
def run(): """Run custom platform specific init scripts.""" platform = environment.platform().lower() script_path = os.path.join(environment.get_config_directory(), SCRIPT_DIR, platform + _extension(platform)) if not os.path.exists(script_path): return os.chmod(script_path, 0o750) if script_path.endswith('.ps1'): cmd = 'powershell.exe ' + script_path else: cmd = script_path try: process_handler.run_process(cmd, timeout=1800, need_shell=True, testcase_run=False, ignore_children=True) except Exception: logs.log_error('Failed to execute platform initialization script.')
def filter_binary_path(binary_path): """Filters binary path to provide a local copy.""" if environment.is_android() or environment.is_lkl_job(): return symbols_downloader.filter_binary_path(binary_path) if environment.platform() == 'CHROMEOS': # FIXME: Add code to pull binaries from ChromeOS device. return binary_path if environment.is_chromeos_system_job(): # This conditional is True for ChromeOS system fuzzers that are running on # Linux. Ensure that the binary is always looked for in the chroot and not # in system directories. build_dir = environment.get_value('BUILD_DIR') if not binary_path.startswith(build_dir): # Fixup path so |binary_path| points to a binary in the chroot (probably # a system library). return os.path.join(build_dir, binary_path[1:]) # For Linux and Mac, the binary exists locally. No work to do, # just return the same binary path. return binary_path
def start_if_needed(service): """Start Google Cloud Profiler if |USE_PYTHON_PROFILER| environment variable is set.""" if not environment.get_value('USE_PYTHON_PROFILER'): return True project_id = utils.get_application_id() service_with_platform = '{service}_{platform}'.format( service=service, platform=environment.platform().lower()) try: # Import the package here since it is only needed when profiler is enabled. # Also, this is supported on Linux only. import googlecloudprofiler googlecloudprofiler.start(project_id=project_id, service=service_with_platform) except Exception: logs.log_error('Failed to start the profiler for service %s.' % service_with_platform) return False return True
def find_fuzzer_path(build_directory, fuzzer_name): """Find the fuzzer path with the given name.""" if not build_directory: # Grey-box fuzzers might not have the build directory for a particular job # configuration when doing variant task testing (e.g. Android on-device # fuzz target might not exist on host). In this case, treat it similar to # target not found by returning None. logs.log_warn('No build directory found for fuzzer: %s' % fuzzer_name) return None if environment.platform() == 'FUCHSIA': # Fuchsia targets are not on disk. return fuzzer_name if environment.is_android_kernel(): return os.path.join(build_directory, 'syzkaller', 'bin', 'syz-manager') # TODO(ochang): This is necessary for legacy testcases, which include the # project prefix in arguments. Remove this in the near future. project_name = environment.get_value('PROJECT_NAME') legacy_name_prefix = u'' if project_name: legacy_name_prefix = project_name + u'_' fuzzer_filename = environment.get_executable_filename(fuzzer_name) for root, _, files in shell.walk(build_directory): for filename in files: if (legacy_name_prefix + filename == fuzzer_name or filename == fuzzer_filename): return os.path.join(root, filename) # This is an expected case when doing regression testing with old builds # that do not have that fuzz target. It can also happen when a host sends a # message to an untrusted worker that just restarted and lost information on # build directory. logs.log_warn('Fuzzer: %s not found in build_directory: %s.' % (fuzzer_name, build_directory)) return None
def close_open_file_handles_if_needed(path): """Try to close all open file handle for a specific path.""" if environment.platform() != 'WINDOWS': # Handle closing is only applicable on Windows platform. return resources_directory = environment.get_platform_resources_directory() handle_executable_path = os.path.join(resources_directory, 'handle.exe') handle_output = execute_command('%s -accepteula "%s"' % (handle_executable_path, path)) for line in handle_output.splitlines(): match = HANDLE_OUTPUT_FILE_TYPE_REGEX.match(line) if not match: continue process_id = match.group(1).decode('utf-8') file_handle_id = match.group(2).decode('utf-8') file_path = match.group(3).decode('utf-8') logs.log('Closing file handle id %s for path %s.' % (file_handle_id, file_path)) execute_command('%s -accepteula -c %s -p %s -y' % (handle_executable_path, file_handle_id, process_id))
def _get_gsutil_path(): """Get path to gsutil executable. Returns: Path to gsutil executable on the system. """ gsutil_executable = 'gsutil' if environment.platform() == 'WINDOWS': gsutil_executable += '.cmd' gsutil_directory = environment.get_value('GSUTIL_PATH') if not gsutil_directory: # Try searching the binary in path. gsutil_absolute_path = shell.which(gsutil_executable) if gsutil_absolute_path: return gsutil_absolute_path logs.log_error( 'Cannot locate gsutil in PATH, set GSUTIL_PATH to directory ' 'containing gsutil binary.') return None gsutil_absolute_path = os.path.join(gsutil_directory, gsutil_executable) return gsutil_absolute_path
def get_fuzz_task_payload(platform=None): """Select a fuzzer that can run on this platform.""" if not platform: queue_override = environment.get_value('QUEUE_OVERRIDE') platform = queue_override if queue_override else environment.platform() if environment.is_local_development(): query = data_types.FuzzerJob.query() query = query.filter(data_types.FuzzerJobs.platform == platform) mappings = list(ndb_utils.get_all_from_query(query)) else: query = data_types.FuzzerJobs.query() query = query.filter(data_types.FuzzerJobs.platform == platform) mappings = [] for entity in query: mappings.extend(entity.fuzzer_jobs) if not mappings: return None, None selection = utils.random_weighted_choice(mappings, weight_attribute='actual_weight') return selection.fuzzer, selection.job
def run_platform_init_scripts(): """Run platform specific initialization scripts.""" logs.log('Running platform initialization scripts.') plt = environment.platform() if environment.is_android_emulator(): # Nothing to do here since emulator is not started yet. pass elif environment.is_android(): android_init.run() elif plt == 'CHROMEOS': chromeos_init.run() elif plt == 'FUCHSIA': fuchsia_init.run() elif plt == 'LINUX': linux_init.run() elif plt == 'MAC': mac_init.run() elif plt == 'WINDOWS': windows_init.run() else: raise RuntimeError('Unsupported platform') logs.log('Completed running platform initialization scripts.')
def execute(input_directory, output_directory, fuzzer_name, generation_timeout): """Execute ML RNN generator to produce new inputs. This method should be called inside launcher, to generate a number of new inputs based on ML RNN model. It will fetch ML model from GCS bucket specified in environment variable `CORPUS_BUCKET`. The script to run the model resides in folder `tools/fuzzers/ml/rnn`. Args: input_directory: Seed corpus path. The directory should not be empty. output_directory: The directory to place generated inputs. fuzzer_name: Name of the fuzzer, e.g libpng_read_fuzzer. It indicates the subdirectory in gcs bucket to store models. generation_timeout: Time in seconds for the generator to run. Normally it takes <1s to generate an input, assuming the input length is <4KB. """ if environment.platform() != 'LINUX': logs.log('Unsupported platform for ML RNN generation, skipping.') return # Validate corpus folder. file_count = shell.get_directory_file_count(input_directory) if not file_count: logs.log('Corpus is empty. Skip generation.') return # Number of existing new inputs. They are possibly generated by other # generators. old_corpus_units = shell.get_directory_file_count(output_directory) old_corpus_bytes = shell.get_directory_size(output_directory) # Get model path. model_path = prepare_model_directory(fuzzer_name) if not model_path: return result = run(input_directory, output_directory, model_path, generation_timeout) # Generation process exited abnormally but not caused by timeout, meaning # error occurred during execution. if result.return_code and not result.timed_out: if result.return_code == constants.ExitCode.CORPUS_TOO_SMALL: logs.log_warn( 'ML RNN generation for fuzzer %s aborted due to small corpus.' % fuzzer_name) else: logs.log_error( 'ML RNN generation for fuzzer %s failed with ExitCode = %d.' % (fuzzer_name, result.return_code), output=utils.decode_to_unicode(result.output)) return # Timeout is not error, if we have new units generated. if result.timed_out: logs.log_warn('ML RNN generation for fuzzer %s timed out.' % fuzzer_name) new_corpus_units = (shell.get_directory_file_count(output_directory) - old_corpus_units) new_corpus_bytes = (shell.get_directory_size(output_directory) - old_corpus_bytes) if new_corpus_units: logs.log( 'Added %d new inputs (%d bytes) using ML RNN generator for %s.' % (new_corpus_units, new_corpus_bytes, fuzzer_name)) else: logs.log_error('ML RNN generator did not produce any inputs for %s' % fuzzer_name, output=utils.decode_to_unicode(result.output))