def test_copy_global_to_local_blacklist( self, mock_get_local_blacklist_file_path): """Test copying of global to local blacklist.""" local_blacklist_file_path = os.path.join(self.temp_directory, 'lsan_suppressions.txt') mock_get_local_blacklist_file_path.return_value = local_blacklist_file_path testcase = self._add_dummy_leak_testcase() blacklist_item = leak_blacklist.add_crash_to_global_blacklist_if_needed( testcase) self.assertTrue(blacklist_item.key.get()) # Test that a reproducible leak gets copied to local blacklist file. leak_blacklist.copy_global_to_local_blacklist() blacklist_function = leak_blacklist.get_leak_function_for_blacklist( testcase) expected_lsan_suppression_line = ( leak_blacklist.LSAN_SUPPRESSION_LINE.format( function=blacklist_function)) self.assertTrue(os.path.isfile(local_blacklist_file_path)) self.assertIn(expected_lsan_suppression_line, self._read_test_data(local_blacklist_file_path)) # Test that an excluded reproducible leak is not copied to blacklist file. leak_blacklist.copy_global_to_local_blacklist( excluded_testcase=testcase) self.assertTrue(os.path.isfile(local_blacklist_file_path)) self.assertNotIn(expected_lsan_suppression_line, self._read_test_data(local_blacklist_file_path))
def execute_task(fuzzer_name_and_revision, job_type): """Execute corpus pruning task.""" # TODO(ochang): Remove this once remaining jobs in queue are all processed. if '@' in fuzzer_name_and_revision: full_fuzzer_name, revision = fuzzer_name_and_revision.split('@') revision = revisions.convert_revision_to_integer(revision) else: full_fuzzer_name = fuzzer_name_and_revision revision = 0 fuzz_target = data_handler.get_fuzz_target(full_fuzzer_name) task_name = 'corpus_pruning_%s_%s' % (full_fuzzer_name, job_type) # Get status of last execution. last_execution_metadata = data_handler.get_task_status(task_name) last_execution_failed = (last_execution_metadata and last_execution_metadata.status == data_types.TaskState.ERROR) # Make sure we're the only instance running for the given fuzzer and # job_type. if not data_handler.update_task_status(task_name, data_types.TaskState.STARTED): logs.log('A previous corpus pruning task is still running, exiting.') return # Setup fuzzer and data bundle. if not setup.update_fuzzer_and_data_bundles(fuzz_target.engine): raise CorpusPruningException('Failed to set up fuzzer %s.' % fuzz_target.engine) use_minijail = environment.get_value('USE_MINIJAIL') # TODO(unassigned): Use coverage information for better selection here. cross_pollinate_fuzzers = _get_cross_pollinate_fuzzers( fuzz_target.engine, full_fuzzer_name) context = Context(fuzz_target, cross_pollinate_fuzzers, use_minijail) # Copy global blacklist into local suppressions file if LSan is enabled. is_lsan_enabled = environment.get_value('LSAN') if is_lsan_enabled: # TODO(ochang): Copy this to untrusted worker. leak_blacklist.copy_global_to_local_blacklist() try: result = do_corpus_pruning(context, last_execution_failed, revision) _save_coverage_information(context, result) _process_corpus_crashes(context, result) except CorpusPruningException as e: logs.log_error('Corpus pruning failed: %s.' % str(e)) data_handler.update_task_status(task_name, data_types.TaskState.ERROR) return finally: context.cleanup() data_handler.update_task_status(task_name, data_types.TaskState.FINISHED)
def execute_task(full_fuzzer_name, job_type): """Execute corpus pruning task.""" fuzz_target = data_handler.get_fuzz_target(full_fuzzer_name) task_name = 'corpus_pruning_%s_%s' % (full_fuzzer_name, job_type) revision = 0 # Trunk revision # Get status of last execution. last_execution_metadata = data_handler.get_task_status(task_name) last_execution_failed = (last_execution_metadata and last_execution_metadata.status == data_types.TaskState.ERROR) # Make sure we're the only instance running for the given fuzzer and # job_type. if not data_handler.update_task_status(task_name, data_types.TaskState.STARTED): logs.log('A previous corpus pruning task is still running, exiting.') return # Setup fuzzer and data bundle. if not setup.update_fuzzer_and_data_bundles(fuzz_target.engine): raise CorpusPruningException('Failed to set up fuzzer %s.' % fuzz_target.engine) cross_pollination_method, tag = choose_cross_pollination_strategy( full_fuzzer_name) # TODO(unassigned): Use coverage information for better selection here. cross_pollinate_fuzzers = _get_cross_pollinate_fuzzers( fuzz_target.engine, full_fuzzer_name, cross_pollination_method, tag) context = Context(fuzz_target, cross_pollinate_fuzzers, cross_pollination_method, tag) # Copy global blacklist into local suppressions file if LSan is enabled. is_lsan_enabled = environment.get_value('LSAN') if is_lsan_enabled: # TODO(ochang): Copy this to untrusted worker. leak_blacklist.copy_global_to_local_blacklist() try: result = do_corpus_pruning(context, last_execution_failed, revision) _record_cross_pollination_stats(result.cross_pollination_stats) _save_coverage_information(context, result) _process_corpus_crashes(context, result) except Exception: logs.log_error('Corpus pruning failed.') data_handler.update_task_status(task_name, data_types.TaskState.ERROR) return finally: context.cleanup() data_handler.update_task_status(task_name, data_types.TaskState.FINISHED)
def setup_testcase(testcase, fuzzer_override=None): """Sets up the testcase and needed dependencies like fuzzer, data bundle, etc.""" fuzzer_name = fuzzer_override or testcase.fuzzer_name job_type = testcase.job_type task_name = environment.get_value('TASK_NAME') testcase_fail_wait = environment.get_value('FAIL_WAIT') testcase_id = testcase.key.id() # Clear testcase directories. shell.clear_testcase_directories() # Adjust the test timeout value if this is coming from an user uploaded # testcase. _set_timeout_value_from_user_upload(testcase_id) # Update the fuzzer if necessary in order to get the updated data bundle. if fuzzer_name: try: update_successful = update_fuzzer_and_data_bundles(fuzzer_name) except errors.InvalidFuzzerError: # Close testcase and don't recreate tasks if this fuzzer is invalid. testcase.open = False testcase.fixed = 'NA' testcase.set_metadata('fuzzer_was_deleted', True) logs.log_error('Closed testcase %d with invalid fuzzer %s.' % (testcase_id, fuzzer_name)) error_message = 'Fuzzer %s no longer exists' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) return None, None, None if not update_successful: error_message = 'Unable to setup fuzzer %s' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task(task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # Extract the testcase and any of its resources to the input directory. file_list, input_directory, testcase_file_path = unpack_testcase(testcase) if not file_list: error_message = 'Unable to setup testcase %s' % testcase_file_path data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task(task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # For Android/Fuchsia, we need to sync our local testcases directory with the # one on the device. if environment.platform() == 'ANDROID': _copy_testcase_to_device_and_setup_environment(testcase, testcase_file_path) # Push testcases to worker. if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.push_testcases_to_worker() # Copy global blacklist into local blacklist. is_lsan_enabled = environment.get_value('LSAN') if is_lsan_enabled: # Get local blacklist without this testcase's entry. leak_blacklist.copy_global_to_local_blacklist( excluded_testcase=testcase) prepare_environment_for_testcase(testcase) return file_list, input_directory, testcase_file_path
def setup_testcase(testcase): """Sets up the testcase and needed dependencies like fuzzer, data bundle, etc.""" fuzzer_name = testcase.fuzzer_name job_type = testcase.job_type task_name = environment.get_value('TASK_NAME') testcase_fail_wait = environment.get_value('FAIL_WAIT') testcase_id = testcase.key.id() # Clear testcase directories. shell.clear_testcase_directories() # Setup memory debugging tool environment. environment.reset_current_memory_tool_options( redzone_size=testcase.redzone) # Adjust the test timeout value if this is coming from an user uploaded # testcase. _set_timeout_value_from_user_upload(testcase_id) if task_name == 'minimize': # Allow minimizing with a different fuzzer set up. minimize_fuzzer_override = environment.get_value( 'MINIMIZE_FUZZER_OVERRIDE') fuzzer_name = minimize_fuzzer_override or fuzzer_name # Update the fuzzer if necessary in order to get the updated data bundle. if fuzzer_name: try: update_successful = update_fuzzer_and_data_bundles(fuzzer_name) except errors.InvalidFuzzerError: # Close testcase and don't recreate tasks if this fuzzer is invalid. testcase.open = False testcase.fixed = 'NA' testcase.set_metadata('fuzzer_was_deleted', True) logs.log_error('Closed testcase %d with invalid fuzzer %s.' % (testcase_id, fuzzer_name)) error_message = 'Fuzzer %s no longer exists.' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) return None, None, None if not update_successful: error_message = 'Unable to setup fuzzer %s.' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task(task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # Extract the testcase and any of its resources to the input directory. file_list, input_directory, testcase_file_path = unpack_testcase(testcase) if not file_list: error_message = 'Unable to setup testcase %s.' % testcase_file_path data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task(task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # For Android/Fuchsia, we need to sync our local testcases directory with the # one on the device. if environment.platform() == 'ANDROID': _copy_testcase_to_device_and_setup_environment(testcase, testcase_file_path) if environment.platform() == 'FUCHSIA': fuchsia.device.copy_testcase_to_device(testcase_file_path) # Push testcases to worker. if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.push_testcases_to_worker() # Copy global blacklist into local blacklist. is_lsan_enabled = environment.get_value('LSAN') if is_lsan_enabled: # Get local blacklist without this testcase's entry. leak_blacklist.copy_global_to_local_blacklist( excluded_testcase=testcase) # Setup environment variable for windows size and location properties. # Explicitly use empty string to indicate use of default window properties. if hasattr(testcase, 'window_argument'): environment.set_value('WINDOW_ARG', testcase.window_argument) # Adjust timeout based on the stored multiplier (if available). if hasattr(testcase, 'timeout_multiplier') and testcase.timeout_multiplier: test_timeout = environment.get_value('TEST_TIMEOUT') environment.set_value('TEST_TIMEOUT', int(test_timeout * testcase.timeout_multiplier)) # Override APP_ARGS with minimized arguments (if available). if (hasattr(testcase, 'minimized_arguments') and testcase.minimized_arguments): environment.set_value('APP_ARGS', testcase.minimized_arguments) # Add FUZZ_TARGET to environment if this is a fuzz target testcase. fuzz_target = testcase.get_metadata('fuzzer_binary_name') if fuzz_target: environment.set_value('FUZZ_TARGET', fuzz_target) return file_list, input_directory, testcase_file_path