def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many-statements; settings_path: Path, iteration_seed_override: Optional[int] = None, fuzzing_tool_pattern: Optional[List[FuzzingTool]] = None, allow_no_stack_traces: bool = False, override_sigint: bool = True, use_amber_vulkan_loader: bool = False, active_device_names: Optional[List[str]] = None, update_ignored_crash_signatures_gerrit_cookie: Optional[str] = None, ) -> None: if not fuzzing_tool_pattern: fuzzing_tool_pattern = [FuzzingTool.GLSL_FUZZ] util.update_gcov_environment_variable_if_needed() if override_sigint: interrupt_util.override_sigint() try_get_root_file() settings = settings_util.read_or_create(settings_path) binary_manager = binaries_util.get_default_binary_manager( settings=settings) temp_dir = Path() / "temp" # Note: we use "is not None" so that if the user passes an empty Gerrit cookie, we still try to execute this code. if update_ignored_crash_signatures_gerrit_cookie is not None: git_tool = util.tool_on_path("git") downloaded_graphicsfuzz_tests_dir = ( temp_dir / f"graphicsfuzz_cts_tests_{get_random_name()[:8]}") work_dir = temp_dir / f"graphicsfuzz_cts_run_{get_random_name()[:8]}" download_cts_gf_tests.download_cts_graphicsfuzz_tests( git_tool=git_tool, cookie=update_ignored_crash_signatures_gerrit_cookie, output_tests_dir=downloaded_graphicsfuzz_tests_dir, ) download_cts_gf_tests.extract_shaders( tests_dir=downloaded_graphicsfuzz_tests_dir, binaries=binary_manager) with util.file_open_text(work_dir / "results.csv", "w") as results_out_handle: run_cts_gf_tests.main_helper( tests_dir=downloaded_graphicsfuzz_tests_dir, work_dir=work_dir, binaries=binary_manager, settings=settings, active_devices=devices_util.get_active_devices( settings.device_list), results_out_handle=results_out_handle, updated_settings_output_path=settings_path, ) return active_devices = devices_util.get_active_devices( settings.device_list, active_device_names=active_device_names) # Add host_preprocessor device from device list if it is missing. if not active_devices[0].HasField("preprocess"): for device in settings.device_list.devices: if device.HasField("preprocess"): active_devices.insert(0, device) break # Add host_preprocessor device (from scratch) if it is still missing. if not active_devices[0].HasField("preprocess"): active_devices.insert( 0, Device(name="host_preprocessor", preprocess=DevicePreprocess())) reports_dir = Path() / "reports" fuzz_failures_dir = reports_dir / FUZZ_FAILURES_DIR_NAME references_dir = Path() / REFERENCES_DIR donors_dir = Path() / DONORS_DIR spirv_fuzz_shaders_dir = Path() / "spirv_fuzz_shaders" # Log a warning if there is no tool on the PATH for printing stack traces. prepended = util.prepend_catchsegv_if_available([], log_warning=True) if not allow_no_stack_traces and not prepended: raise AssertionError("Stopping because we cannot get stack traces.") spirv_fuzz_shaders: List[Path] = [] references: List[Path] = [] if FuzzingTool.SPIRV_FUZZ in fuzzing_tool_pattern: check_dir_exists(spirv_fuzz_shaders_dir) spirv_fuzz_shaders = sorted(spirv_fuzz_shaders_dir.rglob("*.json")) if FuzzingTool.GLSL_FUZZ in fuzzing_tool_pattern: check_dir_exists(references_dir) check_dir_exists(donors_dir) # TODO: make GraphicsFuzz find donors recursively. references = sorted(references_dir.rglob("*.json")) # Filter to only include .json files that have at least one shader (.frag, .vert, .comp) file. references = [ ref for ref in references if shader_job_util.get_related_files(ref) ] if use_amber_vulkan_loader: library_path = binary_manager.get_binary_path_by_name( binaries_util.AMBER_VULKAN_LOADER_NAME).path.parent util.add_library_paths_to_environ([library_path], os.environ) fuzzing_tool_index = 0 while True: interrupt_util.interrupt_if_needed() # We have to use "is not None" because the seed could be 0. if iteration_seed_override is not None: iteration_seed = iteration_seed_override else: iteration_seed = secrets.randbits(ITERATION_SEED_BITS) log(f"Iteration seed: {iteration_seed}") random.seed(iteration_seed) staging_name = get_random_name()[:8] staging_dir = temp_dir / staging_name try: util.mkdir_p_new(staging_dir) except FileExistsError: if iteration_seed_override is not None: raise log(f"Staging directory already exists: {str(staging_dir)}") log("Starting new iteration.") continue # Pseudocode: # - Create test_dir(s) in staging directory. # - Run test_dir(s) on all active devices (stop early if appropriate). # - For each test failure on each device, copy the test to reports_dir, adding the device and crash signature. # - Reduce each report (on the given device). # - Produce a summary for each report. fuzzing_tool = fuzzing_tool_pattern[fuzzing_tool_index] fuzzing_tool_index = (fuzzing_tool_index + 1) % len(fuzzing_tool_pattern) if fuzzing_tool == FuzzingTool.SPIRV_FUZZ: fuzz_spirv_test.fuzz_spirv( staging_dir, reports_dir, fuzz_failures_dir, active_devices, spirv_fuzz_shaders, settings, binary_manager, ) elif fuzzing_tool == FuzzingTool.GLSL_FUZZ: fuzz_glsl_test.fuzz_glsl( staging_dir, reports_dir, fuzz_failures_dir, active_devices, references, donors_dir, settings, binary_manager, ) else: raise AssertionError(f"Unknown fuzzing tool: {fuzzing_tool}") if iteration_seed_override is not None: log("Stopping due to iteration_seed") break shutil.rmtree(staging_dir)
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements; parser = argparse.ArgumentParser( description= "Runs GraphicsFuzz AmberScript tests on the active devices listed in " "the settings.json file.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( "--settings", help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) parser.add_argument( "--tests", help= "Path to the directory of AmberScript tests with shaders extracted.", default="graphicsfuzz", ) parser.add_argument( "--update_ignored_crash_signatures", help= "As the tests are run for each device, add any crash signatures to the device's ignored_crash_signatures " "property and write out the updated settings.json file.", action="store_true", ) parser.add_argument( "--results_out", help="Output file path for the CSV results table.", default="results.csv", ) parsed_args = parser.parse_args(sys.argv[1:]) # Args. tests_dir: Path = Path(parsed_args.tests) settings_path: Path = Path(parsed_args.settings) update_ignored_crash_signatures: bool = parsed_args.update_ignored_crash_signatures results_out_path: Path = Path(parsed_args.results_out) # Settings and devices. settings = settings_util.read_or_create(settings_path) active_devices = devices_util.get_active_devices(settings.device_list) # Binaries. binaries = binaries_util.get_default_binary_manager(settings=settings) work_dir = Path( ) / "temp" / f"graphicsfuzz_cts_run_{fuzz.get_random_name()[:8]}" with util.file_open_text(results_out_path, "w") as results_out_handle: main_helper( tests_dir=tests_dir, work_dir=work_dir, binaries=binaries, settings=settings, active_devices=active_devices, results_out_handle=results_out_handle, updated_settings_output_path=( settings_path if update_ignored_crash_signatures else None), )
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements; parser = argparse.ArgumentParser( description="Runs GraphicsFuzz AmberScript tests on the active devices listed in " "the settings.json file.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( "--settings", help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) parser.add_argument( "--tests", help="Path to the directory of AmberScript tests with shaders extracted.", default="graphicsfuzz", ) parser.add_argument( "--update_ignored_signatures", help="As the tests are run for each device, add any crash signatures to the device's ignored_crash_signatures " "property and write out the updated settings.json file.", action="store_true", ) parser.add_argument( "--results_out", help="Output file path for the CSV results table.", default="results.csv", ) parsed_args = parser.parse_args(sys.argv[1:]) # Args. tests_dir: Path = Path(parsed_args.tests) settings_path: Path = Path(parsed_args.settings) update_ignored_signatures: bool = parsed_args.update_ignored_signatures results_out_path: Path = Path(parsed_args.results_out) # Settings and devices. settings = settings_util.read_or_create(settings_path) active_devices = devices_util.get_active_devices(settings.device_list) # Binaries. binaries = binaries_util.get_default_binary_manager(settings=settings) work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}" util.mkdirs_p(work_dir) with util.file_open_text(results_out_path, "w") as results_handle: def write_entry(entry: str) -> None: results_handle.write(entry) results_handle.write(", ") results_handle.flush() def write_newline() -> None: results_handle.write("\n") results_handle.flush() spirv_opt_path: Optional[Path] = None swift_shader_path: Optional[Path] = None amber_path: Optional[Path] = None # Small hack to ensure we have three devices for spirv-opt, each with a different name. main_spirv_opt_device: Optional[Device] = None if active_devices and active_devices[0].name == "host_preprocessor": main_spirv_opt_device = active_devices[0] main_spirv_opt_device.name = SPIRV_OPT_O spirv_opt_custom = Device() spirv_opt_custom.CopyFrom(main_spirv_opt_device) spirv_opt_custom.name = SPIRV_OPT_CUSTOM active_devices.insert(1, spirv_opt_custom) spirv_opt_os = Device() spirv_opt_os.CopyFrom(main_spirv_opt_device) spirv_opt_os.name = SPIRV_OPT_OS active_devices.insert(1, spirv_opt_os) # Enumerate active devices, writing their name and storing binary paths if needed. write_entry("test") for device in active_devices: write_entry(device.name) if device.HasField("preprocess"): spirv_opt_path = binaries.get_binary_path_by_name( binaries_util.SPIRV_OPT_NAME ).path if device.HasField("swift_shader"): swift_shader_path = binaries.get_binary_path_by_name( binaries_util.SWIFT_SHADER_NAME ).path if device.HasField("swift_shader") or device.HasField("host"): amber_path = binaries.get_binary_path_by_name( binaries_util.AMBER_NAME ).path write_newline() # Enumerate tests and devices, writing the results. for test in sorted(tests_dir.glob("*.amber")): test_name = util.remove_end(test.name, ".amber") write_entry(test_name) spirv_shaders = sorted( tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv") ) for device in active_devices: test_run_dir = work_dir / f"{test_name}_{device.name}" util.mkdirs_p(test_run_dir) ignored_signatures_set: Set[str] = set(device.ignored_crash_signatures) with util.file_open_text(test_run_dir / "log.txt", "w") as log_stream: try: gflogging.push_stream_for_logging(log_stream) if device.HasField("preprocess"): # This just means spirv-opt for now. assert spirv_opt_path # noqa assert main_spirv_opt_device # noqa # Pick spirv-opt arguments based on device name. if device.name == SPIRV_OPT_O: spirv_opt_args = ["-O"] elif device.name == SPIRV_OPT_OS: spirv_opt_args = ["-Os"] elif device.name == SPIRV_OPT_CUSTOM: spirv_opt_args = ( spirv_opt_util.OPT_INTERESTING_SUBSET_OF_PASSES ) else: raise AssertionError( f"Can't tell how to run device {device.name}; " f"must be named host_preprocessor and be the first active device." ) # Reset device and ignored_crash_signatures. device = main_spirv_opt_device ignored_signatures_set = set( device.ignored_crash_signatures ) try: for spirv_shader in spirv_shaders: spirv_opt_util.run_spirv_opt_on_spirv_shader( spirv_shader, test_run_dir, spirv_opt_args, spirv_opt_path, ) result_util.write_status( test_run_dir, fuzz.STATUS_SUCCESS, ) except subprocess.CalledProcessError: result_util.write_status( test_run_dir, fuzz.STATUS_TOOL_CRASH, ) except subprocess.TimeoutExpired: result_util.write_status( test_run_dir, fuzz.STATUS_TOOL_TIMEOUT, ) elif device.HasField("shader_compiler"): try: for spirv_shader in spirv_shaders: shader_compiler_util.run_shader( shader_compiler_device=device.shader_compiler, shader_path=spirv_shader, output_dir=test_run_dir, compiler_path=binaries.get_binary_path_by_name( device.shader_compiler.binary ).path, timeout=DEFAULT_TIMEOUT, ) result_util.write_status( test_run_dir, fuzz.STATUS_SUCCESS, ) except subprocess.CalledProcessError: result_util.write_status( test_run_dir, fuzz.STATUS_CRASH, ) except subprocess.TimeoutExpired: result_util.write_status( test_run_dir, fuzz.STATUS_TIMEOUT, ) elif device.HasField("swift_shader"): assert swift_shader_path # noqa assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, icd=swift_shader_path, ) elif device.HasField("host"): assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, custom_launcher=list(device.host.custom_launcher), ) elif device.HasField("android"): android_device.run_amber_on_device( test, test_run_dir, dump_image=False, dump_buffer=False, serial=device.android.serial, ) else: raise AssertionError(f"Unsupported device {device.name}") finally: gflogging.pop_stream_for_logging() status = result_util.get_status(test_run_dir) if status == fuzz.STATUS_SUCCESS: write_entry("P") elif status in (fuzz.STATUS_TIMEOUT, fuzz.STATUS_TOOL_TIMEOUT): write_entry("T") else: write_entry("F") # Update ignored signatures. if ( status in ( fuzz.STATUS_TOOL_CRASH, fuzz.STATUS_CRASH, fuzz.STATUS_UNRESPONSIVE, ) and update_ignored_signatures ): log_contents = util.file_read_text( result_util.get_log_path(test_run_dir) ) signature = signature_util.get_signature_from_log_contents( log_contents ) if signature == signature_util.NO_SIGNATURE: log(f"NOT updating ignored signatures to include {signature}") elif signature in ignored_signatures_set: log(f"Signature is already ignored: {signature}") else: log(f"Adding ignored signature: {signature}") device.ignored_crash_signatures.append(signature) write_newline() if update_ignored_signatures: # Reset main_spirv_opt_device name before writing it back out. if main_spirv_opt_device: main_spirv_opt_device.name = "host_preprocessor" settings_util.write(settings, settings_path)
def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many-statements; settings_path: Path, iteration_seed_override: Optional[int], use_spirv_fuzz: bool, force_no_stack_traces: bool, ) -> None: util.update_gcov_environment_variable_if_needed() try: artifact_util.artifact_path_get_root() except FileNotFoundError: log( "Could not find ROOT file (in the current directory or above) to mark where binaries should be stored. " "Creating a ROOT file in the current directory." ) util.file_write_text(Path(artifact_util.ARTIFACT_ROOT_FILE_NAME), "") settings = settings_util.read_or_create(settings_path) active_devices = devices_util.get_active_devices(settings.device_list) reports_dir = Path() / "reports" fuzz_failures_dir = reports_dir / FUZZ_FAILURES_DIR_NAME temp_dir = Path() / "temp" references_dir = Path() / "references" donors_dir = Path() / "donors" spirv_fuzz_shaders_dir = Path() / "spirv_fuzz_shaders" # Log a warning if there is no tool on the PATH for printing stack traces. prepended = util.prepend_catchsegv_if_available([], log_warning=True) if not force_no_stack_traces and not prepended: raise AssertionError("Stopping because we cannot get stack traces.") spirv_fuzz_shaders: List[Path] = [] references: List[Path] = [] if use_spirv_fuzz: check_dir_exists(spirv_fuzz_shaders_dir) spirv_fuzz_shaders = sorted(spirv_fuzz_shaders_dir.rglob("*.json")) else: check_dir_exists(references_dir) check_dir_exists(donors_dir) # TODO: make GraphicsFuzz find donors recursively. references = sorted(references_dir.rglob("*.json")) # Filter to only include .json files that have at least one shader (.frag, .vert, .comp) file. references = [ ref for ref in references if shader_job_util.get_related_files(ref) ] binary_manager = binaries_util.get_default_binary_manager( settings=settings ).get_child_binary_manager(list(settings.custom_binaries), prepend=True) while True: # We have to use "is not None" because the seed could be 0. if iteration_seed_override is not None: iteration_seed = iteration_seed_override else: iteration_seed = secrets.randbits(ITERATION_SEED_BITS) log(f"Iteration seed: {iteration_seed}") random.seed(iteration_seed) staging_name = get_random_name()[:8] staging_dir = temp_dir / staging_name try: util.mkdir_p_new(staging_dir) except FileExistsError: if iteration_seed_override is not None: raise log(f"Staging directory already exists: {str(staging_dir)}") log(f"Starting new iteration.") continue # Pseudocode: # - Create test_dir(s) in staging directory. # - Run test_dir(s) on all active devices (stop early if appropriate). # - For each test failure on each device, copy the test to reports_dir, adding the device and crash signature. # - Reduce each report (on the given device). # - Produce a summary for each report. if use_spirv_fuzz: fuzz_spirv_test.fuzz_spirv( staging_dir, reports_dir, fuzz_failures_dir, active_devices, spirv_fuzz_shaders, settings, binary_manager, ) else: fuzz_glsl_test.fuzz_glsl( staging_dir, reports_dir, fuzz_failures_dir, active_devices, references, donors_dir, settings, binary_manager, ) shutil.rmtree(staging_dir) if iteration_seed_override is not None: log("Stopping due to iteration_seed") break
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements; parser = argparse.ArgumentParser( description="Runs GraphicsFuzz AmberScript tests on the active devices listed in " "the settings.json file." ) parser.add_argument( "--settings", help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) parser.add_argument( "--tests", help="Path to the directory of AmberScript tests with shaders extracted.", default=str("graphicsfuzz"), ) parsed_args = parser.parse_args(sys.argv[1:]) # Args. tests_dir: Path = Path(parsed_args.tests) settings_path: Path = Path(parsed_args.settings) # Settings and devices. settings = settings_util.read_or_create(settings_path) active_devices = devices_util.get_active_devices(settings.device_list) # Binaries. binaries = binaries_util.get_default_binary_manager(settings=settings) work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}" util.mkdirs_p(work_dir) with util.file_open_text(Path("results.txt"), "w") as log_handle: def write_entry(entry: str) -> None: log_handle.write(entry) log_handle.write(", ") log_handle.flush() def write_newline() -> None: log_handle.write("\n") log_handle.flush() spirv_opt_path: Optional[Path] = None swift_shader_path: Optional[Path] = None amber_path: Optional[Path] = None # Enumerate active devices, writing their name and storing binary paths if needed. write_entry("test") for device in active_devices: if device.name == "host_preprocessor": # We are actually just running spirv-opt on the SPIR-V shaders. write_entry("spirv-opt") else: write_entry(device.name) if device.HasField("preprocess"): spirv_opt_path = binaries.get_binary_path_by_name( binaries_util.SPIRV_OPT_NAME ).path if device.HasField("swift_shader"): swift_shader_path = binaries.get_binary_path_by_name( binaries_util.SWIFT_SHADER_NAME ).path if device.HasField("swift_shader") or device.HasField("host"): amber_path = binaries.get_binary_path_by_name( binaries_util.AMBER_NAME ).path write_newline() # Enumerate tests and devices, writing the results. for test in sorted(tests_dir.glob("*.amber")): test_name = util.remove_end(test.name, ".amber") write_entry(test_name) spirv_shaders = sorted( tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv") ) for device in active_devices: test_run_dir = work_dir / f"{test_name}_{device.name}" util.mkdirs_p(test_run_dir) try: # Confusingly, some functions below will raise on an error; others will write e.g. CRASH to the # STATUS file in the output directory. In the latter case, we update |status|. We check |status| at # the end of this if-else chain and raise fake exceptions if appropriate. status = fuzz.STATUS_SUCCESS if device.HasField("preprocess"): # This just means spirv-op for now. assert spirv_opt_path # noqa for spirv_shader in spirv_shaders: spirv_opt_util.run_spirv_opt_on_spirv_shader( spirv_shader, test_run_dir, ["-O"], spirv_opt_path ) elif device.HasField("shader_compiler"): for spirv_shader in spirv_shaders: shader_compiler_util.run_shader( shader_compiler_device=device.shader_compiler, shader_path=spirv_shader, output_dir=test_run_dir, compiler_path=binaries.get_binary_path_by_name( device.shader_compiler.binary ).path, timeout=DEFAULT_TIMEOUT, ) elif device.HasField("swift_shader"): assert swift_shader_path # noqa assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, icd=swift_shader_path, ) status = result_util.get_status(test_run_dir) elif device.HasField("host"): assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, ) status = result_util.get_status(test_run_dir) elif device.HasField("android"): android_device.run_amber_on_device( test, test_run_dir, dump_image=False, dump_buffer=False, serial=device.android.serial, ) status = result_util.get_status(test_run_dir) else: raise AssertionError(f"Unsupported device {device.name}") if status in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH): raise CalledProcessError(1, "??") if status != fuzz.STATUS_SUCCESS: raise TimeoutExpired("??", fuzz.AMBER_RUN_TIME_LIMIT) write_entry("P") except CalledProcessError: write_entry("F") except TimeoutExpired: write_entry("T") write_newline()