def create_summary_and_reproduce( test_dir: Path, binary_manager: binaries_util.BinaryManager) -> None: util.mkdirs_p(test_dir / "summary") test_metadata = test_util.metadata_read(test_dir) # noinspection PyTypeChecker if test_metadata.HasField("glsl") or test_metadata.HasField("spirv_fuzz"): fuzz_glsl_test.create_summary_and_reproduce(test_dir, binary_manager) else: raise AssertionError("Unrecognized test type")
def maybe_add_report( # pylint: disable=too-many-locals; test_dir: Path, reports_dir: Path, device: Device, settings: Settings) -> Optional[Path]: result_output_dir = test_util.get_results_directory(test_dir, device.name) status = result_util.get_status(result_output_dir) report_subdirectory_name = "" if status == fuzz.STATUS_CRASH: report_subdirectory_name = "crashes" elif status == fuzz.STATUS_TOOL_CRASH: report_subdirectory_name = "tool_crashes" elif status == fuzz.STATUS_UNRESPONSIVE: report_subdirectory_name = "unresponsive" if not report_subdirectory_name: return None log_path = result_util.get_log_path(result_output_dir) log_contents = util.file_read_text(log_path) signature = signature_util.get_signature_from_log_contents(log_contents) signature_dir = reports_dir / report_subdirectory_name / signature util.mkdirs_p(signature_dir) # If the signature_dir contains a NOT_INTERESTING file, then don't bother creating a report. if (signature_dir / "NOT_INTERESTING").exists(): return None if signature != signature_util.BAD_IMAGE_SIGNATURE: # If we have reached the maximum number of crashes per signature for this device, don't create a report. num_duplicates = [ report_dir for report_dir in signature_dir.iterdir() if report_dir.is_dir() and report_dir.name.endswith(f"_{device.name}") ] if len(num_duplicates) >= settings.maximum_duplicate_crashes: return None # We include the device name in the directory name because it is possible that this test crashes on two # different devices but gives the same crash signature in both cases (e.g. for generic signatures # like "compile_error"). This would lead to two test copies having the same path. # It also means we can limit duplicates per device using the directory name. test_dir_in_reports = signature_dir / f"{test_dir.name}_{device.name}" util.copy_dir(test_dir, test_dir_in_reports) if signature != signature_util.BAD_IMAGE_SIGNATURE: # If we found a crash, rename the directories for all shaders other than the variant. Thus, only the variant # shader will run. bad_shader_name = result_util.get_status_bad_shader_name( test_util.get_results_directory(test_dir_in_reports, device.name)) # TODO: Could possibly improve this. Could try scanning the Amber log to figure out which shader failed? if not bad_shader_name: log("WARNING: assuming that the bad shader is the variant") bad_shader_name = test_util.VARIANT_DIR shader_jobs = tool.get_shader_jobs( test_util.get_source_dir(test_dir_in_reports)) found_bad_shader = False for shader_job in shader_jobs: if shader_job.name == bad_shader_name: found_bad_shader = True else: shader_job.shader_job.parent.rename( shader_job.shader_job.parent.parent / f"_{shader_job.name}") check( found_bad_shader, AssertionError( f"Could not find bad shader at: {test_util.get_source_dir(test_dir_in_reports) / bad_shader_name}" ), ) test_metadata = test_util.metadata_read(test_dir_in_reports) test_metadata.crash_signature = signature test_metadata.device.CopyFrom(device) test_metadata.expected_status = status test_util.metadata_write(test_metadata, test_dir_in_reports) return test_dir_in_reports
def fuzz_glsl( # pylint: disable=too-many-locals; staging_dir: Path, reports_dir: Path, fuzz_failures_dir: Path, active_devices: List[Device], references: List[Path], donors_dir: Path, settings: Settings, binary_manager: binaries_util.BinaryManager, ) -> None: staging_name = staging_dir.name template_source_dir = staging_dir / "source_template" # Pick a randomly chosen reference. unprepared_reference_shader_job: Path = random.choice(references) # The "graphicsfuzz-tool" tool is designed to be on your PATH so that e.g. ".bat" will be appended on Windows. # So we use tool_on_path with a custom PATH to get the actual file we want to execute. graphicsfuzz_tool_path = util.tool_on_path( "graphicsfuzz-tool", str( binary_manager.get_binary_path_by_name( "graphicsfuzz-tool").path.parent), ) try: with util.file_open_text(staging_dir / "log.txt", "w") as log_file: try: gflogging.push_stream_for_logging(log_file) # Create the prepared (for Vulkan GLSL) reference. glsl_generate_util.run_prepare_reference( graphicsfuzz_tool_path, unprepared_reference_shader_job, template_source_dir / test_util.REFERENCE_DIR / test_util.SHADER_JOB, legacy_graphics_fuzz_vulkan_arg=settings. legacy_graphics_fuzz_vulkan_arg, ) # Generate the variant (GraphicsFuzz requires the unprepared reference as input). glsl_generate_util.run_generate( graphicsfuzz_tool_path, unprepared_reference_shader_job, donors_dir, template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, seed=str( random.getrandbits( glsl_generate_util.GENERATE_SEED_BITS)), other_args=list(settings.extra_graphics_fuzz_generate_args) if settings.extra_graphics_fuzz_generate_args else None, legacy_graphics_fuzz_vulkan_arg=settings. legacy_graphics_fuzz_vulkan_arg, ) finally: gflogging.pop_stream_for_logging() except subprocess.CalledProcessError: util.mkdirs_p(fuzz_failures_dir) if len(list( fuzz_failures_dir.iterdir())) < settings.maximum_fuzz_failures: util.copy_dir(staging_dir, fuzz_failures_dir / staging_dir.name) return reference_name = unprepared_reference_shader_job.stem stable_shader = reference_name.startswith("stable_") common_spirv_args = list(settings.common_spirv_args) test_dirs = [ make_test( template_source_dir, staging_dir / f"{staging_name}_no_opt_test", spirv_opt_args=None, binary_manager=binary_manager, derived_from=reference_name, stable_shader=stable_shader, common_spirv_args=common_spirv_args, ), make_test( template_source_dir, staging_dir / f"{staging_name}_opt_O_test", spirv_opt_args=["-O"], binary_manager=binary_manager, derived_from=reference_name, stable_shader=stable_shader, common_spirv_args=common_spirv_args, ), ] if not settings.spirv_opt_just_o: test_dirs += [ make_test( template_source_dir, staging_dir / f"{staging_name}_opt_Os_test", spirv_opt_args=["-Os"], binary_manager=binary_manager, derived_from=reference_name, stable_shader=stable_shader, common_spirv_args=common_spirv_args, ), make_test( template_source_dir, staging_dir / f"{staging_name}_opt_rand1_test", spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), binary_manager=binary_manager, derived_from=reference_name, stable_shader=stable_shader, common_spirv_args=common_spirv_args, ), make_test( template_source_dir, staging_dir / f"{staging_name}_opt_rand2_test", spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), binary_manager=binary_manager, derived_from=reference_name, stable_shader=stable_shader, common_spirv_args=common_spirv_args, ), make_test( template_source_dir, staging_dir / f"{staging_name}_opt_rand3_test", spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), binary_manager=binary_manager, derived_from=reference_name, stable_shader=stable_shader, common_spirv_args=common_spirv_args, ), ] for test_dir in test_dirs: interrupt_util.interrupt_if_needed() if handle_test(test_dir, reports_dir, active_devices, binary_manager, settings): # If we generated a report, don't bother trying other optimization combinations. break
def fuzz_spirv( # pylint: disable=too-many-locals; staging_dir: Path, reports_dir: Path, fuzz_failures_dir: Path, active_devices: List[Device], spirv_fuzz_shaders: List[Path], settings: Settings, binary_manager: binaries_util.BinaryManager, ) -> None: staging_name = staging_dir.name template_source_dir = staging_dir / "source_template" reference_spirv_shader_job_orig_path: Path = random.choice( spirv_fuzz_shaders) # Copy in a randomly chosen reference. reference_spirv_shader_job = shader_job_util.copy( reference_spirv_shader_job_orig_path, template_source_dir / test_util.REFERENCE_DIR / test_util.SHADER_JOB, language_suffix=shader_job_util.SUFFIXES_SPIRV_FUZZ_INPUT, ) try: with util.file_open_text(staging_dir / "log.txt", "w") as log_file: try: gflogging.push_stream_for_logging(log_file) spirv_fuzz_util.run_generate_on_shader_job( binary_manager.get_binary_path_by_name("spirv-fuzz").path, reference_spirv_shader_job, template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, donor_shader_job_paths=spirv_fuzz_shaders, seed=str( random.getrandbits( spirv_fuzz_util.GENERATE_SEED_BITS)), other_args=list(settings.extra_spirv_fuzz_generate_args) + list(settings.common_spirv_args), ) finally: gflogging.pop_stream_for_logging() except subprocess.CalledProcessError: util.mkdirs_p(fuzz_failures_dir) if len(list( fuzz_failures_dir.iterdir())) < settings.maximum_fuzz_failures: util.copy_dir(staging_dir, fuzz_failures_dir / staging_dir.name) return reference_name = reference_spirv_shader_job_orig_path.stem stable_shader = reference_name.startswith("stable_") common_spirv_args = list(settings.common_spirv_args) test_dirs = [ make_test( template_source_dir, staging_dir / f"{staging_name}_no_opt_test", spirv_opt_args=None, binary_manager=binary_manager, derived_from=reference_name, stable_shader=stable_shader, common_spirv_args=common_spirv_args, ), make_test( template_source_dir, staging_dir / f"{staging_name}_opt_O_test", spirv_opt_args=["-O"], binary_manager=binary_manager, derived_from=reference_name, stable_shader=stable_shader, common_spirv_args=common_spirv_args, ), make_test( template_source_dir, staging_dir / f"{staging_name}_opt_Os_test", spirv_opt_args=["-Os"], binary_manager=binary_manager, derived_from=reference_name, stable_shader=stable_shader, common_spirv_args=common_spirv_args, ), make_test( template_source_dir, staging_dir / f"{staging_name}_opt_rand1_test", spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), binary_manager=binary_manager, derived_from=reference_name, stable_shader=stable_shader, common_spirv_args=common_spirv_args, ), make_test( template_source_dir, staging_dir / f"{staging_name}_opt_rand2_test", spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), binary_manager=binary_manager, derived_from=reference_name, stable_shader=stable_shader, common_spirv_args=common_spirv_args, ), make_test( template_source_dir, staging_dir / f"{staging_name}_opt_rand3_test", spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), binary_manager=binary_manager, derived_from=reference_name, stable_shader=stable_shader, common_spirv_args=common_spirv_args, ), ] for test_dir in test_dirs: interrupt_util.interrupt_if_needed() if handle_test(test_dir, reports_dir, active_devices, binary_manager, settings): # If we generated a report, don't bother trying other optimization combinations. break
def run_spirv_reduce_or_shrink( # pylint: disable=too-many-locals; source_dir: Path, name_of_shader_job_to_reduce: str, extension_to_reduce: str, output_dir: Path, preserve_semantics: bool, binary_manager: binaries_util.BinaryManager, settings: Settings, ) -> Path: test = test_util.metadata_read_from_source_dir(source_dir) input_shader_job = source_dir / name_of_shader_job_to_reduce / test_util.SHADER_JOB original_spirv_file = input_shader_job.with_suffix( extension_to_reduce + shader_job_util.SUFFIX_SPIRV_ORIG) transformed_spirv_file = input_shader_job.with_suffix( extension_to_reduce + shader_job_util.SUFFIX_SPIRV) transformations_file = input_shader_job.with_suffix( extension_to_reduce + shader_job_util.SUFFIX_TRANSFORMATIONS) util.mkdirs_p(output_dir) final_shader = output_dir / "final.spv" # E.g. transformation_suffix_to_reduce == ".frag.transformations" # E.g. ".frag.??" -> ".frag.spv" shader_suffix_to_override = extension_to_reduce + shader_job_util.SUFFIX_SPIRV if preserve_semantics: cmd = [ str(binary_manager.get_binary_path_by_name("spirv-fuzz").path), str(original_spirv_file), "-o", str(final_shader), f"--shrink={str(transformations_file)}", f"--shrinker-temp-file-prefix={str(output_dir / 'temp_')}", ] cmd += list(settings.extra_spirv_fuzz_shrink_args) cmd += list(test.common_spirv_args) cmd += [ # This ensures the arguments that follow are all positional arguments. "--", "gfauto_interestingness_test", str(source_dir), # --override_shader requires three parameters to follow; the third will be added by spirv-fuzz (the shader.spv file). "--override_shader", name_of_shader_job_to_reduce, shader_suffix_to_override, ] else: cmd = [ str(binary_manager.get_binary_path_by_name("spirv-reduce").path), str(transformed_spirv_file), "-o", str(final_shader), f"--temp-file-prefix={str(output_dir / 'temp_')}", ] cmd += list(settings.extra_spirv_reduce_args) cmd += list(test.common_spirv_args) cmd += [ # This ensures the arguments that follow are all positional arguments. "--", "gfauto_interestingness_test", str(source_dir), # --override_shader requires three parameters to follow; the third will be added by spirv-reduce (the shader.spv file). "--override_shader", name_of_shader_job_to_reduce, shader_suffix_to_override, ] # Log the reduction. with util.file_open_text(output_dir / "command.log", "w") as f: gflogging.push_stream_for_logging(f) try: # The reducer can fail, but it will typically output an exception file, so we can ignore the exit code. subprocess_util.run(cmd, verbose=True, check_exit_code=False) finally: gflogging.pop_stream_for_logging() return final_shader
def main_helper( # pylint: disable=too-many-locals,too-many-branches,too-many-statements; tests_dir: Path, work_dir: Path, binaries: binaries_util.BinaryManager, settings: Settings, active_devices: List[Device], results_out_handle: Optional[TextIO], updated_settings_output_path: Optional[Path], ) -> None: util.mkdirs_p(work_dir) def write_entry(entry: str) -> None: if not results_out_handle: return results_out_handle.write(entry) results_out_handle.write(", ") results_out_handle.flush() def write_newline() -> None: if not results_out_handle: return results_out_handle.write("\n") results_out_handle.flush() spirv_opt_path: Optional[Path] = None swift_shader_path: Optional[Path] = None amber_path: Optional[Path] = None # Small hack to ensure we have three devices for spirv-opt, each with a different name. main_spirv_opt_device: Optional[Device] = None if active_devices and active_devices[0].name == "host_preprocessor": main_spirv_opt_device = active_devices[0] main_spirv_opt_device.name = SPIRV_OPT_O spirv_opt_custom = Device() spirv_opt_custom.CopyFrom(main_spirv_opt_device) spirv_opt_custom.name = SPIRV_OPT_CUSTOM active_devices.insert(1, spirv_opt_custom) spirv_opt_os = Device() spirv_opt_os.CopyFrom(main_spirv_opt_device) spirv_opt_os.name = SPIRV_OPT_OS active_devices.insert(1, spirv_opt_os) # Enumerate active devices, writing their name and storing binary paths if needed. write_entry("test") for device in active_devices: write_entry(device.name) if device.HasField("preprocess"): spirv_opt_path = binaries.get_binary_path_by_name( binaries_util.SPIRV_OPT_NAME).path if device.HasField("swift_shader"): swift_shader_path = binaries.get_binary_path_by_name( binaries_util.SWIFT_SHADER_NAME).path if device.HasField("swift_shader") or device.HasField("host"): amber_path = binaries.get_binary_path_by_name( binaries_util.AMBER_NAME).path write_newline() # Enumerate tests and devices, writing the results. for test in sorted(tests_dir.glob("*.amber")): test_name = util.remove_end(test.name, ".amber") write_entry(test_name) spirv_shaders = sorted( tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv")) for device in active_devices: test_run_dir = work_dir / f"{test_name}_{device.name}" util.mkdirs_p(test_run_dir) ignored_signatures_set: Set[str] = set( device.ignored_crash_signatures) with util.file_open_text(test_run_dir / "log.txt", "w") as log_stream: try: gflogging.push_stream_for_logging(log_stream) if device.HasField("preprocess"): # This just means spirv-opt for now. assert spirv_opt_path # noqa assert main_spirv_opt_device # noqa # Pick spirv-opt arguments based on device name. if device.name == SPIRV_OPT_O: spirv_opt_args = ["-O"] elif device.name == SPIRV_OPT_OS: spirv_opt_args = ["-Os"] elif device.name == SPIRV_OPT_CUSTOM: spirv_opt_args = (spirv_opt_util. OPT_INTERESTING_SUBSET_OF_PASSES) else: raise AssertionError( f"Can't tell how to run device {device.name}; " f"must be named host_preprocessor and be the first active device." ) # Reset device and ignored_crash_signatures. device = main_spirv_opt_device ignored_signatures_set = set( device.ignored_crash_signatures) try: for spirv_shader in spirv_shaders: spirv_opt_util.run_spirv_opt_on_spirv_shader( spirv_shader, test_run_dir, spirv_opt_args, spirv_opt_path, ) result_util.write_status( test_run_dir, fuzz.STATUS_SUCCESS, ) except subprocess.CalledProcessError: result_util.write_status( test_run_dir, fuzz.STATUS_TOOL_CRASH, ) except subprocess.TimeoutExpired: result_util.write_status( test_run_dir, fuzz.STATUS_TOOL_TIMEOUT, ) elif device.HasField("shader_compiler"): try: for spirv_shader in spirv_shaders: shader_compiler_util.run_shader( shader_compiler_device=device. shader_compiler, shader_path=spirv_shader, output_dir=test_run_dir, compiler_path=binaries. get_binary_path_by_name( device.shader_compiler.binary).path, timeout=DEFAULT_TIMEOUT, ) result_util.write_status( test_run_dir, fuzz.STATUS_SUCCESS, ) except subprocess.CalledProcessError: result_util.write_status( test_run_dir, fuzz.STATUS_CRASH, ) except subprocess.TimeoutExpired: result_util.write_status( test_run_dir, fuzz.STATUS_TIMEOUT, ) elif device.HasField("swift_shader"): assert swift_shader_path # noqa assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, icd=swift_shader_path, ) elif device.HasField("host"): assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, custom_launcher=list(device.host.custom_launcher), ) elif device.HasField("android"): android_device.run_amber_on_device( test, test_run_dir, dump_image=False, dump_buffer=False, serial=device.android.serial, ) else: raise AssertionError( f"Unsupported device {device.name}") finally: gflogging.pop_stream_for_logging() status = result_util.get_status(test_run_dir) if status == fuzz.STATUS_SUCCESS: write_entry("P") elif status in (fuzz.STATUS_TIMEOUT, fuzz.STATUS_TOOL_TIMEOUT): write_entry("T") else: write_entry("F") # Update ignored signatures. if (status in ( fuzz.STATUS_TOOL_CRASH, fuzz.STATUS_CRASH, fuzz.STATUS_UNRESPONSIVE, ) and updated_settings_output_path): log_contents = util.file_read_text( result_util.get_log_path(test_run_dir)) signature = signature_util.get_signature_from_log_contents( log_contents) if signature == signature_util.NO_SIGNATURE: log(f"NOT updating ignored signatures to include {signature}" ) elif signature in ignored_signatures_set: log(f"Signature is already ignored: {signature}") else: log(f"Adding ignored signature: {signature}") device.ignored_crash_signatures.append(signature) write_newline() if updated_settings_output_path: # Reset main_spirv_opt_device name before writing it back out. if main_spirv_opt_device: main_spirv_opt_device.name = "host_preprocessor" settings_util.write(settings, updated_settings_output_path)
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements; parser = argparse.ArgumentParser( description="Runs GraphicsFuzz AmberScript tests on the active devices listed in " "the settings.json file.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( "--settings", help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) parser.add_argument( "--tests", help="Path to the directory of AmberScript tests with shaders extracted.", default="graphicsfuzz", ) parser.add_argument( "--update_ignored_signatures", help="As the tests are run for each device, add any crash signatures to the device's ignored_crash_signatures " "property and write out the updated settings.json file.", action="store_true", ) parser.add_argument( "--results_out", help="Output file path for the CSV results table.", default="results.csv", ) parsed_args = parser.parse_args(sys.argv[1:]) # Args. tests_dir: Path = Path(parsed_args.tests) settings_path: Path = Path(parsed_args.settings) update_ignored_signatures: bool = parsed_args.update_ignored_signatures results_out_path: Path = Path(parsed_args.results_out) # Settings and devices. settings = settings_util.read_or_create(settings_path) active_devices = devices_util.get_active_devices(settings.device_list) # Binaries. binaries = binaries_util.get_default_binary_manager(settings=settings) work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}" util.mkdirs_p(work_dir) with util.file_open_text(results_out_path, "w") as results_handle: def write_entry(entry: str) -> None: results_handle.write(entry) results_handle.write(", ") results_handle.flush() def write_newline() -> None: results_handle.write("\n") results_handle.flush() spirv_opt_path: Optional[Path] = None swift_shader_path: Optional[Path] = None amber_path: Optional[Path] = None # Small hack to ensure we have three devices for spirv-opt, each with a different name. main_spirv_opt_device: Optional[Device] = None if active_devices and active_devices[0].name == "host_preprocessor": main_spirv_opt_device = active_devices[0] main_spirv_opt_device.name = SPIRV_OPT_O spirv_opt_custom = Device() spirv_opt_custom.CopyFrom(main_spirv_opt_device) spirv_opt_custom.name = SPIRV_OPT_CUSTOM active_devices.insert(1, spirv_opt_custom) spirv_opt_os = Device() spirv_opt_os.CopyFrom(main_spirv_opt_device) spirv_opt_os.name = SPIRV_OPT_OS active_devices.insert(1, spirv_opt_os) # Enumerate active devices, writing their name and storing binary paths if needed. write_entry("test") for device in active_devices: write_entry(device.name) if device.HasField("preprocess"): spirv_opt_path = binaries.get_binary_path_by_name( binaries_util.SPIRV_OPT_NAME ).path if device.HasField("swift_shader"): swift_shader_path = binaries.get_binary_path_by_name( binaries_util.SWIFT_SHADER_NAME ).path if device.HasField("swift_shader") or device.HasField("host"): amber_path = binaries.get_binary_path_by_name( binaries_util.AMBER_NAME ).path write_newline() # Enumerate tests and devices, writing the results. for test in sorted(tests_dir.glob("*.amber")): test_name = util.remove_end(test.name, ".amber") write_entry(test_name) spirv_shaders = sorted( tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv") ) for device in active_devices: test_run_dir = work_dir / f"{test_name}_{device.name}" util.mkdirs_p(test_run_dir) ignored_signatures_set: Set[str] = set(device.ignored_crash_signatures) with util.file_open_text(test_run_dir / "log.txt", "w") as log_stream: try: gflogging.push_stream_for_logging(log_stream) if device.HasField("preprocess"): # This just means spirv-opt for now. assert spirv_opt_path # noqa assert main_spirv_opt_device # noqa # Pick spirv-opt arguments based on device name. if device.name == SPIRV_OPT_O: spirv_opt_args = ["-O"] elif device.name == SPIRV_OPT_OS: spirv_opt_args = ["-Os"] elif device.name == SPIRV_OPT_CUSTOM: spirv_opt_args = ( spirv_opt_util.OPT_INTERESTING_SUBSET_OF_PASSES ) else: raise AssertionError( f"Can't tell how to run device {device.name}; " f"must be named host_preprocessor and be the first active device." ) # Reset device and ignored_crash_signatures. device = main_spirv_opt_device ignored_signatures_set = set( device.ignored_crash_signatures ) try: for spirv_shader in spirv_shaders: spirv_opt_util.run_spirv_opt_on_spirv_shader( spirv_shader, test_run_dir, spirv_opt_args, spirv_opt_path, ) result_util.write_status( test_run_dir, fuzz.STATUS_SUCCESS, ) except subprocess.CalledProcessError: result_util.write_status( test_run_dir, fuzz.STATUS_TOOL_CRASH, ) except subprocess.TimeoutExpired: result_util.write_status( test_run_dir, fuzz.STATUS_TOOL_TIMEOUT, ) elif device.HasField("shader_compiler"): try: for spirv_shader in spirv_shaders: shader_compiler_util.run_shader( shader_compiler_device=device.shader_compiler, shader_path=spirv_shader, output_dir=test_run_dir, compiler_path=binaries.get_binary_path_by_name( device.shader_compiler.binary ).path, timeout=DEFAULT_TIMEOUT, ) result_util.write_status( test_run_dir, fuzz.STATUS_SUCCESS, ) except subprocess.CalledProcessError: result_util.write_status( test_run_dir, fuzz.STATUS_CRASH, ) except subprocess.TimeoutExpired: result_util.write_status( test_run_dir, fuzz.STATUS_TIMEOUT, ) elif device.HasField("swift_shader"): assert swift_shader_path # noqa assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, icd=swift_shader_path, ) elif device.HasField("host"): assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, custom_launcher=list(device.host.custom_launcher), ) elif device.HasField("android"): android_device.run_amber_on_device( test, test_run_dir, dump_image=False, dump_buffer=False, serial=device.android.serial, ) else: raise AssertionError(f"Unsupported device {device.name}") finally: gflogging.pop_stream_for_logging() status = result_util.get_status(test_run_dir) if status == fuzz.STATUS_SUCCESS: write_entry("P") elif status in (fuzz.STATUS_TIMEOUT, fuzz.STATUS_TOOL_TIMEOUT): write_entry("T") else: write_entry("F") # Update ignored signatures. if ( status in ( fuzz.STATUS_TOOL_CRASH, fuzz.STATUS_CRASH, fuzz.STATUS_UNRESPONSIVE, ) and update_ignored_signatures ): log_contents = util.file_read_text( result_util.get_log_path(test_run_dir) ) signature = signature_util.get_signature_from_log_contents( log_contents ) if signature == signature_util.NO_SIGNATURE: log(f"NOT updating ignored signatures to include {signature}") elif signature in ignored_signatures_set: log(f"Signature is already ignored: {signature}") else: log(f"Adding ignored signature: {signature}") device.ignored_crash_signatures.append(signature) write_newline() if update_ignored_signatures: # Reset main_spirv_opt_device name before writing it back out. if main_spirv_opt_device: main_spirv_opt_device.name = "host_preprocessor" settings_util.write(settings, settings_path)
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements; parser = argparse.ArgumentParser( description="Runs GraphicsFuzz AmberScript tests on the active devices listed in " "the settings.json file." ) parser.add_argument( "--settings", help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) parser.add_argument( "--tests", help="Path to the directory of AmberScript tests with shaders extracted.", default=str("graphicsfuzz"), ) parsed_args = parser.parse_args(sys.argv[1:]) # Args. tests_dir: Path = Path(parsed_args.tests) settings_path: Path = Path(parsed_args.settings) # Settings and devices. settings = settings_util.read_or_create(settings_path) active_devices = devices_util.get_active_devices(settings.device_list) # Binaries. binaries = binaries_util.get_default_binary_manager(settings=settings) work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}" util.mkdirs_p(work_dir) with util.file_open_text(Path("results.txt"), "w") as log_handle: def write_entry(entry: str) -> None: log_handle.write(entry) log_handle.write(", ") log_handle.flush() def write_newline() -> None: log_handle.write("\n") log_handle.flush() spirv_opt_path: Optional[Path] = None swift_shader_path: Optional[Path] = None amber_path: Optional[Path] = None # Enumerate active devices, writing their name and storing binary paths if needed. write_entry("test") for device in active_devices: if device.name == "host_preprocessor": # We are actually just running spirv-opt on the SPIR-V shaders. write_entry("spirv-opt") else: write_entry(device.name) if device.HasField("preprocess"): spirv_opt_path = binaries.get_binary_path_by_name( binaries_util.SPIRV_OPT_NAME ).path if device.HasField("swift_shader"): swift_shader_path = binaries.get_binary_path_by_name( binaries_util.SWIFT_SHADER_NAME ).path if device.HasField("swift_shader") or device.HasField("host"): amber_path = binaries.get_binary_path_by_name( binaries_util.AMBER_NAME ).path write_newline() # Enumerate tests and devices, writing the results. for test in sorted(tests_dir.glob("*.amber")): test_name = util.remove_end(test.name, ".amber") write_entry(test_name) spirv_shaders = sorted( tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv") ) for device in active_devices: test_run_dir = work_dir / f"{test_name}_{device.name}" util.mkdirs_p(test_run_dir) try: # Confusingly, some functions below will raise on an error; others will write e.g. CRASH to the # STATUS file in the output directory. In the latter case, we update |status|. We check |status| at # the end of this if-else chain and raise fake exceptions if appropriate. status = fuzz.STATUS_SUCCESS if device.HasField("preprocess"): # This just means spirv-op for now. assert spirv_opt_path # noqa for spirv_shader in spirv_shaders: spirv_opt_util.run_spirv_opt_on_spirv_shader( spirv_shader, test_run_dir, ["-O"], spirv_opt_path ) elif device.HasField("shader_compiler"): for spirv_shader in spirv_shaders: shader_compiler_util.run_shader( shader_compiler_device=device.shader_compiler, shader_path=spirv_shader, output_dir=test_run_dir, compiler_path=binaries.get_binary_path_by_name( device.shader_compiler.binary ).path, timeout=DEFAULT_TIMEOUT, ) elif device.HasField("swift_shader"): assert swift_shader_path # noqa assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, icd=swift_shader_path, ) status = result_util.get_status(test_run_dir) elif device.HasField("host"): assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, ) status = result_util.get_status(test_run_dir) elif device.HasField("android"): android_device.run_amber_on_device( test, test_run_dir, dump_image=False, dump_buffer=False, serial=device.android.serial, ) status = result_util.get_status(test_run_dir) else: raise AssertionError(f"Unsupported device {device.name}") if status in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH): raise CalledProcessError(1, "??") if status != fuzz.STATUS_SUCCESS: raise TimeoutExpired("??", fuzz.AMBER_RUN_TIME_LIMIT) write_entry("P") except CalledProcessError: write_entry("F") except TimeoutExpired: write_entry("T") write_newline()