def write_default(settings_path: Path) -> Path: settings = Settings() settings.CopyFrom(DEFAULT_SETTINGS) # noinspection PyBroadException try: settings.latest_binary_versions.extend( binaries_util.download_latest_binary_version_numbers()) except Exception: # pylint: disable=broad-except; message = "WARNING: Could not download the latest binary version numbers. We will just use the (older) hardcoded versions." details = traceback.format_exc( ) # noqa: SC100, SC200 (spelling of exc) log(message) log(f"\nDetails:\n{details}\n\n") log(message) settings.latest_binary_versions.extend(binaries_util.DEFAULT_BINARIES) binary_manager = binaries_util.get_default_binary_manager( settings=settings) devices_util.get_device_list(binary_manager, settings.device_list) return write(settings, settings_path)
def main() -> None: parser = argparse.ArgumentParser( description= "Downloads the latest GraphicsFuzz AmberScript tests from vk-gl-cts, " "including those in pending CLs. " "Requires Git. Requires Khronos membership.") parser.add_argument( "gerrit_cookie", help= "The Gerrit cookie used for authentication. Requires Khronos membership. Obtain this as follows. " + GERRIT_COOKIE_INSTRUCTIONS, ) parser.add_argument( "--settings", help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) parsed_args = parser.parse_args(sys.argv[1:]) cookie: str = parsed_args.gerrit_cookie settings_path: Path = Path(parsed_args.settings) # Need git. git_tool = util.tool_on_path("git") settings = settings_util.read_or_create(settings_path) binaries = binaries_util.get_default_binary_manager(settings=settings) tests_dir = Path() / "graphicsfuzz" download_cts_graphicsfuzz_tests(git_tool, cookie, tests_dir) extract_shaders(tests_dir, binaries)
def main() -> None: parser = argparse.ArgumentParser( description= "Downloads the latest GraphicsFuzz AmberScript tests from vk-gl-cts, " "including those in pending CLs. " "Requires Git.") parser.add_argument("gerrit_cookie", help=GERRIT_COOKIE_ARGUMENT_DESCRIPTION) parser.add_argument( "--settings", help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) parsed_args = parser.parse_args(sys.argv[1:]) cookie: str = parsed_args.gerrit_cookie settings_path: Path = Path(parsed_args.settings) # Need git. git_tool = util.tool_on_path("git") settings = settings_util.read_or_create(settings_path) binaries = binaries_util.get_default_binary_manager(settings=settings) download_cts_graphicsfuzz_tests(git_tool, cookie, binaries)
def main() -> None: parser = argparse.ArgumentParser( description="A tool for running a reduction on a source directory." ) parser.add_argument( "source_dir", help="The source directory containing the shaders and the test.json file that describes how to run the test.", ) parser.add_argument( "--output", help="Output directory.", default="reduction_output", ) parser.add_argument( "--settings", help="Path to a settings JSON file for this instance.", default="settings.json", ) parser.add_argument( "--literals_to_uniforms", action="store_true", help="Pass --literals-to-uniforms to glsl-reduce.", ) parsed_args = parser.parse_args(sys.argv[1:]) source_dir = Path(parsed_args.source_dir) output_dir = Path(parsed_args.output) settings = settings_util.read_or_create(Path(parsed_args.settings)) literals_to_uniforms: bool = parsed_args.literals_to_uniforms binary_manager = binaries_util.get_default_binary_manager(settings=settings) test = test_util.metadata_read_from_source_dir(source_dir) if test.HasField("glsl"): if ( literals_to_uniforms and "--literals-to-uniforms" not in settings.extra_graphics_fuzz_reduce_args ): settings.extra_graphics_fuzz_reduce_args.append("--literals-to-uniforms") fuzz_glsl_test.run_reduction( source_dir_to_reduce=source_dir, reduction_output_dir=output_dir, binary_manager=binary_manager, settings=settings, ) elif test.HasField("spirv_fuzz"): fuzz_spirv_test.run_reduction( source_dir_to_reduce=source_dir, reduction_output_dir=output_dir, binary_manager=binary_manager, settings=settings, ) else: raise AssertionError(f"Unknown test type: {test}")
def main() -> None: # pylint: disable=too-many-statements, too-many-locals, too-many-branches; parser = argparse.ArgumentParser( description="Runs AmberScript files on Android devices." ) parser.add_argument( "amber_script_file", help="AmberScript tests to run.", nargs="+", ) parser.add_argument( "--output", help="Output directory.", default="output", ) parser.add_argument( "--settings", help="Path to a settings JSON file for this instance. " "Unlike with gfauto_fuzz, the default value is an empty string, which is ignored. ", default="", ) parser.add_argument( "--serial", help="Android device serial. If left unspecified, the tests will be run on all Android devices.", action="append", ) parsed_args = parser.parse_args(sys.argv[1:]) amber_script_files: List[Path] = [Path(a) for a in parsed_args.amber_script_file] output_path: Path = Path(parsed_args.output) serials: Optional[List[str]] = parsed_args.serial settings_str: str = parsed_args.settings settings = Settings() if settings_str: settings = settings_util.read_or_create(Path(settings_str)) binary_manager = binaries_util.get_default_binary_manager(settings) if not serials: android_devices = android_device.get_all_android_devices( binary_manager, include_device_details=False ) serials = [] for device in android_devices: serials.append(device.android.serial) for amber_script_file in amber_script_files: for serial in serials: android_device.run_amber_on_device( amber_script_file, output_path / serial, dump_image=False, dump_buffer=False, serial=serial, )
def main() -> None: parser = argparse.ArgumentParser( description= "A script that creates a README and bug_report directory for a test and its result_dir." ) parser.add_argument( "source_dir", help="Source directory containing test.json and shaders.") parser.add_argument( "result_dir", help= "Path to the result_dir of a test containing e.g. the intermediate shader files, log.txt, etc.", ) parser.add_argument( "--output_dir", help= "Output directory where the README and bug_report directory will be written.", default=".", ) parsed_args = parser.parse_args(sys.argv[1:]) source_dir = Path(parsed_args.source_dir) result_dir = Path(parsed_args.result_dir) output_dir = Path(parsed_args.output_dir) check_dir_exists(source_dir) check_dir_exists(result_dir) test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) binary_manager = binaries_util.get_default_binary_manager( settings=Settings()).get_child_binary_manager( binary_list=list(test.binaries) + list(test.device.binaries)) check(test.HasField("glsl"), AssertionError("Only glsl tests currently supported")) check( test.device.HasField("preprocess"), AssertionError("Only preprocess device tests currently supported"), ) fuzz_test_util.tool_crash_summary_bug_report_dir(source_dir, result_dir, output_dir, binary_manager)
def main() -> None: parser = argparse.ArgumentParser( description="Updates or creates a settings.json file. " "If the settings file does not exist, a default settings file will be created and the tool will exit. " "If the settings file exists, it is read, updated, and written; command line arguments define the update steps. " ) parser.add_argument( "--settings", help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) parser.add_argument( "--update_binary_versions", help="Update binary versions to the latest available (online).", action="store_true", ) parser.add_argument( "--update_devices", help= 'Update each device in the settings.json file. This mostly just means updating the device_properties field obtained by running "amber -d -V". ' "SwiftShader devices will be updated to use the latest version of SwiftShader and Android devices will update their build_fingerprint field. " "It is recommended that you also pass --update_binary_versions so that the binary versions are updated first. ", action="store_true", ) parsed_args = parser.parse_args(sys.argv[1:]) # Args. settings_path: Path = Path(parsed_args.settings) update_binary_versions: bool = parsed_args.update_binary_versions update_devices: bool = parsed_args.update_devices settings = settings_util.read_or_create(settings_path) if update_binary_versions: del settings.latest_binary_versions[:] settings.latest_binary_versions.extend( binaries_util.download_latest_binary_version_numbers()) if update_devices: binary_manager = binaries_util.get_default_binary_manager(settings) for device in settings.device_list.devices: devices_util.update_device(binary_manager, device) settings_util.write(settings, settings_path)
def main() -> int: parser = argparse.ArgumentParser( description= "Runs a binary given the binary name and settings.json file. " "Use -- to separate args to run_bin and your binary. ") parser.add_argument( "--settings", help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) parser.add_argument( "binary_name", help="The name of the binary to run. E.g. spirv-opt, glslangValidator", type=str, ) parser.add_argument( "arguments", metavar="arguments", type=str, nargs="*", help="The arguments to pass to the binary", ) parsed_args = parser.parse_args(sys.argv[1:]) # Args. settings_path: Path = Path(parsed_args.settings) binary_name: str = parsed_args.binary_name arguments: List[str] = parsed_args.arguments try: settings = settings_util.read_or_create(settings_path) except settings_util.NoSettingsFile: log(f"Settings file {str(settings_path)} was created for you; using this." ) settings = settings_util.read_or_create(settings_path) binary_manager = binaries_util.get_default_binary_manager( settings=settings) cmd = [str(binary_manager.get_binary_path_by_name(binary_name).path)] cmd.extend(arguments) return subprocess.run(cmd, check=False).returncode
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements; parser = argparse.ArgumentParser( description="Runs GraphicsFuzz AmberScript tests on the active devices listed in " "the settings.json file.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( "--settings", help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) parser.add_argument( "--tests", help="Path to the directory of AmberScript tests with shaders extracted.", default="graphicsfuzz", ) parser.add_argument( "--update_ignored_signatures", help="As the tests are run for each device, add any crash signatures to the device's ignored_crash_signatures " "property and write out the updated settings.json file.", action="store_true", ) parser.add_argument( "--results_out", help="Output file path for the CSV results table.", default="results.csv", ) parsed_args = parser.parse_args(sys.argv[1:]) # Args. tests_dir: Path = Path(parsed_args.tests) settings_path: Path = Path(parsed_args.settings) update_ignored_signatures: bool = parsed_args.update_ignored_signatures results_out_path: Path = Path(parsed_args.results_out) # Settings and devices. settings = settings_util.read_or_create(settings_path) active_devices = devices_util.get_active_devices(settings.device_list) # Binaries. binaries = binaries_util.get_default_binary_manager(settings=settings) work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}" util.mkdirs_p(work_dir) with util.file_open_text(results_out_path, "w") as results_handle: def write_entry(entry: str) -> None: results_handle.write(entry) results_handle.write(", ") results_handle.flush() def write_newline() -> None: results_handle.write("\n") results_handle.flush() spirv_opt_path: Optional[Path] = None swift_shader_path: Optional[Path] = None amber_path: Optional[Path] = None # Small hack to ensure we have three devices for spirv-opt, each with a different name. main_spirv_opt_device: Optional[Device] = None if active_devices and active_devices[0].name == "host_preprocessor": main_spirv_opt_device = active_devices[0] main_spirv_opt_device.name = SPIRV_OPT_O spirv_opt_custom = Device() spirv_opt_custom.CopyFrom(main_spirv_opt_device) spirv_opt_custom.name = SPIRV_OPT_CUSTOM active_devices.insert(1, spirv_opt_custom) spirv_opt_os = Device() spirv_opt_os.CopyFrom(main_spirv_opt_device) spirv_opt_os.name = SPIRV_OPT_OS active_devices.insert(1, spirv_opt_os) # Enumerate active devices, writing their name and storing binary paths if needed. write_entry("test") for device in active_devices: write_entry(device.name) if device.HasField("preprocess"): spirv_opt_path = binaries.get_binary_path_by_name( binaries_util.SPIRV_OPT_NAME ).path if device.HasField("swift_shader"): swift_shader_path = binaries.get_binary_path_by_name( binaries_util.SWIFT_SHADER_NAME ).path if device.HasField("swift_shader") or device.HasField("host"): amber_path = binaries.get_binary_path_by_name( binaries_util.AMBER_NAME ).path write_newline() # Enumerate tests and devices, writing the results. for test in sorted(tests_dir.glob("*.amber")): test_name = util.remove_end(test.name, ".amber") write_entry(test_name) spirv_shaders = sorted( tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv") ) for device in active_devices: test_run_dir = work_dir / f"{test_name}_{device.name}" util.mkdirs_p(test_run_dir) ignored_signatures_set: Set[str] = set(device.ignored_crash_signatures) with util.file_open_text(test_run_dir / "log.txt", "w") as log_stream: try: gflogging.push_stream_for_logging(log_stream) if device.HasField("preprocess"): # This just means spirv-opt for now. assert spirv_opt_path # noqa assert main_spirv_opt_device # noqa # Pick spirv-opt arguments based on device name. if device.name == SPIRV_OPT_O: spirv_opt_args = ["-O"] elif device.name == SPIRV_OPT_OS: spirv_opt_args = ["-Os"] elif device.name == SPIRV_OPT_CUSTOM: spirv_opt_args = ( spirv_opt_util.OPT_INTERESTING_SUBSET_OF_PASSES ) else: raise AssertionError( f"Can't tell how to run device {device.name}; " f"must be named host_preprocessor and be the first active device." ) # Reset device and ignored_crash_signatures. device = main_spirv_opt_device ignored_signatures_set = set( device.ignored_crash_signatures ) try: for spirv_shader in spirv_shaders: spirv_opt_util.run_spirv_opt_on_spirv_shader( spirv_shader, test_run_dir, spirv_opt_args, spirv_opt_path, ) result_util.write_status( test_run_dir, fuzz.STATUS_SUCCESS, ) except subprocess.CalledProcessError: result_util.write_status( test_run_dir, fuzz.STATUS_TOOL_CRASH, ) except subprocess.TimeoutExpired: result_util.write_status( test_run_dir, fuzz.STATUS_TOOL_TIMEOUT, ) elif device.HasField("shader_compiler"): try: for spirv_shader in spirv_shaders: shader_compiler_util.run_shader( shader_compiler_device=device.shader_compiler, shader_path=spirv_shader, output_dir=test_run_dir, compiler_path=binaries.get_binary_path_by_name( device.shader_compiler.binary ).path, timeout=DEFAULT_TIMEOUT, ) result_util.write_status( test_run_dir, fuzz.STATUS_SUCCESS, ) except subprocess.CalledProcessError: result_util.write_status( test_run_dir, fuzz.STATUS_CRASH, ) except subprocess.TimeoutExpired: result_util.write_status( test_run_dir, fuzz.STATUS_TIMEOUT, ) elif device.HasField("swift_shader"): assert swift_shader_path # noqa assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, icd=swift_shader_path, ) elif device.HasField("host"): assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, custom_launcher=list(device.host.custom_launcher), ) elif device.HasField("android"): android_device.run_amber_on_device( test, test_run_dir, dump_image=False, dump_buffer=False, serial=device.android.serial, ) else: raise AssertionError(f"Unsupported device {device.name}") finally: gflogging.pop_stream_for_logging() status = result_util.get_status(test_run_dir) if status == fuzz.STATUS_SUCCESS: write_entry("P") elif status in (fuzz.STATUS_TIMEOUT, fuzz.STATUS_TOOL_TIMEOUT): write_entry("T") else: write_entry("F") # Update ignored signatures. if ( status in ( fuzz.STATUS_TOOL_CRASH, fuzz.STATUS_CRASH, fuzz.STATUS_UNRESPONSIVE, ) and update_ignored_signatures ): log_contents = util.file_read_text( result_util.get_log_path(test_run_dir) ) signature = signature_util.get_signature_from_log_contents( log_contents ) if signature == signature_util.NO_SIGNATURE: log(f"NOT updating ignored signatures to include {signature}") elif signature in ignored_signatures_set: log(f"Signature is already ignored: {signature}") else: log(f"Adding ignored signature: {signature}") device.ignored_crash_signatures.append(signature) write_newline() if update_ignored_signatures: # Reset main_spirv_opt_device name before writing it back out. if main_spirv_opt_device: main_spirv_opt_device.name = "host_preprocessor" settings_util.write(settings, settings_path)
def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many-statements; settings_path: Path, iteration_seed_override: Optional[int] = None, fuzzing_tool_pattern: Optional[List[FuzzingTool]] = None, allow_no_stack_traces: bool = False, override_sigint: bool = True, use_amber_vulkan_loader: bool = False, active_device_names: Optional[List[str]] = None, update_ignored_crash_signatures_gerrit_cookie: Optional[str] = None, ) -> None: if not fuzzing_tool_pattern: fuzzing_tool_pattern = [FuzzingTool.GLSL_FUZZ] util.update_gcov_environment_variable_if_needed() if override_sigint: interrupt_util.override_sigint() try_get_root_file() settings = settings_util.read_or_create(settings_path) binary_manager = binaries_util.get_default_binary_manager( settings=settings) temp_dir = Path() / "temp" # Note: we use "is not None" so that if the user passes an empty Gerrit cookie, we still try to execute this code. if update_ignored_crash_signatures_gerrit_cookie is not None: git_tool = util.tool_on_path("git") downloaded_graphicsfuzz_tests_dir = ( temp_dir / f"graphicsfuzz_cts_tests_{get_random_name()[:8]}") work_dir = temp_dir / f"graphicsfuzz_cts_run_{get_random_name()[:8]}" download_cts_gf_tests.download_cts_graphicsfuzz_tests( git_tool=git_tool, cookie=update_ignored_crash_signatures_gerrit_cookie, output_tests_dir=downloaded_graphicsfuzz_tests_dir, ) download_cts_gf_tests.extract_shaders( tests_dir=downloaded_graphicsfuzz_tests_dir, binaries=binary_manager) with util.file_open_text(work_dir / "results.csv", "w") as results_out_handle: run_cts_gf_tests.main_helper( tests_dir=downloaded_graphicsfuzz_tests_dir, work_dir=work_dir, binaries=binary_manager, settings=settings, active_devices=devices_util.get_active_devices( settings.device_list), results_out_handle=results_out_handle, updated_settings_output_path=settings_path, ) return active_devices = devices_util.get_active_devices( settings.device_list, active_device_names=active_device_names) # Add host_preprocessor device from device list if it is missing. if not active_devices[0].HasField("preprocess"): for device in settings.device_list.devices: if device.HasField("preprocess"): active_devices.insert(0, device) break # Add host_preprocessor device (from scratch) if it is still missing. if not active_devices[0].HasField("preprocess"): active_devices.insert( 0, Device(name="host_preprocessor", preprocess=DevicePreprocess())) reports_dir = Path() / "reports" fuzz_failures_dir = reports_dir / FUZZ_FAILURES_DIR_NAME references_dir = Path() / REFERENCES_DIR donors_dir = Path() / DONORS_DIR spirv_fuzz_shaders_dir = Path() / "spirv_fuzz_shaders" # Log a warning if there is no tool on the PATH for printing stack traces. prepended = util.prepend_catchsegv_if_available([], log_warning=True) if not allow_no_stack_traces and not prepended: raise AssertionError("Stopping because we cannot get stack traces.") spirv_fuzz_shaders: List[Path] = [] references: List[Path] = [] if FuzzingTool.SPIRV_FUZZ in fuzzing_tool_pattern: check_dir_exists(spirv_fuzz_shaders_dir) spirv_fuzz_shaders = sorted(spirv_fuzz_shaders_dir.rglob("*.json")) if FuzzingTool.GLSL_FUZZ in fuzzing_tool_pattern: check_dir_exists(references_dir) check_dir_exists(donors_dir) # TODO: make GraphicsFuzz find donors recursively. references = sorted(references_dir.rglob("*.json")) # Filter to only include .json files that have at least one shader (.frag, .vert, .comp) file. references = [ ref for ref in references if shader_job_util.get_related_files(ref) ] if use_amber_vulkan_loader: library_path = binary_manager.get_binary_path_by_name( binaries_util.AMBER_VULKAN_LOADER_NAME).path.parent util.add_library_paths_to_environ([library_path], os.environ) fuzzing_tool_index = 0 while True: interrupt_util.interrupt_if_needed() # We have to use "is not None" because the seed could be 0. if iteration_seed_override is not None: iteration_seed = iteration_seed_override else: iteration_seed = secrets.randbits(ITERATION_SEED_BITS) log(f"Iteration seed: {iteration_seed}") random.seed(iteration_seed) staging_name = get_random_name()[:8] staging_dir = temp_dir / staging_name try: util.mkdir_p_new(staging_dir) except FileExistsError: if iteration_seed_override is not None: raise log(f"Staging directory already exists: {str(staging_dir)}") log("Starting new iteration.") continue # Pseudocode: # - Create test_dir(s) in staging directory. # - Run test_dir(s) on all active devices (stop early if appropriate). # - For each test failure on each device, copy the test to reports_dir, adding the device and crash signature. # - Reduce each report (on the given device). # - Produce a summary for each report. fuzzing_tool = fuzzing_tool_pattern[fuzzing_tool_index] fuzzing_tool_index = (fuzzing_tool_index + 1) % len(fuzzing_tool_pattern) if fuzzing_tool == FuzzingTool.SPIRV_FUZZ: fuzz_spirv_test.fuzz_spirv( staging_dir, reports_dir, fuzz_failures_dir, active_devices, spirv_fuzz_shaders, settings, binary_manager, ) elif fuzzing_tool == FuzzingTool.GLSL_FUZZ: fuzz_glsl_test.fuzz_glsl( staging_dir, reports_dir, fuzz_failures_dir, active_devices, references, donors_dir, settings, binary_manager, ) else: raise AssertionError(f"Unknown fuzzing tool: {fuzzing_tool}") if iteration_seed_override is not None: log("Stopping due to iteration_seed") break shutil.rmtree(staging_dir)
def glsl_shader_job_wrong_image_to_amber_script_for_google_cts( source_dir: Path, output_amber: Path, work_dir: Path, short_description: str, comment_text: str, copyright_year: str, extra_commands: str, is_coverage_gap: bool = False, ) -> Path: """Converts a GLSL shader job of a wrong image case to an Amber script suitable for adding to the CTS.""" check( not short_description.endswith("."), AssertionError("Short description should not end with period."), ) check( "because shader" not in comment_text, AssertionError( 'In comment_text: change "because shader" to "because the shader"' ), ) shader_jobs = get_shader_jobs(source_dir) test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) binary_manager = binaries_util.get_default_binary_manager( settings=Settings() ).get_child_binary_manager( binary_list=list(test.device.binaries) + list(test.binaries) ) spirv_opt_args = list(test.glsl.spirv_opt_args) or None spirv_opt_hash: Optional[str] = None if spirv_opt_args: spirv_opt_hash = binary_manager.get_binary_by_name( binaries_util.SPIRV_OPT_NAME ).version # Compile all shader jobs shader_job_files = [ amber_converter.ShaderJobFile( shader_job.name, compile_shader_job( shader_job.name, shader_job.shader_job, work_dir / shader_job.name, binary_manager, spirv_opt_args=spirv_opt_args, skip_validation=test.skip_validation, common_spirv_args=list(test.common_spirv_args), ).spirv_asm_shader_job, shader_job.shader_job, "", ) for shader_job in shader_jobs ] reference_asm_spirv_job: Optional[amber_converter.ShaderJobFile] = None if shader_job_files[0].name_prefix == test_util.REFERENCE_DIR: reference_asm_spirv_job = shader_job_files[0] del shader_job_files[0] return amber_converter.spirv_asm_shader_job_to_amber_script( amber_converter.ShaderJobFileBasedAmberTest( reference_asm_spirv_job=reference_asm_spirv_job, variants_asm_spirv_job=shader_job_files, ), output_amber, amber_converter.AmberfySettings( copyright_header_text=get_copyright_header_google(copyright_year), add_graphics_fuzz_comment=True, short_description=short_description, comment_text=comment_text, use_default_fence_timeout=True, spirv_opt_args=spirv_opt_args, spirv_opt_hash=spirv_opt_hash, extra_commands=extra_commands, is_coverage_gap=is_coverage_gap, ), )
def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many-statements; settings_path: Path, iteration_seed_override: Optional[int], use_spirv_fuzz: bool, force_no_stack_traces: bool, ) -> None: util.update_gcov_environment_variable_if_needed() try: artifact_util.artifact_path_get_root() except FileNotFoundError: log( "Could not find ROOT file (in the current directory or above) to mark where binaries should be stored. " "Creating a ROOT file in the current directory." ) util.file_write_text(Path(artifact_util.ARTIFACT_ROOT_FILE_NAME), "") settings = settings_util.read_or_create(settings_path) active_devices = devices_util.get_active_devices(settings.device_list) reports_dir = Path() / "reports" fuzz_failures_dir = reports_dir / FUZZ_FAILURES_DIR_NAME temp_dir = Path() / "temp" references_dir = Path() / "references" donors_dir = Path() / "donors" spirv_fuzz_shaders_dir = Path() / "spirv_fuzz_shaders" # Log a warning if there is no tool on the PATH for printing stack traces. prepended = util.prepend_catchsegv_if_available([], log_warning=True) if not force_no_stack_traces and not prepended: raise AssertionError("Stopping because we cannot get stack traces.") spirv_fuzz_shaders: List[Path] = [] references: List[Path] = [] if use_spirv_fuzz: check_dir_exists(spirv_fuzz_shaders_dir) spirv_fuzz_shaders = sorted(spirv_fuzz_shaders_dir.rglob("*.json")) else: check_dir_exists(references_dir) check_dir_exists(donors_dir) # TODO: make GraphicsFuzz find donors recursively. references = sorted(references_dir.rglob("*.json")) # Filter to only include .json files that have at least one shader (.frag, .vert, .comp) file. references = [ ref for ref in references if shader_job_util.get_related_files(ref) ] binary_manager = binaries_util.get_default_binary_manager( settings=settings ).get_child_binary_manager(list(settings.custom_binaries), prepend=True) while True: # We have to use "is not None" because the seed could be 0. if iteration_seed_override is not None: iteration_seed = iteration_seed_override else: iteration_seed = secrets.randbits(ITERATION_SEED_BITS) log(f"Iteration seed: {iteration_seed}") random.seed(iteration_seed) staging_name = get_random_name()[:8] staging_dir = temp_dir / staging_name try: util.mkdir_p_new(staging_dir) except FileExistsError: if iteration_seed_override is not None: raise log(f"Staging directory already exists: {str(staging_dir)}") log(f"Starting new iteration.") continue # Pseudocode: # - Create test_dir(s) in staging directory. # - Run test_dir(s) on all active devices (stop early if appropriate). # - For each test failure on each device, copy the test to reports_dir, adding the device and crash signature. # - Reduce each report (on the given device). # - Produce a summary for each report. if use_spirv_fuzz: fuzz_spirv_test.fuzz_spirv( staging_dir, reports_dir, fuzz_failures_dir, active_devices, spirv_fuzz_shaders, settings, binary_manager, ) else: fuzz_glsl_test.fuzz_glsl( staging_dir, reports_dir, fuzz_failures_dir, active_devices, references, donors_dir, settings, binary_manager, ) shutil.rmtree(staging_dir) if iteration_seed_override is not None: log("Stopping due to iteration_seed") break
def main() -> None: # pylint: disable=too-many-locals; parser = argparse.ArgumentParser( description="Generates an AmberScript test from a shader job.") parser.add_argument( "shader_job", help="The input .json shader job file.", ) parser.add_argument( "--output", help="Output directory.", default="output", ) parser.add_argument( "--spirv_opt_args", help= "Arguments for spirv-opt as a space-separated string, or an empty string to skip running spirv-opt.", default="", ) parser.add_argument( "--settings", help= "Path to a settings JSON file for this instance. The file will be generated if needed. ", default="settings.json", ) parsed_args = parser.parse_args(sys.argv[1:]) shader_job: Path = Path(parsed_args.shader_job) out_dir: Path = Path(parsed_args.output) spirv_opt_args_str: str = parsed_args.spirv_opt_args settings_path: Path = Path(parsed_args.settings) spirv_opt_args: List[str] = [] if spirv_opt_args_str: spirv_opt_args = spirv_opt_args_str.split(" ") settings = settings_util.read_or_create(settings_path) binary_manager = binaries_util.get_default_binary_manager(settings) staging_dir = out_dir / "staging" template_source_dir = staging_dir / "source_template" test_dir = staging_dir / "test" run_output_dir: Path = out_dir / "run" # Remove stale directories. if staging_dir.is_dir(): shutil.rmtree(staging_dir) if run_output_dir.is_dir(): shutil.rmtree(run_output_dir) # Create source template and call |make_test|. if shader_job_util.get_related_suffixes_that_exist( shader_job, language_suffix=[shader_job_util.SUFFIX_SPIRV]): # This is a SPIR-V shader job. shader_job_util.copy( shader_job, template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, language_suffix=shader_job_util.SUFFIXES_SPIRV_FUZZ_INPUT, ) fuzz_spirv_amber_test.make_test( template_source_dir, test_dir, spirv_opt_args=spirv_opt_args, binary_manager=binary_manager, derived_from=shader_job.stem, stable_shader=False, common_spirv_args=list(settings.common_spirv_args), ) elif shader_job_util.get_related_suffixes_that_exist( shader_job, language_suffix=[shader_job_util.SUFFIX_GLSL]): # This is a GLSL shader job. # The "graphicsfuzz-tool" tool is designed to be on your PATH so that e.g. ".bat" will be appended on Windows. # So we use tool_on_path with a custom PATH to get the actual file we want to execute. graphicsfuzz_tool_path = util.tool_on_path( "graphicsfuzz-tool", str( binary_manager.get_binary_path_by_name( "graphicsfuzz-tool").path.parent), ) with util.file_open_text(staging_dir / "log.txt", "w") as log_file: try: gflogging.push_stream_for_logging(log_file) # Create the prepared (for Vulkan GLSL) reference. glsl_generate_util.run_prepare_reference( graphicsfuzz_tool_path, shader_job, template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, legacy_graphics_fuzz_vulkan_arg=settings. legacy_graphics_fuzz_vulkan_arg, ) finally: gflogging.pop_stream_for_logging() fuzz_glsl_amber_test.make_test( template_source_dir, test_dir, spirv_opt_args=spirv_opt_args, binary_manager=binary_manager, derived_from=shader_job.stem, stable_shader=False, common_spirv_args=list(settings.common_spirv_args), ) else: raise AssertionError( "Unexpected shader job type; expected GLSL or SPIR-V shaders.") preprocessor_cache = util.CommandCache() fuzz_test_util.run_shader_job( source_dir=test_util.get_source_dir(test_dir), output_dir=run_output_dir, binary_manager=binary_manager, device=Device(host=DeviceHost()), preprocessor_cache=preprocessor_cache, stop_after_amber=True, )
def main() -> None: # pylint: disable=too-many-statements, too-many-locals, too-many-branches; parser = argparse.ArgumentParser( description="Interestingness test that runs a test using Amber, " "calculates the crash signature based on the result, and returns 0 " "if the signature matches the expected crash signature.") parser.add_argument( "source_dir", help= "The source directory containing the shaders and the test.json file that describes how to run the test.", ) parser.add_argument( "--override_shader_job", nargs=2, metavar=("shader_job_name", "shader_job_json"), help= 'Override one of the shader jobs. E.g.: "--override_shader_job variant temp/variant.json". Note that ' "the output directory will be set to shader_job_json/ (with the .json extension removed) by default in this case. ", ) parser.add_argument( "--override_shader", nargs=3, metavar=("shader_name", "suffix", "shader_path"), help= 'Override one of the shaders. E.g.: "--override_shader variant .frag.spv temp/my_shader.spv". Note that ' "the output directory will be set to shader_path/ (with the .spv extension removed) by default in this case. ", ) parser.add_argument( "--use_default_binaries", help="Use the latest binaries, ignoring those defined in the test.json. " "Implies --fallback_binaries. Passing --settings is recommended to ensure the latest binaries are used.", action="store_true", ) parser.add_argument( "--fallback_binaries", help= "Fallback to the latest binaries if they are not defined in the test.json.", action="store_true", ) parser.add_argument( "--output", help= "Output directory. Required unless --override_shader[_job] is used; see --override_shader[_job] for details.", default=None, ) parser.add_argument( "--settings", help="Path to a settings JSON file for this instance. " "Unlike with gfauto_fuzz, the default value is an empty string, which is ignored. " "You only need to use a settings file if you pass --use_default_binaries and you want to use the latest binary versions. " 'In this case, use e.g. "--settings settings.json" so that a default settings file is generated with the latest binary version numbers ' "and then run gfauto_interestingness_test again to use those latest binaries.", default="", ) parsed_args = parser.parse_args(sys.argv[1:]) source_dir: Path = Path(parsed_args.source_dir) override_shader_job: Optional[Tuple[str, str]] = parsed_args.override_shader_job override_shader: Optional[Tuple[str, str, str]] = parsed_args.override_shader settings_str: str = parsed_args.settings settings = Settings() if settings_str: settings = settings_util.read_or_create(Path(settings_str)) use_default_binaries: bool = parsed_args.use_default_binaries fallback_binaries: bool = parsed_args.fallback_binaries or use_default_binaries output: Path if parsed_args.output: output = Path(parsed_args.output) elif override_shader_job: output = Path(override_shader_job[1]).with_suffix("") elif override_shader: output = Path(override_shader[2]).with_suffix("") else: raise AssertionError( "Need --output or --override_shader[_job] parameter.") binary_manager = binaries_util.get_default_binary_manager( settings=settings) if not fallback_binaries: binary_manager = binary_manager.get_child_binary_manager( binary_list=[]) shader_job_overrides: List[tool.NameAndShaderJob] = [] if override_shader_job: shader_job_overrides.append( tool.NameAndShaderJob(name=override_shader_job[0], shader_job=Path(override_shader_job[1]))) shader_overrides: tool.ShaderJobNameToShaderOverridesMap = {} if override_shader: override = tool.ShaderPathWithNameAndSuffix( name=override_shader[0], suffix=override_shader[1], path=Path(override_shader[2]), ) shader_overrides[override.name] = {override.suffix: override} # E.g. shader_overrides == # { # "variant": { # ".frag.spv": ShaderPathWithNameAndSuffix("variant", ".frag.spv", Path("path/to/shader.frag.spv")) # } # } # We don't need to read this to run the shader, but we need it afterwards anyway. test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) output_dir = fuzz_glsl_test.run_shader_job( source_dir=source_dir, output_dir=output, binary_manager=binary_manager, test=test, ignore_test_and_device_binaries=use_default_binaries, shader_job_overrides=shader_job_overrides, shader_job_shader_overrides=shader_overrides, ) log(f"gfauto_interestingness_test: finished running {str(source_dir)} in {str(output_dir)}." ) if override_shader_job: log(f"The {override_shader_job[0]} shader was overridden with {override_shader_job[1]}" ) status = result_util.get_status(output_dir) if test.expected_status: log("") log(f"Expected status: {test.expected_status}") log(f"Actual status: {status}") log_contents = util.file_read_text(result_util.get_log_path(output_dir)) signature = signature_util.get_signature_from_log_contents(log_contents) log("") log(f"Expected signature: {test.crash_signature}") log(f"Actual signature: {signature}") log("") # The |crash_regex_override| overrides all other checks. if test.crash_regex_override: log(f"Testing crash_regex_override: {test.crash_regex_override}") override_pattern: Pattern[str] = re.compile(test.crash_regex_override, re.DOTALL) match: Optional[Match[str]] = override_pattern.fullmatch(log_contents) if match: log("Match!") log("Interesting") sys.exit(0) else: log("No match; not interesting") sys.exit(1) if test.expected_status: if status != test.expected_status: log("status != expected_status; not interesting") sys.exit(1) else: # There is no expected status given, so just assume it needs to be one of the "bad" statuses: if status not in ( fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH, fuzz.STATUS_UNRESPONSIVE, ): log("shader run did not fail; not interesting.") sys.exit(1) if signature != test.crash_signature: log("signature != crash_signature; not interesting") sys.exit(1) log("Interesting!")
def fuzz_and_reduce_bug( active_device: str, seed: int, check_result: Callable[[], None], settings: Optional[Settings] = None, ignored_signatures: Optional[List[str]] = None, ) -> None: """ Fuzz, find a bug, reduce it. Linux only. """ # Test only works on Linux. if util.get_platform() != "Linux": return here = util.norm_path(Path(__file__).absolute()).parent temp_dir: Path = here.parent / "temp" assert temp_dir.is_dir() os.chdir(temp_dir) # Create ROOT file in temp/ if needed. fuzz.try_get_root_file() work_dir = temp_dir / fuzz.get_random_name()[:8] util.mkdir_p_new(work_dir) os.chdir(work_dir) log(f"Changed to {str(work_dir)}") if settings is None: settings = Settings() settings.CopyFrom(settings_util.DEFAULT_SETTINGS) settings.device_list.CopyFrom( DeviceList( active_device_names=[active_device], devices=[ Device( name="amdllpc", shader_compiler=DeviceShaderCompiler( binary="amdllpc", args=[ "-gfxip=9.0.0", "-verify-ir", "-auto-layout-desc" ], ), binaries=[ Binary( name="amdllpc", tags=["Release"], version="c21d76dceaf26361f9b6b3838a955ec3301506b5", ), ], ), Device( name="swift_shader", swift_shader=DeviceSwiftShader(), binaries=[ Binary( name="swift_shader_icd", tags=["Release"], version="6d69aae0e1ab49190ea46cd1c999fd3d02e016b9", ), ], ignored_crash_signatures=ignored_signatures, ), ], )) spirv_tools_version = "983b5b4fccea17cab053de24d51403efb4829158" settings.latest_binary_versions.extend([ Binary( name="glslangValidator", tags=["Release"], version="1afa2b8cc57b92c6b769eb44a6854510b6921a0b", ), Binary(name="spirv-opt", tags=["Release"], version=spirv_tools_version), Binary(name="spirv-dis", tags=["Release"], version=spirv_tools_version), Binary(name="spirv-as", tags=["Release"], version=spirv_tools_version), Binary(name="spirv-val", tags=["Release"], version=spirv_tools_version), Binary(name="spirv-fuzz", tags=["Release"], version=spirv_tools_version), Binary(name="spirv-reduce", tags=["Release"], version=spirv_tools_version), Binary( name="graphicsfuzz-tool", tags=[], version="7b143bcb3ad38b64ddc17d132886636b229b6684", ), ]) # Add default binaries; the ones above have priority. settings.latest_binary_versions.extend(binaries_util.DEFAULT_BINARIES) settings.extra_graphics_fuzz_generate_args.append("--small") settings.extra_graphics_fuzz_generate_args.append("--single-pass") settings.extra_graphics_fuzz_reduce_args.append("--max-steps") settings.extra_graphics_fuzz_reduce_args.append("2") settings_util.write(settings, settings_util.DEFAULT_SETTINGS_FILE_PATH) # Add shaders. binary_manager = binaries_util.get_default_binary_manager(settings) graphicsfuzz_tool = binary_manager.get_binary_path_by_name( "graphicsfuzz-tool") sample_shaders_path: Path = graphicsfuzz_tool.path.parent.parent.parent / "shaders" / "samples" / "310es" util.copy_dir(sample_shaders_path, Path() / fuzz.REFERENCES_DIR) util.copy_dir(sample_shaders_path, Path() / fuzz.DONORS_DIR) fuzz.main_helper( settings_path=settings_util.DEFAULT_SETTINGS_FILE_PATH, iteration_seed_override=seed, override_sigint=False, use_amber_vulkan_loader=True, ) check_result() os.chdir(here) shutil.rmtree(work_dir)
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements; parser = argparse.ArgumentParser( description= "Runs GraphicsFuzz AmberScript tests on the active devices listed in " "the settings.json file.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( "--settings", help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) parser.add_argument( "--tests", help= "Path to the directory of AmberScript tests with shaders extracted.", default="graphicsfuzz", ) parser.add_argument( "--update_ignored_crash_signatures", help= "As the tests are run for each device, add any crash signatures to the device's ignored_crash_signatures " "property and write out the updated settings.json file.", action="store_true", ) parser.add_argument( "--results_out", help="Output file path for the CSV results table.", default="results.csv", ) parsed_args = parser.parse_args(sys.argv[1:]) # Args. tests_dir: Path = Path(parsed_args.tests) settings_path: Path = Path(parsed_args.settings) update_ignored_crash_signatures: bool = parsed_args.update_ignored_crash_signatures results_out_path: Path = Path(parsed_args.results_out) # Settings and devices. settings = settings_util.read_or_create(settings_path) active_devices = devices_util.get_active_devices(settings.device_list) # Binaries. binaries = binaries_util.get_default_binary_manager(settings=settings) work_dir = Path( ) / "temp" / f"graphicsfuzz_cts_run_{fuzz.get_random_name()[:8]}" with util.file_open_text(results_out_path, "w") as results_out_handle: main_helper( tests_dir=tests_dir, work_dir=work_dir, binaries=binaries, settings=settings, active_devices=active_devices, results_out_handle=results_out_handle, updated_settings_output_path=( settings_path if update_ignored_crash_signatures else None), )
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements; parser = argparse.ArgumentParser( description="Runs GraphicsFuzz AmberScript tests on the active devices listed in " "the settings.json file." ) parser.add_argument( "--settings", help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) parser.add_argument( "--tests", help="Path to the directory of AmberScript tests with shaders extracted.", default=str("graphicsfuzz"), ) parsed_args = parser.parse_args(sys.argv[1:]) # Args. tests_dir: Path = Path(parsed_args.tests) settings_path: Path = Path(parsed_args.settings) # Settings and devices. settings = settings_util.read_or_create(settings_path) active_devices = devices_util.get_active_devices(settings.device_list) # Binaries. binaries = binaries_util.get_default_binary_manager(settings=settings) work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}" util.mkdirs_p(work_dir) with util.file_open_text(Path("results.txt"), "w") as log_handle: def write_entry(entry: str) -> None: log_handle.write(entry) log_handle.write(", ") log_handle.flush() def write_newline() -> None: log_handle.write("\n") log_handle.flush() spirv_opt_path: Optional[Path] = None swift_shader_path: Optional[Path] = None amber_path: Optional[Path] = None # Enumerate active devices, writing their name and storing binary paths if needed. write_entry("test") for device in active_devices: if device.name == "host_preprocessor": # We are actually just running spirv-opt on the SPIR-V shaders. write_entry("spirv-opt") else: write_entry(device.name) if device.HasField("preprocess"): spirv_opt_path = binaries.get_binary_path_by_name( binaries_util.SPIRV_OPT_NAME ).path if device.HasField("swift_shader"): swift_shader_path = binaries.get_binary_path_by_name( binaries_util.SWIFT_SHADER_NAME ).path if device.HasField("swift_shader") or device.HasField("host"): amber_path = binaries.get_binary_path_by_name( binaries_util.AMBER_NAME ).path write_newline() # Enumerate tests and devices, writing the results. for test in sorted(tests_dir.glob("*.amber")): test_name = util.remove_end(test.name, ".amber") write_entry(test_name) spirv_shaders = sorted( tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv") ) for device in active_devices: test_run_dir = work_dir / f"{test_name}_{device.name}" util.mkdirs_p(test_run_dir) try: # Confusingly, some functions below will raise on an error; others will write e.g. CRASH to the # STATUS file in the output directory. In the latter case, we update |status|. We check |status| at # the end of this if-else chain and raise fake exceptions if appropriate. status = fuzz.STATUS_SUCCESS if device.HasField("preprocess"): # This just means spirv-op for now. assert spirv_opt_path # noqa for spirv_shader in spirv_shaders: spirv_opt_util.run_spirv_opt_on_spirv_shader( spirv_shader, test_run_dir, ["-O"], spirv_opt_path ) elif device.HasField("shader_compiler"): for spirv_shader in spirv_shaders: shader_compiler_util.run_shader( shader_compiler_device=device.shader_compiler, shader_path=spirv_shader, output_dir=test_run_dir, compiler_path=binaries.get_binary_path_by_name( device.shader_compiler.binary ).path, timeout=DEFAULT_TIMEOUT, ) elif device.HasField("swift_shader"): assert swift_shader_path # noqa assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, icd=swift_shader_path, ) status = result_util.get_status(test_run_dir) elif device.HasField("host"): assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, amber_path=amber_path, dump_image=False, dump_buffer=False, ) status = result_util.get_status(test_run_dir) elif device.HasField("android"): android_device.run_amber_on_device( test, test_run_dir, dump_image=False, dump_buffer=False, serial=device.android.serial, ) status = result_util.get_status(test_run_dir) else: raise AssertionError(f"Unsupported device {device.name}") if status in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH): raise CalledProcessError(1, "??") if status != fuzz.STATUS_SUCCESS: raise TimeoutExpired("??", fuzz.AMBER_RUN_TIME_LIMIT) write_entry("P") except CalledProcessError: write_entry("F") except TimeoutExpired: write_entry("T") write_newline()