def run_reduction( source_dir_to_reduce: Path, reduction_output_dir: Path, binary_manager: binaries_util.BinaryManager, settings: Settings, ) -> Path: test = test_util.metadata_read_from_source_dir(source_dir_to_reduce) reduced_source_dir = source_dir_to_reduce reduced_source_dir = run_reduction_part( reduction_part_output_dir=reduction_output_dir / "1", source_dir_to_reduce=reduced_source_dir, preserve_semantics=True, binary_manager=binary_manager, settings=settings, ) if (test.crash_signature != signature_util.BAD_IMAGE_SIGNATURE and not settings.skip_semantics_changing_reduction): reduced_source_dir = run_reduction_part( reduction_part_output_dir=reduction_output_dir / "2", source_dir_to_reduce=reduced_source_dir, preserve_semantics=False, binary_manager=binary_manager, settings=settings, ) # Create and return a symlink to the "best" reduction part directory. return util.make_directory_symlink( new_symlink_file_path=reduction_output_dir / fuzz.BEST_REDUCTION_NAME, existing_dir=reduced_source_dir.parent, )
def main() -> None: parser = argparse.ArgumentParser( description="A tool for running a reduction on a source directory." ) parser.add_argument( "source_dir", help="The source directory containing the shaders and the test.json file that describes how to run the test.", ) parser.add_argument( "--output", help="Output directory.", default="reduction_output", ) parser.add_argument( "--settings", help="Path to a settings JSON file for this instance.", default="settings.json", ) parser.add_argument( "--literals_to_uniforms", action="store_true", help="Pass --literals-to-uniforms to glsl-reduce.", ) parsed_args = parser.parse_args(sys.argv[1:]) source_dir = Path(parsed_args.source_dir) output_dir = Path(parsed_args.output) settings = settings_util.read_or_create(Path(parsed_args.settings)) literals_to_uniforms: bool = parsed_args.literals_to_uniforms binary_manager = binaries_util.get_default_binary_manager(settings=settings) test = test_util.metadata_read_from_source_dir(source_dir) if test.HasField("glsl"): if ( literals_to_uniforms and "--literals-to-uniforms" not in settings.extra_graphics_fuzz_reduce_args ): settings.extra_graphics_fuzz_reduce_args.append("--literals-to-uniforms") fuzz_glsl_test.run_reduction( source_dir_to_reduce=source_dir, reduction_output_dir=output_dir, binary_manager=binary_manager, settings=settings, ) elif test.HasField("spirv_fuzz"): fuzz_spirv_test.run_reduction( source_dir_to_reduce=source_dir, reduction_output_dir=output_dir, binary_manager=binary_manager, settings=settings, ) else: raise AssertionError(f"Unknown test type: {test}")
def create_spirv_fuzz_variant_2( source_dir: Path, binary_manager: binaries_util.BinaryManager, settings: Settings, ) -> Optional[Path]: """ Replays all transformations except the last to get variant_2. Replays all transformations except the last to get a variant_2 shader job, such that variant <-> variant_2 are likely even more similar than reference <-> variant. |source_dir| must be a spirv_fuzz test. """ test_metadata: Test = test_util.metadata_read_from_source_dir(source_dir) check(test_metadata.HasField("spirv_fuzz"), AssertionError("Not a spirv_fuzz test")) variant_shader_job = source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB variant_2_shader_job = ( source_dir / f"{test_util.VARIANT_DIR}_2" / test_util.SHADER_JOB ) if not variant_shader_job.is_file(): log( f"Skip generating variant_2 for {str(source_dir)} because the variant shader job was not found." ) return None if variant_2_shader_job.is_file(): log( f"Skip generating variant_2 for {str(source_dir)} because variant_2 shader job already exists." ) return None return spirv_fuzz_util.run_replay_on_shader_job( spirv_fuzz_path=binary_manager.get_binary_path_by_name( binaries_util.SPIRV_FUZZ_NAME ).path, variant_shader_job_json=variant_shader_job, output_shader_job_json=variant_2_shader_job, other_args=list(settings.common_spirv_args), )
def run_reduction_part( reduction_part_output_dir: Path, source_dir_to_reduce: Path, preserve_semantics: bool, binary_manager: binaries_util.BinaryManager, settings: Settings, ) -> Path: test = test_util.metadata_read_from_source_dir(source_dir_to_reduce) if not test.device or not test.device.name: raise AssertionError( f"Cannot reduce {str(source_dir_to_reduce)}; " f"device must be specified in {str(test_util.get_metadata_path_from_source_dir(source_dir_to_reduce))}" ) if not test.crash_signature: raise AssertionError( f"Cannot reduce {str(source_dir_to_reduce)} because there is no crash string specified." ) shader_jobs = tool.get_shader_jobs(source_dir_to_reduce) # TODO: if needed, this could become a parameter to this function. name_of_shader_to_reduce = shader_jobs[0].name if len(shader_jobs) > 1: check( len(shader_jobs) == 2 and shader_jobs[1].name == test_util.VARIANT_DIR, AssertionError( "Can only reduce tests with shader jobs reference and variant, or just variant." ), ) name_of_shader_to_reduce = shader_jobs[1].name reduction_work_variant_dir = run_glsl_reduce( source_dir=source_dir_to_reduce, name_of_shader_to_reduce=name_of_shader_to_reduce, output_dir=test_util.get_reduction_work_directory( reduction_part_output_dir, name_of_shader_to_reduce), binary_manager=binary_manager, preserve_semantics=preserve_semantics, extra_args=list(settings.extra_graphics_fuzz_reduce_args) if settings.extra_graphics_fuzz_reduce_args else None, ) final_reduced_shader_job_path = get_final_reduced_shader_job_path( reduction_work_variant_dir) check( final_reduced_shader_job_path.exists(), ReductionFailedError("Reduction failed.", reduction_work_variant_dir), ) # Finally, create the output source_dir. util.copy_dir(source_dir_to_reduce, test_util.get_source_dir(reduction_part_output_dir)) shader_job_util.copy( final_reduced_shader_job_path, test_util.get_shader_job_path(reduction_part_output_dir, name_of_shader_to_reduce), ) if not settings.keep_reduction_work: shutil.rmtree(reduction_work_variant_dir) return test_util.get_source_dir(reduction_part_output_dir)
def run_reduction( source_dir_to_reduce: Path, reduction_output_dir: Path, binary_manager: binaries_util.BinaryManager, settings: Settings, ) -> Path: test = test_util.metadata_read_from_source_dir(source_dir_to_reduce) shader_jobs = tool.get_shader_jobs(source_dir_to_reduce) # TODO: if needed, this could become a parameter to this function. shader_job_to_reduce = shader_jobs[0] if len(shader_jobs) > 1: check( len(shader_jobs) == 2 and shader_jobs[1].name == test_util.VARIANT_DIR, AssertionError( "Can only reduce tests with shader jobs reference and variant, or just variant." ), ) shader_job_to_reduce = shader_jobs[1] shader_transformation_suffixes = shader_job_util.get_related_suffixes_that_exist( shader_job_to_reduce.shader_job, language_suffix=(shader_job_util.SUFFIX_TRANSFORMATIONS, ), ) shader_spv_suffixes = shader_job_util.get_related_suffixes_that_exist( shader_job_to_reduce.shader_job, language_suffix=(shader_job_util.SUFFIX_SPIRV, )) reduced_source_dir = source_dir_to_reduce for index, suffix in enumerate(shader_transformation_suffixes): # E.g. .frag.transformations -> .frag extension_to_reduce = str(Path(suffix).with_suffix("")) reduced_source_dir = run_reduction_part( reduction_part_output_dir=reduction_output_dir / f"0_{index}_{suffix.split('.')[1]}", source_dir_to_reduce=reduced_source_dir, shader_job_name_to_reduce=shader_job_to_reduce.name, extension_to_reduce=extension_to_reduce, preserve_semantics=True, binary_manager=binary_manager, settings=settings, ) if (test.crash_signature != signature_util.BAD_IMAGE_SIGNATURE and not settings.skip_spirv_reduce): for index, suffix in enumerate(shader_spv_suffixes): # E.g. .frag.spv -> .frag extension_to_reduce = str(Path(suffix).with_suffix("")) reduced_source_dir = run_reduction_part( reduction_part_output_dir=reduction_output_dir / f"1_{index}_{suffix.split('.')[1]}", source_dir_to_reduce=reduced_source_dir, shader_job_name_to_reduce=shader_job_to_reduce.name, extension_to_reduce=extension_to_reduce, preserve_semantics=False, binary_manager=binary_manager, settings=settings, ) # Create and return a symlink to the "best" reduction. return util.make_directory_symlink( new_symlink_file_path=reduction_output_dir / fuzz.BEST_REDUCTION_NAME, existing_dir=reduced_source_dir.parent, )
def run_reduction_part( reduction_part_output_dir: Path, source_dir_to_reduce: Path, shader_job_name_to_reduce: str, extension_to_reduce: str, preserve_semantics: bool, binary_manager: binaries_util.BinaryManager, settings: Settings, ) -> Path: test = test_util.metadata_read_from_source_dir(source_dir_to_reduce) check( bool(test.device and test.device.name), AssertionError( f"Cannot reduce {str(source_dir_to_reduce)}; device must be specified" ), ) check( bool(test.crash_signature), AssertionError( f"Cannot reduce {str(source_dir_to_reduce)} because there is no crash string specified." ), ) output_dir = test_util.get_reduction_work_directory( reduction_part_output_dir, shader_job_name_to_reduce) final_shader_path = run_spirv_reduce_or_shrink( source_dir=source_dir_to_reduce, name_of_shader_job_to_reduce=shader_job_name_to_reduce, extension_to_reduce=extension_to_reduce, output_dir=output_dir, preserve_semantics=preserve_semantics, binary_manager=binary_manager, settings=settings, ) check( final_shader_path.exists(), fuzz_glsl_test.ReductionFailedError("Reduction failed.", output_dir), ) # Finally, create the source_dir so the returned directory can be used as a test_dir. # Copy the original source directory. util.copy_dir(source_dir_to_reduce, test_util.get_source_dir(reduction_part_output_dir)) # And then replace the shader. # Destination file. E.g. reductions/source/variant/shader.frag.spv output_shader_prefix = ( test_util.get_source_dir(reduction_part_output_dir) / shader_job_name_to_reduce / test_util.SHADER_JOB).with_suffix(extension_to_reduce + shader_job_util.SUFFIX_SPIRV) util.copy_file( final_shader_path.with_suffix(shader_job_util.SUFFIX_SPIRV), output_shader_prefix.with_suffix(shader_job_util.SUFFIX_SPIRV), ) if preserve_semantics: util.copy_file( final_shader_path.with_suffix( shader_job_util.SUFFIX_TRANSFORMATIONS), output_shader_prefix.with_suffix( shader_job_util.SUFFIX_TRANSFORMATIONS), ) util.copy_file( final_shader_path.with_suffix( shader_job_util.SUFFIX_TRANSFORMATIONS_JSON), output_shader_prefix.with_suffix( shader_job_util.SUFFIX_TRANSFORMATIONS_JSON), ) return test_util.get_source_dir(reduction_part_output_dir)
def run_spirv_reduce_or_shrink( # pylint: disable=too-many-locals; source_dir: Path, name_of_shader_job_to_reduce: str, extension_to_reduce: str, output_dir: Path, preserve_semantics: bool, binary_manager: binaries_util.BinaryManager, settings: Settings, ) -> Path: test = test_util.metadata_read_from_source_dir(source_dir) input_shader_job = source_dir / name_of_shader_job_to_reduce / test_util.SHADER_JOB original_spirv_file = input_shader_job.with_suffix( extension_to_reduce + shader_job_util.SUFFIX_SPIRV_ORIG) transformed_spirv_file = input_shader_job.with_suffix( extension_to_reduce + shader_job_util.SUFFIX_SPIRV) transformations_file = input_shader_job.with_suffix( extension_to_reduce + shader_job_util.SUFFIX_TRANSFORMATIONS) util.mkdirs_p(output_dir) final_shader = output_dir / "final.spv" # E.g. transformation_suffix_to_reduce == ".frag.transformations" # E.g. ".frag.??" -> ".frag.spv" shader_suffix_to_override = extension_to_reduce + shader_job_util.SUFFIX_SPIRV if preserve_semantics: cmd = [ str(binary_manager.get_binary_path_by_name("spirv-fuzz").path), str(original_spirv_file), "-o", str(final_shader), f"--shrink={str(transformations_file)}", f"--shrinker-temp-file-prefix={str(output_dir / 'temp_')}", ] cmd += list(settings.extra_spirv_fuzz_shrink_args) cmd += list(test.common_spirv_args) cmd += [ # This ensures the arguments that follow are all positional arguments. "--", "gfauto_interestingness_test", str(source_dir), # --override_shader requires three parameters to follow; the third will be added by spirv-fuzz (the shader.spv file). "--override_shader", name_of_shader_job_to_reduce, shader_suffix_to_override, ] else: cmd = [ str(binary_manager.get_binary_path_by_name("spirv-reduce").path), str(transformed_spirv_file), "-o", str(final_shader), f"--temp-file-prefix={str(output_dir / 'temp_')}", ] cmd += list(settings.extra_spirv_reduce_args) cmd += list(test.common_spirv_args) cmd += [ # This ensures the arguments that follow are all positional arguments. "--", "gfauto_interestingness_test", str(source_dir), # --override_shader requires three parameters to follow; the third will be added by spirv-reduce (the shader.spv file). "--override_shader", name_of_shader_job_to_reduce, shader_suffix_to_override, ] # Log the reduction. with util.file_open_text(output_dir / "command.log", "w") as f: gflogging.push_stream_for_logging(f) try: # The reducer can fail, but it will typically output an exception file, so we can ignore the exit code. subprocess_util.run(cmd, verbose=True, check_exit_code=False) finally: gflogging.pop_stream_for_logging() return final_shader