def get_binary_list_from_test_metadata(test_json_path: Path) -> List[Binary]: test_metadata = test_util.metadata_read_from_path(test_json_path) result: List[Binary] = [] if test_metadata.device: result.extend(test_metadata.device.binaries) result.extend(test_metadata.binaries) return result
def main() -> None: parser = argparse.ArgumentParser( description= "A script that creates a README and bug_report directory for a test and its result_dir." ) parser.add_argument( "source_dir", help="Source directory containing test.json and shaders.") parser.add_argument( "result_dir", help= "Path to the result_dir of a test containing e.g. the intermediate shader files, log.txt, etc.", ) parser.add_argument( "--output_dir", help= "Output directory where the README and bug_report directory will be written.", default=".", ) parsed_args = parser.parse_args(sys.argv[1:]) source_dir = Path(parsed_args.source_dir) result_dir = Path(parsed_args.result_dir) output_dir = Path(parsed_args.output_dir) check_dir_exists(source_dir) check_dir_exists(result_dir) test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) binary_manager = binaries_util.get_default_binary_manager( settings=Settings()).get_child_binary_manager( binary_list=list(test.binaries) + list(test.device.binaries)) check(test.HasField("glsl"), AssertionError("Only glsl tests currently supported")) check( test.device.HasField("preprocess"), AssertionError("Only preprocess device tests currently supported"), ) fuzz_test_util.tool_crash_summary_bug_report_dir(source_dir, result_dir, output_dir, binary_manager)
def update_test_json(test_json: Path) -> Path: test = test_util.metadata_read_from_path(test_json) for test_binary in itertools.chain(test.binaries, test.device.binaries): # type: Binary for default_binary in binaries_util.DEFAULT_BINARIES: if (test_binary.name == default_binary.name and test_binary.version != default_binary.version): log(f"Updating version: {test_binary.version} -> {default_binary.version}" ) test_binary.version = default_binary.version break return test_util.metadata_write_to_path(test, test_json)
def tool_crash_summary_bug_report_dir( # pylint: disable=too-many-locals; reduced_glsl_source_dir: Path, variant_reduced_glsl_result_dir: Path, output_dir: Path, binary_manager: binaries_util.BinaryManager, ) -> Optional[Path]: # Create a simple script and README. shader_job = reduced_glsl_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB if not shader_job.is_file(): return None test_metadata: Test = test_util.metadata_read_from_path( reduced_glsl_source_dir / test_util.TEST_METADATA) shader_files = shader_job_util.get_related_files( shader_job, shader_job_util.EXT_ALL, (shader_job_util.SUFFIX_GLSL, shader_job_util.SUFFIX_SPIRV), ) check( len(shader_files) > 0, AssertionError(f"Need at least one shader for {shader_job}"), ) shader_extension = shader_files[0].suffix bug_report_dir = util.copy_dir(variant_reduced_glsl_result_dir, output_dir / "bug_report") shader_files = sorted(bug_report_dir.rglob("shader.*")) glsl_files = [ shader_file for shader_file in shader_files if shader_file.suffix == shader_extension ] asm_files = [ shader_file for shader_file in shader_files if shader_file.name.endswith(shader_extension + shader_job_util.SUFFIX_ASM_SPIRV) ] spv_files = [ shader_file for shader_file in shader_files if shader_file.name.endswith(shader_extension + shader_job_util.SUFFIX_SPIRV) ] readme = "\n\n" readme += ( "Issue found using [GraphicsFuzz](https://github.com/google/graphicsfuzz).\n\n" ) readme += "Tool versions:\n\n" # noinspection PyTypeChecker if test_metadata.HasField("glsl"): readme += f"* glslangValidator commit hash: {binary_manager.get_binary_by_name(binaries_util.GLSLANG_VALIDATOR_NAME).version}\n" if test_metadata.glsl.spirv_opt_args or test_metadata.spirv_fuzz.spirv_opt_args: readme += f"* spirv-opt commit hash: {binary_manager.get_binary_by_name(binaries_util.SPIRV_OPT_NAME).version}\n" readme += "\nTo reproduce:\n\n" readme += f"`glslangValidator -V shader{shader_extension} -o shader{shader_extension}.spv`\n\n" if (test_metadata.HasField("glsl") and spv_files and not test_metadata.glsl.spirv_opt_args): # GLSL was converted to SPIR-V, and spirv-opt was not run, so indicate that we should validate the SPIR-V. readme += f"`spirv-val shader{shader_extension}.spv`\n\n" if test_metadata.glsl.spirv_opt_args or test_metadata.spirv_fuzz.spirv_opt_args: readme += f"`spirv-opt shader{shader_extension}.spv -o temp.spv --validate-after-all {' '.join(test_metadata.glsl.spirv_opt_args)}`\n\n" files_to_list = glsl_files + spv_files + asm_files files_to_list.sort() files_to_show = glsl_files + asm_files files_to_show.sort() readme += "The following shader files are included in the attached archive, some of which are also shown inline below:\n\n" for file_to_list in files_to_list: short_path = file_to_list.relative_to(bug_report_dir).as_posix() readme += f"* {short_path}\n" for file_to_show in files_to_show: short_path = file_to_show.relative_to(bug_report_dir).as_posix() file_contents = util.file_read_text(file_to_show) readme += f"\n{short_path}:\n\n" readme += f"```\n{file_contents}\n```\n" util.file_write_text(output_dir / "README.md", readme) return bug_report_dir
def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branches, too-many-locals, too-many-statements; source_dir: Path, output_dir: Path, binary_manager: binaries_util.BinaryManager, test: Optional[Test] = None, device: Optional[Device] = None, ignore_test_and_device_binaries: bool = False, shader_job_overrides: Iterable[tool.NameAndShaderJob] = (), shader_job_shader_overrides: Optional[ tool.ShaderJobNameToShaderOverridesMap] = None, ) -> Path: if not shader_job_shader_overrides: shader_job_shader_overrides = {} with util.file_open_text(output_dir / "log.txt", "w") as log_file: try: gflogging.push_stream_for_logging(log_file) # TODO: Find amber path. NDK or host. # TODO: If Amber is going to be used, check if Amber can use Vulkan debug layers now, and if not, pass that # info down via a bool. if not test: test = test_util.metadata_read_from_path( source_dir / test_util.TEST_METADATA) if not device: device = test.device log(f"Running test on device:\n{device.name}") # We will create a binary_manager child with a restricted set of binaries so that we only use the binaries # specified in the test and by the device; if some required binaries are not specified by the test nor the # device, there will be an error instead of falling back to our default binaries. But we keep a reference to # the parent so we can still access certain "test-independent" binaries like Amber. binary_manager_parent = binary_manager if not ignore_test_and_device_binaries: binary_manager = binary_manager.get_child_binary_manager( list(device.binaries) + list(test.binaries)) spirv_opt_hash: Optional[str] = None spirv_opt_args: Optional[List[str]] = None if test.glsl.spirv_opt_args or test.spirv_fuzz.spirv_opt_args: spirv_opt_hash = binary_manager.get_binary_by_name( binaries_util.SPIRV_OPT_NAME).version spirv_opt_args = (list(test.glsl.spirv_opt_args) if test.glsl.spirv_opt_args else list( test.spirv_fuzz.spirv_opt_args)) shader_jobs = tool.get_shader_jobs(source_dir, overrides=shader_job_overrides) combined_spirv_shader_jobs: List[tool.SpirvCombinedShaderJob] = [] for shader_job in shader_jobs: try: shader_overrides = shader_job_shader_overrides.get( shader_job.name, None) combined_spirv_shader_jobs.append( tool.compile_shader_job( name=shader_job.name, input_json=shader_job.shader_job, work_dir=output_dir / shader_job.name, binary_paths=binary_manager, spirv_opt_args=spirv_opt_args, shader_overrides=shader_overrides, )) except subprocess.CalledProcessError: result_util.write_status(output_dir, fuzz.STATUS_TOOL_CRASH, shader_job.name) return output_dir except subprocess.TimeoutExpired: result_util.write_status(output_dir, fuzz.STATUS_TOOL_TIMEOUT, shader_job.name) return output_dir # Device types: |preprocess| and |shader_compiler| don't need an AmberScript file. # noinspection PyTypeChecker if device.HasField("preprocess"): # The "preprocess" device type just needs to get this far, so this is a success. result_util.write_status(output_dir, fuzz.STATUS_SUCCESS) return output_dir # noinspection PyTypeChecker if device.HasField("shader_compiler"): for combined_spirv_shader_job in combined_spirv_shader_jobs: try: shader_compiler_util.run_shader_job( device.shader_compiler, combined_spirv_shader_job.spirv_shader_job, output_dir, binary_manager=binary_manager, ) except subprocess.CalledProcessError: result_util.write_status( output_dir, fuzz.STATUS_CRASH, combined_spirv_shader_job.name, ) return output_dir except subprocess.TimeoutExpired: result_util.write_status( output_dir, fuzz.STATUS_TIMEOUT, combined_spirv_shader_job.name, ) return output_dir # The shader compiler succeeded on all files; this is a success. result_util.write_status(output_dir, fuzz.STATUS_SUCCESS) return output_dir # Other device types need an AmberScript file. amber_converter_shader_job_files = [ amber_converter.ShaderJobFile( name_prefix=combined_spirv_shader_job.name, asm_spirv_shader_job_json=combined_spirv_shader_job. spirv_asm_shader_job, glsl_source_json=combined_spirv_shader_job. glsl_source_shader_job, processing_info="", ) for combined_spirv_shader_job in combined_spirv_shader_jobs ] # Check if the first is the reference shader; if so, pull it out into its own variable. reference: Optional[amber_converter.ShaderJobFile] = None variants = amber_converter_shader_job_files if (amber_converter_shader_job_files[0].name_prefix == test_util.REFERENCE_DIR): reference = amber_converter_shader_job_files[0] variants = variants[1:] elif len(variants) > 1: raise AssertionError( "More than one variant, but no reference. This is unexpected." ) amber_script_file = amber_converter.spirv_asm_shader_job_to_amber_script( shader_job_file_amber_test=amber_converter. ShaderJobFileBasedAmberTest(reference_asm_spirv_job=reference, variants_asm_spirv_job=variants), output_amber_script_file_path=output_dir / "test.amber", amberfy_settings=amber_converter.AmberfySettings( spirv_opt_args=spirv_opt_args, spirv_opt_hash=spirv_opt_hash), ) is_compute = bool( shader_job_util.get_related_files( combined_spirv_shader_jobs[0].spirv_shader_job, [shader_job_util.EXT_COMP], )) # noinspection PyTypeChecker if device.HasField("host") or device.HasField("swift_shader"): icd: Optional[Path] = None # noinspection PyTypeChecker if device.HasField("swift_shader"): icd = binary_manager.get_binary_path_by_name( binaries_util.SWIFT_SHADER_NAME).path # Run the test on the host using Amber. host_device_util.run_amber( amber_script_file, output_dir, amber_path=binary_manager_parent.get_binary_path_by_name( binaries_util.AMBER_NAME).path, dump_image=(not is_compute), dump_buffer=is_compute, icd=icd, ) return output_dir # noinspection PyTypeChecker if device.HasField("android"): android_device.run_amber_on_device( amber_script_file, output_dir, dump_image=(not is_compute), dump_buffer=is_compute, serial=device.android.serial, ) return output_dir # TODO: For a remote device (which we will probably need to support), use log_a_file to output the # "amber_log.txt" file. raise AssertionError(f"Unhandled device type:\n{str(device)}") finally: gflogging.pop_stream_for_logging()
def main() -> None: # pylint: disable=too-many-statements, too-many-locals, too-many-branches; parser = argparse.ArgumentParser( description="Interestingness test that runs a test using Amber, " "calculates the crash signature based on the result, and returns 0 " "if the signature matches the expected crash signature.") parser.add_argument( "source_dir", help= "The source directory containing the shaders and the test.json file that describes how to run the test.", ) parser.add_argument( "--override_shader_job", nargs=2, metavar=("shader_job_name", "shader_job_json"), help= 'Override one of the shader jobs. E.g.: "--override_shader_job variant temp/variant.json". Note that ' "the output directory will be set to shader_job_json/ (with the .json extension removed) by default in this case. ", ) parser.add_argument( "--override_shader", nargs=3, metavar=("shader_name", "suffix", "shader_path"), help= 'Override one of the shaders. E.g.: "--override_shader variant .frag.spv temp/my_shader.spv". Note that ' "the output directory will be set to shader_path/ (with the .spv extension removed) by default in this case. ", ) parser.add_argument( "--use_default_binaries", help="Use the latest binaries, ignoring those defined in the test.json. " "Implies --fallback_binaries. Passing --settings is recommended to ensure the latest binaries are used.", action="store_true", ) parser.add_argument( "--fallback_binaries", help= "Fallback to the latest binaries if they are not defined in the test.json.", action="store_true", ) parser.add_argument( "--output", help= "Output directory. Required unless --override_shader[_job] is used; see --override_shader[_job] for details.", default=None, ) parser.add_argument( "--settings", help="Path to a settings JSON file for this instance. " "Unlike with gfauto_fuzz, the default value is an empty string, which is ignored. " "You only need to use a settings file if you pass --use_default_binaries and you want to use the latest binary versions. " 'In this case, use e.g. "--settings settings.json" so that a default settings file is generated with the latest binary version numbers ' "and then run gfauto_interestingness_test again to use those latest binaries.", default="", ) parsed_args = parser.parse_args(sys.argv[1:]) source_dir: Path = Path(parsed_args.source_dir) override_shader_job: Optional[Tuple[str, str]] = parsed_args.override_shader_job override_shader: Optional[Tuple[str, str, str]] = parsed_args.override_shader settings_str: str = parsed_args.settings settings = Settings() if settings_str: settings = settings_util.read_or_create(Path(settings_str)) use_default_binaries: bool = parsed_args.use_default_binaries fallback_binaries: bool = parsed_args.fallback_binaries or use_default_binaries output: Path if parsed_args.output: output = Path(parsed_args.output) elif override_shader_job: output = Path(override_shader_job[1]).with_suffix("") elif override_shader: output = Path(override_shader[2]).with_suffix("") else: raise AssertionError( "Need --output or --override_shader[_job] parameter.") binary_manager = binaries_util.get_default_binary_manager( settings=settings) if not fallback_binaries: binary_manager = binary_manager.get_child_binary_manager( binary_list=[]) shader_job_overrides: List[tool.NameAndShaderJob] = [] if override_shader_job: shader_job_overrides.append( tool.NameAndShaderJob(name=override_shader_job[0], shader_job=Path(override_shader_job[1]))) shader_overrides: tool.ShaderJobNameToShaderOverridesMap = {} if override_shader: override = tool.ShaderPathWithNameAndSuffix( name=override_shader[0], suffix=override_shader[1], path=Path(override_shader[2]), ) shader_overrides[override.name] = {override.suffix: override} # E.g. shader_overrides == # { # "variant": { # ".frag.spv": ShaderPathWithNameAndSuffix("variant", ".frag.spv", Path("path/to/shader.frag.spv")) # } # } # We don't need to read this to run the shader, but we need it afterwards anyway. test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) output_dir = fuzz_glsl_test.run_shader_job( source_dir=source_dir, output_dir=output, binary_manager=binary_manager, test=test, ignore_test_and_device_binaries=use_default_binaries, shader_job_overrides=shader_job_overrides, shader_job_shader_overrides=shader_overrides, ) log(f"gfauto_interestingness_test: finished running {str(source_dir)} in {str(output_dir)}." ) if override_shader_job: log(f"The {override_shader_job[0]} shader was overridden with {override_shader_job[1]}" ) status = result_util.get_status(output_dir) if test.expected_status: log("") log(f"Expected status: {test.expected_status}") log(f"Actual status: {status}") log_contents = util.file_read_text(result_util.get_log_path(output_dir)) signature = signature_util.get_signature_from_log_contents(log_contents) log("") log(f"Expected signature: {test.crash_signature}") log(f"Actual signature: {signature}") log("") # The |crash_regex_override| overrides all other checks. if test.crash_regex_override: log(f"Testing crash_regex_override: {test.crash_regex_override}") override_pattern: Pattern[str] = re.compile(test.crash_regex_override, re.DOTALL) match: Optional[Match[str]] = override_pattern.fullmatch(log_contents) if match: log("Match!") log("Interesting") sys.exit(0) else: log("No match; not interesting") sys.exit(1) if test.expected_status: if status != test.expected_status: log("status != expected_status; not interesting") sys.exit(1) else: # There is no expected status given, so just assume it needs to be one of the "bad" statuses: if status not in ( fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH, fuzz.STATUS_UNRESPONSIVE, ): log("shader run did not fail; not interesting.") sys.exit(1) if signature != test.crash_signature: log("signature != crash_signature; not interesting") sys.exit(1) log("Interesting!")
def glsl_shader_job_wrong_image_to_amber_script_for_google_cts( source_dir: Path, output_amber: Path, work_dir: Path, short_description: str, comment_text: str, copyright_year: str, extra_commands: str, is_coverage_gap: bool = False, ) -> Path: """Converts a GLSL shader job of a wrong image case to an Amber script suitable for adding to the CTS.""" check( not short_description.endswith("."), AssertionError("Short description should not end with period."), ) check( "because shader" not in comment_text, AssertionError( 'In comment_text: change "because shader" to "because the shader"' ), ) shader_jobs = get_shader_jobs(source_dir) test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) binary_manager = binaries_util.get_default_binary_manager( settings=Settings() ).get_child_binary_manager( binary_list=list(test.device.binaries) + list(test.binaries) ) spirv_opt_args = list(test.glsl.spirv_opt_args) or None spirv_opt_hash: Optional[str] = None if spirv_opt_args: spirv_opt_hash = binary_manager.get_binary_by_name( binaries_util.SPIRV_OPT_NAME ).version # Compile all shader jobs shader_job_files = [ amber_converter.ShaderJobFile( shader_job.name, compile_shader_job( shader_job.name, shader_job.shader_job, work_dir / shader_job.name, binary_manager, spirv_opt_args=spirv_opt_args, skip_validation=test.skip_validation, common_spirv_args=list(test.common_spirv_args), ).spirv_asm_shader_job, shader_job.shader_job, "", ) for shader_job in shader_jobs ] reference_asm_spirv_job: Optional[amber_converter.ShaderJobFile] = None if shader_job_files[0].name_prefix == test_util.REFERENCE_DIR: reference_asm_spirv_job = shader_job_files[0] del shader_job_files[0] return amber_converter.spirv_asm_shader_job_to_amber_script( amber_converter.ShaderJobFileBasedAmberTest( reference_asm_spirv_job=reference_asm_spirv_job, variants_asm_spirv_job=shader_job_files, ), output_amber, amber_converter.AmberfySettings( copyright_header_text=get_copyright_header_google(copyright_year), add_graphics_fuzz_comment=True, short_description=short_description, comment_text=comment_text, use_default_fence_timeout=True, spirv_opt_args=spirv_opt_args, spirv_opt_hash=spirv_opt_hash, extra_commands=extra_commands, is_coverage_gap=is_coverage_gap, ), )