def run_spirv_opt_on_spirv_shader_job(
input_spirv_shader_job_json_file_path: pathlib.Path,
output_spirv_shader_job_json_file_path: pathlib.Path,
spirv_opt_args: List[str],
spirv_opt_file_path: Optional[pathlib.Path] = None,
spirv_opt_no_validate_after_all: bool = False,
preprocessor_cache: Optional[util.CommandCache] = None,
) -> pathlib.Path:
if not spirv_opt_file_path:
spirv_opt_file_path = util.tool_on_path(binaries_util.SPIRV_OPT_NAME)
shader_files = shader_job_util.get_related_files(
input_spirv_shader_job_json_file_path,
language_suffix=[shader_job_util.SUFFIX_SPIRV],
)
util.copy_file(input_spirv_shader_job_json_file_path,
output_spirv_shader_job_json_file_path)
for shader_file in shader_files:
run_spirv_opt_on_spirv_shader(
shader_file,
output_spirv_shader_job_json_file_path.parent,
spirv_opt_args,
spirv_opt_file_path,
spirv_opt_no_validate_after_all,
preprocessor_cache=preprocessor_cache,
)
return output_spirv_shader_job_json_file_path
def run_glslang_glsl_to_spirv_job(
glsl_shader_job_json_file_path: pathlib.Path,
spirv_shader_job_json_file_path: pathlib.Path,
glslang_validator_file_path: Optional[pathlib.Path] = None,
) -> pathlib.Path:
if not glslang_validator_file_path:
glslang_validator_file_path = util.tool_on_path(
binaries_util.GLSLANG_VALIDATOR_NAME
)
glsl_shader_files = shader_job_util.get_related_files(
glsl_shader_job_json_file_path
)
util.copy_file(glsl_shader_job_json_file_path, spirv_shader_job_json_file_path)
for glsl_shader_file in glsl_shader_files:
run_glslang_glsl_shader_to_spirv_shader(
glsl_shader_file,
spirv_shader_job_json_file_path.parent,
glslang_validator_file_path,
)
return spirv_shader_job_json_file_path
def run_generate(
spirv_fuzz_path: Path,
reference_shader_spv: Path,
output_shader_spv: Path,
donors_list_path: Path,
seed: Optional[str] = None,
other_args: Optional[List[str]] = None,
) -> Path:
util.check(
output_shader_spv.suffix == shader_job_util.SUFFIX_SPIRV,
AssertionError(f"Expected {str(output_shader_spv)} to end with .spv"),
)
util.file_mkdirs_parent(output_shader_spv)
cmd = [
str(spirv_fuzz_path),
str(reference_shader_spv),
"-o",
str(output_shader_spv),
f"--donors={str(donors_list_path)}",
"--fuzzer-pass-validation",
]
if seed:
cmd.append(f"--seed={seed}")
if other_args:
cmd.extend(other_args)
subprocess_util.run(cmd)
# reference.spv -> output.spv_orig
util.copy_file(
reference_shader_spv,
output_shader_spv.with_suffix(shader_job_util.SUFFIX_SPIRV_ORIG),
)
# reference.spv.facts -> output.spv.facts
source_facts_path = reference_shader_spv.with_suffix(
shader_job_util.SUFFIX_FACTS)
dest_facts_path = output_shader_spv.with_suffix(
shader_job_util.SUFFIX_FACTS)
if source_facts_path.exists():
util.copy_file(source_facts_path, dest_facts_path)
return output_shader_spv
def run_replay(
spirv_fuzz_path: Path,
variant_shader_spv: Path,
output_shader_spv: Path,
other_args: Optional[List[str]] = None,
) -> Path:
"""Replays all transformations except the last to get a similar variant shader."""
util.check(
output_shader_spv.suffix == shader_job_util.SUFFIX_SPIRV,
AssertionError(f"Expected {str(output_shader_spv)} to end with .spv"),
)
util.file_mkdirs_parent(output_shader_spv)
# Copy shader.<STAGE>.facts.
if variant_shader_spv.with_suffix(shader_job_util.SUFFIX_FACTS).is_file():
util.copy_file(
variant_shader_spv.with_suffix(shader_job_util.SUFFIX_FACTS),
output_shader_spv.with_suffix(shader_job_util.SUFFIX_FACTS),
)
# Copy shader.<STAGE>.spv_orig.
orig_spv = util.copy_file(
variant_shader_spv.with_suffix(shader_job_util.SUFFIX_SPIRV_ORIG),
output_shader_spv.with_suffix(shader_job_util.SUFFIX_SPIRV_ORIG),
)
transformations = variant_shader_spv.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS)
cmd = [
str(spirv_fuzz_path),
str(orig_spv),
"-o",
str(output_shader_spv),
f"--replay={str(transformations)}",
"--replay-range=-1", # replays all transformations except the last
]
if other_args:
cmd.extend(other_args)
subprocess_util.run(cmd)
return output_shader_spv
def copy(
shader_job_file_path: pathlib.Path,
output_shader_job_file_path: pathlib.Path,
extensions: Union[Tuple[str, ...], List[str]] = EXT_ALL,
language_suffix: Union[Tuple[str, ...], List[str]] = (SUFFIX_GLSL, ),
) -> pathlib.Path:
util.copy_file(shader_job_file_path, output_shader_job_file_path)
suffixes_that_exist = get_related_suffixes_that_exist(
shader_job_file_path, extensions, language_suffix)
for suffix in suffixes_that_exist:
util.copy_file(
shader_job_file_path.with_suffix(suffix),
output_shader_job_file_path.with_suffix(suffix),
)
return output_shader_job_file_path
def run_generate_on_shader_job(
spirv_fuzz_path: Path,
reference_shader_json: Path,
output_shader_json: Path,
donor_shader_job_paths: Optional[List[Path]] = None,
seed: Optional[str] = None,
other_args: Optional[List[str]] = None,
) -> Path:
if donor_shader_job_paths is None:
donor_shader_job_paths = []
util.copy_file(reference_shader_json, output_shader_json)
suffixes_that_exist = shader_job_util.get_related_suffixes_that_exist(
reference_shader_json, shader_job_util.EXT_ALL,
[shader_job_util.SUFFIX_SPIRV])
for suffix in suffixes_that_exist:
# Create a donors list file "donors.{suffix}.txt" containing the file paths to all relevant donor .spv shaders.
donor_list_contents = ""
for donor_shader_job_path in donor_shader_job_paths:
donor_shader_path = donor_shader_job_path.with_suffix(suffix)
if donor_shader_path.exists():
donor_list_contents += f"{str(donor_shader_path)}\n"
donors_list_path = util.file_write_text(
reference_shader_json.parent / f"donors{suffix}.txt",
donor_list_contents)
run_generate(
spirv_fuzz_path,
reference_shader_json.with_suffix(suffix),
output_shader_json.with_suffix(suffix),
donors_list_path=donors_list_path,
seed=seed,
other_args=other_args,
)
return output_shader_json
def run_spirv_shader_job_to_spirv_asm_shader_job(
input_spirv_job_json_file_path: pathlib.Path,
output_spirv_job_json_file_path: pathlib.Path,
spirv_dis_file_path: Optional[pathlib.Path] = None,
) -> pathlib.Path:
if not spirv_dis_file_path:
spirv_dis_file_path = util.tool_on_path(binaries_util.SPIRV_DIS_NAME)
shader_files = shader_job_util.get_related_files(
input_spirv_job_json_file_path,
language_suffix=[shader_job_util.SUFFIX_SPIRV])
util.copy_file(input_spirv_job_json_file_path,
output_spirv_job_json_file_path)
for shader_file in shader_files:
run_spirv_dis_on_spirv_shader(shader_file,
output_spirv_job_json_file_path.parent,
spirv_dis_file_path)
return output_spirv_job_json_file_path
def run_replay_on_shader_job(
spirv_fuzz_path: Path,
variant_shader_job_json: Path,
output_shader_job_json: Path,
other_args: Optional[List[str]] = None,
) -> Path:
"""Replays all transformations except the last on all shaders to get a similar variant shader job."""
util.copy_file(variant_shader_job_json, output_shader_job_json)
suffixes_that_exist = shader_job_util.get_related_suffixes_that_exist(
variant_shader_job_json, shader_job_util.EXT_ALL,
[shader_job_util.SUFFIX_SPIRV])
for suffix in suffixes_that_exist:
run_replay(
spirv_fuzz_path,
variant_shader_job_json.with_suffix(suffix),
output_shader_job_json.with_suffix(suffix),
other_args=other_args,
)
return output_shader_job_json
def run_generate_on_shader_job(
spirv_fuzz_path: Path,
reference_shader_json: Path,
output_shader_json: Path,
seed: Optional[str] = None,
other_args: Optional[List[str]] = None,
) -> Path:
util.copy_file(reference_shader_json, output_shader_json)
suffixes_that_exist = shader_job_util.get_related_suffixes_that_exist(
reference_shader_json, shader_job_util.EXT_ALL,
[shader_job_util.SUFFIX_SPIRV])
for suffix in suffixes_that_exist:
run_generate(
spirv_fuzz_path,
reference_shader_json.with_suffix(suffix),
output_shader_json.with_suffix(suffix),
seed,
other_args,
)
return output_shader_json
def run_reduction_part(
reduction_part_output_dir: Path,
source_dir_to_reduce: Path,
shader_job_name_to_reduce: str,
extension_to_reduce: str,
preserve_semantics: bool,
binary_manager: binaries_util.BinaryManager,
settings: Settings,
) -> Path:
test = test_util.metadata_read_from_source_dir(source_dir_to_reduce)
check(
bool(test.device and test.device.name),
AssertionError(
f"Cannot reduce {str(source_dir_to_reduce)}; device must be specified"
),
)
check(
bool(test.crash_signature),
AssertionError(
f"Cannot reduce {str(source_dir_to_reduce)} because there is no crash string specified."
),
)
output_dir = test_util.get_reduction_work_directory(
reduction_part_output_dir, shader_job_name_to_reduce)
final_shader_path = run_spirv_reduce_or_shrink(
source_dir=source_dir_to_reduce,
name_of_shader_job_to_reduce=shader_job_name_to_reduce,
extension_to_reduce=extension_to_reduce,
output_dir=output_dir,
preserve_semantics=preserve_semantics,
binary_manager=binary_manager,
settings=settings,
)
check(
final_shader_path.exists(),
fuzz_glsl_test.ReductionFailedError("Reduction failed.", output_dir),
)
# Finally, create the source_dir so the returned directory can be used as a test_dir.
# Copy the original source directory.
util.copy_dir(source_dir_to_reduce,
test_util.get_source_dir(reduction_part_output_dir))
# And then replace the shader.
# Destination file. E.g. reductions/source/variant/shader.frag.spv
output_shader_prefix = (
test_util.get_source_dir(reduction_part_output_dir) /
shader_job_name_to_reduce /
test_util.SHADER_JOB).with_suffix(extension_to_reduce +
shader_job_util.SUFFIX_SPIRV)
util.copy_file(
final_shader_path.with_suffix(shader_job_util.SUFFIX_SPIRV),
output_shader_prefix.with_suffix(shader_job_util.SUFFIX_SPIRV),
)
if preserve_semantics:
util.copy_file(
final_shader_path.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS),
output_shader_prefix.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS),
)
util.copy_file(
final_shader_path.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS_JSON),
output_shader_prefix.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS_JSON),
)
return test_util.get_source_dir(reduction_part_output_dir)
def run_reduction(
test_dir_reduction_output: Path,
test_dir_to_reduce: Path,
shader_job_name_to_reduce: str,
extension_to_reduce: str,
preserve_semantics: bool,
binary_manager: binaries_util.BinaryManager,
reduction_name: str = "reduction1",
) -> Path:
test = test_util.metadata_read(test_dir_to_reduce)
check(
bool(test.device and test.device.name),
AssertionError(
f"Cannot reduce {str(test_dir_to_reduce)}; "
f"device must be specified in {str(test_util.get_metadata_path(test_dir_to_reduce))}"
),
)
check(
bool(test.crash_signature),
AssertionError(
f"Cannot reduce {str(test_dir_to_reduce)} because there is no crash string specified."
),
)
# E.g. reports/crashes/no_signature/d50c96e8_opt_rand2_test_phone_ABC/results/phone_ABC/reductions/1
# Will contain work/ and source/
reduced_test_dir = test_util.get_reduced_test_dir(
test_dir_reduction_output, test.device.name, reduction_name)
source_dir = test_util.get_source_dir(test_dir_to_reduce)
output_dir = test_util.get_reduction_work_directory(
reduced_test_dir, shader_job_name_to_reduce)
if preserve_semantics:
final_shader_path = run_spirv_reduce_or_shrink(
source_dir=source_dir,
name_of_shader_job_to_reduce=shader_job_name_to_reduce,
extension_to_reduce=extension_to_reduce,
output_dir=output_dir,
preserve_semantics=preserve_semantics,
binary_manager=binary_manager,
)
else:
final_shader_path = run_spirv_reduce_or_shrink(
source_dir=source_dir,
name_of_shader_job_to_reduce=shader_job_name_to_reduce,
extension_to_reduce=extension_to_reduce,
output_dir=output_dir,
preserve_semantics=preserve_semantics,
binary_manager=binary_manager,
)
check(
final_shader_path.exists(),
ReductionFailedError("Reduction failed.", reduction_name, output_dir),
)
# Finally, create the source_dir so the returned directory can be used as a test_dir.
# Copy the original source directory.
util.copy_dir(source_dir, test_util.get_source_dir(reduced_test_dir))
# And then replace the shader.
# Destination file. E.g. reductions/source/variant/shader.frag.spv
output_shader_prefix = (
test_util.get_source_dir(reduced_test_dir) /
shader_job_name_to_reduce /
test_util.SHADER_JOB).with_suffix(extension_to_reduce +
shader_job_util.SUFFIX_SPIRV)
util.copy_file(
final_shader_path.with_suffix(shader_job_util.SUFFIX_SPIRV),
output_shader_prefix.with_suffix(shader_job_util.SUFFIX_SPIRV),
)
if preserve_semantics:
util.copy_file(
final_shader_path.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS),
output_shader_prefix.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS),
)
util.copy_file(
final_shader_path.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS_JSON),
output_shader_prefix.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS_JSON),
)
return reduced_test_dir
