def run_reduction(
source_dir_to_reduce: Path,
reduction_output_dir: Path,
binary_manager: binaries_util.BinaryManager,
settings: Settings,
) -> Path:
test = test_util.metadata_read_from_source_dir(source_dir_to_reduce)
reduced_source_dir = source_dir_to_reduce
reduced_source_dir = run_reduction_part(
reduction_part_output_dir=reduction_output_dir / "1",
source_dir_to_reduce=reduced_source_dir,
preserve_semantics=True,
binary_manager=binary_manager,
settings=settings,
)
if (test.crash_signature != signature_util.BAD_IMAGE_SIGNATURE
and not settings.skip_semantics_changing_reduction):
reduced_source_dir = run_reduction_part(
reduction_part_output_dir=reduction_output_dir / "2",
source_dir_to_reduce=reduced_source_dir,
preserve_semantics=False,
binary_manager=binary_manager,
settings=settings,
)
# Create and return a symlink to the "best" reduction part directory.
return util.make_directory_symlink(
new_symlink_file_path=reduction_output_dir / fuzz.BEST_REDUCTION_NAME,
existing_dir=reduced_source_dir.parent,
)
def main() -> None:
parser = argparse.ArgumentParser(
description="A tool for running a reduction on a source directory."
)
parser.add_argument(
"source_dir",
help="The source directory containing the shaders and the test.json file that describes how to run the test.",
)
parser.add_argument(
"--output", help="Output directory.", default="reduction_output",
)
parser.add_argument(
"--settings",
help="Path to a settings JSON file for this instance.",
default="settings.json",
)
parser.add_argument(
"--literals_to_uniforms",
action="store_true",
help="Pass --literals-to-uniforms to glsl-reduce.",
)
parsed_args = parser.parse_args(sys.argv[1:])
source_dir = Path(parsed_args.source_dir)
output_dir = Path(parsed_args.output)
settings = settings_util.read_or_create(Path(parsed_args.settings))
literals_to_uniforms: bool = parsed_args.literals_to_uniforms
binary_manager = binaries_util.get_default_binary_manager(settings=settings)
test = test_util.metadata_read_from_source_dir(source_dir)
if test.HasField("glsl"):
if (
literals_to_uniforms
and "--literals-to-uniforms" not in settings.extra_graphics_fuzz_reduce_args
):
settings.extra_graphics_fuzz_reduce_args.append("--literals-to-uniforms")
fuzz_glsl_test.run_reduction(
source_dir_to_reduce=source_dir,
reduction_output_dir=output_dir,
binary_manager=binary_manager,
settings=settings,
)
elif test.HasField("spirv_fuzz"):
fuzz_spirv_test.run_reduction(
source_dir_to_reduce=source_dir,
reduction_output_dir=output_dir,
binary_manager=binary_manager,
settings=settings,
)
else:
raise AssertionError(f"Unknown test type: {test}")
def create_spirv_fuzz_variant_2(
source_dir: Path, binary_manager: binaries_util.BinaryManager, settings: Settings,
) -> Optional[Path]:
"""
Replays all transformations except the last to get variant_2.
Replays all transformations except the last to get a variant_2 shader job, such that variant <-> variant_2 are
likely even more similar than reference <-> variant.
|source_dir| must be a spirv_fuzz test.
"""
test_metadata: Test = test_util.metadata_read_from_source_dir(source_dir)
check(test_metadata.HasField("spirv_fuzz"), AssertionError("Not a spirv_fuzz test"))
variant_shader_job = source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB
variant_2_shader_job = (
source_dir / f"{test_util.VARIANT_DIR}_2" / test_util.SHADER_JOB
)
if not variant_shader_job.is_file():
log(
f"Skip generating variant_2 for {str(source_dir)} because the variant shader job was not found."
)
return None
if variant_2_shader_job.is_file():
log(
f"Skip generating variant_2 for {str(source_dir)} because variant_2 shader job already exists."
)
return None
return spirv_fuzz_util.run_replay_on_shader_job(
spirv_fuzz_path=binary_manager.get_binary_path_by_name(
binaries_util.SPIRV_FUZZ_NAME
).path,
variant_shader_job_json=variant_shader_job,
output_shader_job_json=variant_2_shader_job,
other_args=list(settings.common_spirv_args),
)
def run_reduction_part(
reduction_part_output_dir: Path,
source_dir_to_reduce: Path,
preserve_semantics: bool,
binary_manager: binaries_util.BinaryManager,
settings: Settings,
) -> Path:
test = test_util.metadata_read_from_source_dir(source_dir_to_reduce)
if not test.device or not test.device.name:
raise AssertionError(
f"Cannot reduce {str(source_dir_to_reduce)}; "
f"device must be specified in {str(test_util.get_metadata_path_from_source_dir(source_dir_to_reduce))}"
)
if not test.crash_signature:
raise AssertionError(
f"Cannot reduce {str(source_dir_to_reduce)} because there is no crash string specified."
)
shader_jobs = tool.get_shader_jobs(source_dir_to_reduce)
# TODO: if needed, this could become a parameter to this function.
name_of_shader_to_reduce = shader_jobs[0].name
if len(shader_jobs) > 1:
check(
len(shader_jobs) == 2
and shader_jobs[1].name == test_util.VARIANT_DIR,
AssertionError(
"Can only reduce tests with shader jobs reference and variant, or just variant."
),
)
name_of_shader_to_reduce = shader_jobs[1].name
reduction_work_variant_dir = run_glsl_reduce(
source_dir=source_dir_to_reduce,
name_of_shader_to_reduce=name_of_shader_to_reduce,
output_dir=test_util.get_reduction_work_directory(
reduction_part_output_dir, name_of_shader_to_reduce),
binary_manager=binary_manager,
preserve_semantics=preserve_semantics,
extra_args=list(settings.extra_graphics_fuzz_reduce_args)
if settings.extra_graphics_fuzz_reduce_args else None,
)
final_reduced_shader_job_path = get_final_reduced_shader_job_path(
reduction_work_variant_dir)
check(
final_reduced_shader_job_path.exists(),
ReductionFailedError("Reduction failed.", reduction_work_variant_dir),
)
# Finally, create the output source_dir.
util.copy_dir(source_dir_to_reduce,
test_util.get_source_dir(reduction_part_output_dir))
shader_job_util.copy(
final_reduced_shader_job_path,
test_util.get_shader_job_path(reduction_part_output_dir,
name_of_shader_to_reduce),
)
if not settings.keep_reduction_work:
shutil.rmtree(reduction_work_variant_dir)
return test_util.get_source_dir(reduction_part_output_dir)
def run_reduction(
source_dir_to_reduce: Path,
reduction_output_dir: Path,
binary_manager: binaries_util.BinaryManager,
settings: Settings,
) -> Path:
test = test_util.metadata_read_from_source_dir(source_dir_to_reduce)
shader_jobs = tool.get_shader_jobs(source_dir_to_reduce)
# TODO: if needed, this could become a parameter to this function.
shader_job_to_reduce = shader_jobs[0]
if len(shader_jobs) > 1:
check(
len(shader_jobs) == 2
and shader_jobs[1].name == test_util.VARIANT_DIR,
AssertionError(
"Can only reduce tests with shader jobs reference and variant, or just variant."
),
)
shader_job_to_reduce = shader_jobs[1]
shader_transformation_suffixes = shader_job_util.get_related_suffixes_that_exist(
shader_job_to_reduce.shader_job,
language_suffix=(shader_job_util.SUFFIX_TRANSFORMATIONS, ),
)
shader_spv_suffixes = shader_job_util.get_related_suffixes_that_exist(
shader_job_to_reduce.shader_job,
language_suffix=(shader_job_util.SUFFIX_SPIRV, ))
reduced_source_dir = source_dir_to_reduce
for index, suffix in enumerate(shader_transformation_suffixes):
# E.g. .frag.transformations -> .frag
extension_to_reduce = str(Path(suffix).with_suffix(""))
reduced_source_dir = run_reduction_part(
reduction_part_output_dir=reduction_output_dir /
f"0_{index}_{suffix.split('.')[1]}",
source_dir_to_reduce=reduced_source_dir,
shader_job_name_to_reduce=shader_job_to_reduce.name,
extension_to_reduce=extension_to_reduce,
preserve_semantics=True,
binary_manager=binary_manager,
settings=settings,
)
if (test.crash_signature != signature_util.BAD_IMAGE_SIGNATURE
and not settings.skip_spirv_reduce):
for index, suffix in enumerate(shader_spv_suffixes):
# E.g. .frag.spv -> .frag
extension_to_reduce = str(Path(suffix).with_suffix(""))
reduced_source_dir = run_reduction_part(
reduction_part_output_dir=reduction_output_dir /
f"1_{index}_{suffix.split('.')[1]}",
source_dir_to_reduce=reduced_source_dir,
shader_job_name_to_reduce=shader_job_to_reduce.name,
extension_to_reduce=extension_to_reduce,
preserve_semantics=False,
binary_manager=binary_manager,
settings=settings,
)
# Create and return a symlink to the "best" reduction.
return util.make_directory_symlink(
new_symlink_file_path=reduction_output_dir / fuzz.BEST_REDUCTION_NAME,
existing_dir=reduced_source_dir.parent,
)
def run_reduction_part(
reduction_part_output_dir: Path,
source_dir_to_reduce: Path,
shader_job_name_to_reduce: str,
extension_to_reduce: str,
preserve_semantics: bool,
binary_manager: binaries_util.BinaryManager,
settings: Settings,
) -> Path:
test = test_util.metadata_read_from_source_dir(source_dir_to_reduce)
check(
bool(test.device and test.device.name),
AssertionError(
f"Cannot reduce {str(source_dir_to_reduce)}; device must be specified"
),
)
check(
bool(test.crash_signature),
AssertionError(
f"Cannot reduce {str(source_dir_to_reduce)} because there is no crash string specified."
),
)
output_dir = test_util.get_reduction_work_directory(
reduction_part_output_dir, shader_job_name_to_reduce)
final_shader_path = run_spirv_reduce_or_shrink(
source_dir=source_dir_to_reduce,
name_of_shader_job_to_reduce=shader_job_name_to_reduce,
extension_to_reduce=extension_to_reduce,
output_dir=output_dir,
preserve_semantics=preserve_semantics,
binary_manager=binary_manager,
settings=settings,
)
check(
final_shader_path.exists(),
fuzz_glsl_test.ReductionFailedError("Reduction failed.", output_dir),
)
# Finally, create the source_dir so the returned directory can be used as a test_dir.
# Copy the original source directory.
util.copy_dir(source_dir_to_reduce,
test_util.get_source_dir(reduction_part_output_dir))
# And then replace the shader.
# Destination file. E.g. reductions/source/variant/shader.frag.spv
output_shader_prefix = (
test_util.get_source_dir(reduction_part_output_dir) /
shader_job_name_to_reduce /
test_util.SHADER_JOB).with_suffix(extension_to_reduce +
shader_job_util.SUFFIX_SPIRV)
util.copy_file(
final_shader_path.with_suffix(shader_job_util.SUFFIX_SPIRV),
output_shader_prefix.with_suffix(shader_job_util.SUFFIX_SPIRV),
)
if preserve_semantics:
util.copy_file(
final_shader_path.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS),
output_shader_prefix.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS),
)
util.copy_file(
final_shader_path.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS_JSON),
output_shader_prefix.with_suffix(
shader_job_util.SUFFIX_TRANSFORMATIONS_JSON),
)
return test_util.get_source_dir(reduction_part_output_dir)
def run_spirv_reduce_or_shrink( # pylint: disable=too-many-locals;
source_dir: Path,
name_of_shader_job_to_reduce: str,
extension_to_reduce: str,
output_dir: Path,
preserve_semantics: bool,
binary_manager: binaries_util.BinaryManager,
settings: Settings,
) -> Path:
test = test_util.metadata_read_from_source_dir(source_dir)
input_shader_job = source_dir / name_of_shader_job_to_reduce / test_util.SHADER_JOB
original_spirv_file = input_shader_job.with_suffix(
extension_to_reduce + shader_job_util.SUFFIX_SPIRV_ORIG)
transformed_spirv_file = input_shader_job.with_suffix(
extension_to_reduce + shader_job_util.SUFFIX_SPIRV)
transformations_file = input_shader_job.with_suffix(
extension_to_reduce + shader_job_util.SUFFIX_TRANSFORMATIONS)
util.mkdirs_p(output_dir)
final_shader = output_dir / "final.spv"
# E.g. transformation_suffix_to_reduce == ".frag.transformations"
# E.g. ".frag.??" -> ".frag.spv"
shader_suffix_to_override = extension_to_reduce + shader_job_util.SUFFIX_SPIRV
if preserve_semantics:
cmd = [
str(binary_manager.get_binary_path_by_name("spirv-fuzz").path),
str(original_spirv_file),
"-o",
str(final_shader),
f"--shrink={str(transformations_file)}",
f"--shrinker-temp-file-prefix={str(output_dir / 'temp_')}",
]
cmd += list(settings.extra_spirv_fuzz_shrink_args)
cmd += list(test.common_spirv_args)
cmd += [
# This ensures the arguments that follow are all positional arguments.
"--",
"gfauto_interestingness_test",
str(source_dir),
# --override_shader requires three parameters to follow; the third will be added by spirv-fuzz (the shader.spv file).
"--override_shader",
name_of_shader_job_to_reduce,
shader_suffix_to_override,
]
else:
cmd = [
str(binary_manager.get_binary_path_by_name("spirv-reduce").path),
str(transformed_spirv_file),
"-o",
str(final_shader),
f"--temp-file-prefix={str(output_dir / 'temp_')}",
]
cmd += list(settings.extra_spirv_reduce_args)
cmd += list(test.common_spirv_args)
cmd += [
# This ensures the arguments that follow are all positional arguments.
"--",
"gfauto_interestingness_test",
str(source_dir),
# --override_shader requires three parameters to follow; the third will be added by spirv-reduce (the shader.spv file).
"--override_shader",
name_of_shader_job_to_reduce,
shader_suffix_to_override,
]
# Log the reduction.
with util.file_open_text(output_dir / "command.log", "w") as f:
gflogging.push_stream_for_logging(f)
try:
# The reducer can fail, but it will typically output an exception file, so we can ignore the exit code.
subprocess_util.run(cmd, verbose=True, check_exit_code=False)
finally:
gflogging.pop_stream_for_logging()
return final_shader
