def create_summary_and_reproduce(
test_dir: Path, binary_manager: binaries_util.BinaryManager) -> None:
util.mkdirs_p(test_dir / "summary")
test_metadata = test_util.metadata_read(test_dir)
# noinspection PyTypeChecker
if test_metadata.HasField("glsl") or test_metadata.HasField("spirv_fuzz"):
fuzz_glsl_test.create_summary_and_reproduce(test_dir, binary_manager)
else:
raise AssertionError("Unrecognized test type")
def maybe_add_report( # pylint: disable=too-many-locals;
test_dir: Path, reports_dir: Path, device: Device,
settings: Settings) -> Optional[Path]:
result_output_dir = test_util.get_results_directory(test_dir, device.name)
status = result_util.get_status(result_output_dir)
report_subdirectory_name = ""
if status == fuzz.STATUS_CRASH:
report_subdirectory_name = "crashes"
elif status == fuzz.STATUS_TOOL_CRASH:
report_subdirectory_name = "tool_crashes"
elif status == fuzz.STATUS_UNRESPONSIVE:
report_subdirectory_name = "unresponsive"
if not report_subdirectory_name:
return None
log_path = result_util.get_log_path(result_output_dir)
log_contents = util.file_read_text(log_path)
signature = signature_util.get_signature_from_log_contents(log_contents)
signature_dir = reports_dir / report_subdirectory_name / signature
util.mkdirs_p(signature_dir)
# If the signature_dir contains a NOT_INTERESTING file, then don't bother creating a report.
if (signature_dir / "NOT_INTERESTING").exists():
return None
if signature != signature_util.BAD_IMAGE_SIGNATURE:
# If we have reached the maximum number of crashes per signature for this device, don't create a report.
num_duplicates = [
report_dir for report_dir in signature_dir.iterdir() if
report_dir.is_dir() and report_dir.name.endswith(f"_{device.name}")
]
if len(num_duplicates) >= settings.maximum_duplicate_crashes:
return None
# We include the device name in the directory name because it is possible that this test crashes on two
# different devices but gives the same crash signature in both cases (e.g. for generic signatures
# like "compile_error"). This would lead to two test copies having the same path.
# It also means we can limit duplicates per device using the directory name.
test_dir_in_reports = signature_dir / f"{test_dir.name}_{device.name}"
util.copy_dir(test_dir, test_dir_in_reports)
if signature != signature_util.BAD_IMAGE_SIGNATURE:
# If we found a crash, rename the directories for all shaders other than the variant. Thus, only the variant
# shader will run.
bad_shader_name = result_util.get_status_bad_shader_name(
test_util.get_results_directory(test_dir_in_reports, device.name))
# TODO: Could possibly improve this. Could try scanning the Amber log to figure out which shader failed?
if not bad_shader_name:
log("WARNING: assuming that the bad shader is the variant")
bad_shader_name = test_util.VARIANT_DIR
shader_jobs = tool.get_shader_jobs(
test_util.get_source_dir(test_dir_in_reports))
found_bad_shader = False
for shader_job in shader_jobs:
if shader_job.name == bad_shader_name:
found_bad_shader = True
else:
shader_job.shader_job.parent.rename(
shader_job.shader_job.parent.parent /
f"_{shader_job.name}")
check(
found_bad_shader,
AssertionError(
f"Could not find bad shader at: {test_util.get_source_dir(test_dir_in_reports) / bad_shader_name}"
),
)
test_metadata = test_util.metadata_read(test_dir_in_reports)
test_metadata.crash_signature = signature
test_metadata.device.CopyFrom(device)
test_metadata.expected_status = status
test_util.metadata_write(test_metadata, test_dir_in_reports)
return test_dir_in_reports
def fuzz_glsl( # pylint: disable=too-many-locals;
staging_dir: Path,
reports_dir: Path,
fuzz_failures_dir: Path,
active_devices: List[Device],
references: List[Path],
donors_dir: Path,
settings: Settings,
binary_manager: binaries_util.BinaryManager,
) -> None:
staging_name = staging_dir.name
template_source_dir = staging_dir / "source_template"
# Pick a randomly chosen reference.
unprepared_reference_shader_job: Path = random.choice(references)
# The "graphicsfuzz-tool" tool is designed to be on your PATH so that e.g. ".bat" will be appended on Windows.
# So we use tool_on_path with a custom PATH to get the actual file we want to execute.
graphicsfuzz_tool_path = util.tool_on_path(
"graphicsfuzz-tool",
str(
binary_manager.get_binary_path_by_name(
"graphicsfuzz-tool").path.parent),
)
try:
with util.file_open_text(staging_dir / "log.txt", "w") as log_file:
try:
gflogging.push_stream_for_logging(log_file)
# Create the prepared (for Vulkan GLSL) reference.
glsl_generate_util.run_prepare_reference(
graphicsfuzz_tool_path,
unprepared_reference_shader_job,
template_source_dir / test_util.REFERENCE_DIR /
test_util.SHADER_JOB,
legacy_graphics_fuzz_vulkan_arg=settings.
legacy_graphics_fuzz_vulkan_arg,
)
# Generate the variant (GraphicsFuzz requires the unprepared reference as input).
glsl_generate_util.run_generate(
graphicsfuzz_tool_path,
unprepared_reference_shader_job,
donors_dir,
template_source_dir / test_util.VARIANT_DIR /
test_util.SHADER_JOB,
seed=str(
random.getrandbits(
glsl_generate_util.GENERATE_SEED_BITS)),
other_args=list(settings.extra_graphics_fuzz_generate_args)
if settings.extra_graphics_fuzz_generate_args else None,
legacy_graphics_fuzz_vulkan_arg=settings.
legacy_graphics_fuzz_vulkan_arg,
)
finally:
gflogging.pop_stream_for_logging()
except subprocess.CalledProcessError:
util.mkdirs_p(fuzz_failures_dir)
if len(list(
fuzz_failures_dir.iterdir())) < settings.maximum_fuzz_failures:
util.copy_dir(staging_dir, fuzz_failures_dir / staging_dir.name)
return
reference_name = unprepared_reference_shader_job.stem
stable_shader = reference_name.startswith("stable_")
common_spirv_args = list(settings.common_spirv_args)
test_dirs = [
make_test(
template_source_dir,
staging_dir / f"{staging_name}_no_opt_test",
spirv_opt_args=None,
binary_manager=binary_manager,
derived_from=reference_name,
stable_shader=stable_shader,
common_spirv_args=common_spirv_args,
),
make_test(
template_source_dir,
staging_dir / f"{staging_name}_opt_O_test",
spirv_opt_args=["-O"],
binary_manager=binary_manager,
derived_from=reference_name,
stable_shader=stable_shader,
common_spirv_args=common_spirv_args,
),
]
if not settings.spirv_opt_just_o:
test_dirs += [
make_test(
template_source_dir,
staging_dir / f"{staging_name}_opt_Os_test",
spirv_opt_args=["-Os"],
binary_manager=binary_manager,
derived_from=reference_name,
stable_shader=stable_shader,
common_spirv_args=common_spirv_args,
),
make_test(
template_source_dir,
staging_dir / f"{staging_name}_opt_rand1_test",
spirv_opt_args=spirv_opt_util.random_spirv_opt_args(),
binary_manager=binary_manager,
derived_from=reference_name,
stable_shader=stable_shader,
common_spirv_args=common_spirv_args,
),
make_test(
template_source_dir,
staging_dir / f"{staging_name}_opt_rand2_test",
spirv_opt_args=spirv_opt_util.random_spirv_opt_args(),
binary_manager=binary_manager,
derived_from=reference_name,
stable_shader=stable_shader,
common_spirv_args=common_spirv_args,
),
make_test(
template_source_dir,
staging_dir / f"{staging_name}_opt_rand3_test",
spirv_opt_args=spirv_opt_util.random_spirv_opt_args(),
binary_manager=binary_manager,
derived_from=reference_name,
stable_shader=stable_shader,
common_spirv_args=common_spirv_args,
),
]
for test_dir in test_dirs:
interrupt_util.interrupt_if_needed()
if handle_test(test_dir, reports_dir, active_devices, binary_manager,
settings):
# If we generated a report, don't bother trying other optimization combinations.
break
def fuzz_spirv( # pylint: disable=too-many-locals;
staging_dir: Path,
reports_dir: Path,
fuzz_failures_dir: Path,
active_devices: List[Device],
spirv_fuzz_shaders: List[Path],
settings: Settings,
binary_manager: binaries_util.BinaryManager,
) -> None:
staging_name = staging_dir.name
template_source_dir = staging_dir / "source_template"
reference_spirv_shader_job_orig_path: Path = random.choice(
spirv_fuzz_shaders)
# Copy in a randomly chosen reference.
reference_spirv_shader_job = shader_job_util.copy(
reference_spirv_shader_job_orig_path,
template_source_dir / test_util.REFERENCE_DIR / test_util.SHADER_JOB,
language_suffix=shader_job_util.SUFFIXES_SPIRV_FUZZ_INPUT,
)
try:
with util.file_open_text(staging_dir / "log.txt", "w") as log_file:
try:
gflogging.push_stream_for_logging(log_file)
spirv_fuzz_util.run_generate_on_shader_job(
binary_manager.get_binary_path_by_name("spirv-fuzz").path,
reference_spirv_shader_job,
template_source_dir / test_util.VARIANT_DIR /
test_util.SHADER_JOB,
donor_shader_job_paths=spirv_fuzz_shaders,
seed=str(
random.getrandbits(
spirv_fuzz_util.GENERATE_SEED_BITS)),
other_args=list(settings.extra_spirv_fuzz_generate_args) +
list(settings.common_spirv_args),
)
finally:
gflogging.pop_stream_for_logging()
except subprocess.CalledProcessError:
util.mkdirs_p(fuzz_failures_dir)
if len(list(
fuzz_failures_dir.iterdir())) < settings.maximum_fuzz_failures:
util.copy_dir(staging_dir, fuzz_failures_dir / staging_dir.name)
return
reference_name = reference_spirv_shader_job_orig_path.stem
stable_shader = reference_name.startswith("stable_")
common_spirv_args = list(settings.common_spirv_args)
test_dirs = [
make_test(
template_source_dir,
staging_dir / f"{staging_name}_no_opt_test",
spirv_opt_args=None,
binary_manager=binary_manager,
derived_from=reference_name,
stable_shader=stable_shader,
common_spirv_args=common_spirv_args,
),
make_test(
template_source_dir,
staging_dir / f"{staging_name}_opt_O_test",
spirv_opt_args=["-O"],
binary_manager=binary_manager,
derived_from=reference_name,
stable_shader=stable_shader,
common_spirv_args=common_spirv_args,
),
make_test(
template_source_dir,
staging_dir / f"{staging_name}_opt_Os_test",
spirv_opt_args=["-Os"],
binary_manager=binary_manager,
derived_from=reference_name,
stable_shader=stable_shader,
common_spirv_args=common_spirv_args,
),
make_test(
template_source_dir,
staging_dir / f"{staging_name}_opt_rand1_test",
spirv_opt_args=spirv_opt_util.random_spirv_opt_args(),
binary_manager=binary_manager,
derived_from=reference_name,
stable_shader=stable_shader,
common_spirv_args=common_spirv_args,
),
make_test(
template_source_dir,
staging_dir / f"{staging_name}_opt_rand2_test",
spirv_opt_args=spirv_opt_util.random_spirv_opt_args(),
binary_manager=binary_manager,
derived_from=reference_name,
stable_shader=stable_shader,
common_spirv_args=common_spirv_args,
),
make_test(
template_source_dir,
staging_dir / f"{staging_name}_opt_rand3_test",
spirv_opt_args=spirv_opt_util.random_spirv_opt_args(),
binary_manager=binary_manager,
derived_from=reference_name,
stable_shader=stable_shader,
common_spirv_args=common_spirv_args,
),
]
for test_dir in test_dirs:
interrupt_util.interrupt_if_needed()
if handle_test(test_dir, reports_dir, active_devices, binary_manager,
settings):
# If we generated a report, don't bother trying other optimization combinations.
break
def run_spirv_reduce_or_shrink( # pylint: disable=too-many-locals;
source_dir: Path,
name_of_shader_job_to_reduce: str,
extension_to_reduce: str,
output_dir: Path,
preserve_semantics: bool,
binary_manager: binaries_util.BinaryManager,
settings: Settings,
) -> Path:
test = test_util.metadata_read_from_source_dir(source_dir)
input_shader_job = source_dir / name_of_shader_job_to_reduce / test_util.SHADER_JOB
original_spirv_file = input_shader_job.with_suffix(
extension_to_reduce + shader_job_util.SUFFIX_SPIRV_ORIG)
transformed_spirv_file = input_shader_job.with_suffix(
extension_to_reduce + shader_job_util.SUFFIX_SPIRV)
transformations_file = input_shader_job.with_suffix(
extension_to_reduce + shader_job_util.SUFFIX_TRANSFORMATIONS)
util.mkdirs_p(output_dir)
final_shader = output_dir / "final.spv"
# E.g. transformation_suffix_to_reduce == ".frag.transformations"
# E.g. ".frag.??" -> ".frag.spv"
shader_suffix_to_override = extension_to_reduce + shader_job_util.SUFFIX_SPIRV
if preserve_semantics:
cmd = [
str(binary_manager.get_binary_path_by_name("spirv-fuzz").path),
str(original_spirv_file),
"-o",
str(final_shader),
f"--shrink={str(transformations_file)}",
f"--shrinker-temp-file-prefix={str(output_dir / 'temp_')}",
]
cmd += list(settings.extra_spirv_fuzz_shrink_args)
cmd += list(test.common_spirv_args)
cmd += [
# This ensures the arguments that follow are all positional arguments.
"--",
"gfauto_interestingness_test",
str(source_dir),
# --override_shader requires three parameters to follow; the third will be added by spirv-fuzz (the shader.spv file).
"--override_shader",
name_of_shader_job_to_reduce,
shader_suffix_to_override,
]
else:
cmd = [
str(binary_manager.get_binary_path_by_name("spirv-reduce").path),
str(transformed_spirv_file),
"-o",
str(final_shader),
f"--temp-file-prefix={str(output_dir / 'temp_')}",
]
cmd += list(settings.extra_spirv_reduce_args)
cmd += list(test.common_spirv_args)
cmd += [
# This ensures the arguments that follow are all positional arguments.
"--",
"gfauto_interestingness_test",
str(source_dir),
# --override_shader requires three parameters to follow; the third will be added by spirv-reduce (the shader.spv file).
"--override_shader",
name_of_shader_job_to_reduce,
shader_suffix_to_override,
]
# Log the reduction.
with util.file_open_text(output_dir / "command.log", "w") as f:
gflogging.push_stream_for_logging(f)
try:
# The reducer can fail, but it will typically output an exception file, so we can ignore the exit code.
subprocess_util.run(cmd, verbose=True, check_exit_code=False)
finally:
gflogging.pop_stream_for_logging()
return final_shader
def main_helper( # pylint: disable=too-many-locals,too-many-branches,too-many-statements;
tests_dir: Path,
work_dir: Path,
binaries: binaries_util.BinaryManager,
settings: Settings,
active_devices: List[Device],
results_out_handle: Optional[TextIO],
updated_settings_output_path: Optional[Path],
) -> None:
util.mkdirs_p(work_dir)
def write_entry(entry: str) -> None:
if not results_out_handle:
return
results_out_handle.write(entry)
results_out_handle.write(", ")
results_out_handle.flush()
def write_newline() -> None:
if not results_out_handle:
return
results_out_handle.write("\n")
results_out_handle.flush()
spirv_opt_path: Optional[Path] = None
swift_shader_path: Optional[Path] = None
amber_path: Optional[Path] = None
# Small hack to ensure we have three devices for spirv-opt, each with a different name.
main_spirv_opt_device: Optional[Device] = None
if active_devices and active_devices[0].name == "host_preprocessor":
main_spirv_opt_device = active_devices[0]
main_spirv_opt_device.name = SPIRV_OPT_O
spirv_opt_custom = Device()
spirv_opt_custom.CopyFrom(main_spirv_opt_device)
spirv_opt_custom.name = SPIRV_OPT_CUSTOM
active_devices.insert(1, spirv_opt_custom)
spirv_opt_os = Device()
spirv_opt_os.CopyFrom(main_spirv_opt_device)
spirv_opt_os.name = SPIRV_OPT_OS
active_devices.insert(1, spirv_opt_os)
# Enumerate active devices, writing their name and storing binary paths if needed.
write_entry("test")
for device in active_devices:
write_entry(device.name)
if device.HasField("preprocess"):
spirv_opt_path = binaries.get_binary_path_by_name(
binaries_util.SPIRV_OPT_NAME).path
if device.HasField("swift_shader"):
swift_shader_path = binaries.get_binary_path_by_name(
binaries_util.SWIFT_SHADER_NAME).path
if device.HasField("swift_shader") or device.HasField("host"):
amber_path = binaries.get_binary_path_by_name(
binaries_util.AMBER_NAME).path
write_newline()
# Enumerate tests and devices, writing the results.
for test in sorted(tests_dir.glob("*.amber")):
test_name = util.remove_end(test.name, ".amber")
write_entry(test_name)
spirv_shaders = sorted(
tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv"))
for device in active_devices:
test_run_dir = work_dir / f"{test_name}_{device.name}"
util.mkdirs_p(test_run_dir)
ignored_signatures_set: Set[str] = set(
device.ignored_crash_signatures)
with util.file_open_text(test_run_dir / "log.txt",
"w") as log_stream:
try:
gflogging.push_stream_for_logging(log_stream)
if device.HasField("preprocess"):
# This just means spirv-opt for now.
assert spirv_opt_path # noqa
assert main_spirv_opt_device # noqa
# Pick spirv-opt arguments based on device name.
if device.name == SPIRV_OPT_O:
spirv_opt_args = ["-O"]
elif device.name == SPIRV_OPT_OS:
spirv_opt_args = ["-Os"]
elif device.name == SPIRV_OPT_CUSTOM:
spirv_opt_args = (spirv_opt_util.
OPT_INTERESTING_SUBSET_OF_PASSES)
else:
raise AssertionError(
f"Can't tell how to run device {device.name}; "
f"must be named host_preprocessor and be the first active device."
)
# Reset device and ignored_crash_signatures.
device = main_spirv_opt_device
ignored_signatures_set = set(
device.ignored_crash_signatures)
try:
for spirv_shader in spirv_shaders:
spirv_opt_util.run_spirv_opt_on_spirv_shader(
spirv_shader,
test_run_dir,
spirv_opt_args,
spirv_opt_path,
)
result_util.write_status(
test_run_dir,
fuzz.STATUS_SUCCESS,
)
except subprocess.CalledProcessError:
result_util.write_status(
test_run_dir,
fuzz.STATUS_TOOL_CRASH,
)
except subprocess.TimeoutExpired:
result_util.write_status(
test_run_dir,
fuzz.STATUS_TOOL_TIMEOUT,
)
elif device.HasField("shader_compiler"):
try:
for spirv_shader in spirv_shaders:
shader_compiler_util.run_shader(
shader_compiler_device=device.
shader_compiler,
shader_path=spirv_shader,
output_dir=test_run_dir,
compiler_path=binaries.
get_binary_path_by_name(
device.shader_compiler.binary).path,
timeout=DEFAULT_TIMEOUT,
)
result_util.write_status(
test_run_dir,
fuzz.STATUS_SUCCESS,
)
except subprocess.CalledProcessError:
result_util.write_status(
test_run_dir,
fuzz.STATUS_CRASH,
)
except subprocess.TimeoutExpired:
result_util.write_status(
test_run_dir,
fuzz.STATUS_TIMEOUT,
)
elif device.HasField("swift_shader"):
assert swift_shader_path # noqa
assert amber_path # noqa
host_device_util.run_amber(
test,
test_run_dir,
amber_path=amber_path,
dump_image=False,
dump_buffer=False,
icd=swift_shader_path,
)
elif device.HasField("host"):
assert amber_path # noqa
host_device_util.run_amber(
test,
test_run_dir,
amber_path=amber_path,
dump_image=False,
dump_buffer=False,
custom_launcher=list(device.host.custom_launcher),
)
elif device.HasField("android"):
android_device.run_amber_on_device(
test,
test_run_dir,
dump_image=False,
dump_buffer=False,
serial=device.android.serial,
)
else:
raise AssertionError(
f"Unsupported device {device.name}")
finally:
gflogging.pop_stream_for_logging()
status = result_util.get_status(test_run_dir)
if status == fuzz.STATUS_SUCCESS:
write_entry("P")
elif status in (fuzz.STATUS_TIMEOUT, fuzz.STATUS_TOOL_TIMEOUT):
write_entry("T")
else:
write_entry("F")
# Update ignored signatures.
if (status in (
fuzz.STATUS_TOOL_CRASH,
fuzz.STATUS_CRASH,
fuzz.STATUS_UNRESPONSIVE,
) and updated_settings_output_path):
log_contents = util.file_read_text(
result_util.get_log_path(test_run_dir))
signature = signature_util.get_signature_from_log_contents(
log_contents)
if signature == signature_util.NO_SIGNATURE:
log(f"NOT updating ignored signatures to include {signature}"
)
elif signature in ignored_signatures_set:
log(f"Signature is already ignored: {signature}")
else:
log(f"Adding ignored signature: {signature}")
device.ignored_crash_signatures.append(signature)
write_newline()
if updated_settings_output_path:
# Reset main_spirv_opt_device name before writing it back out.
if main_spirv_opt_device:
main_spirv_opt_device.name = "host_preprocessor"
settings_util.write(settings, updated_settings_output_path)
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements;
parser = argparse.ArgumentParser(
description="Runs GraphicsFuzz AmberScript tests on the active devices listed in "
"the settings.json file.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
"--settings",
help="Path to the settings JSON file for this instance.",
default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH),
)
parser.add_argument(
"--tests",
help="Path to the directory of AmberScript tests with shaders extracted.",
default="graphicsfuzz",
)
parser.add_argument(
"--update_ignored_signatures",
help="As the tests are run for each device, add any crash signatures to the device's ignored_crash_signatures "
"property and write out the updated settings.json file.",
action="store_true",
)
parser.add_argument(
"--results_out",
help="Output file path for the CSV results table.",
default="results.csv",
)
parsed_args = parser.parse_args(sys.argv[1:])
# Args.
tests_dir: Path = Path(parsed_args.tests)
settings_path: Path = Path(parsed_args.settings)
update_ignored_signatures: bool = parsed_args.update_ignored_signatures
results_out_path: Path = Path(parsed_args.results_out)
# Settings and devices.
settings = settings_util.read_or_create(settings_path)
active_devices = devices_util.get_active_devices(settings.device_list)
# Binaries.
binaries = binaries_util.get_default_binary_manager(settings=settings)
work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}"
util.mkdirs_p(work_dir)
with util.file_open_text(results_out_path, "w") as results_handle:
def write_entry(entry: str) -> None:
results_handle.write(entry)
results_handle.write(", ")
results_handle.flush()
def write_newline() -> None:
results_handle.write("\n")
results_handle.flush()
spirv_opt_path: Optional[Path] = None
swift_shader_path: Optional[Path] = None
amber_path: Optional[Path] = None
# Small hack to ensure we have three devices for spirv-opt, each with a different name.
main_spirv_opt_device: Optional[Device] = None
if active_devices and active_devices[0].name == "host_preprocessor":
main_spirv_opt_device = active_devices[0]
main_spirv_opt_device.name = SPIRV_OPT_O
spirv_opt_custom = Device()
spirv_opt_custom.CopyFrom(main_spirv_opt_device)
spirv_opt_custom.name = SPIRV_OPT_CUSTOM
active_devices.insert(1, spirv_opt_custom)
spirv_opt_os = Device()
spirv_opt_os.CopyFrom(main_spirv_opt_device)
spirv_opt_os.name = SPIRV_OPT_OS
active_devices.insert(1, spirv_opt_os)
# Enumerate active devices, writing their name and storing binary paths if needed.
write_entry("test")
for device in active_devices:
write_entry(device.name)
if device.HasField("preprocess"):
spirv_opt_path = binaries.get_binary_path_by_name(
binaries_util.SPIRV_OPT_NAME
).path
if device.HasField("swift_shader"):
swift_shader_path = binaries.get_binary_path_by_name(
binaries_util.SWIFT_SHADER_NAME
).path
if device.HasField("swift_shader") or device.HasField("host"):
amber_path = binaries.get_binary_path_by_name(
binaries_util.AMBER_NAME
).path
write_newline()
# Enumerate tests and devices, writing the results.
for test in sorted(tests_dir.glob("*.amber")):
test_name = util.remove_end(test.name, ".amber")
write_entry(test_name)
spirv_shaders = sorted(
tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv")
)
for device in active_devices:
test_run_dir = work_dir / f"{test_name}_{device.name}"
util.mkdirs_p(test_run_dir)
ignored_signatures_set: Set[str] = set(device.ignored_crash_signatures)
with util.file_open_text(test_run_dir / "log.txt", "w") as log_stream:
try:
gflogging.push_stream_for_logging(log_stream)
if device.HasField("preprocess"):
# This just means spirv-opt for now.
assert spirv_opt_path # noqa
assert main_spirv_opt_device # noqa
# Pick spirv-opt arguments based on device name.
if device.name == SPIRV_OPT_O:
spirv_opt_args = ["-O"]
elif device.name == SPIRV_OPT_OS:
spirv_opt_args = ["-Os"]
elif device.name == SPIRV_OPT_CUSTOM:
spirv_opt_args = (
spirv_opt_util.OPT_INTERESTING_SUBSET_OF_PASSES
)
else:
raise AssertionError(
f"Can't tell how to run device {device.name}; "
f"must be named host_preprocessor and be the first active device."
)
# Reset device and ignored_crash_signatures.
device = main_spirv_opt_device
ignored_signatures_set = set(
device.ignored_crash_signatures
)
try:
for spirv_shader in spirv_shaders:
spirv_opt_util.run_spirv_opt_on_spirv_shader(
spirv_shader,
test_run_dir,
spirv_opt_args,
spirv_opt_path,
)
result_util.write_status(
test_run_dir, fuzz.STATUS_SUCCESS,
)
except subprocess.CalledProcessError:
result_util.write_status(
test_run_dir, fuzz.STATUS_TOOL_CRASH,
)
except subprocess.TimeoutExpired:
result_util.write_status(
test_run_dir, fuzz.STATUS_TOOL_TIMEOUT,
)
elif device.HasField("shader_compiler"):
try:
for spirv_shader in spirv_shaders:
shader_compiler_util.run_shader(
shader_compiler_device=device.shader_compiler,
shader_path=spirv_shader,
output_dir=test_run_dir,
compiler_path=binaries.get_binary_path_by_name(
device.shader_compiler.binary
).path,
timeout=DEFAULT_TIMEOUT,
)
result_util.write_status(
test_run_dir, fuzz.STATUS_SUCCESS,
)
except subprocess.CalledProcessError:
result_util.write_status(
test_run_dir, fuzz.STATUS_CRASH,
)
except subprocess.TimeoutExpired:
result_util.write_status(
test_run_dir, fuzz.STATUS_TIMEOUT,
)
elif device.HasField("swift_shader"):
assert swift_shader_path # noqa
assert amber_path # noqa
host_device_util.run_amber(
test,
test_run_dir,
amber_path=amber_path,
dump_image=False,
dump_buffer=False,
icd=swift_shader_path,
)
elif device.HasField("host"):
assert amber_path # noqa
host_device_util.run_amber(
test,
test_run_dir,
amber_path=amber_path,
dump_image=False,
dump_buffer=False,
custom_launcher=list(device.host.custom_launcher),
)
elif device.HasField("android"):
android_device.run_amber_on_device(
test,
test_run_dir,
dump_image=False,
dump_buffer=False,
serial=device.android.serial,
)
else:
raise AssertionError(f"Unsupported device {device.name}")
finally:
gflogging.pop_stream_for_logging()
status = result_util.get_status(test_run_dir)
if status == fuzz.STATUS_SUCCESS:
write_entry("P")
elif status in (fuzz.STATUS_TIMEOUT, fuzz.STATUS_TOOL_TIMEOUT):
write_entry("T")
else:
write_entry("F")
# Update ignored signatures.
if (
status
in (
fuzz.STATUS_TOOL_CRASH,
fuzz.STATUS_CRASH,
fuzz.STATUS_UNRESPONSIVE,
)
and update_ignored_signatures
):
log_contents = util.file_read_text(
result_util.get_log_path(test_run_dir)
)
signature = signature_util.get_signature_from_log_contents(
log_contents
)
if signature == signature_util.NO_SIGNATURE:
log(f"NOT updating ignored signatures to include {signature}")
elif signature in ignored_signatures_set:
log(f"Signature is already ignored: {signature}")
else:
log(f"Adding ignored signature: {signature}")
device.ignored_crash_signatures.append(signature)
write_newline()
if update_ignored_signatures:
# Reset main_spirv_opt_device name before writing it back out.
if main_spirv_opt_device:
main_spirv_opt_device.name = "host_preprocessor"
settings_util.write(settings, settings_path)
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements;
parser = argparse.ArgumentParser(
description="Runs GraphicsFuzz AmberScript tests on the active devices listed in "
"the settings.json file."
)
parser.add_argument(
"--settings",
help="Path to the settings JSON file for this instance.",
default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH),
)
parser.add_argument(
"--tests",
help="Path to the directory of AmberScript tests with shaders extracted.",
default=str("graphicsfuzz"),
)
parsed_args = parser.parse_args(sys.argv[1:])
# Args.
tests_dir: Path = Path(parsed_args.tests)
settings_path: Path = Path(parsed_args.settings)
# Settings and devices.
settings = settings_util.read_or_create(settings_path)
active_devices = devices_util.get_active_devices(settings.device_list)
# Binaries.
binaries = binaries_util.get_default_binary_manager(settings=settings)
work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}"
util.mkdirs_p(work_dir)
with util.file_open_text(Path("results.txt"), "w") as log_handle:
def write_entry(entry: str) -> None:
log_handle.write(entry)
log_handle.write(", ")
log_handle.flush()
def write_newline() -> None:
log_handle.write("\n")
log_handle.flush()
spirv_opt_path: Optional[Path] = None
swift_shader_path: Optional[Path] = None
amber_path: Optional[Path] = None
# Enumerate active devices, writing their name and storing binary paths if needed.
write_entry("test")
for device in active_devices:
if device.name == "host_preprocessor":
# We are actually just running spirv-opt on the SPIR-V shaders.
write_entry("spirv-opt")
else:
write_entry(device.name)
if device.HasField("preprocess"):
spirv_opt_path = binaries.get_binary_path_by_name(
binaries_util.SPIRV_OPT_NAME
).path
if device.HasField("swift_shader"):
swift_shader_path = binaries.get_binary_path_by_name(
binaries_util.SWIFT_SHADER_NAME
).path
if device.HasField("swift_shader") or device.HasField("host"):
amber_path = binaries.get_binary_path_by_name(
binaries_util.AMBER_NAME
).path
write_newline()
# Enumerate tests and devices, writing the results.
for test in sorted(tests_dir.glob("*.amber")):
test_name = util.remove_end(test.name, ".amber")
write_entry(test_name)
spirv_shaders = sorted(
tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv")
)
for device in active_devices:
test_run_dir = work_dir / f"{test_name}_{device.name}"
util.mkdirs_p(test_run_dir)
try:
# Confusingly, some functions below will raise on an error; others will write e.g. CRASH to the
# STATUS file in the output directory. In the latter case, we update |status|. We check |status| at
# the end of this if-else chain and raise fake exceptions if appropriate.
status = fuzz.STATUS_SUCCESS
if device.HasField("preprocess"):
# This just means spirv-op for now.
assert spirv_opt_path # noqa
for spirv_shader in spirv_shaders:
spirv_opt_util.run_spirv_opt_on_spirv_shader(
spirv_shader, test_run_dir, ["-O"], spirv_opt_path
)
elif device.HasField("shader_compiler"):
for spirv_shader in spirv_shaders:
shader_compiler_util.run_shader(
shader_compiler_device=device.shader_compiler,
shader_path=spirv_shader,
output_dir=test_run_dir,
compiler_path=binaries.get_binary_path_by_name(
device.shader_compiler.binary
).path,
timeout=DEFAULT_TIMEOUT,
)
elif device.HasField("swift_shader"):
assert swift_shader_path # noqa
assert amber_path # noqa
host_device_util.run_amber(
test,
test_run_dir,
amber_path=amber_path,
dump_image=False,
dump_buffer=False,
icd=swift_shader_path,
)
status = result_util.get_status(test_run_dir)
elif device.HasField("host"):
assert amber_path # noqa
host_device_util.run_amber(
test,
test_run_dir,
amber_path=amber_path,
dump_image=False,
dump_buffer=False,
)
status = result_util.get_status(test_run_dir)
elif device.HasField("android"):
android_device.run_amber_on_device(
test,
test_run_dir,
dump_image=False,
dump_buffer=False,
serial=device.android.serial,
)
status = result_util.get_status(test_run_dir)
else:
raise AssertionError(f"Unsupported device {device.name}")
if status in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH):
raise CalledProcessError(1, "??")
if status != fuzz.STATUS_SUCCESS:
raise TimeoutExpired("??", fuzz.AMBER_RUN_TIME_LIMIT)
write_entry("P")
except CalledProcessError:
write_entry("F")
except TimeoutExpired:
write_entry("T")
write_newline()
