def write_default(settings_path: Path) -> Path:
settings = Settings()
settings.CopyFrom(DEFAULT_SETTINGS)
# noinspection PyBroadException
try:
settings.latest_binary_versions.extend(
binaries_util.download_latest_binary_version_numbers())
except Exception: # pylint: disable=broad-except;
message = "WARNING: Could not download the latest binary version numbers. We will just use the (older) hardcoded versions."
details = traceback.format_exc(
) # noqa: SC100, SC200 (spelling of exc)
log(message)
log(f"\nDetails:\n{details}\n\n")
log(message)
settings.latest_binary_versions.extend(binaries_util.DEFAULT_BINARIES)
binary_manager = binaries_util.get_default_binary_manager(
settings=settings)
devices_util.get_device_list(binary_manager, settings.device_list)
return write(settings, settings_path)
def main() -> None:
parser = argparse.ArgumentParser(
description=
"Downloads the latest GraphicsFuzz AmberScript tests from vk-gl-cts, "
"including those in pending CLs. "
"Requires Git. Requires Khronos membership.")
parser.add_argument(
"gerrit_cookie",
help=
"The Gerrit cookie used for authentication. Requires Khronos membership. Obtain this as follows. "
+ GERRIT_COOKIE_INSTRUCTIONS,
)
parser.add_argument(
"--settings",
help="Path to the settings JSON file for this instance.",
default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH),
)
parsed_args = parser.parse_args(sys.argv[1:])
cookie: str = parsed_args.gerrit_cookie
settings_path: Path = Path(parsed_args.settings)
# Need git.
git_tool = util.tool_on_path("git")
settings = settings_util.read_or_create(settings_path)
binaries = binaries_util.get_default_binary_manager(settings=settings)
tests_dir = Path() / "graphicsfuzz"
download_cts_graphicsfuzz_tests(git_tool, cookie, tests_dir)
extract_shaders(tests_dir, binaries)
def main() -> None:
parser = argparse.ArgumentParser(
description=
"Downloads the latest GraphicsFuzz AmberScript tests from vk-gl-cts, "
"including those in pending CLs. "
"Requires Git.")
parser.add_argument("gerrit_cookie",
help=GERRIT_COOKIE_ARGUMENT_DESCRIPTION)
parser.add_argument(
"--settings",
help="Path to the settings JSON file for this instance.",
default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH),
)
parsed_args = parser.parse_args(sys.argv[1:])
cookie: str = parsed_args.gerrit_cookie
settings_path: Path = Path(parsed_args.settings)
# Need git.
git_tool = util.tool_on_path("git")
settings = settings_util.read_or_create(settings_path)
binaries = binaries_util.get_default_binary_manager(settings=settings)
download_cts_graphicsfuzz_tests(git_tool, cookie, binaries)
def main() -> None:
parser = argparse.ArgumentParser(
description="A tool for running a reduction on a source directory."
)
parser.add_argument(
"source_dir",
help="The source directory containing the shaders and the test.json file that describes how to run the test.",
)
parser.add_argument(
"--output", help="Output directory.", default="reduction_output",
)
parser.add_argument(
"--settings",
help="Path to a settings JSON file for this instance.",
default="settings.json",
)
parser.add_argument(
"--literals_to_uniforms",
action="store_true",
help="Pass --literals-to-uniforms to glsl-reduce.",
)
parsed_args = parser.parse_args(sys.argv[1:])
source_dir = Path(parsed_args.source_dir)
output_dir = Path(parsed_args.output)
settings = settings_util.read_or_create(Path(parsed_args.settings))
literals_to_uniforms: bool = parsed_args.literals_to_uniforms
binary_manager = binaries_util.get_default_binary_manager(settings=settings)
test = test_util.metadata_read_from_source_dir(source_dir)
if test.HasField("glsl"):
if (
literals_to_uniforms
and "--literals-to-uniforms" not in settings.extra_graphics_fuzz_reduce_args
):
settings.extra_graphics_fuzz_reduce_args.append("--literals-to-uniforms")
fuzz_glsl_test.run_reduction(
source_dir_to_reduce=source_dir,
reduction_output_dir=output_dir,
binary_manager=binary_manager,
settings=settings,
)
elif test.HasField("spirv_fuzz"):
fuzz_spirv_test.run_reduction(
source_dir_to_reduce=source_dir,
reduction_output_dir=output_dir,
binary_manager=binary_manager,
settings=settings,
)
else:
raise AssertionError(f"Unknown test type: {test}")
def main() -> None: # pylint: disable=too-many-statements, too-many-locals, too-many-branches;
parser = argparse.ArgumentParser(
description="Runs AmberScript files on Android devices."
)
parser.add_argument(
"amber_script_file", help="AmberScript tests to run.", nargs="+",
)
parser.add_argument(
"--output", help="Output directory.", default="output",
)
parser.add_argument(
"--settings",
help="Path to a settings JSON file for this instance. "
"Unlike with gfauto_fuzz, the default value is an empty string, which is ignored. ",
default="",
)
parser.add_argument(
"--serial",
help="Android device serial. If left unspecified, the tests will be run on all Android devices.",
action="append",
)
parsed_args = parser.parse_args(sys.argv[1:])
amber_script_files: List[Path] = [Path(a) for a in parsed_args.amber_script_file]
output_path: Path = Path(parsed_args.output)
serials: Optional[List[str]] = parsed_args.serial
settings_str: str = parsed_args.settings
settings = Settings()
if settings_str:
settings = settings_util.read_or_create(Path(settings_str))
binary_manager = binaries_util.get_default_binary_manager(settings)
if not serials:
android_devices = android_device.get_all_android_devices(
binary_manager, include_device_details=False
)
serials = []
for device in android_devices:
serials.append(device.android.serial)
for amber_script_file in amber_script_files:
for serial in serials:
android_device.run_amber_on_device(
amber_script_file,
output_path / serial,
dump_image=False,
dump_buffer=False,
serial=serial,
)
def main() -> None:
parser = argparse.ArgumentParser(
description=
"A script that creates a README and bug_report directory for a test and its result_dir."
)
parser.add_argument(
"source_dir",
help="Source directory containing test.json and shaders.")
parser.add_argument(
"result_dir",
help=
"Path to the result_dir of a test containing e.g. the intermediate shader files, log.txt, etc.",
)
parser.add_argument(
"--output_dir",
help=
"Output directory where the README and bug_report directory will be written.",
default=".",
)
parsed_args = parser.parse_args(sys.argv[1:])
source_dir = Path(parsed_args.source_dir)
result_dir = Path(parsed_args.result_dir)
output_dir = Path(parsed_args.output_dir)
check_dir_exists(source_dir)
check_dir_exists(result_dir)
test = test_util.metadata_read_from_path(source_dir /
test_util.TEST_METADATA)
binary_manager = binaries_util.get_default_binary_manager(
settings=Settings()).get_child_binary_manager(
binary_list=list(test.binaries) + list(test.device.binaries))
check(test.HasField("glsl"),
AssertionError("Only glsl tests currently supported"))
check(
test.device.HasField("preprocess"),
AssertionError("Only preprocess device tests currently supported"),
)
fuzz_test_util.tool_crash_summary_bug_report_dir(source_dir, result_dir,
output_dir,
binary_manager)
def main() -> None:
parser = argparse.ArgumentParser(
description="Updates or creates a settings.json file. "
"If the settings file does not exist, a default settings file will be created and the tool will exit. "
"If the settings file exists, it is read, updated, and written; command line arguments define the update steps. "
)
parser.add_argument(
"--settings",
help="Path to the settings JSON file for this instance.",
default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH),
)
parser.add_argument(
"--update_binary_versions",
help="Update binary versions to the latest available (online).",
action="store_true",
)
parser.add_argument(
"--update_devices",
help=
'Update each device in the settings.json file. This mostly just means updating the device_properties field obtained by running "amber -d -V". '
"SwiftShader devices will be updated to use the latest version of SwiftShader and Android devices will update their build_fingerprint field. "
"It is recommended that you also pass --update_binary_versions so that the binary versions are updated first. ",
action="store_true",
)
parsed_args = parser.parse_args(sys.argv[1:])
# Args.
settings_path: Path = Path(parsed_args.settings)
update_binary_versions: bool = parsed_args.update_binary_versions
update_devices: bool = parsed_args.update_devices
settings = settings_util.read_or_create(settings_path)
if update_binary_versions:
del settings.latest_binary_versions[:]
settings.latest_binary_versions.extend(
binaries_util.download_latest_binary_version_numbers())
if update_devices:
binary_manager = binaries_util.get_default_binary_manager(settings)
for device in settings.device_list.devices:
devices_util.update_device(binary_manager, device)
settings_util.write(settings, settings_path)
def main() -> int:
parser = argparse.ArgumentParser(
description=
"Runs a binary given the binary name and settings.json file. "
"Use -- to separate args to run_bin and your binary. ")
parser.add_argument(
"--settings",
help="Path to the settings JSON file for this instance.",
default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH),
)
parser.add_argument(
"binary_name",
help="The name of the binary to run. E.g. spirv-opt, glslangValidator",
type=str,
)
parser.add_argument(
"arguments",
metavar="arguments",
type=str,
nargs="*",
help="The arguments to pass to the binary",
)
parsed_args = parser.parse_args(sys.argv[1:])
# Args.
settings_path: Path = Path(parsed_args.settings)
binary_name: str = parsed_args.binary_name
arguments: List[str] = parsed_args.arguments
try:
settings = settings_util.read_or_create(settings_path)
except settings_util.NoSettingsFile:
log(f"Settings file {str(settings_path)} was created for you; using this."
)
settings = settings_util.read_or_create(settings_path)
binary_manager = binaries_util.get_default_binary_manager(
settings=settings)
cmd = [str(binary_manager.get_binary_path_by_name(binary_name).path)]
cmd.extend(arguments)
return subprocess.run(cmd, check=False).returncode
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements;
parser = argparse.ArgumentParser(
description="Runs GraphicsFuzz AmberScript tests on the active devices listed in "
"the settings.json file.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
"--settings",
help="Path to the settings JSON file for this instance.",
default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH),
)
parser.add_argument(
"--tests",
help="Path to the directory of AmberScript tests with shaders extracted.",
default="graphicsfuzz",
)
parser.add_argument(
"--update_ignored_signatures",
help="As the tests are run for each device, add any crash signatures to the device's ignored_crash_signatures "
"property and write out the updated settings.json file.",
action="store_true",
)
parser.add_argument(
"--results_out",
help="Output file path for the CSV results table.",
default="results.csv",
)
parsed_args = parser.parse_args(sys.argv[1:])
# Args.
tests_dir: Path = Path(parsed_args.tests)
settings_path: Path = Path(parsed_args.settings)
update_ignored_signatures: bool = parsed_args.update_ignored_signatures
results_out_path: Path = Path(parsed_args.results_out)
# Settings and devices.
settings = settings_util.read_or_create(settings_path)
active_devices = devices_util.get_active_devices(settings.device_list)
# Binaries.
binaries = binaries_util.get_default_binary_manager(settings=settings)
work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}"
util.mkdirs_p(work_dir)
with util.file_open_text(results_out_path, "w") as results_handle:
def write_entry(entry: str) -> None:
results_handle.write(entry)
results_handle.write(", ")
results_handle.flush()
def write_newline() -> None:
results_handle.write("\n")
results_handle.flush()
spirv_opt_path: Optional[Path] = None
swift_shader_path: Optional[Path] = None
amber_path: Optional[Path] = None
# Small hack to ensure we have three devices for spirv-opt, each with a different name.
main_spirv_opt_device: Optional[Device] = None
if active_devices and active_devices[0].name == "host_preprocessor":
main_spirv_opt_device = active_devices[0]
main_spirv_opt_device.name = SPIRV_OPT_O
spirv_opt_custom = Device()
spirv_opt_custom.CopyFrom(main_spirv_opt_device)
spirv_opt_custom.name = SPIRV_OPT_CUSTOM
active_devices.insert(1, spirv_opt_custom)
spirv_opt_os = Device()
spirv_opt_os.CopyFrom(main_spirv_opt_device)
spirv_opt_os.name = SPIRV_OPT_OS
active_devices.insert(1, spirv_opt_os)
# Enumerate active devices, writing their name and storing binary paths if needed.
write_entry("test")
for device in active_devices:
write_entry(device.name)
if device.HasField("preprocess"):
spirv_opt_path = binaries.get_binary_path_by_name(
binaries_util.SPIRV_OPT_NAME
).path
if device.HasField("swift_shader"):
swift_shader_path = binaries.get_binary_path_by_name(
binaries_util.SWIFT_SHADER_NAME
).path
if device.HasField("swift_shader") or device.HasField("host"):
amber_path = binaries.get_binary_path_by_name(
binaries_util.AMBER_NAME
).path
write_newline()
# Enumerate tests and devices, writing the results.
for test in sorted(tests_dir.glob("*.amber")):
test_name = util.remove_end(test.name, ".amber")
write_entry(test_name)
spirv_shaders = sorted(
tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv")
)
for device in active_devices:
test_run_dir = work_dir / f"{test_name}_{device.name}"
util.mkdirs_p(test_run_dir)
ignored_signatures_set: Set[str] = set(device.ignored_crash_signatures)
with util.file_open_text(test_run_dir / "log.txt", "w") as log_stream:
try:
gflogging.push_stream_for_logging(log_stream)
if device.HasField("preprocess"):
# This just means spirv-opt for now.
assert spirv_opt_path # noqa
assert main_spirv_opt_device # noqa
# Pick spirv-opt arguments based on device name.
if device.name == SPIRV_OPT_O:
spirv_opt_args = ["-O"]
elif device.name == SPIRV_OPT_OS:
spirv_opt_args = ["-Os"]
elif device.name == SPIRV_OPT_CUSTOM:
spirv_opt_args = (
spirv_opt_util.OPT_INTERESTING_SUBSET_OF_PASSES
)
else:
raise AssertionError(
f"Can't tell how to run device {device.name}; "
f"must be named host_preprocessor and be the first active device."
)
# Reset device and ignored_crash_signatures.
device = main_spirv_opt_device
ignored_signatures_set = set(
device.ignored_crash_signatures
)
try:
for spirv_shader in spirv_shaders:
spirv_opt_util.run_spirv_opt_on_spirv_shader(
spirv_shader,
test_run_dir,
spirv_opt_args,
spirv_opt_path,
)
result_util.write_status(
test_run_dir, fuzz.STATUS_SUCCESS,
)
except subprocess.CalledProcessError:
result_util.write_status(
test_run_dir, fuzz.STATUS_TOOL_CRASH,
)
except subprocess.TimeoutExpired:
result_util.write_status(
test_run_dir, fuzz.STATUS_TOOL_TIMEOUT,
)
elif device.HasField("shader_compiler"):
try:
for spirv_shader in spirv_shaders:
shader_compiler_util.run_shader(
shader_compiler_device=device.shader_compiler,
shader_path=spirv_shader,
output_dir=test_run_dir,
compiler_path=binaries.get_binary_path_by_name(
device.shader_compiler.binary
).path,
timeout=DEFAULT_TIMEOUT,
)
result_util.write_status(
test_run_dir, fuzz.STATUS_SUCCESS,
)
except subprocess.CalledProcessError:
result_util.write_status(
test_run_dir, fuzz.STATUS_CRASH,
)
except subprocess.TimeoutExpired:
result_util.write_status(
test_run_dir, fuzz.STATUS_TIMEOUT,
)
elif device.HasField("swift_shader"):
assert swift_shader_path # noqa
assert amber_path # noqa
host_device_util.run_amber(
test,
test_run_dir,
amber_path=amber_path,
dump_image=False,
dump_buffer=False,
icd=swift_shader_path,
)
elif device.HasField("host"):
assert amber_path # noqa
host_device_util.run_amber(
test,
test_run_dir,
amber_path=amber_path,
dump_image=False,
dump_buffer=False,
custom_launcher=list(device.host.custom_launcher),
)
elif device.HasField("android"):
android_device.run_amber_on_device(
test,
test_run_dir,
dump_image=False,
dump_buffer=False,
serial=device.android.serial,
)
else:
raise AssertionError(f"Unsupported device {device.name}")
finally:
gflogging.pop_stream_for_logging()
status = result_util.get_status(test_run_dir)
if status == fuzz.STATUS_SUCCESS:
write_entry("P")
elif status in (fuzz.STATUS_TIMEOUT, fuzz.STATUS_TOOL_TIMEOUT):
write_entry("T")
else:
write_entry("F")
# Update ignored signatures.
if (
status
in (
fuzz.STATUS_TOOL_CRASH,
fuzz.STATUS_CRASH,
fuzz.STATUS_UNRESPONSIVE,
)
and update_ignored_signatures
):
log_contents = util.file_read_text(
result_util.get_log_path(test_run_dir)
)
signature = signature_util.get_signature_from_log_contents(
log_contents
)
if signature == signature_util.NO_SIGNATURE:
log(f"NOT updating ignored signatures to include {signature}")
elif signature in ignored_signatures_set:
log(f"Signature is already ignored: {signature}")
else:
log(f"Adding ignored signature: {signature}")
device.ignored_crash_signatures.append(signature)
write_newline()
if update_ignored_signatures:
# Reset main_spirv_opt_device name before writing it back out.
if main_spirv_opt_device:
main_spirv_opt_device.name = "host_preprocessor"
settings_util.write(settings, settings_path)
def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many-statements;
settings_path: Path,
iteration_seed_override: Optional[int] = None,
fuzzing_tool_pattern: Optional[List[FuzzingTool]] = None,
allow_no_stack_traces: bool = False,
override_sigint: bool = True,
use_amber_vulkan_loader: bool = False,
active_device_names: Optional[List[str]] = None,
update_ignored_crash_signatures_gerrit_cookie: Optional[str] = None,
) -> None:
if not fuzzing_tool_pattern:
fuzzing_tool_pattern = [FuzzingTool.GLSL_FUZZ]
util.update_gcov_environment_variable_if_needed()
if override_sigint:
interrupt_util.override_sigint()
try_get_root_file()
settings = settings_util.read_or_create(settings_path)
binary_manager = binaries_util.get_default_binary_manager(
settings=settings)
temp_dir = Path() / "temp"
# Note: we use "is not None" so that if the user passes an empty Gerrit cookie, we still try to execute this code.
if update_ignored_crash_signatures_gerrit_cookie is not None:
git_tool = util.tool_on_path("git")
downloaded_graphicsfuzz_tests_dir = (
temp_dir / f"graphicsfuzz_cts_tests_{get_random_name()[:8]}")
work_dir = temp_dir / f"graphicsfuzz_cts_run_{get_random_name()[:8]}"
download_cts_gf_tests.download_cts_graphicsfuzz_tests(
git_tool=git_tool,
cookie=update_ignored_crash_signatures_gerrit_cookie,
output_tests_dir=downloaded_graphicsfuzz_tests_dir,
)
download_cts_gf_tests.extract_shaders(
tests_dir=downloaded_graphicsfuzz_tests_dir,
binaries=binary_manager)
with util.file_open_text(work_dir / "results.csv",
"w") as results_out_handle:
run_cts_gf_tests.main_helper(
tests_dir=downloaded_graphicsfuzz_tests_dir,
work_dir=work_dir,
binaries=binary_manager,
settings=settings,
active_devices=devices_util.get_active_devices(
settings.device_list),
results_out_handle=results_out_handle,
updated_settings_output_path=settings_path,
)
return
active_devices = devices_util.get_active_devices(
settings.device_list, active_device_names=active_device_names)
# Add host_preprocessor device from device list if it is missing.
if not active_devices[0].HasField("preprocess"):
for device in settings.device_list.devices:
if device.HasField("preprocess"):
active_devices.insert(0, device)
break
# Add host_preprocessor device (from scratch) if it is still missing.
if not active_devices[0].HasField("preprocess"):
active_devices.insert(
0, Device(name="host_preprocessor", preprocess=DevicePreprocess()))
reports_dir = Path() / "reports"
fuzz_failures_dir = reports_dir / FUZZ_FAILURES_DIR_NAME
references_dir = Path() / REFERENCES_DIR
donors_dir = Path() / DONORS_DIR
spirv_fuzz_shaders_dir = Path() / "spirv_fuzz_shaders"
# Log a warning if there is no tool on the PATH for printing stack traces.
prepended = util.prepend_catchsegv_if_available([], log_warning=True)
if not allow_no_stack_traces and not prepended:
raise AssertionError("Stopping because we cannot get stack traces.")
spirv_fuzz_shaders: List[Path] = []
references: List[Path] = []
if FuzzingTool.SPIRV_FUZZ in fuzzing_tool_pattern:
check_dir_exists(spirv_fuzz_shaders_dir)
spirv_fuzz_shaders = sorted(spirv_fuzz_shaders_dir.rglob("*.json"))
if FuzzingTool.GLSL_FUZZ in fuzzing_tool_pattern:
check_dir_exists(references_dir)
check_dir_exists(donors_dir)
# TODO: make GraphicsFuzz find donors recursively.
references = sorted(references_dir.rglob("*.json"))
# Filter to only include .json files that have at least one shader (.frag, .vert, .comp) file.
references = [
ref for ref in references if shader_job_util.get_related_files(ref)
]
if use_amber_vulkan_loader:
library_path = binary_manager.get_binary_path_by_name(
binaries_util.AMBER_VULKAN_LOADER_NAME).path.parent
util.add_library_paths_to_environ([library_path], os.environ)
fuzzing_tool_index = 0
while True:
interrupt_util.interrupt_if_needed()
# We have to use "is not None" because the seed could be 0.
if iteration_seed_override is not None:
iteration_seed = iteration_seed_override
else:
iteration_seed = secrets.randbits(ITERATION_SEED_BITS)
log(f"Iteration seed: {iteration_seed}")
random.seed(iteration_seed)
staging_name = get_random_name()[:8]
staging_dir = temp_dir / staging_name
try:
util.mkdir_p_new(staging_dir)
except FileExistsError:
if iteration_seed_override is not None:
raise
log(f"Staging directory already exists: {str(staging_dir)}")
log("Starting new iteration.")
continue
# Pseudocode:
# - Create test_dir(s) in staging directory.
# - Run test_dir(s) on all active devices (stop early if appropriate).
# - For each test failure on each device, copy the test to reports_dir, adding the device and crash signature.
# - Reduce each report (on the given device).
# - Produce a summary for each report.
fuzzing_tool = fuzzing_tool_pattern[fuzzing_tool_index]
fuzzing_tool_index = (fuzzing_tool_index +
1) % len(fuzzing_tool_pattern)
if fuzzing_tool == FuzzingTool.SPIRV_FUZZ:
fuzz_spirv_test.fuzz_spirv(
staging_dir,
reports_dir,
fuzz_failures_dir,
active_devices,
spirv_fuzz_shaders,
settings,
binary_manager,
)
elif fuzzing_tool == FuzzingTool.GLSL_FUZZ:
fuzz_glsl_test.fuzz_glsl(
staging_dir,
reports_dir,
fuzz_failures_dir,
active_devices,
references,
donors_dir,
settings,
binary_manager,
)
else:
raise AssertionError(f"Unknown fuzzing tool: {fuzzing_tool}")
if iteration_seed_override is not None:
log("Stopping due to iteration_seed")
break
shutil.rmtree(staging_dir)
def glsl_shader_job_wrong_image_to_amber_script_for_google_cts(
source_dir: Path,
output_amber: Path,
work_dir: Path,
short_description: str,
comment_text: str,
copyright_year: str,
extra_commands: str,
is_coverage_gap: bool = False,
) -> Path:
"""Converts a GLSL shader job of a wrong image case to an Amber script suitable for adding to the CTS."""
check(
not short_description.endswith("."),
AssertionError("Short description should not end with period."),
)
check(
"because shader" not in comment_text,
AssertionError(
'In comment_text: change "because shader" to "because the shader"'
),
)
shader_jobs = get_shader_jobs(source_dir)
test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA)
binary_manager = binaries_util.get_default_binary_manager(
settings=Settings()
).get_child_binary_manager(
binary_list=list(test.device.binaries) + list(test.binaries)
)
spirv_opt_args = list(test.glsl.spirv_opt_args) or None
spirv_opt_hash: Optional[str] = None
if spirv_opt_args:
spirv_opt_hash = binary_manager.get_binary_by_name(
binaries_util.SPIRV_OPT_NAME
).version
# Compile all shader jobs
shader_job_files = [
amber_converter.ShaderJobFile(
shader_job.name,
compile_shader_job(
shader_job.name,
shader_job.shader_job,
work_dir / shader_job.name,
binary_manager,
spirv_opt_args=spirv_opt_args,
skip_validation=test.skip_validation,
common_spirv_args=list(test.common_spirv_args),
).spirv_asm_shader_job,
shader_job.shader_job,
"",
)
for shader_job in shader_jobs
]
reference_asm_spirv_job: Optional[amber_converter.ShaderJobFile] = None
if shader_job_files[0].name_prefix == test_util.REFERENCE_DIR:
reference_asm_spirv_job = shader_job_files[0]
del shader_job_files[0]
return amber_converter.spirv_asm_shader_job_to_amber_script(
amber_converter.ShaderJobFileBasedAmberTest(
reference_asm_spirv_job=reference_asm_spirv_job,
variants_asm_spirv_job=shader_job_files,
),
output_amber,
amber_converter.AmberfySettings(
copyright_header_text=get_copyright_header_google(copyright_year),
add_graphics_fuzz_comment=True,
short_description=short_description,
comment_text=comment_text,
use_default_fence_timeout=True,
spirv_opt_args=spirv_opt_args,
spirv_opt_hash=spirv_opt_hash,
extra_commands=extra_commands,
is_coverage_gap=is_coverage_gap,
),
)
def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many-statements;
settings_path: Path,
iteration_seed_override: Optional[int],
use_spirv_fuzz: bool,
force_no_stack_traces: bool,
) -> None:
util.update_gcov_environment_variable_if_needed()
try:
artifact_util.artifact_path_get_root()
except FileNotFoundError:
log(
"Could not find ROOT file (in the current directory or above) to mark where binaries should be stored. "
"Creating a ROOT file in the current directory."
)
util.file_write_text(Path(artifact_util.ARTIFACT_ROOT_FILE_NAME), "")
settings = settings_util.read_or_create(settings_path)
active_devices = devices_util.get_active_devices(settings.device_list)
reports_dir = Path() / "reports"
fuzz_failures_dir = reports_dir / FUZZ_FAILURES_DIR_NAME
temp_dir = Path() / "temp"
references_dir = Path() / "references"
donors_dir = Path() / "donors"
spirv_fuzz_shaders_dir = Path() / "spirv_fuzz_shaders"
# Log a warning if there is no tool on the PATH for printing stack traces.
prepended = util.prepend_catchsegv_if_available([], log_warning=True)
if not force_no_stack_traces and not prepended:
raise AssertionError("Stopping because we cannot get stack traces.")
spirv_fuzz_shaders: List[Path] = []
references: List[Path] = []
if use_spirv_fuzz:
check_dir_exists(spirv_fuzz_shaders_dir)
spirv_fuzz_shaders = sorted(spirv_fuzz_shaders_dir.rglob("*.json"))
else:
check_dir_exists(references_dir)
check_dir_exists(donors_dir)
# TODO: make GraphicsFuzz find donors recursively.
references = sorted(references_dir.rglob("*.json"))
# Filter to only include .json files that have at least one shader (.frag, .vert, .comp) file.
references = [
ref for ref in references if shader_job_util.get_related_files(ref)
]
binary_manager = binaries_util.get_default_binary_manager(
settings=settings
).get_child_binary_manager(list(settings.custom_binaries), prepend=True)
while True:
# We have to use "is not None" because the seed could be 0.
if iteration_seed_override is not None:
iteration_seed = iteration_seed_override
else:
iteration_seed = secrets.randbits(ITERATION_SEED_BITS)
log(f"Iteration seed: {iteration_seed}")
random.seed(iteration_seed)
staging_name = get_random_name()[:8]
staging_dir = temp_dir / staging_name
try:
util.mkdir_p_new(staging_dir)
except FileExistsError:
if iteration_seed_override is not None:
raise
log(f"Staging directory already exists: {str(staging_dir)}")
log(f"Starting new iteration.")
continue
# Pseudocode:
# - Create test_dir(s) in staging directory.
# - Run test_dir(s) on all active devices (stop early if appropriate).
# - For each test failure on each device, copy the test to reports_dir, adding the device and crash signature.
# - Reduce each report (on the given device).
# - Produce a summary for each report.
if use_spirv_fuzz:
fuzz_spirv_test.fuzz_spirv(
staging_dir,
reports_dir,
fuzz_failures_dir,
active_devices,
spirv_fuzz_shaders,
settings,
binary_manager,
)
else:
fuzz_glsl_test.fuzz_glsl(
staging_dir,
reports_dir,
fuzz_failures_dir,
active_devices,
references,
donors_dir,
settings,
binary_manager,
)
shutil.rmtree(staging_dir)
if iteration_seed_override is not None:
log("Stopping due to iteration_seed")
break
def main() -> None: # pylint: disable=too-many-locals;
parser = argparse.ArgumentParser(
description="Generates an AmberScript test from a shader job.")
parser.add_argument(
"shader_job",
help="The input .json shader job file.",
)
parser.add_argument(
"--output",
help="Output directory.",
default="output",
)
parser.add_argument(
"--spirv_opt_args",
help=
"Arguments for spirv-opt as a space-separated string, or an empty string to skip running spirv-opt.",
default="",
)
parser.add_argument(
"--settings",
help=
"Path to a settings JSON file for this instance. The file will be generated if needed. ",
default="settings.json",
)
parsed_args = parser.parse_args(sys.argv[1:])
shader_job: Path = Path(parsed_args.shader_job)
out_dir: Path = Path(parsed_args.output)
spirv_opt_args_str: str = parsed_args.spirv_opt_args
settings_path: Path = Path(parsed_args.settings)
spirv_opt_args: List[str] = []
if spirv_opt_args_str:
spirv_opt_args = spirv_opt_args_str.split(" ")
settings = settings_util.read_or_create(settings_path)
binary_manager = binaries_util.get_default_binary_manager(settings)
staging_dir = out_dir / "staging"
template_source_dir = staging_dir / "source_template"
test_dir = staging_dir / "test"
run_output_dir: Path = out_dir / "run"
# Remove stale directories.
if staging_dir.is_dir():
shutil.rmtree(staging_dir)
if run_output_dir.is_dir():
shutil.rmtree(run_output_dir)
# Create source template and call |make_test|.
if shader_job_util.get_related_suffixes_that_exist(
shader_job, language_suffix=[shader_job_util.SUFFIX_SPIRV]):
# This is a SPIR-V shader job.
shader_job_util.copy(
shader_job,
template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB,
language_suffix=shader_job_util.SUFFIXES_SPIRV_FUZZ_INPUT,
)
fuzz_spirv_amber_test.make_test(
template_source_dir,
test_dir,
spirv_opt_args=spirv_opt_args,
binary_manager=binary_manager,
derived_from=shader_job.stem,
stable_shader=False,
common_spirv_args=list(settings.common_spirv_args),
)
elif shader_job_util.get_related_suffixes_that_exist(
shader_job, language_suffix=[shader_job_util.SUFFIX_GLSL]):
# This is a GLSL shader job.
# The "graphicsfuzz-tool" tool is designed to be on your PATH so that e.g. ".bat" will be appended on Windows.
# So we use tool_on_path with a custom PATH to get the actual file we want to execute.
graphicsfuzz_tool_path = util.tool_on_path(
"graphicsfuzz-tool",
str(
binary_manager.get_binary_path_by_name(
"graphicsfuzz-tool").path.parent),
)
with util.file_open_text(staging_dir / "log.txt", "w") as log_file:
try:
gflogging.push_stream_for_logging(log_file)
# Create the prepared (for Vulkan GLSL) reference.
glsl_generate_util.run_prepare_reference(
graphicsfuzz_tool_path,
shader_job,
template_source_dir / test_util.VARIANT_DIR /
test_util.SHADER_JOB,
legacy_graphics_fuzz_vulkan_arg=settings.
legacy_graphics_fuzz_vulkan_arg,
)
finally:
gflogging.pop_stream_for_logging()
fuzz_glsl_amber_test.make_test(
template_source_dir,
test_dir,
spirv_opt_args=spirv_opt_args,
binary_manager=binary_manager,
derived_from=shader_job.stem,
stable_shader=False,
common_spirv_args=list(settings.common_spirv_args),
)
else:
raise AssertionError(
"Unexpected shader job type; expected GLSL or SPIR-V shaders.")
preprocessor_cache = util.CommandCache()
fuzz_test_util.run_shader_job(
source_dir=test_util.get_source_dir(test_dir),
output_dir=run_output_dir,
binary_manager=binary_manager,
device=Device(host=DeviceHost()),
preprocessor_cache=preprocessor_cache,
stop_after_amber=True,
)
def main() -> None: # pylint: disable=too-many-statements, too-many-locals, too-many-branches;
parser = argparse.ArgumentParser(
description="Interestingness test that runs a test using Amber, "
"calculates the crash signature based on the result, and returns 0 "
"if the signature matches the expected crash signature.")
parser.add_argument(
"source_dir",
help=
"The source directory containing the shaders and the test.json file that describes how to run the test.",
)
parser.add_argument(
"--override_shader_job",
nargs=2,
metavar=("shader_job_name", "shader_job_json"),
help=
'Override one of the shader jobs. E.g.: "--override_shader_job variant temp/variant.json". Note that '
"the output directory will be set to shader_job_json/ (with the .json extension removed) by default in this case. ",
)
parser.add_argument(
"--override_shader",
nargs=3,
metavar=("shader_name", "suffix", "shader_path"),
help=
'Override one of the shaders. E.g.: "--override_shader variant .frag.spv temp/my_shader.spv". Note that '
"the output directory will be set to shader_path/ (with the .spv extension removed) by default in this case. ",
)
parser.add_argument(
"--use_default_binaries",
help="Use the latest binaries, ignoring those defined in the test.json. "
"Implies --fallback_binaries. Passing --settings is recommended to ensure the latest binaries are used.",
action="store_true",
)
parser.add_argument(
"--fallback_binaries",
help=
"Fallback to the latest binaries if they are not defined in the test.json.",
action="store_true",
)
parser.add_argument(
"--output",
help=
"Output directory. Required unless --override_shader[_job] is used; see --override_shader[_job] for details.",
default=None,
)
parser.add_argument(
"--settings",
help="Path to a settings JSON file for this instance. "
"Unlike with gfauto_fuzz, the default value is an empty string, which is ignored. "
"You only need to use a settings file if you pass --use_default_binaries and you want to use the latest binary versions. "
'In this case, use e.g. "--settings settings.json" so that a default settings file is generated with the latest binary version numbers '
"and then run gfauto_interestingness_test again to use those latest binaries.",
default="",
)
parsed_args = parser.parse_args(sys.argv[1:])
source_dir: Path = Path(parsed_args.source_dir)
override_shader_job: Optional[Tuple[str,
str]] = parsed_args.override_shader_job
override_shader: Optional[Tuple[str, str,
str]] = parsed_args.override_shader
settings_str: str = parsed_args.settings
settings = Settings()
if settings_str:
settings = settings_util.read_or_create(Path(settings_str))
use_default_binaries: bool = parsed_args.use_default_binaries
fallback_binaries: bool = parsed_args.fallback_binaries or use_default_binaries
output: Path
if parsed_args.output:
output = Path(parsed_args.output)
elif override_shader_job:
output = Path(override_shader_job[1]).with_suffix("")
elif override_shader:
output = Path(override_shader[2]).with_suffix("")
else:
raise AssertionError(
"Need --output or --override_shader[_job] parameter.")
binary_manager = binaries_util.get_default_binary_manager(
settings=settings)
if not fallback_binaries:
binary_manager = binary_manager.get_child_binary_manager(
binary_list=[])
shader_job_overrides: List[tool.NameAndShaderJob] = []
if override_shader_job:
shader_job_overrides.append(
tool.NameAndShaderJob(name=override_shader_job[0],
shader_job=Path(override_shader_job[1])))
shader_overrides: tool.ShaderJobNameToShaderOverridesMap = {}
if override_shader:
override = tool.ShaderPathWithNameAndSuffix(
name=override_shader[0],
suffix=override_shader[1],
path=Path(override_shader[2]),
)
shader_overrides[override.name] = {override.suffix: override}
# E.g. shader_overrides ==
# {
# "variant": {
# ".frag.spv": ShaderPathWithNameAndSuffix("variant", ".frag.spv", Path("path/to/shader.frag.spv"))
# }
# }
# We don't need to read this to run the shader, but we need it afterwards anyway.
test = test_util.metadata_read_from_path(source_dir /
test_util.TEST_METADATA)
output_dir = fuzz_glsl_test.run_shader_job(
source_dir=source_dir,
output_dir=output,
binary_manager=binary_manager,
test=test,
ignore_test_and_device_binaries=use_default_binaries,
shader_job_overrides=shader_job_overrides,
shader_job_shader_overrides=shader_overrides,
)
log(f"gfauto_interestingness_test: finished running {str(source_dir)} in {str(output_dir)}."
)
if override_shader_job:
log(f"The {override_shader_job[0]} shader was overridden with {override_shader_job[1]}"
)
status = result_util.get_status(output_dir)
if test.expected_status:
log("")
log(f"Expected status: {test.expected_status}")
log(f"Actual status: {status}")
log_contents = util.file_read_text(result_util.get_log_path(output_dir))
signature = signature_util.get_signature_from_log_contents(log_contents)
log("")
log(f"Expected signature: {test.crash_signature}")
log(f"Actual signature: {signature}")
log("")
# The |crash_regex_override| overrides all other checks.
if test.crash_regex_override:
log(f"Testing crash_regex_override: {test.crash_regex_override}")
override_pattern: Pattern[str] = re.compile(test.crash_regex_override,
re.DOTALL)
match: Optional[Match[str]] = override_pattern.fullmatch(log_contents)
if match:
log("Match!")
log("Interesting")
sys.exit(0)
else:
log("No match; not interesting")
sys.exit(1)
if test.expected_status:
if status != test.expected_status:
log("status != expected_status; not interesting")
sys.exit(1)
else:
# There is no expected status given, so just assume it needs to be one of the "bad" statuses:
if status not in (
fuzz.STATUS_CRASH,
fuzz.STATUS_TOOL_CRASH,
fuzz.STATUS_UNRESPONSIVE,
):
log("shader run did not fail; not interesting.")
sys.exit(1)
if signature != test.crash_signature:
log("signature != crash_signature; not interesting")
sys.exit(1)
log("Interesting!")
def fuzz_and_reduce_bug(
active_device: str,
seed: int,
check_result: Callable[[], None],
settings: Optional[Settings] = None,
ignored_signatures: Optional[List[str]] = None,
) -> None:
"""
Fuzz, find a bug, reduce it.
Linux only.
"""
# Test only works on Linux.
if util.get_platform() != "Linux":
return
here = util.norm_path(Path(__file__).absolute()).parent
temp_dir: Path = here.parent / "temp"
assert temp_dir.is_dir()
os.chdir(temp_dir)
# Create ROOT file in temp/ if needed.
fuzz.try_get_root_file()
work_dir = temp_dir / fuzz.get_random_name()[:8]
util.mkdir_p_new(work_dir)
os.chdir(work_dir)
log(f"Changed to {str(work_dir)}")
if settings is None:
settings = Settings()
settings.CopyFrom(settings_util.DEFAULT_SETTINGS)
settings.device_list.CopyFrom(
DeviceList(
active_device_names=[active_device],
devices=[
Device(
name="amdllpc",
shader_compiler=DeviceShaderCompiler(
binary="amdllpc",
args=[
"-gfxip=9.0.0", "-verify-ir", "-auto-layout-desc"
],
),
binaries=[
Binary(
name="amdllpc",
tags=["Release"],
version="c21d76dceaf26361f9b6b3838a955ec3301506b5",
),
],
),
Device(
name="swift_shader",
swift_shader=DeviceSwiftShader(),
binaries=[
Binary(
name="swift_shader_icd",
tags=["Release"],
version="6d69aae0e1ab49190ea46cd1c999fd3d02e016b9",
),
],
ignored_crash_signatures=ignored_signatures,
),
],
))
spirv_tools_version = "983b5b4fccea17cab053de24d51403efb4829158"
settings.latest_binary_versions.extend([
Binary(
name="glslangValidator",
tags=["Release"],
version="1afa2b8cc57b92c6b769eb44a6854510b6921a0b",
),
Binary(name="spirv-opt", tags=["Release"],
version=spirv_tools_version),
Binary(name="spirv-dis", tags=["Release"],
version=spirv_tools_version),
Binary(name="spirv-as", tags=["Release"], version=spirv_tools_version),
Binary(name="spirv-val", tags=["Release"],
version=spirv_tools_version),
Binary(name="spirv-fuzz",
tags=["Release"],
version=spirv_tools_version),
Binary(name="spirv-reduce",
tags=["Release"],
version=spirv_tools_version),
Binary(
name="graphicsfuzz-tool",
tags=[],
version="7b143bcb3ad38b64ddc17d132886636b229b6684",
),
])
# Add default binaries; the ones above have priority.
settings.latest_binary_versions.extend(binaries_util.DEFAULT_BINARIES)
settings.extra_graphics_fuzz_generate_args.append("--small")
settings.extra_graphics_fuzz_generate_args.append("--single-pass")
settings.extra_graphics_fuzz_reduce_args.append("--max-steps")
settings.extra_graphics_fuzz_reduce_args.append("2")
settings_util.write(settings, settings_util.DEFAULT_SETTINGS_FILE_PATH)
# Add shaders.
binary_manager = binaries_util.get_default_binary_manager(settings)
graphicsfuzz_tool = binary_manager.get_binary_path_by_name(
"graphicsfuzz-tool")
sample_shaders_path: Path = graphicsfuzz_tool.path.parent.parent.parent / "shaders" / "samples" / "310es"
util.copy_dir(sample_shaders_path, Path() / fuzz.REFERENCES_DIR)
util.copy_dir(sample_shaders_path, Path() / fuzz.DONORS_DIR)
fuzz.main_helper(
settings_path=settings_util.DEFAULT_SETTINGS_FILE_PATH,
iteration_seed_override=seed,
override_sigint=False,
use_amber_vulkan_loader=True,
)
check_result()
os.chdir(here)
shutil.rmtree(work_dir)
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements;
parser = argparse.ArgumentParser(
description=
"Runs GraphicsFuzz AmberScript tests on the active devices listed in "
"the settings.json file.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
"--settings",
help="Path to the settings JSON file for this instance.",
default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH),
)
parser.add_argument(
"--tests",
help=
"Path to the directory of AmberScript tests with shaders extracted.",
default="graphicsfuzz",
)
parser.add_argument(
"--update_ignored_crash_signatures",
help=
"As the tests are run for each device, add any crash signatures to the device's ignored_crash_signatures "
"property and write out the updated settings.json file.",
action="store_true",
)
parser.add_argument(
"--results_out",
help="Output file path for the CSV results table.",
default="results.csv",
)
parsed_args = parser.parse_args(sys.argv[1:])
# Args.
tests_dir: Path = Path(parsed_args.tests)
settings_path: Path = Path(parsed_args.settings)
update_ignored_crash_signatures: bool = parsed_args.update_ignored_crash_signatures
results_out_path: Path = Path(parsed_args.results_out)
# Settings and devices.
settings = settings_util.read_or_create(settings_path)
active_devices = devices_util.get_active_devices(settings.device_list)
# Binaries.
binaries = binaries_util.get_default_binary_manager(settings=settings)
work_dir = Path(
) / "temp" / f"graphicsfuzz_cts_run_{fuzz.get_random_name()[:8]}"
with util.file_open_text(results_out_path, "w") as results_out_handle:
main_helper(
tests_dir=tests_dir,
work_dir=work_dir,
binaries=binaries,
settings=settings,
active_devices=active_devices,
results_out_handle=results_out_handle,
updated_settings_output_path=(
settings_path if update_ignored_crash_signatures else None),
)
def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements;
parser = argparse.ArgumentParser(
description="Runs GraphicsFuzz AmberScript tests on the active devices listed in "
"the settings.json file."
)
parser.add_argument(
"--settings",
help="Path to the settings JSON file for this instance.",
default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH),
)
parser.add_argument(
"--tests",
help="Path to the directory of AmberScript tests with shaders extracted.",
default=str("graphicsfuzz"),
)
parsed_args = parser.parse_args(sys.argv[1:])
# Args.
tests_dir: Path = Path(parsed_args.tests)
settings_path: Path = Path(parsed_args.settings)
# Settings and devices.
settings = settings_util.read_or_create(settings_path)
active_devices = devices_util.get_active_devices(settings.device_list)
# Binaries.
binaries = binaries_util.get_default_binary_manager(settings=settings)
work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}"
util.mkdirs_p(work_dir)
with util.file_open_text(Path("results.txt"), "w") as log_handle:
def write_entry(entry: str) -> None:
log_handle.write(entry)
log_handle.write(", ")
log_handle.flush()
def write_newline() -> None:
log_handle.write("\n")
log_handle.flush()
spirv_opt_path: Optional[Path] = None
swift_shader_path: Optional[Path] = None
amber_path: Optional[Path] = None
# Enumerate active devices, writing their name and storing binary paths if needed.
write_entry("test")
for device in active_devices:
if device.name == "host_preprocessor":
# We are actually just running spirv-opt on the SPIR-V shaders.
write_entry("spirv-opt")
else:
write_entry(device.name)
if device.HasField("preprocess"):
spirv_opt_path = binaries.get_binary_path_by_name(
binaries_util.SPIRV_OPT_NAME
).path
if device.HasField("swift_shader"):
swift_shader_path = binaries.get_binary_path_by_name(
binaries_util.SWIFT_SHADER_NAME
).path
if device.HasField("swift_shader") or device.HasField("host"):
amber_path = binaries.get_binary_path_by_name(
binaries_util.AMBER_NAME
).path
write_newline()
# Enumerate tests and devices, writing the results.
for test in sorted(tests_dir.glob("*.amber")):
test_name = util.remove_end(test.name, ".amber")
write_entry(test_name)
spirv_shaders = sorted(
tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv")
)
for device in active_devices:
test_run_dir = work_dir / f"{test_name}_{device.name}"
util.mkdirs_p(test_run_dir)
try:
# Confusingly, some functions below will raise on an error; others will write e.g. CRASH to the
# STATUS file in the output directory. In the latter case, we update |status|. We check |status| at
# the end of this if-else chain and raise fake exceptions if appropriate.
status = fuzz.STATUS_SUCCESS
if device.HasField("preprocess"):
# This just means spirv-op for now.
assert spirv_opt_path # noqa
for spirv_shader in spirv_shaders:
spirv_opt_util.run_spirv_opt_on_spirv_shader(
spirv_shader, test_run_dir, ["-O"], spirv_opt_path
)
elif device.HasField("shader_compiler"):
for spirv_shader in spirv_shaders:
shader_compiler_util.run_shader(
shader_compiler_device=device.shader_compiler,
shader_path=spirv_shader,
output_dir=test_run_dir,
compiler_path=binaries.get_binary_path_by_name(
device.shader_compiler.binary
).path,
timeout=DEFAULT_TIMEOUT,
)
elif device.HasField("swift_shader"):
assert swift_shader_path # noqa
assert amber_path # noqa
host_device_util.run_amber(
test,
test_run_dir,
amber_path=amber_path,
dump_image=False,
dump_buffer=False,
icd=swift_shader_path,
)
status = result_util.get_status(test_run_dir)
elif device.HasField("host"):
assert amber_path # noqa
host_device_util.run_amber(
test,
test_run_dir,
amber_path=amber_path,
dump_image=False,
dump_buffer=False,
)
status = result_util.get_status(test_run_dir)
elif device.HasField("android"):
android_device.run_amber_on_device(
test,
test_run_dir,
dump_image=False,
dump_buffer=False,
serial=device.android.serial,
)
status = result_util.get_status(test_run_dir)
else:
raise AssertionError(f"Unsupported device {device.name}")
if status in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH):
raise CalledProcessError(1, "??")
if status != fuzz.STATUS_SUCCESS:
raise TimeoutExpired("??", fuzz.AMBER_RUN_TIME_LIMIT)
write_entry("P")
except CalledProcessError:
write_entry("F")
except TimeoutExpired:
write_entry("T")
write_newline()
