def get_binary_path(self, binary: Binary) -> Path:
# Special case: allow the path to be specified in the binary object itself for testing purposes:
if binary.path:
return Path(binary.path)
# Try resolved cache first.
result = self._resolved_paths.get(binary.SerializePartialToString())
if result:
return result
log(f"Finding path of binary:\n{binary}")
# Try list (cache) of binary artifacts on disk.
result = self._get_binary_path_from_binary_artifacts(binary)
if result:
return result
# Try online.
wrapped_recipe = get_github_release_recipe(binary)
# Execute the recipe to download the binaries.
artifact_util.artifact_execute_recipe_if_needed(
wrapped_recipe.path, {wrapped_recipe.path: wrapped_recipe.recipe})
# Add to binary artifacts list (cache).
self._binary_artifacts.append((
wrapped_recipe.recipe.download_and_extract_archive_set.archive_set,
wrapped_recipe.path,
))
# Now we should be able to find it in the binary artifacts list.
result = self._get_binary_path_from_binary_artifacts(binary)
check(
bool(result),
AssertionError(
f"Could not find:\n{binary} even though we just added it:\n{wrapped_recipe}"
),
)
assert result # noqa
return result
def download_latest_binary_version_numbers() -> List[Binary]:
log("Downloading the latest binary version numbers...")
# Deep copy of DEFAULT_BINARIES.
binaries: List[Binary] = []
for binary in DEFAULT_BINARIES:
new_binary = Binary()
new_binary.CopyFrom(binary)
binaries.append(new_binary)
# Update version numbers.
for binary in binaries:
project_name = binary_name_to_project_name(binary.name)
binary.version = _download_latest_version_number(project_name)
return binaries
def _get_built_in_binary_recipe_from_build_github_repo(
project_name: str,
version_hash: str,
build_version_hash: str,
platform_suffixes: List[str],
tools: List[ToolNameAndPath],
) -> List[recipe_wrap.RecipeWrap]:
result: List[recipe_wrap.RecipeWrap] = []
for platform_suffix in platform_suffixes:
tags: List[str] = []
add_common_tags_from_platform_suffix(tags, platform_suffix)
binaries = [
Binary(
name=binary.name,
tags=tags,
path=(
f"{project_name}/{(binary.subpath + '.exe') if 'Windows' in tags and binary.add_exe_on_windows else binary.subpath}"
),
version=version_hash,
)
for binary in tools
]
result.append(
recipe_wrap.RecipeWrap(
f"{BUILT_IN_BINARY_RECIPES_PATH_PREFIX}/{project_name}_{version_hash}_{platform_suffix}",
Recipe(
download_and_extract_archive_set=RecipeDownloadAndExtractArchiveSet(
archive_set=ArchiveSet(
archives=[
Archive(
url=f"https://github.com/paulthomson/build-{project_name}/releases/download/github/paulthomson/build-{project_name}/{build_version_hash}/build-{project_name}-{build_version_hash}-{platform_suffix}.zip",
output_file=f"{project_name}.zip",
output_directory=project_name,
)
],
binaries=binaries,
)
)
),
)
)
return result
def _get_binary_path_from_binary_artifacts(
self, binary: Binary) -> Optional[Path]:
binary_tags = set(binary.tags)
binary_tags.add(self._platform)
for (archive_set, artifact_path) in self._binary_artifacts:
for artifact_binary in archive_set.binaries: # type: Binary
if artifact_binary.name != binary.name:
continue
if artifact_binary.version != binary.version:
continue
recipe_binary_tags = set(artifact_binary.tags)
if not binary_tags.issubset(recipe_binary_tags):
continue
artifact_util.artifact_execute_recipe_if_needed(
artifact_path, self._built_in_binary_recipes)
result = artifact_util.artifact_get_inner_file_path(
artifact_binary.path, artifact_path)
self._resolved_paths[
binary.SerializePartialToString()] = result
return result
return None
def get_github_release_recipe( # pylint: disable=too-many-branches,too-many-statements;
binary: Binary, ) -> recipe_wrap.RecipeWrap:
project_name = binary_name_to_project_name(binary.name)
if project_name == "graphicsfuzz":
# Special case:
platform = util.get_platform() # Not used.
tags = PLATFORMS[:] # All host platforms.
repo_name = f"gfbuild-{project_name}"
version = binary.version
artifact_name = f"gfbuild-{project_name}-{version}"
elif project_name == "amber" and binary.name in ("amber_apk",
"amber_apk_test"):
# Special case:
platform = util.get_platform() # Not used.
tags = PLATFORMS[:] # All host platforms.
tags.append("Debug")
repo_name = f"gfbuild-{project_name}"
version = binary.version
artifact_name = f"gfbuild-{project_name}-{version}-android_apk"
else:
# Normal case:
platform = get_platform_from_binary(binary)
config = get_config_from_binary(binary)
arch = "x64"
tags = [platform, config, arch]
repo_name = f"gfbuild-{project_name}"
version = binary.version
artifact_name = f"gfbuild-{project_name}-{version}-{platform}_{arch}_{config}"
recipe = recipe_wrap.RecipeWrap(
path=f"{BUILT_IN_BINARY_RECIPES_PATH_PREFIX}/{artifact_name}",
recipe=Recipe(
download_and_extract_archive_set=
RecipeDownloadAndExtractArchiveSet(archive_set=ArchiveSet(
archives=[
Archive(
url=
f"https://github.com/google/{repo_name}/releases/download/github/google/{repo_name}/{version}/{artifact_name}.zip",
output_file=f"{project_name}.zip",
output_directory=f"{project_name}",
)
],
binaries=[],
))),
)
executable_suffix = ".exe" if platform == "Windows" else ""
if project_name == "glslang":
binaries = [
Binary(
name="glslangValidator",
tags=tags,
path=f"{project_name}/bin/glslangValidator{executable_suffix}",
version=version,
)
]
elif project_name == "SPIRV-Tools":
binaries = [
Binary(
name="spirv-opt",
tags=tags,
path=f"{project_name}/bin/spirv-opt{executable_suffix}",
version=version,
),
Binary(
name="spirv-as",
tags=tags,
path=f"{project_name}/bin/spirv-as{executable_suffix}",
version=version,
),
Binary(
name="spirv-dis",
tags=tags,
path=f"{project_name}/bin/spirv-dis{executable_suffix}",
version=version,
),
Binary(
name="spirv-val",
tags=tags,
path=f"{project_name}/bin/spirv-val{executable_suffix}",
version=version,
),
Binary(
name="spirv-fuzz",
tags=tags,
path=f"{project_name}/bin/spirv-fuzz{executable_suffix}",
version=version,
),
Binary(
name="spirv-reduce",
tags=tags,
path=f"{project_name}/bin/spirv-reduce{executable_suffix}",
version=version,
),
]
elif project_name == "swiftshader":
binaries = [
Binary(
name="swift_shader_icd",
tags=tags,
path=f"{project_name}/lib/vk_swiftshader_icd.json",
version=version,
)
]
elif project_name == "amber":
if binary.name in ("amber_apk", "amber_apk_test"):
binaries = [
Binary(
name="amber_apk",
tags=tags,
path=f"{project_name}/amber.apk",
version=version,
),
Binary(
name="amber_apk_test",
tags=tags,
path=f"{project_name}/amber-test.apk",
version=version,
),
]
else:
if platform == "Linux":
vulkan_loader_name = "libvulkan.so.1"
elif platform == "Mac":
vulkan_loader_name = "libvulkan.1.dylib"
elif platform == "Windows":
vulkan_loader_name = "vulkan-1.dll"
else:
raise AssertionError(
f"Unknown platform for Amber's Vulkan loader: {platform}")
binaries = [
Binary(
name="amber",
tags=tags,
path=f"{project_name}/bin/amber{executable_suffix}",
version=version,
),
Binary(
name=AMBER_VULKAN_LOADER_NAME,
tags=tags,
path=f"{project_name}/lib/{vulkan_loader_name}",
version=version,
),
]
elif project_name == "graphicsfuzz":
binaries = [
Binary(
name="graphicsfuzz-tool",
tags=tags,
path=f"{project_name}/python/drivers/graphicsfuzz-tool",
version=version,
)
]
elif project_name == "llpc":
if platform != "Linux":
raise AssertionError("amdllpc is only available on Linux")
binaries = [
Binary(
name="amdllpc",
tags=tags,
path=f"{project_name}/bin/amdllpc{executable_suffix}",
version=version,
)
]
else:
raise AssertionError(f"Unknown project name: {project_name}")
recipe.recipe.download_and_extract_archive_set.archive_set.binaries.extend(
binaries)
return recipe
def get_graphics_fuzz_121() -> List[recipe_wrap.RecipeWrap]:
return [
recipe_wrap.RecipeWrap(
f"{BUILT_IN_BINARY_RECIPES_PATH_PREFIX}/graphicsfuzz_v1.2.1",
Recipe(
download_and_extract_archive_set=
RecipeDownloadAndExtractArchiveSet(archive_set=ArchiveSet(
archives=[
Archive(
url=
"https://github.com/google/graphicsfuzz/releases/download/v1.2.1/graphicsfuzz.zip",
output_file="graphicsfuzz.zip",
output_directory="graphicsfuzz",
)
],
binaries=[
#
# glslangValidator
Binary(
name="glslangValidator",
tags=["Linux", "x64", "Release"],
path="graphicsfuzz/bin/Linux/glslangValidator",
version="40c16ec0b3ad03fc170f1369a58e7bbe662d82cd",
),
Binary(
name="glslangValidator",
tags=["Windows", "x64", "Release"],
path=
"graphicsfuzz/bin/Windows/glslangValidator.exe",
version="40c16ec0b3ad03fc170f1369a58e7bbe662d82cd",
),
Binary(
name="glslangValidator",
tags=["Mac", "x64", "Release"],
path="graphicsfuzz/bin/Mac/glslangValidator",
version="40c16ec0b3ad03fc170f1369a58e7bbe662d82cd",
),
#
# spirv-opt
Binary(
name="spirv-opt",
tags=[
"Linux",
"x64",
"Release",
SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG,
],
path="graphicsfuzz/bin/Linux/spirv-opt",
version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed",
),
Binary(
name="spirv-opt",
tags=[
"Windows",
"x64",
"Release",
SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG,
],
path="graphicsfuzz/bin/Windows/spirv-opt.exe",
version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed",
),
Binary(
name="spirv-opt",
tags=[
"Mac",
"x64",
"Release",
SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG,
],
path="graphicsfuzz/bin/Mac/spirv-opt",
version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed",
),
#
# spirv-dis
Binary(
name="spirv-dis",
tags=["Linux", "x64", "Release"],
path="graphicsfuzz/bin/Linux/spirv-dis",
version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed",
),
Binary(
name="spirv-dis",
tags=["Windows", "x64", "Release"],
path="graphicsfuzz/bin/Windows/spirv-dis.exe",
version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed",
),
Binary(
name="spirv-dis",
tags=["Mac", "x64", "Release"],
path="graphicsfuzz/bin/Mac/spirv-dis",
version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed",
),
#
# spirv-as
Binary(
name="spirv-as",
tags=["Linux", "x64", "Release"],
path="graphicsfuzz/bin/Linux/spirv-as",
version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed",
),
Binary(
name="spirv-as",
tags=["Windows", "x64", "Release"],
path="graphicsfuzz/bin/Windows/spirv-as.exe",
version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed",
),
Binary(
name="spirv-as",
tags=["Mac", "x64", "Release"],
path="graphicsfuzz/bin/Mac/spirv-as",
version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed",
),
#
# spirv-val
Binary(
name="spirv-val",
tags=["Linux", "x64", "Release"],
path="graphicsfuzz/bin/Linux/spirv-val",
version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed",
),
Binary(
name="spirv-val",
tags=["Windows", "x64", "Release"],
path="graphicsfuzz/bin/Windows/spirv-val.exe",
version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed",
),
Binary(
name="spirv-val",
tags=["Mac", "x64", "Release"],
path="graphicsfuzz/bin/Mac/spirv-val",
version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed",
),
],
))),
)
]
"Mac_x64_Release",
]
PLATFORM_SUFFIXES_RELWITHDEBINFO = [
"Linux_x64_RelWithDebInfo",
"Windows_x64_RelWithDebInfo",
"Mac_x64_RelWithDebInfo",
]
DEFAULT_SPIRV_TOOLS_VERSION = "983b5b4fccea17cab053de24d51403efb4829158"
DEFAULT_AMBER_VERSION = "298b24a4379658949bcc5256f9b54678613208c1"
DEFAULT_BINARIES = [
Binary(
name="glslangValidator",
tags=["Debug"],
version="1afa2b8cc57b92c6b769eb44a6854510b6921a0b",
),
Binary(name="spirv-opt",
tags=["Debug"],
version=DEFAULT_SPIRV_TOOLS_VERSION),
Binary(name="spirv-dis",
tags=["Debug"],
version=DEFAULT_SPIRV_TOOLS_VERSION),
Binary(name="spirv-as",
tags=["Debug"],
version=DEFAULT_SPIRV_TOOLS_VERSION),
Binary(name="spirv-val",
tags=["Debug"],
version=DEFAULT_SPIRV_TOOLS_VERSION),
Binary(name="spirv-fuzz",
def fuzz_and_reduce_bug(
active_device: str,
seed: int,
check_result: Callable[[], None],
settings: Optional[Settings] = None,
ignored_signatures: Optional[List[str]] = None,
) -> None:
"""
Fuzz, find a bug, reduce it.
Linux only.
"""
# Test only works on Linux.
if util.get_platform() != "Linux":
return
here = util.norm_path(Path(__file__).absolute()).parent
temp_dir: Path = here.parent / "temp"
assert temp_dir.is_dir()
os.chdir(temp_dir)
# Create ROOT file in temp/ if needed.
fuzz.try_get_root_file()
work_dir = temp_dir / fuzz.get_random_name()[:8]
util.mkdir_p_new(work_dir)
os.chdir(work_dir)
log(f"Changed to {str(work_dir)}")
if settings is None:
settings = Settings()
settings.CopyFrom(settings_util.DEFAULT_SETTINGS)
settings.device_list.CopyFrom(
DeviceList(
active_device_names=[active_device],
devices=[
Device(
name="amdllpc",
shader_compiler=DeviceShaderCompiler(
binary="amdllpc",
args=[
"-gfxip=9.0.0", "-verify-ir", "-auto-layout-desc"
],
),
binaries=[
Binary(
name="amdllpc",
tags=["Release"],
version="c21d76dceaf26361f9b6b3838a955ec3301506b5",
),
],
),
Device(
name="swift_shader",
swift_shader=DeviceSwiftShader(),
binaries=[
Binary(
name="swift_shader_icd",
tags=["Release"],
version="6d69aae0e1ab49190ea46cd1c999fd3d02e016b9",
),
],
ignored_crash_signatures=ignored_signatures,
),
],
))
spirv_tools_version = "983b5b4fccea17cab053de24d51403efb4829158"
settings.latest_binary_versions.extend([
Binary(
name="glslangValidator",
tags=["Release"],
version="1afa2b8cc57b92c6b769eb44a6854510b6921a0b",
),
Binary(name="spirv-opt", tags=["Release"],
version=spirv_tools_version),
Binary(name="spirv-dis", tags=["Release"],
version=spirv_tools_version),
Binary(name="spirv-as", tags=["Release"], version=spirv_tools_version),
Binary(name="spirv-val", tags=["Release"],
version=spirv_tools_version),
Binary(name="spirv-fuzz",
tags=["Release"],
version=spirv_tools_version),
Binary(name="spirv-reduce",
tags=["Release"],
version=spirv_tools_version),
Binary(
name="graphicsfuzz-tool",
tags=[],
version="7b143bcb3ad38b64ddc17d132886636b229b6684",
),
])
# Add default binaries; the ones above have priority.
settings.latest_binary_versions.extend(binaries_util.DEFAULT_BINARIES)
settings.extra_graphics_fuzz_generate_args.append("--small")
settings.extra_graphics_fuzz_generate_args.append("--single-pass")
settings.extra_graphics_fuzz_reduce_args.append("--max-steps")
settings.extra_graphics_fuzz_reduce_args.append("2")
settings_util.write(settings, settings_util.DEFAULT_SETTINGS_FILE_PATH)
# Add shaders.
binary_manager = binaries_util.get_default_binary_manager(settings)
graphicsfuzz_tool = binary_manager.get_binary_path_by_name(
"graphicsfuzz-tool")
sample_shaders_path: Path = graphicsfuzz_tool.path.parent.parent.parent / "shaders" / "samples" / "310es"
util.copy_dir(sample_shaders_path, Path() / fuzz.REFERENCES_DIR)
util.copy_dir(sample_shaders_path, Path() / fuzz.DONORS_DIR)
fuzz.main_helper(
settings_path=settings_util.DEFAULT_SETTINGS_FILE_PATH,
iteration_seed_override=seed,
override_sigint=False,
use_amber_vulkan_loader=True,
)
check_result()
os.chdir(here)
shutil.rmtree(work_dir)
"Linux_x64_Release",
"Windows_x64_Release",
"Mac_x64_Release",
]
PLATFORM_SUFFIXES_RELWITHDEBINFO = [
"Linux_x64_RelWithDebInfo",
"Windows_x64_RelWithDebInfo",
"Mac_x64_RelWithDebInfo",
]
DEFAULT_SPIRV_TOOLS_VERSION = "f1e5cd73f658abcc23ee96d78f2dc27c4b7028c1"
DEFAULT_BINARIES = [
Binary(
name="glslangValidator",
tags=["Debug"],
version="18d6b6b63e9adc2aa2cce1ce85d1c348f9475118",
),
Binary(name="spirv-opt",
tags=["Debug"],
version=DEFAULT_SPIRV_TOOLS_VERSION),
Binary(name="spirv-dis",
tags=["Debug"],
version=DEFAULT_SPIRV_TOOLS_VERSION),
Binary(name="spirv-as",
tags=["Debug"],
version=DEFAULT_SPIRV_TOOLS_VERSION),
Binary(name="spirv-val",
tags=["Debug"],
version=DEFAULT_SPIRV_TOOLS_VERSION),
Binary(name="spirv-fuzz",