def test_unrecognized_match(self) -> None:
with self.assertRaises(UnrecognizedMatchError):
match(EnumMatchTest.Test.pig, { # type: ignore[type-var]
EnumMatchTest.Test.dog: "woof",
EnumMatchTest.Test.cat: "meow",
EnumMatchTest.Test.pig: "oink",
"horse": "neigh",
})
def test_valid_match(self) -> None:
match_mapping = {
EnumMatchTest.Test.dog: "woof",
EnumMatchTest.Test.cat: "meow",
EnumMatchTest.Test.pig: "oink",
}
self.assertEqual("woof", match(EnumMatchTest.Test.dog, match_mapping))
self.assertEqual("meow", match(EnumMatchTest.Test.cat, match_mapping))
self.assertEqual("oink", match(EnumMatchTest.Test.pig, match_mapping))
def test_ctypes_native_language_interop(self, toolchain_variant):
# TODO: consider making this mock_buildroot/run_pants_with_workdir into a
# PantsRunIntegrationTest method!
with self.mock_buildroot(
dirs_to_copy=[self._binary_interop_target_dir]
) as buildroot, buildroot.pushd():
# Replace strict_deps=False with nothing so we can override it (because target values for this
# option take precedence over subsystem options).
orig_wrapped_math_build = read_file(self._wrapped_math_build_file)
without_strict_deps_wrapped_math_build = re.sub(
"strict_deps=False,", "", orig_wrapped_math_build
)
safe_file_dump(self._wrapped_math_build_file, without_strict_deps_wrapped_math_build)
# This should fail because it does not turn on strict_deps for a target which requires it.
pants_binary_strict_deps_failure = self.run_pants_with_workdir(
command=["binary", self._binary_target_with_interop],
# Explicitly set to True (although this is the default).
config={
"native-build-step": {"toolchain_variant": toolchain_variant.value},
# TODO(#6848): don't make it possible to forget to add the toolchain_variant option!
"native-build-settings": {"strict_deps": True},
},
workdir=os.path.join(buildroot.new_buildroot, ".pants.d"),
)
self.assert_failure(pants_binary_strict_deps_failure)
self.assertIn(
match(
toolchain_variant,
{
ToolchainVariant.gnu: "fatal error: some_math.h: No such file or directory",
ToolchainVariant.llvm: "fatal error: 'some_math.h' file not found",
},
),
pants_binary_strict_deps_failure.stdout_data,
)
# TODO(#6848): we need to provide the libstdc++.so.6.dylib which comes with gcc on osx in the
# DYLD_LIBRARY_PATH during the 'run' goal somehow.
attempt_pants_run = match(
Platform.current,
{Platform.darwin: toolchain_variant == ToolchainVariant.llvm, Platform.linux: True},
)
if attempt_pants_run:
pants_run_interop = self.run_pants(
["-q", "run", self._binary_target_with_interop],
config={
"native-build-step": {"toolchain_variant": toolchain_variant.value},
"native-build-settings": {"strict_deps": True},
},
)
self.assert_success(pants_run_interop)
self.assertEqual("x=3, f(x)=299\n", pants_run_interop.stdout_data)
def test_native_compiler_option_sets_integration(self, toolchain_variant):
"""Test that native compilation includes extra compiler flags from target definitions.
This target uses the ndebug and asdf option sets. If either of these are not present
(disabled), this test will fail.
"""
# TODO(#6848): this fails when run with gcc on osx as it requires gcc's libstdc++.so.6.dylib to
# be available on the runtime library path.
attempt_pants_run = match(
Platform.current,
{Platform.darwin: toolchain_variant == ToolchainVariant.llvm, Platform.linux: True},
)
if not attempt_pants_run:
return
command = ["run", self._binary_target_with_compiler_option_sets]
pants_run = self.run_pants(
command=command,
config={
"native-build-step": {"toolchain_variant": toolchain_variant.value},
"native-build-step.cpp-compile-settings": {
"compiler_option_sets_enabled_args": {"asdf": ["-D_ASDF=1"]},
"compiler_option_sets_disabled_args": {"asdf": ["-D_ASDF=0"]},
},
},
)
self.assert_success(pants_run)
self.assertIn("x=3, f(x)=12600000", pants_run.stdout_data)
def register_options(cls, register):
super().register_options(register)
register(
"--compiler-option-sets",
advanced=True,
default=(),
type=list,
fingerprint=True,
help='The default for the "compiler_option_sets" argument '
"for targets of this language.",
)
register(
"--toolchain-variant",
advanced=True,
default=match(
Platform.current,
{
Platform.darwin: ToolchainVariant.llvm,
Platform.linux: ToolchainVariant.gnu,
},
),
type=ToolchainVariant,
fingerprint=True,
help=
"Whether to use gcc (gnu) or clang (llvm) to compile C and C++. Note that "
"currently, despite the choice of toolchain, all linking is done with binutils "
"ld on Linux, and the XCode CLI Tools on MacOS.",
)
def _zinc_key_for_target(self, target, workflow):
return match(workflow, {
self.JvmCompileWorkflowType.zinc_only: lambda: f'zinc[zinc-only]({target.address.spec})',
self.JvmCompileWorkflowType.zinc_java: lambda: f'zinc[zinc-java]({target.address.spec})',
self.JvmCompileWorkflowType.rsc_and_zinc: lambda: f'zinc[rsc-and-zinc]({target.address.spec})',
self.JvmCompileWorkflowType.outline_and_zinc: lambda: f'zinc[outline-and-zinc]({target.address.spec})',
})()
def register_extra_products_from_contexts(self, targets, compile_contexts):
super().register_extra_products_from_contexts(targets, compile_contexts)
def confify(entries):
return [(conf, e) for e in entries for conf in self._confs]
# Ensure that the jar/rsc jar is on the rsc_mixed_compile_classpath.
for target in targets:
merged_cc = compile_contexts[target]
zinc_cc = merged_cc.zinc_cc
rsc_cc = merged_cc.rsc_cc
# Make sure m.jar is digested if it exists when the target is validated.
if rsc_cc.rsc_jar_file.directory_digest is None and os.path.exists(rsc_cc.rsc_jar_file.path):
relpath = fast_relpath(rsc_cc.rsc_jar_file.path, get_buildroot())
classes_dir_snapshot, = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([relpath]),
get_buildroot(),
Digest.load(relpath),
),
])
rsc_cc.rsc_jar_file.hydrate_missing_directory_digest(classes_dir_snapshot.directory_digest)
if rsc_cc.workflow is not None:
cp_entries = match(rsc_cc.workflow, {
self.JvmCompileWorkflowType.zinc_only: lambda: confify([self._classpath_for_context(zinc_cc)]),
self.JvmCompileWorkflowType.zinc_java: lambda: confify([self._classpath_for_context(zinc_cc)]),
self.JvmCompileWorkflowType.rsc_and_zinc: lambda: confify([rsc_cc.rsc_jar_file]),
self.JvmCompileWorkflowType.outline_and_zinc: lambda: confify([rsc_cc.rsc_jar_file]),
})()
self.context.products.get_data('rsc_mixed_compile_classpath').add_for_target(
target,
cp_entries)
def get_zinc_compiler_classpath(self):
return match(self.execution_strategy, {
# NB: We must use the verbose version of super() here, possibly because of the lambda.
self.ExecutionStrategy.hermetic: lambda: super(RscCompile, self).get_zinc_compiler_classpath(),
self.ExecutionStrategy.subprocess: lambda: super(RscCompile, self).get_zinc_compiler_classpath(),
self.ExecutionStrategy.nailgun: lambda: self._nailgunnable_combined_classpath,
})()
def _assert_dist_and_wheel_identity(self, expected_name, expected_version,
expected_platform, dist_target,
**kwargs):
context, synthetic_target, fingerprint_suffix = self._create_distribution_synthetic_target(
dist_target, **kwargs)
resulting_dist_req = assert_single_element(
synthetic_target.requirements.value)
expected_snapshot_version = f'{expected_version}+{fingerprint_suffix}'
self.assertEquals(f'{expected_name}=={expected_snapshot_version}',
str(resulting_dist_req.requirement))
local_wheel_products = context.products.get('local_wheels')
local_wheel = self.retrieve_single_product_at_target_base(
local_wheel_products, dist_target)
dist, version, platform = name_and_platform(local_wheel)
self.assertEquals(dist, expected_name)
self.assertEquals(version, expected_snapshot_version)
expected_platform = match(
expected_platform, {
BuildLocalPythonDistributionsTestBase.ExpectedPlatformType.any:
"any",
BuildLocalPythonDistributionsTestBase.ExpectedPlatformType.current:
normalized_current_platform(),
})
self.assertEquals(platform, expected_platform)
def test_set_invalid_log_location():
assert os.path.isdir("/does/not/exist") is False
sink = _gen_sink_subclass()
with pytest.raises(ExceptionSink.ExceptionSinkError) as exc:
sink.reset_log_location("/does/not/exist")
assert (
"The provided log location path at '/does/not/exist' is not writable or could not be "
"created"
) in str(exc.value)
# NB: This target is marked with 'platform_specific_behavior' because OSX errors out here at
# creating a new directory with safe_mkdir(), Linux errors out trying to create the directory
# for its log files with safe_open(). This may be due to differences in the filesystems.
# TODO: figure out why we error out at different points here!
with pytest.raises(ExceptionSink.ExceptionSinkError) as exc:
sink.reset_log_location("/")
err_str = {
Platform.darwin: (
"The provided log location path at '/' is not writable or could not be created: "
"[Errno 21] Is a directory: '/'."
),
Platform.linux: (
"Error opening fatal error log streams for log location '/': [Errno 13] Permission "
"denied: '/.pids'"
),
}
assert match(Platform.current, err_str) in str(exc.value)
def test_set_invalid_log_location(self):
self.assertFalse(os.path.isdir("/does/not/exist"))
sink = self._gen_sink_subclass()
with self.assertRaisesWithMessageContaining(
ExceptionSink.ExceptionSinkError,
"The provided exception sink path at '/does/not/exist' is not writable or could not be created",
):
sink.reset_log_location("/does/not/exist")
# NB: This target is marked with 'platform_specific_behavior' because OSX errors out here at
# creating a new directory with safe_mkdir(), Linux errors out trying to create the directory
# for its log files with safe_open(). This may be due to differences in the filesystems.
# TODO: figure out why we error out at different points here!
err_str = match(
Platform.current,
{
Platform.darwin:
"The provided exception sink path at '/' is not writable or could not be created: [Errno 21] Is a directory: '/'.",
Platform.linux:
"Error opening fatal error log streams for log location '/': [Errno 13] Permission denied: '/.pids'",
},
)
with self.assertRaisesWithMessageContaining(
ExceptionSink.ExceptionSinkError, err_str):
sink.reset_log_location("/")
def translate_host_platform(
platform_constraint: PlatformConstraint,
binary_util: BinaryUtil,
) -> HostPlatform:
# This method attempts to provide a uname function to BinaryUtil.host_platform() so that the
# download urls can be calculated. For platforms that are different than the current host, we try
# to "spoof" the most appropriate value.
if Platform.current == Platform.darwin:
darwin_uname: Any = os.uname
linux_uname: Any = lambda: ("linux", None, None, None, "x86_64")
else:
assert Platform.current == Platform.linux
darwin_uname = lambda: (
"darwin",
None,
get_closest_mac_host_platform_pair(),
None,
"x86_64",
)
linux_uname = os.uname
return cast(
HostPlatform,
match(
platform_constraint,
{
PlatformConstraint.none:
lambda: HostPlatform.empty,
PlatformConstraint.darwin:
lambda: binary_util.host_platform(uname=darwin_uname()),
PlatformConstraint.linux:
lambda: binary_util.host_platform(uname=linux_uname()),
},
)(),
)
def _runtool(self, distribution, input_digest, ctx, use_youtline):
if use_youtline:
main = "scala.tools.nsc.Main"
tool_name = "scalac-outliner"
tool_classpath = self._scalac_classpath
# In fact, nailgun should not be used for -Youtline
# in case of self.ExecutionStrategy.nailgun,
# we will force the scalac -Youtline invokation to run via subprocess
nailgun_classpath = self._scalac_classpath
else:
main = "rsc.cli.Main"
tool_name = "rsc"
tool_classpath = self._rsc_classpath
nailgun_classpath = self._nailgunnable_combined_classpath
with self.context.new_workunit(tool_name) as wu:
return match(
self.execution_strategy,
{
self.ExecutionStrategy.hermetic:
lambda: self._runtool_hermetic(
main, tool_name, distribution, input_digest, ctx),
self.ExecutionStrategy.subprocess:
lambda: self.
_runtool_nonhermetic(wu, tool_classpath, main, tool_name,
distribution, ctx),
self.ExecutionStrategy.nailgun:
lambda: self._runtool_nonhermetic(
wu, nailgun_classpath, main, tool_name, distribution,
ctx),
},
)()
def test_ctypes_third_party_integration(self, toolchain_variant):
pants_binary = self.run_pants(
["binary", self._binary_target_with_third_party],
config={
"native-build-step": {
"toolchain_variant": toolchain_variant.value,
},
},
)
self.assert_success(pants_binary)
# TODO(#6848): this fails when run with gcc on osx as it requires gcc's libstdc++.so.6.dylib to
# be available on the runtime library path.
attempt_pants_run = match(
Platform.current,
{
Platform.darwin: toolchain_variant == ToolchainVariant.llvm,
Platform.linux: True,
},
)
if attempt_pants_run:
pants_run = self.run_pants(
["-q", "run", self._binary_target_with_third_party],
config={
"native-build-step": {
"toolchain_variant": toolchain_variant.value,
},
},
)
self.assert_success(pants_run)
self.assertIn("Test worked!\n", pants_run.stdout_data)
def _key_for_target_as_dep(self, target, workflow):
# used for jobs that are either rsc jobs or zinc jobs run against rsc
return match(workflow, {
self.JvmCompileWorkflowType.zinc_only: lambda: self._zinc_key_for_target(target, workflow),
self.JvmCompileWorkflowType.zinc_java: lambda: self._zinc_key_for_target(target, workflow),
self.JvmCompileWorkflowType.rsc_and_zinc: lambda: self._rsc_key_for_target(target),
self.JvmCompileWorkflowType.outline_and_zinc: lambda: self._outline_key_for_target(target),
})()
def generate_url(self, plat: Platform) -> str:
plat_str = match(plat, {
Platform.darwin: "macos",
Platform.linux: "linux"
})
return (
f"https://binaries.pantsbuild.org/bin/grpc_python_plugin/{self.version}/"
f"{plat_str}/x86_64/grpc_python_plugin")
def generate_url(self, plat: Platform) -> str:
plat_str = match(plat, {
Platform.darwin: "osx",
Platform.linux: "linux"
})
return (
f"https://github.com/protocolbuffers/protobuf/releases/download/"
f"v{self.version}/protoc-{self.version}-{plat_str}-x86_64.zip")
def generate_url(self, plat: Platform) -> str:
plat_str = match(plat, {
Platform.darwin: "apple-darwin",
Platform.linux: "unknown-linux"
})
return (
f"https://github.com/boyter/scc/releases/download/v{self.version}/scc-{self.version}-"
f"x86_64-{plat_str}.zip")
def granularity_filter(self) -> TargetFilter:
return match(
self.granularity,
{
TargetGranularity.all_targets: lambda _: True,
TargetGranularity.file_targets: lambda tgt: tgt.address.is_file_target,
TargetGranularity.build_targets: lambda tgt: not tgt.address.is_file_target,
},
)
def as_shared_lib(self, platform):
# TODO: check that the name conforms to some format in the constructor (e.g. no dots?).
return match(
platform,
{
Platform.darwin: f"lib{self.lib_name}.dylib",
Platform.linux: f"lib{self.lib_name}.so",
},
)
def filter_granularity(granularity: TargetGranularity) -> TargetFilter:
return match(
granularity,
{
TargetGranularity.all_targets: lambda _: True,
TargetGranularity.file_targets: lambda tgt: tgt.address.is_file_target,
TargetGranularity.build_targets: lambda tgt: not tgt.address.is_file_target,
},
)
def test_ctypes_native_language_interop(self, toolchain_variant):
# Replace strict_deps=False with nothing so we can override it (because target values for this
# option take precedence over subsystem options).
with self.with_overwritten_file_content(
self._wrapped_math_build_file, lambda c: re.sub(b"strict_deps=False,", b"", c)
):
# This should fail because it does not turn on strict_deps for a target which requires it.
pants_binary_strict_deps_failure = self.run_pants(
command=["binary", self._binary_target_with_interop],
# Explicitly set to True (although this is the default).
config={
"native-build-step": {"toolchain_variant": toolchain_variant.value},
# TODO(#6848): don't make it possible to forget to add the toolchain_variant option!
"native-build-settings": {"strict_deps": True},
},
)
self.assert_failure(pants_binary_strict_deps_failure)
self.assertIn(
match(
toolchain_variant,
{
ToolchainVariant.gnu: "fatal error: some_math.h: No such file or directory",
ToolchainVariant.llvm: "fatal error: 'some_math.h' file not found",
},
),
pants_binary_strict_deps_failure.stdout_data,
)
# TODO(#6848): we need to provide the libstdc++.so.6.dylib which comes with gcc on osx in the
# DYLD_LIBRARY_PATH during the 'run' goal somehow.
attempt_pants_run = match(
Platform.current,
{Platform.darwin: toolchain_variant == ToolchainVariant.llvm, Platform.linux: True},
)
if attempt_pants_run:
pants_run_interop = self.run_pants(
["-q", "run", self._binary_target_with_interop],
config={
"native-build-step": {"toolchain_variant": toolchain_variant.value},
"native-build-settings": {"strict_deps": True},
},
)
self.assert_success(pants_run_interop)
self.assertEqual("x=3, f(x)=299\n", pants_run_interop.stdout_data)
def _common_lib_dirs(self, platform):
lib64_tuples = match(platform, {
Platform.darwin: [],
Platform.linux: [('lib64',)]
})
return self._filemap(lib64_tuples + [
('lib',),
('lib/gcc',),
('lib/gcc/*', self.version()),
])
async def select_libc_objects(
platform: Platform, native_toolchain: NativeToolchain) -> LibcObjects:
# We use lambdas here to avoid searching for libc on osx, where it will fail.
paths = match(
platform, {
Platform.darwin: lambda: [],
Platform.linux:
lambda: native_toolchain._libc_dev.get_libc_objects(),
})()
return LibcObjects(paths)
def _common_lib_dirs(self, platform: Platform):
lib64_tuples: List[Tuple[str, ...]] = match(platform, {
Platform.darwin: [],
Platform.linux: [("lib64", )]
})
files: List[Tuple[str, ...]] = [
*lib64_tuples,
("lib", ),
("lib/gcc", ),
("lib/gcc/*", self.version()),
]
return self._filemap(files)
def linker(self, platform: Platform) -> Linker:
return Linker(
path_entries=self.path_entries,
exe_filename=match(platform, {
Platform.darwin: "ld64.lld",
Platform.linux: "lld",
}),
runtime_library_dirs=(),
linking_library_dirs=(),
extra_args=(),
extra_object_files=(),
)
def _get_jvm_distribution(self):
# TODO We may want to use different jvm distributions depending on what
# java version the target expects to be compiled against.
# See: https://github.com/pantsbuild/pants/issues/6416 for covering using
# different jdks in remote builds.
local_distribution = self._local_jvm_distribution()
return match(
self.execution_strategy, {
self.ExecutionStrategy.subprocess:
lambda: local_distribution,
self.ExecutionStrategy.nailgun:
lambda: local_distribution,
self.ExecutionStrategy.hermetic:
lambda: self._HermeticDistribution('.jdk', local_distribution),
})()
def binary_path_test(self) -> BinaryPathTest | None:
arg = match( # type: ignore[misc]
self,
{
self.sh: None,
self.bash: "--version",
self.dash: None,
self.ksh: "--version",
self.pdksh: None,
self.zsh: "--version",
},
)
if not arg:
return None
return BinaryPathTest((arg, ))
def graph_vertex_color_fmt_str(self) -> Optional[str]:
olive = "0.2214,0.7179,0.8528"
gray = "0.576,0,0.6242"
orange = "0.08,0.5,0.976"
blue = "0.5,1,0.9"
color = match(
self,
{
GraphVertexType.task: blue,
GraphVertexType.inner: None,
GraphVertexType.singleton: olive,
GraphVertexType.intrinsic: gray,
GraphVertexType.param: orange,
},
)
if color is None:
return None
return f'[color="{color}",style=filled]'
def default_values(self):
common_values = {
"name": "foo",
"answer": "42",
"scale": "1.2",
"path": "/a/b/42",
"embed": "/a/b/42::foo",
}
return match(
self.format,
{
ConfigFormat.ini: {
**common_values, "disclaimer": "\nLet it be known\nthat."
},
ConfigFormat.toml: {
**common_values, "disclaimer": "Let it be known\nthat."
},
},
)
