def __init__(self, xz_binary_path, xz_library_path):
# TODO(cosmicexplorer): test these exceptions somewhere!
if not is_executable(xz_binary_path):
raise self.XZArchiverError(
"The path {} does not name an existing executable file. An xz executable must be provided "
"to decompress xz archives."
.format(xz_binary_path))
self._xz_binary_path = xz_binary_path
if not os.path.isdir(xz_library_path):
raise self.XZArchiverError(
"The path {} does not name an existing directory. A directory containing liblzma.so must "
"be provided to decompress xz archives."
.format(xz_library_path))
lib_lzma_dylib = os.path.join(xz_library_path, 'liblzma.so')
if not os.path.isfile(lib_lzma_dylib):
raise self.XZArchiverError(
"The path {} names an existing directory, but it does not contain liblzma.so. A directory "
"containing liblzma.so must be provided to decompress xz archives."
.format(xz_library_path))
self._xz_library_path = xz_library_path
super(XZCompressedTarArchiver, self).__init__('r|', 'tar.xz')
def __init__(self, xz_binary_path, xz_library_path):
# TODO(cosmicexplorer): test these exceptions somewhere!
if not is_executable(xz_binary_path):
raise self.XZArchiverError(
"The path {} does not name an existing executable file. An xz executable must be provided "
"to decompress xz archives."
.format(xz_binary_path))
self._xz_binary_path = xz_binary_path
if not os.path.isdir(xz_library_path):
raise self.XZArchiverError(
"The path {} does not name an existing directory. A directory containing liblzma.so must "
"be provided to decompress xz archives."
.format(xz_library_path))
lib_lzma_dylib = os.path.join(xz_library_path, 'liblzma.so')
if not os.path.isfile(lib_lzma_dylib):
raise self.XZArchiverError(
"The path {} names an existing directory, but it does not contain liblzma.so. A directory "
"containing liblzma.so must be provided to decompress xz archives."
.format(xz_library_path))
self._xz_library_path = xz_library_path
super(XZCompressedTarArchiver, self).__init__('r|', 'tar.xz')
def _check_executables_exist(self):
for filename in self._REQUIRED_TOOLS:
executable_path = os.path.join(self._INSTALL_LOCATION, filename)
if not is_executable(executable_path):
raise self.XCodeToolsUnavailable(
"'{}' is not an executable file, but it is required to build "
"native code on this platform. You may need to install the XCode "
"command line developer tools.".format(executable_path))
def _check_executables_exist(self):
for filename in self._REQUIRED_TOOLS:
executable_path = os.path.join(self._INSTALL_LOCATION, filename)
if not is_executable(executable_path):
raise self.XCodeToolsUnavailable(
"'{}' is not an executable file, but it is required to build "
"native code on this platform. You may need to install the XCode "
"command line developer tools."
.format(executable_path))
def _check_executables_exist(self):
for filename in self._REQUIRED_TOOLS:
executable_path = os.path.join(self._install_location, filename)
if not is_executable(executable_path):
raise self.XCodeToolsUnavailable(
"'{exe}' is not an executable file, but it is required to build "
"native code on this platform. You may need to install the XCode "
"command line developer tools from the Mac App Store. "
"(for file '{exe}' with --xcode_cli_install_location={loc!r})"
.format(exe=filename, loc=self._install_location))
def __init__(self, xz_binary_path):
# TODO: test this exception somewhere!
if not is_executable(xz_binary_path):
raise self.XZArchiverError(
"The path {} does not name an existing executable file. An xz executable must be provided "
"to decompress xz archives.".format(xz_binary_path))
self._xz_binary_path = xz_binary_path
super(XZCompressedTarArchiver, self).__init__('r|', 'tar.xz')
def _do_compile_link(self, compiler, linker, source_file, outfile, output):
intermediate_obj_file_name = f"{outfile}.o"
self._invoke_compiler(compiler, ["-c", source_file, "-o", intermediate_obj_file_name])
self.assertTrue(os.path.isfile(intermediate_obj_file_name))
self._invoke_linker(linker, [intermediate_obj_file_name, "-o", outfile])
self.assertTrue(is_executable(outfile))
program_out = self._invoke_capturing_output([os.path.abspath(outfile)])
self.assertEqual((output + "\n"), program_out)
def _do_compile_link(self, compiler, linker, source_file, outfile, output):
intermediate_obj_file_name = '{}.o'.format(outfile)
self._invoke_compiler(
compiler, ['-c', source_file, '-o', intermediate_obj_file_name])
self.assertTrue(os.path.isfile(intermediate_obj_file_name))
self._invoke_linker(linker,
[intermediate_obj_file_name, '-o', outfile])
self.assertTrue(is_executable(outfile))
program_out = self._invoke_capturing_output([os.path.abspath(outfile)])
self.assertEqual((output + '\n'), program_out)
def __init__(self, xz_binary_path):
# TODO: test this exception somewhere!
if not is_executable(xz_binary_path):
raise self.XZArchiverError(
"The path {} does not name an existing executable file. An xz executable must be provided "
"to decompress xz archives."
.format(xz_binary_path))
self._xz_binary_path = xz_binary_path
super(XZCompressedTarArchiver, self).__init__('r|', 'tar.xz')
def _assert_ctypes_binary_creation(self, toolchain_variant):
with temporary_dir() as tmp_dir:
pants_run = self.run_pants(command=['binary', self._binary_target], config={
GLOBAL_SCOPE_CONFIG_SECTION: {
'pants_distdir': tmp_dir,
},
'native-build-step': {
'toolchain_variant': toolchain_variant,
},
})
self.assert_success(pants_run)
# Check that we have selected the appropriate compilers for our selected toolchain variant,
# for both C and C++ compilation.
# TODO(#6866): don't parse info logs for testing!
for compiler_name in self._compiler_names_for_variant[toolchain_variant]:
self.assertIn("selected compiler exe name: '{}'".format(compiler_name),
pants_run.stdout_data)
for linker_name in self._linker_names_for_variant[toolchain_variant]:
self.assertIn("selected linker exe name: '{}'".format(linker_name),
pants_run.stdout_data)
# Check for the pex and for the wheel produced for our python_dist().
pex = os.path.join(tmp_dir, 'bin.pex')
self.assertTrue(is_executable(pex))
# The + is because we append the target's fingerprint to the version. We test this version
# string in test_build_local_python_distributions.py.
wheel_glob = os.path.join(tmp_dir, 'ctypes_test-0.0.1+*.whl')
wheel_dist_with_path = assert_single_element(glob.glob(wheel_glob))
wheel_dist = re.sub('^{}{}'.format(re.escape(tmp_dir), os.path.sep), '', wheel_dist_with_path)
dist_name, dist_version, wheel_platform = name_and_platform(wheel_dist)
self.assertEqual(dist_name, 'ctypes_test')
contains_current_platform = Platform.create().resolve_platform_specific({
'darwin': lambda: wheel_platform.startswith('macosx'),
'linux': lambda: wheel_platform.startswith('linux'),
})
self.assertTrue(contains_current_platform)
# Verify that the wheel contains our shared libraries.
wheel_files = ZipFile(wheel_dist_with_path).namelist()
dist_versioned_name = '{}-{}.data'.format(dist_name, dist_version)
for shared_lib_filename in ['libasdf-c_ctypes.so', 'libasdf-cpp_ctypes.so']:
full_path_in_wheel = os.path.join(dist_versioned_name, 'data', shared_lib_filename)
self.assertIn(full_path_in_wheel, wheel_files)
# Execute the binary and ensure its output is correct.
binary_run_output = invoke_pex_for_output(pex)
self.assertEqual(b'x=3, f(x)=17\n', binary_run_output)
def _do_compile_link(self, compiler, linker, source_file, outfile, output):
intermediate_obj_file_name = '{}.o'.format(outfile)
self._invoke_compiler(
compiler,
['-c', source_file, '-o', intermediate_obj_file_name])
self.assertTrue(os.path.isfile(intermediate_obj_file_name))
self._invoke_linker(
linker,
[intermediate_obj_file_name, '-o', outfile])
self.assertTrue(is_executable(outfile))
program_out = self._invoke_capturing_output([os.path.abspath(outfile)])
self.assertEqual((output + '\n'), program_out)
def bootstrap(self, interpreter, pex_file_path, extra_reqs=None):
# Caching is done just by checking if the file at the specified path is already executable.
if not is_executable(pex_file_path):
pex_info = PexInfo.default(interpreter=interpreter)
if self.entry_point is not None:
pex_info.entry_point = self.entry_point
with safe_concurrent_creation(pex_file_path) as safe_path:
all_reqs = list(self.base_requirements) + list(extra_reqs or [])
pex_builder = PexBuilderWrapper.Factory.create(
builder=PEXBuilder(interpreter=interpreter, pex_info=pex_info))
pex_builder.add_resolved_requirements(all_reqs, platforms=['current'])
pex_builder.build(safe_path)
return PEX(pex_file_path, interpreter)
def test_ctypes_binary(self):
with temporary_dir() as tmp_dir:
pants_run = self.run_pants(command=['binary', self._binary_target],
config={
GLOBAL_SCOPE_CONFIG_SECTION: {
'pants_distdir': tmp_dir,
}
})
self.assert_success(pants_run)
# Check for the pex and for the wheel produced for our python_dist().
pex = os.path.join(tmp_dir, 'bin.pex')
self.assertTrue(is_executable(pex))
# The + is because we append the target's fingerprint to the version. We test this version
# string in test_build_local_python_distributions.py.
wheel_glob = os.path.join(tmp_dir, 'ctypes_test-0.0.1+*.whl')
wheel_dist_with_path = assert_single_element(glob.glob(wheel_glob))
wheel_dist = re.sub(
'^{}{}'.format(re.escape(tmp_dir), os.path.sep), '',
wheel_dist_with_path)
dist_name, dist_version, wheel_platform = name_and_platform(
wheel_dist)
self.assertEqual(dist_name, 'ctypes_test')
contains_current_platform = Platform.create(
).resolve_platform_specific({
'darwin':
lambda: wheel_platform.startswith('macosx'),
'linux':
lambda: wheel_platform.startswith('linux'),
})
self.assertTrue(contains_current_platform)
# Verify that the wheel contains our shared libraries.
wheel_files = ZipFile(wheel_dist_with_path).namelist()
dist_versioned_name = '{}-{}.data'.format(dist_name, dist_version)
for shared_lib_filename in ['libasdf-c.so', 'libasdf-cpp.so']:
full_path_in_wheel = os.path.join(dist_versioned_name, 'data',
shared_lib_filename)
self.assertIn(full_path_in_wheel, wheel_files)
# Execute the binary and ensure its output is correct.
binary_run_output = invoke_pex_for_output(pex)
self.assertEqual('x=3, f(x)=17\n', binary_run_output)
def bootstrap(self, interpreter, pex_file_path, extra_reqs=None):
# Caching is done just by checking if the file at the specified path is already executable.
if not is_executable(pex_file_path):
pex_info = PexInfo.default(interpreter=interpreter)
if self.entry_point is not None:
pex_info.entry_point = self.entry_point
with safe_concurrent_creation(pex_file_path) as safe_path:
all_reqs = list(self.base_requirements) + list(extra_reqs
or [])
pex_builder = PexBuilderWrapper.Factory.create(
builder=PEXBuilder(interpreter=interpreter,
pex_info=pex_info))
pex_builder.add_resolved_requirements(all_reqs,
platforms=['current'])
pex_builder.build(safe_path)
return PEX(pex_file_path, interpreter)
def test_binary(self):
with temporary_dir() as tmp_dir:
pants_run = self.run_pants(command=['binary', self._binary_target], config={
GLOBAL_SCOPE_CONFIG_SECTION: {
'pants_distdir': tmp_dir,
}
})
self.assert_success(pants_run)
# Check for the pex and for the wheel produced for our python_dist().
pex = os.path.join(tmp_dir, 'bin.pex')
self.assertTrue(is_executable(pex))
wheel_glob = os.path.join(tmp_dir, 'ctypes_test-0.0.1-*.whl')
globbed_wheel = glob.glob(wheel_glob)
self.assertEqual(len(globbed_wheel), 1)
wheel_dist = globbed_wheel[0]
_, _, wheel_platform = name_and_platform(wheel_dist)
contains_current_platform = Platform.create().resolve_platform_specific({
'darwin': lambda: wheel_platform.startswith('macosx'),
'linux': lambda: wheel_platform.startswith('linux'),
})
self.assertTrue(contains_current_platform)
# Verify that the wheel contains our shared libraries.
wheel_files = ZipFile(wheel_dist).namelist()
for shared_lib_filename in ['libasdf-c.so', 'libasdf-cpp.so']:
full_path_in_wheel = os.path.join('ctypes_test-0.0.1.data', 'data', shared_lib_filename)
self.assertIn(full_path_in_wheel, wheel_files)
# Execute the binary and ensure its output is correct.
binary_run_output = invoke_pex_for_output(pex)
self.assertEqual('x=3, f(x)=17\n', binary_run_output)
def test_python_distribution_with_setup_requires(self):
# Validate that setup_requires dependencies are present and functional.
# PANTS_TEST_SETUP_REQUIRES triggers test functionality in this particular setup.py.
with environment_as(PANTS_TEST_SETUP_REQUIRES='1'):
command=['run', '{}:main'.format(self.hello_setup_requires)]
pants_run = self.run_pants(command=command)
command=['run', '{}:main'.format(self.hello_setup_requires)]
pants_run = self.run_pants(command=command)
with temporary_dir() as tmp_dir:
pex = os.path.join(tmp_dir, 'main.pex')
command=[
'--pants-distdir={}'.format(tmp_dir),
'binary',
'{}:main'.format(self.hello_setup_requires),
]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(is_executable(pex))
# Check that the pex runs.
output = subprocess.check_output(pex).decode('utf-8')
self.assertEqual('Hello, world!\n', output)
def test_ctypes_binary_creation(self, toolchain_variant):
"""Create a python_binary() with all native toolchain variants, and test the result."""
with temporary_dir() as tmp_dir:
pants_run = self.run_pants(command=['binary', self._binary_target], config={
GLOBAL_SCOPE_CONFIG_SECTION: {
'pants_distdir': tmp_dir,
},
'native-build-step': {
'toolchain_variant': toolchain_variant.value,
},
})
self.assert_success(pants_run)
# Check that we have selected the appropriate compilers for our selected toolchain variant,
# for both C and C++ compilation.
# TODO(#6866): don't parse info logs for testing! There is a TODO in test_cpp_compile.py
# in the native backend testing to traverse the PATH to find the selected compiler.
compiler_names_to_check = toolchain_variant.resolve_for_enum_variant({
'gnu': ['gcc', 'g++'],
'llvm': ['clang', 'clang++'],
})
for compiler_name in compiler_names_to_check:
self.assertIn("selected compiler exe name: '{}'".format(compiler_name),
pants_run.stdout_data)
# All of our toolchains currently use the C++ compiler's filename as argv[0] for the linker,
# so there is only one name to check.
linker_names_to_check = toolchain_variant.resolve_for_enum_variant({
'gnu': ['g++'],
'llvm': ['clang++'],
})
for linker_name in linker_names_to_check:
self.assertIn("selected linker exe name: '{}'".format(linker_name),
pants_run.stdout_data)
# Check for the pex and for the wheel produced for our python_dist().
pex = os.path.join(tmp_dir, 'bin.pex')
self.assertTrue(is_executable(pex))
# The + is because we append the target's fingerprint to the version. We test this version
# string in test_build_local_python_distributions.py.
wheel_glob = os.path.join(tmp_dir, 'ctypes_test-0.0.1+*.whl')
wheel_dist_with_path = assert_single_element(glob.glob(wheel_glob))
wheel_dist = re.sub('^{}{}'.format(re.escape(tmp_dir), os.path.sep), '', wheel_dist_with_path)
dist_name, dist_version, wheel_platform = name_and_platform(wheel_dist)
self.assertEqual(dist_name, 'ctypes_test')
contains_current_platform = Platform.current.resolve_for_enum_variant({
'darwin': wheel_platform.startswith('macosx'),
'linux': wheel_platform.startswith('linux'),
})
self.assertTrue(contains_current_platform)
# Verify that the wheel contains our shared libraries.
wheel_files = ZipFile(wheel_dist_with_path).namelist()
dist_versioned_name = '{}-{}.data'.format(dist_name, dist_version)
for shared_lib_filename in ['libasdf-c_ctypes.so', 'libasdf-cpp_ctypes.so']:
full_path_in_wheel = os.path.join(dist_versioned_name, 'data', shared_lib_filename)
self.assertIn(full_path_in_wheel, wheel_files)
# Execute the binary and ensure its output is correct.
binary_run_output = invoke_pex_for_output(pex)
self.assertEqual(b'x=3, f(x)=17\n', binary_run_output)
def test_ctypes_binary_creation(self, toolchain_variant):
"""Create a python_binary() with all native toolchain variants, and test the result."""
with temporary_dir() as tmp_dir:
pants_run = self.run_pants(command=['binary', self._binary_target],
config={
GLOBAL_SCOPE_CONFIG_SECTION: {
'pants_distdir': tmp_dir,
},
'native-build-step': {
'toolchain_variant':
toolchain_variant.value,
},
})
self.assert_success(pants_run)
# Check that we have selected the appropriate compilers for our selected toolchain variant,
# for both C and C++ compilation.
# TODO(#6866): don't parse info logs for testing! There is a TODO in test_cpp_compile.py
# in the native backend testing to traverse the PATH to find the selected compiler.
compiler_names_to_check = toolchain_variant.match({
ToolchainVariant.gnu: ['gcc', 'g++'],
ToolchainVariant.llvm: ['clang', 'clang++'],
})
for compiler_name in compiler_names_to_check:
self.assertIn(
"selected compiler exe name: '{}'".format(compiler_name),
pants_run.stdout_data)
# All of our toolchains currently use the C++ compiler's filename as argv[0] for the linker,
# so there is only one name to check.
linker_names_to_check = toolchain_variant.match({
ToolchainVariant.gnu: ['g++'],
ToolchainVariant.llvm: ['clang++'],
})
for linker_name in linker_names_to_check:
self.assertIn(
"selected linker exe name: '{}'".format(linker_name),
pants_run.stdout_data)
# Check for the pex and for the wheel produced for our python_dist().
pex = os.path.join(tmp_dir, 'bin.pex')
self.assertTrue(is_executable(pex))
# The + is because we append the target's fingerprint to the version. We test this version
# string in test_build_local_python_distributions.py.
wheel_glob = os.path.join(tmp_dir, 'ctypes_test-0.0.1+*.whl')
wheel_dist_with_path = assert_single_element(glob.glob(wheel_glob))
wheel_dist = re.sub(
'^{}{}'.format(re.escape(tmp_dir), os.path.sep), '',
wheel_dist_with_path)
dist_name, dist_version, wheel_platform = name_and_platform(
wheel_dist)
self.assertEqual(dist_name, 'ctypes_test')
contains_current_platform = Platform.current.match({
Platform.darwin:
wheel_platform.startswith('macosx'),
Platform.linux:
wheel_platform.startswith('linux'),
})
self.assertTrue(contains_current_platform)
# Verify that the wheel contains our shared libraries.
wheel_files = ZipFile(wheel_dist_with_path).namelist()
dist_versioned_name = '{}-{}.data'.format(dist_name, dist_version)
for shared_lib_filename in [
'libasdf-c_ctypes.so', 'libasdf-cpp_ctypes.so'
]:
full_path_in_wheel = os.path.join(dist_versioned_name, 'data',
shared_lib_filename)
self.assertIn(full_path_in_wheel, wheel_files)
# Execute the binary and ensure its output is correct.
binary_run_output = invoke_pex_for_output(pex)
self.assertEqual(b'x=3, f(x)=17\n', binary_run_output)
def test_ctypes_binary_creation(self, toolchain_variant):
"""Create a python_binary() with all native toolchain variants, and test the result."""
with temporary_dir() as tmp_dir:
pants_run = self.run_pants(
command=["binary", self._binary_target],
config={
GLOBAL_SCOPE_CONFIG_SECTION: {"pants_distdir": tmp_dir},
"native-build-step": {"toolchain_variant": toolchain_variant.value},
},
)
self.assert_success(pants_run)
# Check that we have selected the appropriate compilers for our selected toolchain variant,
# for both C and C++ compilation.
# TODO(#6866): don't parse info logs for testing! There is a TODO in test_cpp_compile.py
# in the native backend testing to traverse the PATH to find the selected compiler.
compiler_names_to_check = match(
toolchain_variant,
{
ToolchainVariant.gnu: ["gcc", "g++"],
ToolchainVariant.llvm: ["clang", "clang++"],
},
)
for compiler_name in compiler_names_to_check:
self.assertIn(
f"selected compiler exe name: '{compiler_name}'", pants_run.stdout_data
)
# All of our toolchains currently use the C++ compiler's filename as argv[0] for the linker,
# so there is only one name to check.
linker_names_to_check = match(
toolchain_variant,
{ToolchainVariant.gnu: ["g++"], ToolchainVariant.llvm: ["clang++"]},
)
for linker_name in linker_names_to_check:
self.assertIn(f"selected linker exe name: '{linker_name}'", pants_run.stdout_data)
# Check for the pex and for the wheel produced for our python_dist().
pex = os.path.join(tmp_dir, "bin.pex")
self.assertTrue(is_executable(pex))
# The + is because we append the target's fingerprint to the version. We test this version
# string in test_build_local_python_distributions.py.
wheel_glob = os.path.join(tmp_dir, "ctypes_test-0.0.1+*.whl")
wheel_dist_with_path = assert_single_element(glob.glob(wheel_glob))
wheel_dist = re.sub(f"^{re.escape(tmp_dir)}{os.path.sep}", "", wheel_dist_with_path)
dist_name, dist_version, wheel_platform = name_and_platform(wheel_dist)
self.assertEqual(dist_name, "ctypes_test")
contains_current_platform = match(
Platform.current,
{
Platform.darwin: wheel_platform.startswith("macosx"),
Platform.linux: wheel_platform.startswith("linux"),
},
)
self.assertTrue(contains_current_platform)
# Verify that the wheel contains our shared libraries.
wheel_files = ZipFile(wheel_dist_with_path).namelist()
dist_versioned_name = f"{dist_name}-{dist_version}.data"
for shared_lib_filename in ["libasdf-c_ctypes.so", "libasdf-cpp_ctypes.so"]:
full_path_in_wheel = os.path.join(dist_versioned_name, "data", shared_lib_filename)
self.assertIn(full_path_in_wheel, wheel_files)
# Execute the binary and ensure its output is correct.
binary_run_output = invoke_pex_for_output(pex)
self.assertEqual(b"x=3, f(x)=17\n", binary_run_output)