def _get_existing_subdirs(self, subdir_name):
# TODO(#6143): We should attempt to use ParseSearchDirs here or find some documentation on which
# directories we should be adding to the include path, and why. If we do need to manually
# specify paths, use ArchiveFileMapper instead of doing all that logic over again in this file.
maybe_subdirs = [os.path.join(pfx, subdir_name) for pfx in self._all_existing_install_prefixes]
existing_dirs = [existing_dir for existing_dir in maybe_subdirs if is_readable_dir(existing_dir)]
required_files_for_dir = self._REQUIRED_FILES.get(subdir_name)
if required_files_for_dir:
for fname in required_files_for_dir:
found = False
for subdir in existing_dirs:
full_path = os.path.join(subdir, fname)
if os.path.isfile(full_path):
found = True
continue
if not found:
raise self.XCodeToolsUnavailable(
"File '{fname}' in subdirectory '{subdir_name}' does not exist at any of the specified "
"prefixes. This file is required to build native code on this platform. You may need "
"to install the XCode command line developer tools from the Mac App Store.\n\n"
"If the XCode tools are installed and you are still seeing this message, please file "
"an issue at https://github.com/pantsbuild/pants/issues/new describing your "
"OSX environment and which file could not be found.\n"
"The existing install prefixes were: {pfxs}. These can be extended with "
"--{scope}-install-prefixes."
.format(fname=fname,
subdir_name=subdir_name,
pfxs=self._all_existing_install_prefixes,
scope=self.get_options_scope_equivalent_flag_component()))
return existing_dirs
def include_dirs(self):
base_inc_dirs = self._get_existing_subdirs('include')
all_inc_dirs = base_inc_dirs
for d in base_inc_dirs:
secure_inc_dir = os.path.join(d, 'secure')
if is_readable_dir(secure_inc_dir):
all_inc_dirs.append(secure_inc_dir)
return all_inc_dirs
def select(self):
unpacked_path = super().select()
# The archive from releases.llvm.org wraps the extracted content into a directory one level
# deeper, but the one from our S3 does not. We account for both here.
children = os.listdir(unpacked_path)
if len(children) == 1:
llvm_base_dir = os.path.join(unpacked_path, children[0])
assert(is_readable_dir(llvm_base_dir))
return llvm_base_dir
return unpacked_path
def select(self):
unpacked_path = super(LLVM, self).select()
# The archive from releases.llvm.org wraps the extracted content into a directory one level
# deeper, but the one from our S3 does not. We account for both here.
children = os.listdir(unpacked_path)
if len(children) == 1:
llvm_base_dir = os.path.join(unpacked_path, children[0])
assert(is_readable_dir(llvm_base_dir))
return llvm_base_dir
return unpacked_path
def include_dirs(self):
base_inc_dirs = self._get_existing_subdirs('include')
all_inc_dirs = base_inc_dirs
for d in base_inc_dirs:
# TODO: figure out what this directory does and why it's not already found by this compiler.
secure_inc_dir = os.path.join(d, 'secure')
if is_readable_dir(secure_inc_dir):
all_inc_dirs.append(secure_inc_dir)
return all_inc_dirs
def include_dirs(self):
base_inc_dirs = self._get_existing_subdirs('include')
all_inc_dirs = base_inc_dirs
for d in base_inc_dirs:
# FIXME: what does this directory do?
secure_inc_dir = os.path.join(d, 'secure')
if is_readable_dir(secure_inc_dir):
all_inc_dirs.append(secure_inc_dir)
return all_inc_dirs
def include_dirs(self):
base_inc_dirs = self._get_existing_subdirs('include')
all_inc_dirs = base_inc_dirs
for d in base_inc_dirs:
# FIXME: what does this directory do?
secure_inc_dir = os.path.join(d, 'secure')
if is_readable_dir(secure_inc_dir):
all_inc_dirs.append(secure_inc_dir)
return all_inc_dirs
def _filter_existing_dirs(self, dir_candidates, compiler_exe):
real_dirs = OrderedSet()
for maybe_existing_dir in dir_candidates:
# Could use a `seen_dir_paths` set if we want to avoid pinging the fs for duplicate entries.
if is_readable_dir(maybe_existing_dir):
real_dirs.add(os.path.realpath(maybe_existing_dir))
else:
logger.debug("non-existent or non-accessible directory at {} while "
"parsing directories from {}"
.format(maybe_existing_dir, compiler_exe))
return list(real_dirs)
def _install_yarnpkg(self):
"""Install the Yarnpkg distribution from pants support binaries.
:returns: The Yarnpkg distribution bin path.
:rtype: string
"""
yarnpkg_package_path = YarnpkgDistribution.scoped_instance(self).select()
yarnpkg_bin_path = os.path.join(yarnpkg_package_path, "dist", "bin")
if not is_readable_dir(yarnpkg_bin_path):
# The binary was pulled from yarn's Github release page and not our S3,
# in which case it's installed under a different directory.
return os.path.join(yarnpkg_package_path, os.listdir(yarnpkg_package_path)[0], "bin")
return yarnpkg_bin_path
def _filter_existing_dirs(self, dir_candidates, compiler_exe):
real_dirs = OrderedSet()
for maybe_existing_dir in dir_candidates:
# Could use a `seen_dir_paths` set if we want to avoid pinging the fs for duplicate entries.
if is_readable_dir(maybe_existing_dir):
real_dirs.add(os.path.realpath(maybe_existing_dir))
else:
logger.debug(
"non-existent or non-accessible directory at {} while "
"parsing directories from {}".format(
maybe_existing_dir, compiler_exe))
return list(real_dirs)
def get_compiler_library_dirs(self, compiler_exe, env=None):
all_dir_candidates = self._parse_libraries_from_compiler_search_dirs(compiler_exe, env=env)
real_lib_dirs = OrderedSet()
for lib_dir_path in all_dir_candidates:
# Could use a `seen_dir_paths` set if we want to avoid pinging the fs for duplicate entries.
if is_readable_dir(lib_dir_path):
real_lib_dirs.add(os.path.realpath(lib_dir_path))
else:
logger.debug("non-existent or non-accessible program directory at {} while "
"parsing libraries from {}"
.format(lib_dir_path, compiler_exe))
return list(real_lib_dirs)
def _install_node(self):
"""Install the Node distribution from pants support binaries.
:returns: The Node distribution bin path.
:rtype: string
"""
node_package_path = self.select()
# Todo: https://github.com/pantsbuild/pants/issues/4431
# This line depends on repacked node distribution.
# Should change it from 'node/bin' to 'dist/bin'
node_bin_path = os.path.join(node_package_path, 'node', 'bin')
if not is_readable_dir(node_bin_path):
# The binary was pulled from nodejs and not our S3, in which
# case it's installed under a different directory.
return os.path.join(node_package_path, os.listdir(node_package_path)[0], 'bin')
return node_bin_path
def __new__(cls, zef_requirements, into_dir):
# Non-empty iterable of requirements.
if not zef_requirements:
raise self.make_type_error(
"Error: no zef requirements were provided to install into dir '{}'."
.format(into_dir))
if not is_readable_dir(into_dir) and os.access(into_dir, os.W_OK):
raise self.make_type_error(
"Error: provided directory path '{}' does not exist or is not writable."
.format(into_dir))
# TODO: Remove the need to specify the containing class explicitly!
return super(Zef.ZefInstallRequest,
cls).__new__(cls, tuple(zef_requirements),
binary_type(into_dir))
def _install_node(self):
"""Install the Node distribution from pants support binaries.
:returns: The Node distribution bin path.
:rtype: string
"""
node_package_path = self.select()
# Todo: https://github.com/pantsbuild/pants/issues/4431
# This line depends on repacked node distribution.
# Should change it from 'node/bin' to 'dist/bin'
node_bin_path = os.path.join(node_package_path, "node", "bin")
if not is_readable_dir(node_bin_path):
# The binary was pulled from nodejs and not our S3, in which
# case it's installed under a different directory.
return os.path.join(node_package_path, os.listdir(node_package_path)[0], "bin")
return node_bin_path
def test_select_argv(self):
"""Test invoking binary_util.py as a standalone script."""
with temporary_dir() as tmp_dir:
config_file_loc = os.path.join(tmp_dir, 'pants.ini')
safe_file_dump(config_file_loc, payload="""\
[GLOBAL]
allow_external_binary_tool_downloads: True
pants_bootstrapdir: {}
""".format(tmp_dir))
expected_output_glob = os.path.join(
tmp_dir, 'bin', 'cmake', '*', '*', '3.9.5', 'cmake')
with environment_as(PANTS_CONFIG_FILES='[{!r}]'.format(config_file_loc)):
# Ignore the first argument, as per sys.argv.
output_file = select(['_', 'cmake', '3.9.5', 'cmake.tar.gz'])
self.assertTrue(is_readable_dir(output_file))
realized_glob = assert_single_element(glob.glob(expected_output_glob))
self.assertEqual(os.path.realpath(output_file), os.path.realpath(realized_glob))
def test_select_argv(self):
"""Test invoking binary_util.py as a standalone script."""
with temporary_dir() as tmp_dir:
config_file_loc = os.path.join(tmp_dir, 'pants.ini')
safe_file_dump(config_file_loc, payload="""\
[GLOBAL]
allow_external_binary_tool_downloads: True
pants_bootstrapdir: {}
""".format(tmp_dir))
expected_output_glob = os.path.join(
tmp_dir, 'bin', 'cmake', '*', '*', '3.9.5', 'cmake')
with environment_as(PANTS_CONFIG_FILES='[{!r}]'.format(config_file_loc)):
# Ignore the first argument, as per sys.argv.
output_file = select(['_', 'cmake', '3.9.5', 'cmake.tar.gz'])
self.assertTrue(is_readable_dir(output_file))
realized_glob = assert_single_element(glob.glob(expected_output_glob))
self.assertEqual(os.path.realpath(output_file), os.path.realpath(realized_glob))
def build_tool_into(self, tool_name, version):
known_dirname = '{}-{}'.format(tool_name, version)
expected_dir = os.path.join(self.select(), known_dirname)
# Build it.
if not os.path.isdir(expected_dir):
logger.info("building tool '{}' at version '{}'...".format(
tool_name, version))
output = self._run_rakudobrew_command(
['build', tool_name, version])
logger.info(
"output from building tool '{}' at version '{}':\n{}".format(
tool_name, version, output))
if not is_readable_dir(expected_dir):
raise self.RakudoBrewBootstrapError(
"The expected directory after building tool '{}' at version '{}' is '{}', but that path "
"does not exist or is not writable.".format(
tool_name, version, expected_dir))
return expected_dir
def _get_existing_subdirs(self, subdir_name):
# TODO(#6143): We should attempt to use ParseSearchDirs here or find some documentation on which
# directories we should be adding to the include path, and why. If we do need to manually
# specify paths, use ArchiveFileMapper instead of doing all that logic over again in this file.
maybe_subdirs = [
os.path.join(pfx, subdir_name)
for pfx in self._all_existing_install_prefixes
]
existing_dirs = [
existing_dir for existing_dir in maybe_subdirs
if is_readable_dir(existing_dir)
]
required_files_for_dir = self._REQUIRED_FILES.get(subdir_name)
if required_files_for_dir:
for fname in required_files_for_dir:
found = False
for subdir in existing_dirs:
full_path = os.path.join(subdir, fname)
if os.path.isfile(full_path):
found = True
continue
if not found:
raise self.XCodeToolsUnavailable(
"File '{fname}' in subdirectory '{subdir_name}' does not exist at any of the specified "
"prefixes. This file is required to build native code on this platform. You may need "
"to install the XCode command line developer tools from the Mac App Store.\n\n"
"If the XCode tools are installed and you are still seeing this message, please file "
"an issue at https://github.com/pantsbuild/pants/issues/new describing your "
"OSX environment and which file could not be found.\n"
"The existing install prefixes were: {pfxs}. These can be extended with "
"--{scope}-install-prefixes.".format(
fname=fname,
subdir_name=subdir_name,
pfxs=self._all_existing_install_prefixes,
scope=self.
get_options_scope_equivalent_flag_component()))
return existing_dirs
def test_select_argv(self):
"""Test invoking binary_util.py as a standalone script."""
with temporary_dir() as tmp_dir:
config_file_loc = os.path.join(tmp_dir, "pants.toml")
safe_file_dump(
config_file_loc,
payload=f"""\
[GLOBAL]
allow_external_binary_tool_downloads = true
pants_bootstrapdir = "{tmp_dir}"
""",
)
expected_output_glob = os.path.join(tmp_dir, "bin", "cmake", "*",
"*", "3.9.5", "cmake")
with environment_as(PANTS_CONFIG_FILES=f"[{config_file_loc!r}]"):
# Ignore the first argument, as per sys.argv.
output_file = select(["_", "cmake", "3.9.5", "cmake.tar.gz"])
self.assertTrue(is_readable_dir(output_file))
realized_glob = assert_single_element(
glob.glob(expected_output_glob))
self.assertEqual(os.path.realpath(output_file),
os.path.realpath(realized_glob))
def _all_existing_install_prefixes(self): return [pfx for pfx in self.get_options().install_prefixes if is_readable_dir(pfx)]
def _all_existing_install_prefixes(self):
return [
pfx for pfx in self.get_options().install_prefixes
if is_readable_dir(pfx)
]
