def _get_existing_subdirs(self, subdir_name): # TODO(#6143): We should attempt to use ParseSearchDirs here or find some documentation on which # directories we should be adding to the include path, and why. If we do need to manually # specify paths, use ArchiveFileMapper instead of doing all that logic over again in this file. maybe_subdirs = [os.path.join(pfx, subdir_name) for pfx in self._all_existing_install_prefixes] existing_dirs = [existing_dir for existing_dir in maybe_subdirs if is_readable_dir(existing_dir)] required_files_for_dir = self._REQUIRED_FILES.get(subdir_name) if required_files_for_dir: for fname in required_files_for_dir: found = False for subdir in existing_dirs: full_path = os.path.join(subdir, fname) if os.path.isfile(full_path): found = True continue if not found: raise self.XCodeToolsUnavailable( "File '{fname}' in subdirectory '{subdir_name}' does not exist at any of the specified " "prefixes. This file is required to build native code on this platform. You may need " "to install the XCode command line developer tools from the Mac App Store.\n\n" "If the XCode tools are installed and you are still seeing this message, please file " "an issue at https://github.com/pantsbuild/pants/issues/new describing your " "OSX environment and which file could not be found.\n" "The existing install prefixes were: {pfxs}. These can be extended with " "--{scope}-install-prefixes." .format(fname=fname, subdir_name=subdir_name, pfxs=self._all_existing_install_prefixes, scope=self.get_options_scope_equivalent_flag_component())) return existing_dirs
def include_dirs(self): base_inc_dirs = self._get_existing_subdirs('include') all_inc_dirs = base_inc_dirs for d in base_inc_dirs: secure_inc_dir = os.path.join(d, 'secure') if is_readable_dir(secure_inc_dir): all_inc_dirs.append(secure_inc_dir) return all_inc_dirs
def select(self): unpacked_path = super().select() # The archive from releases.llvm.org wraps the extracted content into a directory one level # deeper, but the one from our S3 does not. We account for both here. children = os.listdir(unpacked_path) if len(children) == 1: llvm_base_dir = os.path.join(unpacked_path, children[0]) assert(is_readable_dir(llvm_base_dir)) return llvm_base_dir return unpacked_path
def select(self): unpacked_path = super(LLVM, self).select() # The archive from releases.llvm.org wraps the extracted content into a directory one level # deeper, but the one from our S3 does not. We account for both here. children = os.listdir(unpacked_path) if len(children) == 1: llvm_base_dir = os.path.join(unpacked_path, children[0]) assert(is_readable_dir(llvm_base_dir)) return llvm_base_dir return unpacked_path
def include_dirs(self): base_inc_dirs = self._get_existing_subdirs('include') all_inc_dirs = base_inc_dirs for d in base_inc_dirs: # TODO: figure out what this directory does and why it's not already found by this compiler. secure_inc_dir = os.path.join(d, 'secure') if is_readable_dir(secure_inc_dir): all_inc_dirs.append(secure_inc_dir) return all_inc_dirs
def include_dirs(self): base_inc_dirs = self._get_existing_subdirs('include') all_inc_dirs = base_inc_dirs for d in base_inc_dirs: # FIXME: what does this directory do? secure_inc_dir = os.path.join(d, 'secure') if is_readable_dir(secure_inc_dir): all_inc_dirs.append(secure_inc_dir) return all_inc_dirs
def _filter_existing_dirs(self, dir_candidates, compiler_exe): real_dirs = OrderedSet() for maybe_existing_dir in dir_candidates: # Could use a `seen_dir_paths` set if we want to avoid pinging the fs for duplicate entries. if is_readable_dir(maybe_existing_dir): real_dirs.add(os.path.realpath(maybe_existing_dir)) else: logger.debug("non-existent or non-accessible directory at {} while " "parsing directories from {}" .format(maybe_existing_dir, compiler_exe)) return list(real_dirs)
def _install_yarnpkg(self): """Install the Yarnpkg distribution from pants support binaries. :returns: The Yarnpkg distribution bin path. :rtype: string """ yarnpkg_package_path = YarnpkgDistribution.scoped_instance(self).select() yarnpkg_bin_path = os.path.join(yarnpkg_package_path, "dist", "bin") if not is_readable_dir(yarnpkg_bin_path): # The binary was pulled from yarn's Github release page and not our S3, # in which case it's installed under a different directory. return os.path.join(yarnpkg_package_path, os.listdir(yarnpkg_package_path)[0], "bin") return yarnpkg_bin_path
def _filter_existing_dirs(self, dir_candidates, compiler_exe): real_dirs = OrderedSet() for maybe_existing_dir in dir_candidates: # Could use a `seen_dir_paths` set if we want to avoid pinging the fs for duplicate entries. if is_readable_dir(maybe_existing_dir): real_dirs.add(os.path.realpath(maybe_existing_dir)) else: logger.debug( "non-existent or non-accessible directory at {} while " "parsing directories from {}".format( maybe_existing_dir, compiler_exe)) return list(real_dirs)
def get_compiler_library_dirs(self, compiler_exe, env=None): all_dir_candidates = self._parse_libraries_from_compiler_search_dirs(compiler_exe, env=env) real_lib_dirs = OrderedSet() for lib_dir_path in all_dir_candidates: # Could use a `seen_dir_paths` set if we want to avoid pinging the fs for duplicate entries. if is_readable_dir(lib_dir_path): real_lib_dirs.add(os.path.realpath(lib_dir_path)) else: logger.debug("non-existent or non-accessible program directory at {} while " "parsing libraries from {}" .format(lib_dir_path, compiler_exe)) return list(real_lib_dirs)
def _install_node(self): """Install the Node distribution from pants support binaries. :returns: The Node distribution bin path. :rtype: string """ node_package_path = self.select() # Todo: https://github.com/pantsbuild/pants/issues/4431 # This line depends on repacked node distribution. # Should change it from 'node/bin' to 'dist/bin' node_bin_path = os.path.join(node_package_path, 'node', 'bin') if not is_readable_dir(node_bin_path): # The binary was pulled from nodejs and not our S3, in which # case it's installed under a different directory. return os.path.join(node_package_path, os.listdir(node_package_path)[0], 'bin') return node_bin_path
def __new__(cls, zef_requirements, into_dir): # Non-empty iterable of requirements. if not zef_requirements: raise self.make_type_error( "Error: no zef requirements were provided to install into dir '{}'." .format(into_dir)) if not is_readable_dir(into_dir) and os.access(into_dir, os.W_OK): raise self.make_type_error( "Error: provided directory path '{}' does not exist or is not writable." .format(into_dir)) # TODO: Remove the need to specify the containing class explicitly! return super(Zef.ZefInstallRequest, cls).__new__(cls, tuple(zef_requirements), binary_type(into_dir))
def _install_node(self): """Install the Node distribution from pants support binaries. :returns: The Node distribution bin path. :rtype: string """ node_package_path = self.select() # Todo: https://github.com/pantsbuild/pants/issues/4431 # This line depends on repacked node distribution. # Should change it from 'node/bin' to 'dist/bin' node_bin_path = os.path.join(node_package_path, "node", "bin") if not is_readable_dir(node_bin_path): # The binary was pulled from nodejs and not our S3, in which # case it's installed under a different directory. return os.path.join(node_package_path, os.listdir(node_package_path)[0], "bin") return node_bin_path
def test_select_argv(self): """Test invoking binary_util.py as a standalone script.""" with temporary_dir() as tmp_dir: config_file_loc = os.path.join(tmp_dir, 'pants.ini') safe_file_dump(config_file_loc, payload="""\ [GLOBAL] allow_external_binary_tool_downloads: True pants_bootstrapdir: {} """.format(tmp_dir)) expected_output_glob = os.path.join( tmp_dir, 'bin', 'cmake', '*', '*', '3.9.5', 'cmake') with environment_as(PANTS_CONFIG_FILES='[{!r}]'.format(config_file_loc)): # Ignore the first argument, as per sys.argv. output_file = select(['_', 'cmake', '3.9.5', 'cmake.tar.gz']) self.assertTrue(is_readable_dir(output_file)) realized_glob = assert_single_element(glob.glob(expected_output_glob)) self.assertEqual(os.path.realpath(output_file), os.path.realpath(realized_glob))
def build_tool_into(self, tool_name, version): known_dirname = '{}-{}'.format(tool_name, version) expected_dir = os.path.join(self.select(), known_dirname) # Build it. if not os.path.isdir(expected_dir): logger.info("building tool '{}' at version '{}'...".format( tool_name, version)) output = self._run_rakudobrew_command( ['build', tool_name, version]) logger.info( "output from building tool '{}' at version '{}':\n{}".format( tool_name, version, output)) if not is_readable_dir(expected_dir): raise self.RakudoBrewBootstrapError( "The expected directory after building tool '{}' at version '{}' is '{}', but that path " "does not exist or is not writable.".format( tool_name, version, expected_dir)) return expected_dir
def _get_existing_subdirs(self, subdir_name): # TODO(#6143): We should attempt to use ParseSearchDirs here or find some documentation on which # directories we should be adding to the include path, and why. If we do need to manually # specify paths, use ArchiveFileMapper instead of doing all that logic over again in this file. maybe_subdirs = [ os.path.join(pfx, subdir_name) for pfx in self._all_existing_install_prefixes ] existing_dirs = [ existing_dir for existing_dir in maybe_subdirs if is_readable_dir(existing_dir) ] required_files_for_dir = self._REQUIRED_FILES.get(subdir_name) if required_files_for_dir: for fname in required_files_for_dir: found = False for subdir in existing_dirs: full_path = os.path.join(subdir, fname) if os.path.isfile(full_path): found = True continue if not found: raise self.XCodeToolsUnavailable( "File '{fname}' in subdirectory '{subdir_name}' does not exist at any of the specified " "prefixes. This file is required to build native code on this platform. You may need " "to install the XCode command line developer tools from the Mac App Store.\n\n" "If the XCode tools are installed and you are still seeing this message, please file " "an issue at https://github.com/pantsbuild/pants/issues/new describing your " "OSX environment and which file could not be found.\n" "The existing install prefixes were: {pfxs}. These can be extended with " "--{scope}-install-prefixes.".format( fname=fname, subdir_name=subdir_name, pfxs=self._all_existing_install_prefixes, scope=self. get_options_scope_equivalent_flag_component())) return existing_dirs
def test_select_argv(self): """Test invoking binary_util.py as a standalone script.""" with temporary_dir() as tmp_dir: config_file_loc = os.path.join(tmp_dir, "pants.toml") safe_file_dump( config_file_loc, payload=f"""\ [GLOBAL] allow_external_binary_tool_downloads = true pants_bootstrapdir = "{tmp_dir}" """, ) expected_output_glob = os.path.join(tmp_dir, "bin", "cmake", "*", "*", "3.9.5", "cmake") with environment_as(PANTS_CONFIG_FILES=f"[{config_file_loc!r}]"): # Ignore the first argument, as per sys.argv. output_file = select(["_", "cmake", "3.9.5", "cmake.tar.gz"]) self.assertTrue(is_readable_dir(output_file)) realized_glob = assert_single_element( glob.glob(expected_output_glob)) self.assertEqual(os.path.realpath(output_file), os.path.realpath(realized_glob))
def _all_existing_install_prefixes(self): return [pfx for pfx in self.get_options().install_prefixes if is_readable_dir(pfx)]
def _all_existing_install_prefixes(self): return [ pfx for pfx in self.get_options().install_prefixes if is_readable_dir(pfx) ]