class SystemPath(object): global_search = attr.ib(default=True) paths = attr.ib( default=attr.Factory(defaultdict) ) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] _executables = attr.ib(default=attr.Factory(list)) # type: List[PathEntry] _python_executables = attr.ib( default=attr.Factory(dict) ) # type: Dict[str, PathEntry] path_order = attr.ib(default=attr.Factory(list)) # type: List[str] python_version_dict = attr.ib() # type: DefaultDict[Tuple, List[PythonVersion]] only_python = attr.ib(default=False, type=bool) pyenv_finder = attr.ib(default=None) # type: Optional[PythonFinder] asdf_finder = attr.ib(default=None) # type: Optional[PythonFinder] windows_finder = attr.ib(default=None) # type: Optional[WindowsFinder] system = attr.ib(default=False, type=bool) _version_dict = attr.ib( default=attr.Factory(defaultdict) ) # type: DefaultDict[Tuple, List[PathEntry]] ignore_unsupported = attr.ib(default=False, type=bool) __finders = attr.ib( default=attr.Factory(dict) ) # type: Dict[str, Union[WindowsFinder, PythonFinder]] def _register_finder(self, finder_name, finder): # type: (str, Union[WindowsFinder, PythonFinder]) -> "SystemPath" if finder_name not in self.__finders: self.__finders[finder_name] = finder return self def clear_caches(self): for key in ["executables", "python_executables", "version_dict", "path_entries"]: if key in self.__dict__: del self.__dict__[key] for finder in list(self.__finders.keys()): del self.__finders[finder] self.__finders = {} return attr.evolve( self, executables=[], python_executables={}, python_version_dict=defaultdict(list), version_dict=defaultdict(list), pyenv_finder=None, windows_finder=None, asdf_finder=None, path_order=[], paths=defaultdict(PathEntry), ) def __del__(self): for key in ["executables", "python_executables", "version_dict", "path_entries"]: try: del self.__dict__[key] except KeyError: pass for finder in list(self.__finders.keys()): del self.__finders[finder] self.__finders = {} self._python_executables = {} self._executables = [] self.python_version_dict = defaultdict(list) self._version_dict = defaultdict(list) self.path_order = [] self.pyenv_finder = None self.asdf_finder = None self.paths = defaultdict(PathEntry) self.__finders = {} @property def finders(self): # type: () -> List[str] return [k for k in self.__finders.keys()] @staticmethod def check_for_pyenv(): return PYENV_INSTALLED or os.path.exists(normalize_path(PYENV_ROOT)) @staticmethod def check_for_asdf(): return ASDF_INSTALLED or os.path.exists(normalize_path(ASDF_DATA_DIR)) @python_version_dict.default def create_python_version_dict(self): # type: () -> DefaultDict[Tuple, List[PythonVersion]] return defaultdict(list) @cached_property def executables(self): # type: () -> List[PathEntry] self.executables = [ p for p in chain(*(child.children.values() for child in self.paths.values())) if p.is_executable ] return self.executables @cached_property def python_executables(self): # type: () -> Dict[str, PathEntry] python_executables = {} for child in self.paths.values(): if child.pythons: python_executables.update(dict(child.pythons)) for finder_name, finder in self.__finders.items(): if finder.pythons: python_executables.update(dict(finder.pythons)) self._python_executables = python_executables return self._python_executables @cached_property def version_dict(self): # type: () -> DefaultDict[Tuple, List[PathEntry]] self._version_dict = defaultdict( list ) # type: DefaultDict[Tuple, List[PathEntry]] for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): if finder_name == "windows": if entry not in self._version_dict[version]: self._version_dict[version].append(entry) continue if entry not in self._version_dict[version] and entry.is_python: self._version_dict[version].append(entry) for p, entry in self.python_executables.items(): version = entry.as_python # type: PythonVersion if not version: continue if not isinstance(version, tuple): version = version.version_tuple if version and entry not in self._version_dict[version]: self._version_dict[version].append(entry) return self._version_dict def _run_setup(self): # type: () -> "SystemPath" if not self.__class__ == SystemPath: return self new_instance = self path_order = new_instance.path_order[:] path_entries = self.paths.copy() if self.global_search and "PATH" in os.environ: path_order = path_order + os.environ["PATH"].split(os.pathsep) path_order = list(dedup(path_order)) path_instances = [ ensure_path(p.strip('"')) for p in path_order if not any( is_in_path(normalize_path(str(p)), normalize_path(shim)) for shim in SHIM_PATHS ) ] path_entries.update( { p.as_posix(): PathEntry.create( path=p.absolute(), is_root=True, only_python=self.only_python ) for p in path_instances if p.exists() } ) new_instance = attr.evolve( new_instance, path_order=[p.as_posix() for p in path_instances if p.exists()], paths=path_entries, ) if os.name == "nt" and "windows" not in self.finders: new_instance = new_instance._setup_windows() #: slice in pyenv if self.check_for_pyenv() and "pyenv" not in self.finders: new_instance = new_instance._setup_pyenv() #: slice in asdf if self.check_for_asdf() and "asdf" not in self.finders: new_instance = new_instance._setup_asdf() venv = os.environ.get("VIRTUAL_ENV") if os.name == "nt": bin_dir = "Scripts" else: bin_dir = "bin" if venv and (new_instance.system or new_instance.global_search): p = ensure_path(venv) path_order = [(p / bin_dir).as_posix()] + new_instance.path_order new_instance = attr.evolve(new_instance, path_order=path_order) paths = new_instance.paths.copy() paths[p] = new_instance.get_path(p.joinpath(bin_dir)) new_instance = attr.evolve(new_instance, paths=paths) if new_instance.system: syspath = Path(sys.executable) syspath_bin = syspath.parent if syspath_bin.name != bin_dir and syspath_bin.joinpath(bin_dir).exists(): syspath_bin = syspath_bin / bin_dir path_order = [syspath_bin.as_posix()] + new_instance.path_order paths = new_instance.paths.copy() paths[syspath_bin] = PathEntry.create( path=syspath_bin, is_root=True, only_python=False ) new_instance = attr.evolve(new_instance, path_order=path_order, paths=paths) return new_instance def _get_last_instance(self, path): # type: (str) -> int reversed_paths = reversed(self.path_order) paths = [normalize_path(p) for p in reversed_paths] normalized_target = normalize_path(path) last_instance = next(iter(p for p in paths if normalized_target in p), None) if last_instance is None: raise ValueError("No instance found on path for target: {0!s}".format(path)) path_index = self.path_order.index(last_instance) return path_index def _slice_in_paths(self, start_idx, paths): # type: (int, List[Path]) -> "SystemPath" before_path = [] # type: List[str] after_path = [] # type: List[str] if start_idx == 0: after_path = self.path_order[:] elif start_idx == -1: before_path = self.path_order[:] else: before_path = self.path_order[: start_idx + 1] after_path = self.path_order[start_idx + 2 :] path_order = before_path + [p.as_posix() for p in paths] + after_path if path_order == self.path_order: return self return attr.evolve(self, path_order=path_order) def _remove_path(self, path): # type: (str) -> "SystemPath" path_copy = [p for p in reversed(self.path_order[:])] new_order = [] target = normalize_path(path) path_map = {normalize_path(pth): pth for pth in self.paths.keys()} new_paths = self.paths.copy() if target in path_map: del new_paths[path_map[target]] for current_path in path_copy: normalized = normalize_path(current_path) if normalized != target: new_order.append(normalized) new_order = [ensure_path(p).as_posix() for p in reversed(new_order)] return attr.evolve(self, path_order=new_order, paths=new_paths) def _setup_asdf(self): # type: () -> "SystemPath" if "asdf" in self.finders and self.asdf_finder is not None: return self from .python import PythonFinder os_path = os.environ["PATH"].split(os.pathsep) asdf_finder = PythonFinder.create( root=ASDF_DATA_DIR, ignore_unsupported=True, sort_function=parse_asdf_version_order, version_glob_path="installs/python/*", ) asdf_index = None try: asdf_index = self._get_last_instance(ASDF_DATA_DIR) except ValueError: asdf_index = 0 if is_in_path(next(iter(os_path), ""), ASDF_DATA_DIR) else -1 if asdf_index is None: # we are in a virtualenv without global pyenv on the path, so we should # not write pyenv to the path here return self # * These are the root paths for the finder _ = [p for p in asdf_finder.roots] new_instance = self._slice_in_paths(asdf_index, [asdf_finder.root]) paths = self.paths.copy() paths[asdf_finder.root] = asdf_finder paths.update(asdf_finder.roots) return ( attr.evolve(new_instance, paths=paths, asdf_finder=asdf_finder) ._remove_path(normalize_path(os.path.join(ASDF_DATA_DIR, "shims"))) ._register_finder("asdf", asdf_finder) ) def reload_finder(self, finder_name): # type: (str) -> "SystemPath" if finder_name is None: raise TypeError("Must pass a string as the name of the target finder") finder_attr = "{0}_finder".format(finder_name) setup_attr = "_setup_{0}".format(finder_name) try: current_finder = getattr(self, finder_attr) # type: Any except AttributeError: raise ValueError("Must pass a valid finder to reload.") try: setup_fn = getattr(self, setup_attr) except AttributeError: raise ValueError("Finder has no valid setup function: %s" % finder_name) if current_finder is None: # TODO: This is called 'reload', should we load a new finder for the first # time here? lets just skip that for now to avoid unallowed finders pass if (finder_name == "pyenv" and not PYENV_INSTALLED) or ( finder_name == "asdf" and not ASDF_INSTALLED ): # Don't allow loading of finders that aren't explicitly 'installed' as it were return self setattr(self, finder_attr, None) if finder_name in self.__finders: del self.__finders[finder_name] return setup_fn() def _setup_pyenv(self): # type: () -> "SystemPath" if "pyenv" in self.finders and self.pyenv_finder is not None: return self from .python import PythonFinder os_path = os.environ["PATH"].split(os.pathsep) pyenv_finder = PythonFinder.create( root=PYENV_ROOT, sort_function=parse_pyenv_version_order, version_glob_path="versions/*", ignore_unsupported=self.ignore_unsupported, ) pyenv_index = None try: pyenv_index = self._get_last_instance(PYENV_ROOT) except ValueError: pyenv_index = 0 if is_in_path(next(iter(os_path), ""), PYENV_ROOT) else -1 if pyenv_index is None: # we are in a virtualenv without global pyenv on the path, so we should # not write pyenv to the path here return self # * These are the root paths for the finder _ = [p for p in pyenv_finder.roots] new_instance = self._slice_in_paths(pyenv_index, [pyenv_finder.root]) paths = new_instance.paths.copy() paths[pyenv_finder.root] = pyenv_finder paths.update(pyenv_finder.roots) return ( attr.evolve(new_instance, paths=paths, pyenv_finder=pyenv_finder) ._remove_path(os.path.join(PYENV_ROOT, "shims")) ._register_finder("pyenv", pyenv_finder) ) def _setup_windows(self): # type: () -> "SystemPath" if "windows" in self.finders and self.windows_finder is not None: return self from .windows import WindowsFinder windows_finder = WindowsFinder.create() root_paths = (p for p in windows_finder.paths if p.is_root) path_addition = [p.path.as_posix() for p in root_paths] new_path_order = self.path_order[:] + path_addition new_paths = self.paths.copy() new_paths.update({p.path: p for p in root_paths}) return attr.evolve( self, windows_finder=windows_finder, path_order=new_path_order, paths=new_paths, )._register_finder("windows", windows_finder) def get_path(self, path): # type: (Union[str, Path]) -> PathType if path is None: raise TypeError("A path must be provided in order to generate a path entry.") path = ensure_path(path) _path = self.paths.get(path) if not _path: _path = self.paths.get(path.as_posix()) if not _path and path.as_posix() in self.path_order and path.exists(): _path = PathEntry.create( path=path.absolute(), is_root=True, only_python=self.only_python ) self.paths[path.as_posix()] = _path if not _path: raise ValueError("Path not found or generated: {0!r}".format(path)) return _path def _get_paths(self): # type: () -> Generator[Union[PathType, WindowsFinder], None, None] for path in self.path_order: try: entry = self.get_path(path) except ValueError: continue else: yield entry @cached_property def path_entries(self): # type: () -> List[Union[PathType, WindowsFinder]] paths = list(self._get_paths()) return paths def find_all(self, executable): # type: (str) -> List[Union[PathEntry, FinderType]] """ Search the path for an executable. Return all copies. :param executable: Name of the executable :type executable: str :returns: List[PathEntry] """ sub_which = operator.methodcaller("which", executable) filtered = (sub_which(self.get_path(k)) for k in self.path_order) return list(filtered) def which(self, executable): # type: (str) -> Union[PathEntry, None] """ Search for an executable on the path. :param executable: Name of the executable to be located. :type executable: str :returns: :class:`~pythonfinder.models.PathEntry` object. """ sub_which = operator.methodcaller("which", executable) filtered = (sub_which(self.get_path(k)) for k in self.path_order) return next(iter(f for f in filtered if f is not None), None) def _filter_paths(self, finder): # type: (Callable) -> Iterator for path in self._get_paths(): if path is None: continue python_versions = finder(path) if python_versions is not None: for python in python_versions: if python is not None: yield python def _get_all_pythons(self, finder): # type: (Callable) -> Iterator for python in self._filter_paths(finder): if python is not None and python.is_python: yield python def get_pythons(self, finder): # type: (Callable) -> Iterator sort_key = operator.attrgetter("as_python.version_sort") pythons = [entry for entry in self._get_all_pythons(finder)] for python in sorted(pythons, key=sort_key, reverse=True): if python is not None: yield python def find_all_python_versions( self, major=None, # type: Optional[Union[str, int]] minor=None, # type: Optional[int] patch=None, # type: Optional[int] pre=None, # type: Optional[bool] dev=None, # type: Optional[bool] arch=None, # type: Optional[str] name=None, # type: Optional[str] ): # type (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. :type major: int :param int minor: Minor python version to search for, defaults to None :param int patch: Patch python version to search for, defaults to None :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ sub_finder = operator.methodcaller( "find_all_python_versions", major, minor, patch, pre, dev, arch, name ) alternate_sub_finder = None if major and not (minor or patch or pre or dev or arch or name): alternate_sub_finder = operator.methodcaller( "find_all_python_versions", None, None, None, None, None, None, major ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version values = list(self.get_pythons(sub_finder)) if not values and alternate_sub_finder is not None: values = list(self.get_pythons(alternate_sub_finder)) return values def find_python_version( self, major=None, # type: Optional[Union[str, int]] minor=None, # type: Optional[Union[str, int]] patch=None, # type: Optional[Union[str, int]] pre=None, # type: Optional[bool] dev=None, # type: Optional[bool] arch=None, # type: Optional[str] name=None, # type: Optional[str] sort_by_path=False, # type: bool ): # type: (...) -> PathEntry """Search for a specific python version on the path. :param major: Major python version to search for. :type major: int :param int minor: Minor python version to search for, defaults to None :param int patch: Patch python version to search for, defaults to None :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :param bool sort_by_path: Whether to sort by path -- default sort is by version(default: False) :return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. :rtype: :class:`~pythonfinder.models.PathEntry` """ major, minor, patch, name = split_version_and_name(major, minor, patch, name) sub_finder = operator.methodcaller( "find_python_version", major, minor, patch, pre, dev, arch, name ) alternate_sub_finder = None if name and not (minor or patch or pre or dev or arch or major): alternate_sub_finder = operator.methodcaller( "find_all_python_versions", None, None, None, None, None, None, name ) if major and minor and patch: _tuple_pre = pre if pre is not None else False _tuple_dev = dev if dev is not None else False version_tuple = (major, minor, patch, _tuple_pre, _tuple_dev) version_tuple_pre = (major, minor, patch, True, False) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version if sort_by_path: paths = [self.get_path(k) for k in self.path_order] for path in paths: found_version = sub_finder(path) if found_version: return found_version if alternate_sub_finder: for path in paths: found_version = alternate_sub_finder(path) if found_version: return found_version ver = next(iter(self.get_pythons(sub_finder)), None) if not ver and alternate_sub_finder is not None: ver = next(iter(self.get_pythons(alternate_sub_finder)), None) if ver: if ver.as_python.version_tuple[:5] in self.python_version_dict: self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) else: self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver] return ver @classmethod def create( cls, path=None, # type: str system=False, # type: bool only_python=False, # type: bool global_search=True, # type: bool ignore_unsupported=True, # type: bool ): # type: (...) -> SystemPath """Create a new :class:`pythonfinder.models.SystemPath` instance. :param path: Search path to prepend when searching, defaults to None :param path: str, optional :param bool system: Whether to use the running python by default instead of searching, defaults to False :param bool only_python: Whether to search only for python executables, defaults to False :param bool ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True :return: A new :class:`pythonfinder.models.SystemPath` instance. :rtype: :class:`pythonfinder.models.SystemPath` """ path_entries = defaultdict( PathEntry ) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] paths = [] # type: List[str] if ignore_unsupported: os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = fs_str("1") if global_search: if "PATH" in os.environ: paths = os.environ["PATH"].split(os.pathsep) path_order = [] # type: List[str] if path: path_order = [path] path_instance = ensure_path(path) path_entries.update( { path_instance.as_posix(): PathEntry.create( path=path_instance.absolute(), is_root=True, only_python=only_python, ) } ) paths = [path] + paths paths = [p for p in paths if not any(is_in_path(p, shim) for shim in SHIM_PATHS)] _path_objects = [ensure_path(p.strip('"')) for p in paths] path_entries.update( { p.as_posix(): PathEntry.create( path=p.absolute(), is_root=True, only_python=only_python ) for p in _path_objects if p.exists() } ) instance = cls( paths=path_entries, path_order=path_order, only_python=only_python, system=system, global_search=global_search, ignore_unsupported=ignore_unsupported, ) instance = instance._run_setup() return instance
class WindowsFinder(BaseFinder): paths = attr.ib(default=attr.Factory(list), type=list) version_list = attr.ib(default=attr.Factory(list), type=list) _versions = attr.ib() # type: DefaultDict[Tuple, PathEntry] _pythons = attr.ib() # type: DefaultDict[str, PathEntry] def find_all_python_versions( self, major=None, # type: Optional[Union[str, int]] minor=None, # type: Optional[int] patch=None, # type: Optional[int] pre=None, # type: Optional[bool] dev=None, # type: Optional[bool] arch=None, # type: Optional[str] name=None, # type: Optional[str] ): # type (...) -> List[PathEntry] version_matcher = operator.methodcaller("matches", major, minor, patch, pre, dev, arch, python_name=name) pythons = [py for py in self.version_list if version_matcher(py)] version_sort = operator.attrgetter("version_sort") return [ c.comes_from for c in sorted(pythons, key=version_sort, reverse=True) if c.comes_from ] def find_python_version( self, major=None, # type: Optional[Union[str, int]] minor=None, # type: Optional[int] patch=None, # type: Optional[int] pre=None, # type: Optional[bool] dev=None, # type: Optional[bool] arch=None, # type: Optional[str] name=None, # type: Optional[str] ): # type: (...) -> Optional[PathEntry] return next( iter(v for v in self.find_all_python_versions( major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name, )), None, ) @_versions.default def get_versions(self): # type: () -> DefaultDict[Tuple, PathEntry] versions = defaultdict( PathEntry) # type: DefaultDict[Tuple, PathEntry] from pipenv.vendor.pythonfinder._vendor.pep514tools import environment as pep514env env_versions = pep514env.findall() path = None for version_object in env_versions: install_path = getattr(version_object.info, "install_path", None) name = getattr(version_object, "tag", None) company = getattr(version_object, "company", None) if install_path is None: continue try: path = ensure_path(install_path.__getattr__("")) except AttributeError: continue if not path.exists(): continue try: py_version = PythonVersion.from_windows_launcher( version_object, name=name, company=company) except (InvalidPythonVersion, AttributeError): continue if py_version is None: continue self.version_list.append(py_version) python_path = (py_version.comes_from.path if py_version.comes_from else py_version.executable) python_kwargs = { python_path: py_version } if python_path is not None else {} base_dir = PathEntry.create(path, is_root=True, only_python=True, pythons=python_kwargs) versions[py_version.version_tuple[:5]] = base_dir self.paths.append(base_dir) return versions @property def versions(self): # type: () -> DefaultDict[Tuple, PathEntry] if not self._versions: self._versions = self.get_versions() return self._versions @_pythons.default def get_pythons(self): # type: () -> DefaultDict[str, PathEntry] pythons = defaultdict() # type: DefaultDict[str, PathEntry] for version in self.version_list: _path = ensure_path(version.comes_from.path) pythons[_path.as_posix()] = version.comes_from return pythons @property def pythons(self): # type: () -> DefaultDict[str, PathEntry] return self._pythons @pythons.setter def pythons(self, value): # type: (DefaultDict[str, PathEntry]) -> None self._pythons = value @classmethod def create(cls, *args, **kwargs): # type: (Type[FinderType], Any, Any) -> FinderType return cls()
class BasePath(object): path = attr.ib(default=None) # type: Path _children = attr.ib(default=attr.Factory(dict), order=False) # type: Dict[str, PathEntry] only_python = attr.ib(default=False) # type: bool name = attr.ib(type=str) _py_version = attr.ib(default=None, order=False) # type: Optional[PythonVersion] _pythons = attr.ib(default=attr.Factory(defaultdict), order=False) # type: DefaultDict[str, PathEntry] _is_dir = attr.ib(default=None, order=False) # type: Optional[bool] _is_executable = attr.ib(default=None, order=False) # type: Optional[bool] _is_python = attr.ib(default=None, order=False) # type: Optional[bool] def __str__(self): # type: () -> str return fs_str("{0}".format(self.path.as_posix())) def __lt__(self, other): # type: ("BasePath") -> bool return self.path.as_posix() < other.path.as_posix() def __lte__(self, other): # type: ("BasePath") -> bool return self.path.as_posix() <= other.path.as_posix() def __gt__(self, other): # type: ("BasePath") -> bool return self.path.as_posix() > other.path.as_posix() def __gte__(self, other): # type: ("BasePath") -> bool return self.path.as_posix() >= other.path.as_posix() def which(self, name): # type: (str) -> Optional[PathEntry] """Search in this path for an executable. :param executable: The name of an executable to search for. :type executable: str :returns: :class:`~pythonfinder.models.PathEntry` instance. """ valid_names = [name] + [ "{0}.{1}".format(name, ext).lower() if ext else "{0}".format(name).lower() for ext in KNOWN_EXTS ] children = self.children found = None if self.path is not None: found = next( (children[(self.path / child).as_posix()] for child in valid_names if (self.path / child).as_posix() in children), None, ) return found def __del__(self): for key in ["_is_dir", "_is_python", "_is_executable", "_py_version"]: if getattr(self, key, None): try: delattr(self, key) except Exception: print("failed deleting key: {0}".format(key)) self._children = {} for key in list(self._pythons.keys()): del self._pythons[key] self._pythons = None self._py_version = None self.path = None @property def children(self): # type: () -> Dict[str, PathEntry] if not self.is_dir: return {} return self._children @property def as_python(self): # type: () -> PythonVersion py_version = None if self.py_version: return self.py_version if not self.is_dir and self.is_python: try: from .python import PythonVersion py_version = PythonVersion.from_path( # type: ignore path=self, name=self.name) except (ValueError, InvalidPythonVersion): pass if py_version is None: pass self.py_version = py_version return py_version # type: ignore @name.default def get_name(self): # type: () -> Optional[str] if self.path: return self.path.name return None @property def is_dir(self): # type: () -> bool if self._is_dir is None: if not self.path: ret_val = False try: ret_val = self.path.is_dir() except OSError: ret_val = False self._is_dir = ret_val return self._is_dir @is_dir.setter def is_dir(self, val): # type: (bool) -> None self._is_dir = val @is_dir.deleter def is_dir(self): # type: () -> None self._is_dir = None @property def is_executable(self): # type: () -> bool if self._is_executable is None: if not self.path: self._is_executable = False else: self._is_executable = path_is_known_executable(self.path) return self._is_executable @is_executable.setter def is_executable(self, val): # type: (bool) -> None self._is_executable = val @is_executable.deleter def is_executable(self): # type: () -> None self._is_executable = None @property def is_python(self): # type: () -> bool if self._is_python is None: if not self.path: self._is_python = False else: self._is_python = self.is_executable and (looks_like_python( self.path.name)) return self._is_python @is_python.setter def is_python(self, val): # type: (bool) -> None self._is_python = val @is_python.deleter def is_python(self): # type: () -> None self._is_python = None def get_py_version(self): # type: () -> Optional[PythonVersion] from ..environment import IGNORE_UNSUPPORTED if self.is_dir: return None if self.is_python: py_version = None from .python import PythonVersion try: py_version = PythonVersion.from_path( # type: ignore path=self, name=self.name) except (InvalidPythonVersion, ValueError): py_version = None except Exception: if not IGNORE_UNSUPPORTED: raise return py_version return None @property def py_version(self): # type: () -> Optional[PythonVersion] if not self._py_version: py_version = self.get_py_version() self._py_version = py_version else: py_version = self._py_version return py_version @py_version.setter def py_version(self, val): # type: (Optional[PythonVersion]) -> None self._py_version = val @py_version.deleter def py_version(self): # type: () -> None self._py_version = None def _iter_pythons(self): # type: () -> Iterator if self.is_dir: for entry in self.children.values(): if entry is None: continue elif entry.is_dir: for python in entry._iter_pythons(): yield python elif entry.is_python and entry.as_python is not None: yield entry elif self.is_python and self.as_python is not None: yield self # type: ignore @property def pythons(self): # type: () -> DefaultDict[Union[str, Path], PathEntry] if not self._pythons: from .path import PathEntry self._pythons = defaultdict(PathEntry) for python in self._iter_pythons(): python_path = python.path.as_posix() # type: ignore self._pythons[python_path] = python return self._pythons def __iter__(self): # type: () -> Iterator for entry in self.children.values(): yield entry def __next__(self): # type: () -> Generator return next(iter(self)) def next(self): # type: () -> Generator return self.__next__() def find_all_python_versions( self, major=None, # type: Optional[Union[str, int]] minor=None, # type: Optional[int] patch=None, # type: Optional[int] pre=None, # type: Optional[bool] dev=None, # type: Optional[bool] arch=None, # type: Optional[str] name=None, # type: Optional[str] ): # type: (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. :type major: int :param int minor: Minor python version to search for, defaults to None :param int patch: Patch python version to search for, defaults to None :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ call_method = "find_all_python_versions" if self.is_dir else "find_python_version" sub_finder = operator.methodcaller(call_method, major, minor, patch, pre, dev, arch, name) if not self.is_dir: return sub_finder(self) unnested = [sub_finder(path) for path in expand_paths(self)] version_sort = operator.attrgetter("as_python.version_sort") unnested = [ p for p in unnested if p is not None and p.as_python is not None ] paths = sorted(unnested, key=version_sort, reverse=True) return list(paths) def find_python_version( self, major=None, # type: Optional[Union[str, int]] minor=None, # type: Optional[int] patch=None, # type: Optional[int] pre=None, # type: Optional[bool] dev=None, # type: Optional[bool] arch=None, # type: Optional[str] name=None, # type: Optional[str] ): # type: (...) -> Optional[PathEntry] """Search or self for the specified Python version and return the first match. :param major: Major version number. :type major: int :param int minor: Minor python version to search for, defaults to None :param int patch: Patch python version to search for, defaults to None :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ version_matcher = operator.methodcaller("matches", major, minor, patch, pre, dev, arch, python_name=name) if not self.is_dir: if self.is_python and self.as_python and version_matcher( self.py_version): return self # type: ignore matching_pythons = [ [entry, entry.as_python.version_sort] for entry in self._iter_pythons() if (entry is not None and entry.as_python is not None and version_matcher(entry.py_version)) ] results = sorted(matching_pythons, key=operator.itemgetter(1, 0), reverse=True) return next(iter(r[0] for r in results if r is not None), None)
class PythonFinder(BaseFinder, BasePath): root = attr.ib(default=None, validator=optional_instance_of(Path), type=Path) # should come before versions, because its value is used in versions's default initializer. #: Whether to ignore any paths which raise exceptions and are not actually python ignore_unsupported = attr.ib(default=True, type=bool) #: Glob path for python versions off of the root directory version_glob_path = attr.ib(default="versions/*", type=str) #: The function to use to sort version order when returning an ordered verion set sort_function = attr.ib(default=None) # type: Callable #: The root locations used for discovery roots = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) #: List of paths discovered during search paths = attr.ib(type=list) #: shim directory shim_dir = attr.ib(default="shims", type=str) #: Versions discovered in the specified paths _versions = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) _pythons = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) def __del__(self): # type: () -> None self._versions = defaultdict() self._pythons = defaultdict() self.roots = defaultdict() self.paths = [] @property def expanded_paths(self): # type: () -> Generator return ( path for path in unnest(p for p in self.versions.values()) if path is not None ) @property def is_pyenv(self): # type: () -> bool return is_in_path(str(self.root), PYENV_ROOT) @property def is_asdf(self): # type: () -> bool return is_in_path(str(self.root), ASDF_DATA_DIR) def get_version_order(self): # type: () -> List[Path] version_paths = [ p for p in self.root.glob(self.version_glob_path) if not (p.parent.name == "envs" or p.name == "envs") ] versions = {v.name: v for v in version_paths} version_order = [] # type: List[Path] if self.is_pyenv: version_order = [ versions[v] for v in parse_pyenv_version_order() if v in versions ] elif self.is_asdf: version_order = [ versions[v] for v in parse_asdf_version_order() if v in versions ] for version in version_order: if version in version_paths: version_paths.remove(version) if version_order: version_order += version_paths else: version_order = version_paths return version_order def get_bin_dir(self, base): # type: (Union[Path, str]) -> Path if isinstance(base, six.string_types): base = Path(base) if os.name == "nt": return base return base / "bin" @classmethod def version_from_bin_dir(cls, entry): # type: (PathEntry) -> Optional[PathEntry] py_version = None py_version = next(iter(entry.find_all_python_versions()), None) return py_version def _iter_version_bases(self): # type: () -> Iterator[Tuple[Path, PathEntry]] from .path import PathEntry for p in self.get_version_order(): bin_dir = self.get_bin_dir(p) if bin_dir.exists() and bin_dir.is_dir(): entry = PathEntry.create( path=bin_dir.absolute(), only_python=False, name=p.name, is_root=True ) self.roots[p] = entry yield (p, entry) def _iter_versions(self): # type: () -> Iterator[Tuple[Path, PathEntry, Tuple]] for base_path, entry in self._iter_version_bases(): version = None version_entry = None try: version = PythonVersion.parse(entry.name) except (ValueError, InvalidPythonVersion): version_entry = next(iter(entry.find_all_python_versions()), None) if version is None: if not self.ignore_unsupported: raise continue if version_entry is not None: version = version_entry.py_version.as_dict() except Exception: if not self.ignore_unsupported: raise logger.warning( "Unsupported Python version %r, ignoring...", base_path.name, exc_info=True, ) continue if version is not None: version_tuple = ( version.get("major"), version.get("minor"), version.get("patch"), version.get("is_prerelease"), version.get("is_devrelease"), version.get("is_debug"), ) yield (base_path, entry, version_tuple) @property def versions(self): # type: () -> DefaultDict[Tuple, PathEntry] if not self._versions: for _, entry, version_tuple in self._iter_versions(): self._versions[version_tuple] = entry return self._versions def _iter_pythons(self): # type: () -> Iterator for path, entry, version_tuple in self._iter_versions(): if path.as_posix() in self._pythons: yield self._pythons[path.as_posix()] elif version_tuple not in self.versions: for python in entry.find_all_python_versions(): yield python else: yield self.versions[version_tuple] @paths.default def get_paths(self): # type: () -> List[PathEntry] _paths = [base for _, base in self._iter_version_bases()] return _paths @property def pythons(self): # type: () -> DefaultDict[str, PathEntry] if not self._pythons: from .path import PathEntry self._pythons = defaultdict(PathEntry) # type: DefaultDict[str, PathEntry] for python in self._iter_pythons(): python_path = python.path.as_posix() # type: ignore self._pythons[python_path] = python return self._pythons @pythons.setter def pythons(self, value): # type: (DefaultDict[str, PathEntry]) -> None self._pythons = value def get_pythons(self): # type: () -> DefaultDict[str, PathEntry] return self.pythons @overload @classmethod def create(cls, root, sort_function, version_glob_path=None, ignore_unsupported=True): # type: (str, Callable, Optional[str], bool) -> PythonFinder root = ensure_path(root) if not version_glob_path: version_glob_path = "versions/*" return cls( root=root, path=root, ignore_unsupported=ignore_unsupported, # type: ignore sort_function=sort_function, version_glob_path=version_glob_path, ) def find_all_python_versions( self, major=None, # type: Optional[Union[str, int]] minor=None, # type: Optional[int] patch=None, # type: Optional[int] pre=None, # type: Optional[bool] dev=None, # type: Optional[bool] arch=None, # type: Optional[str] name=None, # type: Optional[str] ): # type: (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. :type major: int :param int minor: Minor python version to search for, defaults to None :param int patch: Patch python version to search for, defaults to None :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ call_method = "find_all_python_versions" if self.is_dir else "find_python_version" sub_finder = operator.methodcaller( call_method, major, minor, patch, pre, dev, arch, name ) if not any([major, minor, patch, name]): pythons = [ next(iter(py for py in base.find_all_python_versions()), None) for _, base in self._iter_version_bases() ] else: pythons = [sub_finder(path) for path in self.paths] pythons = expand_paths(pythons, True) version_sort = operator.attrgetter("as_python.version_sort") paths = [ p for p in sorted(pythons, key=version_sort, reverse=True) if p is not None ] return paths def find_python_version( self, major=None, # type: Optional[Union[str, int]] minor=None, # type: Optional[int] patch=None, # type: Optional[int] pre=None, # type: Optional[bool] dev=None, # type: Optional[bool] arch=None, # type: Optional[str] name=None, # type: Optional[str] ): # type: (...) -> Optional[PathEntry] """Search or self for the specified Python version and return the first match. :param major: Major version number. :type major: int :param int minor: Minor python version to search for, defaults to None :param int patch: Patch python version to search for, defaults to None :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ sub_finder = operator.methodcaller( "find_python_version", major, minor, patch, pre, dev, arch, name ) version_sort = operator.attrgetter("as_python.version_sort") unnested = [sub_finder(self.roots[path]) for path in self.roots] unnested = [ p for p in unnested if p is not None and p.is_python and p.as_python is not None ] paths = sorted(list(unnested), key=version_sort, reverse=True) return next(iter(p for p in paths if p is not None), None) def which(self, name): # type: (str) -> Optional[PathEntry] """Search in this path for an executable. :param executable: The name of an executable to search for. :type executable: str :returns: :class:`~pythonfinder.models.PathEntry` instance. """ matches = (p.which(name) for p in self.paths) non_empty_match = next(iter(m for m in matches if m is not None), None) return non_empty_match
class Lockfile(object): path = attr.ib(validator=optional_instance_of(Path), type=Path) _requirements = attr.ib(default=attr.Factory(list), type=list) _dev_requirements = attr.ib(default=attr.Factory(list), type=list) projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) _lockfile = attr.ib(validator=is_lockfile, type=plette.lockfiles.Lockfile) newlines = attr.ib(default=DEFAULT_NEWLINES, type=str) @path.default def _get_path(self): return Path(os.curdir).joinpath("Pipfile.lock").absolute() @projectfile.default def _get_projectfile(self): return self.load_projectfile(self.path) @_lockfile.default def _get_lockfile(self): return self.projectfile.model @property def lockfile(self): return self._lockfile @property def section_keys(self): return ["default", "develop"] @property def extended_keys(self): return [ k for k in itertools.product(self.section_keys, ["", "vcs", "editable"]) ] def get(self, k): return self.__getitem__(k) def __contains__(self, k): check_lockfile = k in self.extended_keys or self.lockfile.__contains__( k) if check_lockfile: return True return super(Lockfile, self).__contains__(k) def __setitem__(self, k, v): lockfile = self._lockfile lockfile.__setitem__(k, v) def __getitem__(self, k, *args, **kwargs): retval = None lockfile = self._lockfile section = None pkg_type = None try: retval = lockfile[k] except KeyError: if "-" in k: section, _, pkg_type = k.rpartition("-") vals = getattr(lockfile.get(section, {}), "_data", {}) if pkg_type == "vcs": retval = {k: v for k, v in vals.items() if is_vcs(v)} elif pkg_type == "editable": retval = {k: v for k, v in vals.items() if is_editable(v)} if retval is None: raise else: retval = getattr(retval, "_data", retval) return retval def __getattr__(self, k, *args, **kwargs): retval = None lockfile = super(Lockfile, self).__getattribute__("_lockfile") try: return super(Lockfile, self).__getattribute__(k) except AttributeError: retval = getattr(lockfile, k, None) if retval is not None: return retval return super(Lockfile, self).__getattribute__(k, *args, **kwargs) def get_deps(self, dev=False, only=True): deps = {} if dev: deps.update(self.develop._data) if only: return deps deps = merge_items([deps, self.default._data]) return deps @classmethod def read_projectfile(cls, path): """Read the specified project file and provide an interface for writing/updating. :param str path: Path to the target file. :return: A project file with the model and location for interaction :rtype: :class:`~requirementslib.models.project.ProjectFile` """ pf = ProjectFile.read(path, plette.lockfiles.Lockfile, invalid_ok=True) return pf @classmethod def lockfile_from_pipfile(cls, pipfile_path): from .pipfile import Pipfile if os.path.isfile(pipfile_path): if not os.path.isabs(pipfile_path): pipfile_path = os.path.abspath(pipfile_path) pipfile = Pipfile.load(os.path.dirname(pipfile_path)) return plette.lockfiles.Lockfile.with_meta_from(pipfile._pipfile) raise PipfileNotFound(pipfile_path) @classmethod def load_projectfile(cls, path, create=True, data=None): """Given a path, load or create the necessary lockfile. :param str path: Path to the project root or lockfile :param bool create: Whether to create the lockfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the lockfile doesn't exist and ``create=False`` :return: A project file instance for the supplied project :rtype: :class:`~requirementslib.models.project.ProjectFile` """ if not path: path = os.curdir path = Path(path).absolute() project_path = path if path.is_dir() else path.parent lockfile_path = path if path.is_file( ) else project_path / "Pipfile.lock" if not project_path.exists(): raise OSError("Project does not exist: %s" % project_path.as_posix()) elif not lockfile_path.exists() and not create: raise FileNotFoundError("Lockfile does not exist: %s" % lockfile_path.as_posix()) projectfile = cls.read_projectfile(lockfile_path.as_posix()) if not lockfile_path.exists(): if not data: path_str = lockfile_path.as_posix() if path_str[-5:] == ".lock": pipfile = Path(path_str[:-5]) else: pipfile = project_path.joinpath("Pipfile") lf = cls.lockfile_from_pipfile(pipfile) else: lf = plette.lockfiles.Lockfile(data) projectfile.model = lf return projectfile @classmethod def from_data(cls, path, data, meta_from_project=True): """Create a new lockfile instance from a dictionary. :param str path: Path to the project root. :param dict data: Data to load into the lockfile. :param bool meta_from_project: Attempt to populate the meta section from the project root, default True. """ if path is None: raise MissingParameter("path") if data is None: raise MissingParameter("data") if not isinstance(data, dict): raise TypeError("Expecting a dictionary for parameter 'data'") path = os.path.abspath(str(path)) if os.path.isdir(path): project_path = path elif not os.path.isdir(path) and os.path.isdir(os.path.dirname(path)): project_path = os.path.dirname(path) pipfile_path = os.path.join(project_path, "Pipfile") lockfile_path = os.path.join(project_path, "Pipfile.lock") if meta_from_project: lockfile = cls.lockfile_from_pipfile(pipfile_path) lockfile.update(data) else: lockfile = plette.lockfiles.Lockfile(data) projectfile = ProjectFile(line_ending=DEFAULT_NEWLINES, location=lockfile_path, model=lockfile) return cls( projectfile=projectfile, lockfile=lockfile, newlines=projectfile.line_ending, path=Path(projectfile.location), ) @classmethod def load(cls, path, create=True): """Create a new lockfile instance. :param project_path: Path to project root or lockfile :type project_path: str or :class:`pathlib.Path` :param str lockfile_name: Name of the lockfile in the project root directory :param pipfile_path: Path to the project pipfile :type pipfile_path: :class:`pathlib.Path` :returns: A new lockfile representing the supplied project paths :rtype: :class:`~requirementslib.models.lockfile.Lockfile` """ try: projectfile = cls.load_projectfile(path, create=create) except JSONDecodeError: path = os.path.abspath(path) path = Path( os.path.join(path, "Pipfile.lock") if os.path. isdir(path) else path) formatted_path = path.as_posix() backup_path = "%s.bak" % formatted_path LockfileCorruptException.show(formatted_path, backup_path=backup_path) path.rename(backup_path) cls.load(formatted_path, create=True) lockfile_path = Path(projectfile.location) creation_args = { "projectfile": projectfile, "lockfile": projectfile.model, "newlines": projectfile.line_ending, "path": lockfile_path, } return cls(**creation_args) @classmethod def create(cls, path, create=True): return cls.load(path, create=create) @property def develop(self): return self._lockfile.develop @property def default(self): return self._lockfile.default def get_requirements(self, dev=True, only=False): """Produces a generator which generates requirements from the desired section. :param bool dev: Indicates whether to use dev requirements, defaults to False :return: Requirements from the relevant the relevant pipfile :rtype: :class:`~requirementslib.models.requirements.Requirement` """ deps = self.get_deps(dev=dev, only=only) for k, v in deps.items(): yield Requirement.from_pipfile(k, v) @property def dev_requirements(self): if not self._dev_requirements: self._dev_requirements = list( self.get_requirements(dev=True, only=True)) return self._dev_requirements @property def requirements(self): if not self._requirements: self._requirements = list( self.get_requirements(dev=False, only=True)) return self._requirements @property def dev_requirements_list(self): return [{ name: entry._data } for name, entry in self._lockfile.develop.items()] @property def requirements_list(self): return [{ name: entry._data } for name, entry in self._lockfile.default.items()] def write(self): self.projectfile.model = copy.deepcopy(self._lockfile) self.projectfile.write() def as_requirements(self, include_hashes=False, dev=False): """Returns a list of requirements in pip-style format.""" lines = [] section = self.dev_requirements if dev else self.requirements for req in section: kwargs = {"include_hashes": include_hashes} if req.editable: kwargs["include_markers"] = False r = req.as_line(**kwargs) lines.append(r.strip()) return lines
class AbstractDependency(object): name = attr.ib() # type: STRING_TYPE specifiers = attr.ib() markers = attr.ib() candidates = attr.ib() requirement = attr.ib() parent = attr.ib() finder = attr.ib() dep_dict = attr.ib(default=attr.Factory(dict)) @property def version_set(self): """Return the set of versions for the candidates in this abstract dependency. :return: A set of matching versions :rtype: set(str) """ if len(self.candidates) == 1: return set() return set( packaging.version.parse(version_from_ireq(c)) for c in self.candidates) def compatible_versions(self, other): """Find compatible version numbers between this abstract dependency and another one. :param other: An abstract dependency to compare with. :type other: :class:`~requirementslib.models.dependency.AbstractDependency` :return: A set of compatible version strings :rtype: set(str) """ if len(self.candidates) == 1 and next(iter(self.candidates)).editable: return self elif len(other.candidates) == 1 and next(iter( other.candidates)).editable: return other return self.version_set & other.version_set def compatible_abstract_dep(self, other): """Merge this abstract dependency with another one. Return the result of the merge as a new abstract dependency. :param other: An abstract dependency to merge with :type other: :class:`~requirementslib.models.dependency.AbstractDependency` :return: A new, combined abstract dependency :rtype: :class:`~requirementslib.models.dependency.AbstractDependency` """ from .requirements import Requirement if len(self.candidates) == 1 and next(iter(self.candidates)).editable: return self elif len(other.candidates) == 1 and next(iter( other.candidates)).editable: return other new_specifiers = self.specifiers & other.specifiers markers = set(self.markers) if self.markers else set() if other.markers: markers.add(other.markers) new_markers = None if markers: new_markers = packaging.markers.Marker(" or ".join( str(m) for m in sorted(markers))) new_ireq = copy.deepcopy(self.requirement.ireq) new_ireq.req.specifier = new_specifiers new_ireq.req.marker = new_markers new_requirement = Requirement.from_line(format_requirement(new_ireq)) compatible_versions = self.compatible_versions(other) if isinstance(compatible_versions, AbstractDependency): return compatible_versions candidates = [ c for c in self.candidates if packaging.version.parse( version_from_ireq(c)) in compatible_versions ] dep_dict = {} candidate_strings = [format_requirement(c) for c in candidates] for c in candidate_strings: if c in self.dep_dict: dep_dict[c] = self.dep_dict.get(c) return AbstractDependency( name=self.name, specifiers=new_specifiers, markers=new_markers, candidates=candidates, requirement=new_requirement, parent=self.parent, dep_dict=dep_dict, finder=self.finder, ) def get_deps(self, candidate): """Get the dependencies of the supplied candidate. :param candidate: An installrequirement :type candidate: :class:`~pipenv.patched.notpip._internal.req.req_install.InstallRequirement` :return: A list of abstract dependencies :rtype: list[:class:`~requirementslib.models.dependency.AbstractDependency`] """ key = format_requirement(candidate) if key not in self.dep_dict: from .requirements import Requirement req = Requirement.from_line(key) req = req.merge_markers(self.markers) self.dep_dict[key] = req.get_abstract_dependencies() return self.dep_dict[key] @classmethod def from_requirement(cls, requirement, parent=None): """Creates a new :class:`~requirementslib.models.dependency.AbstractDependency` from a :class:`~requirementslib.models.requirements.Requirement` object. This class is used to find all candidates matching a given set of specifiers and a given requirement. :param requirement: A requirement for resolution :type requirement: :class:`~requirementslib.models.requirements.Requirement` object. """ name = requirement.normalized_name specifiers = requirement.ireq.specifier if not requirement.editable else "" markers = requirement.ireq.markers extras = requirement.ireq.extras is_pinned = is_pinned_requirement(requirement.ireq) is_constraint = bool(parent) _, finder = get_finder(sources=None) candidates = [] if not is_pinned and not requirement.editable: for r in requirement.find_all_matches(finder=finder): req = make_install_requirement( name, r.version, extras=extras, markers=markers, constraint=is_constraint, ) req.req.link = getattr(r, "location", getattr(r, "link", None)) req.parent = parent candidates.append(req) candidates = sorted( set(candidates), key=lambda k: packaging.version.parse(version_from_ireq(k) ), ) else: candidates = [requirement.ireq] return cls( name=name, specifiers=specifiers, markers=markers, candidates=candidates, requirement=requirement, parent=parent, finder=finder, ) @classmethod def from_string(cls, line, parent=None): from .requirements import Requirement req = Requirement.from_line(line) abstract_dep = cls.from_requirement(req, parent=parent) return abstract_dep
class Pipfile(object): path = attr.ib(validator=is_path, type=Path) projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) _pipfile = attr.ib(type=PipfileLoader) _pyproject = attr.ib(default=attr.Factory(tomlkit.document), type=tomlkit.toml_document.TOMLDocument) build_system = attr.ib(default=attr.Factory(dict), type=dict) _requirements = attr.ib(default=attr.Factory(list), type=list) _dev_requirements = attr.ib(default=attr.Factory(list), type=list) @path.default def _get_path(self): # type: () -> Path return Path(os.curdir).absolute() @projectfile.default def _get_projectfile(self): # type: () -> ProjectFile return self.load_projectfile(os.curdir, create=False) @_pipfile.default def _get_pipfile(self): # type: () -> Union[plette.pipfiles.Pipfile, PipfileLoader] return self.projectfile.model @property def root(self): return self.path.parent @property def extended_keys(self): return [ k for k in itertools.product(("packages", "dev-packages"), ("", "vcs", "editable")) ] @property def pipfile(self): # type: () -> Union[PipfileLoader, plette.pipfiles.Pipfile] return self._pipfile def get_deps(self, dev=False, only=True): # type: (bool, bool) -> Dict[Text, Dict[Text, Union[List[Text], Text]]] deps = {} # type: Dict[Text, Dict[Text, Union[List[Text], Text]]] if dev: deps.update(dict(self.pipfile._data.get("dev-packages", {}))) if only: return deps return tomlkit_value_to_python( merge_items([deps, dict(self.pipfile._data.get("packages", {}))])) def get(self, k): # type: (Text) -> Any return self.__getitem__(k) def __contains__(self, k): # type: (Text) -> bool check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) if check_pipfile: return True return False def __getitem__(self, k, *args, **kwargs): # type: ignore retval = None pipfile = self._pipfile section = None pkg_type = None try: retval = pipfile[k] except KeyError: if "-" in k: section, _, pkg_type = k.rpartition("-") vals = getattr(pipfile.get(section, {}), "_data", {}) vals = tomlkit_value_to_python(vals) if pkg_type == "vcs": retval = {k: v for k, v in vals.items() if is_vcs(v)} elif pkg_type == "editable": retval = {k: v for k, v in vals.items() if is_editable(v)} if retval is None: raise else: retval = getattr(retval, "_data", retval) return retval def __getattr__(self, k, *args, **kwargs): # type: ignore retval = None pipfile = super(Pipfile, self).__getattribute__("_pipfile") try: retval = super(Pipfile, self).__getattribute__(k) except AttributeError: retval = getattr(pipfile, k, None) if retval is not None: return retval return super(Pipfile, self).__getattribute__(k, *args, **kwargs) @property def requires_python(self): # type: () -> bool return getattr( self._pipfile.requires, "python_version", getattr(self._pipfile.requires, "python_full_version", None), ) @property def allow_prereleases(self): # type: () -> bool return self._pipfile.get("pipenv", {}).get("allow_prereleases", False) @classmethod def read_projectfile(cls, path): # type: (Text) -> ProjectFile """Read the specified project file and provide an interface for writing/updating. :param Text path: Path to the target file. :return: A project file with the model and location for interaction :rtype: :class:`~requirementslib.models.project.ProjectFile` """ pf = ProjectFile.read(path, PipfileLoader, invalid_ok=True) return pf @classmethod def load_projectfile(cls, path, create=False): # type: (Text, bool) -> ProjectFile """Given a path, load or create the necessary pipfile. :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` :return: A project file instance for the supplied project :rtype: :class:`~requirementslib.models.project.ProjectFile` """ if not path: raise RuntimeError( "Must pass a path to classmethod 'Pipfile.load'") if not isinstance(path, Path): path = Path(path).absolute() pipfile_path = path if path.is_file() else path.joinpath("Pipfile") project_path = pipfile_path.parent if not project_path.exists(): raise FileNotFoundError("%s is not a valid project path!" % path) elif not pipfile_path.exists() or not pipfile_path.is_file(): if not create: raise RequirementError("%s is not a valid Pipfile" % pipfile_path) return cls.read_projectfile(pipfile_path.as_posix()) @classmethod def load(cls, path, create=False): # type: (Text, bool) -> Pipfile """Given a path, load or create the necessary pipfile. :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` :return: A pipfile instance pointing at the supplied project :rtype:: class:`~requirementslib.models.pipfile.Pipfile` """ projectfile = cls.load_projectfile(path, create=create) pipfile = projectfile.model creation_args = { "projectfile": projectfile, "pipfile": pipfile, "path": Path(projectfile.location), } return cls(**creation_args) def write(self): # type: () -> None self.projectfile.model = copy.deepcopy(self._pipfile) self.projectfile.write() @property def dev_packages(self): # type: () -> List[Requirement] return self.dev_requirements @property def packages(self): # type: () -> List[Requirement] return self.requirements @property def dev_requirements(self): # type: () -> List[Requirement] if not self._dev_requirements: packages = tomlkit_value_to_python( self.pipfile.get("dev-packages", {})) self._dev_requirements = [ Requirement.from_pipfile(k, v) for k, v in packages.items() if v is not None ] return self._dev_requirements @property def requirements(self): # type: () -> List[Requirement] if not self._requirements: packages = tomlkit_value_to_python(self.pipfile.get( "packages", {})) self._requirements = [ Requirement.from_pipfile(k, v) for k, v in packages.items() if v is not None ] return self._requirements def _read_pyproject(self): # type: () -> None pyproject = self.path.parent.joinpath("pyproject.toml") if pyproject.exists(): self._pyproject = tomlkit.loads(pyproject.read_text()) build_system = self._pyproject.get("build-system", None) if build_system and not build_system.get("build_backend"): build_system[ "build-backend"] = "setuptools.build_meta:__legacy__" elif not build_system or not build_system.get("requires"): build_system = { "requires": ["setuptools>=40.8", "wheel"], "build-backend": "setuptools.build_meta:__legacy__", } self.build_system = build_system @property def build_requires(self): # type: () -> List[Text] if not self.build_system: self._read_pyproject() return self.build_system.get("requires", []) @property def build_backend(self): # type: () -> Text if not self.build_system: self._read_pyproject() return self.build_system.get("build-backend", None)
class DependencyResolver(object): pinned_deps = attr.ib(default=attr.Factory(dict)) #: A dictionary of abstract dependencies by name dep_dict = attr.ib(default=attr.Factory(dict)) #: A dictionary of sets of version numbers that are valid for a candidate currently candidate_dict = attr.ib(default=attr.Factory(dict)) #: A historical record of pins pin_history = attr.ib(default=attr.Factory(dict)) #: Whether to allow prerelease dependencies allow_prereleases = attr.ib(default=False) #: Stores hashes for each dependency hashes = attr.ib(default=attr.Factory(dict)) #: A hash cache hash_cache = attr.ib(default=attr.Factory(HashCache)) #: A finder for searching the index finder = attr.ib(default=None) #: Whether to include hashes even from incompatible wheels include_incompatible_hashes = attr.ib(default=True) #: A cache for storing available canddiates when using all wheels _available_candidates_cache = attr.ib(default=attr.Factory(dict)) @classmethod def create(cls, finder=None, allow_prereleases=False, get_all_hashes=True): if not finder: from .dependencies import get_finder finder_args = [] if allow_prereleases: finder_args.append("--pre") finder = get_finder(*finder_args) creation_kwargs = { "allow_prereleases": allow_prereleases, "include_incompatible_hashes": get_all_hashes, "finder": finder, "hash_cache": HashCache(), } resolver = cls(**creation_kwargs) return resolver @property def dependencies(self): return list(self.dep_dict.values()) @property def resolution(self): return list(self.pinned_deps.values()) def add_abstract_dep(self, dep): """Add an abstract dependency by either creating a new entry or merging with an old one. :param dep: An abstract dependency to add :type dep: :class:`~requirementslib.models.dependency.AbstractDependency` :raises ResolutionError: Raised when the given dependency is not compatible with an existing abstract dependency. """ if dep.name in self.dep_dict: compatible_versions = self.dep_dict[dep.name].compatible_versions(dep) if compatible_versions: self.candidate_dict[dep.name] = compatible_versions self.dep_dict[dep.name] = self.dep_dict[dep.name].compatible_abstract_dep( dep ) else: raise ResolutionError else: self.candidate_dict[dep.name] = dep.version_set self.dep_dict[dep.name] = dep def pin_deps(self): """Pins the current abstract dependencies and adds them to the history dict. Adds any new dependencies to the abstract dependencies already present by merging them together to form new, compatible abstract dependencies. """ for name in list(self.dep_dict.keys()): candidates = self.dep_dict[name].candidates[:] abs_dep = self.dep_dict[name] while candidates: pin = candidates.pop() # Move on from existing pins if the new pin isn't compatible if name in self.pinned_deps: if self.pinned_deps[name].editable: continue old_version = version_from_ireq(self.pinned_deps[name]) if not pin.editable: new_version = version_from_ireq(pin) if ( new_version != old_version and new_version not in self.candidate_dict[name] ): continue pin.parent = abs_dep.parent pin_subdeps = self.dep_dict[name].get_deps(pin) backup = self.dep_dict.copy(), self.candidate_dict.copy() try: for pin_dep in pin_subdeps: self.add_abstract_dep(pin_dep) except ResolutionError: self.dep_dict, self.candidate_dict = backup continue else: self.pinned_deps[name] = pin break def resolve(self, root_nodes, max_rounds=20): """Resolves dependencies using a backtracking resolver and multiple endpoints. Note: this resolver caches aggressively. Runs for *max_rounds* or until any two pinning rounds yield the same outcome. :param root_nodes: A list of the root requirements. :type root_nodes: list[:class:`~requirementslib.models.requirements.Requirement`] :param max_rounds: The max number of resolution rounds, defaults to 20 :param max_rounds: int, optional :raises RuntimeError: Raised when max rounds is exceeded without a resolution. """ if self.dep_dict: raise RuntimeError("Do not use the same resolver more than once") if not self.hash_cache: self.hash_cache = HashCache() # Coerce input into AbstractDependency instances. # We accept str, Requirement, and AbstractDependency as input. from ..utils import log from .dependencies import AbstractDependency for dep in root_nodes: if isinstance(dep, str): dep = AbstractDependency.from_string(dep) elif not isinstance(dep, AbstractDependency): dep = AbstractDependency.from_requirement(dep) self.add_abstract_dep(dep) for round_ in range(max_rounds): self.pin_deps() self.pin_history[round_] = self.pinned_deps.copy() if round_ > 0: previous_round = set(self.pin_history[round_ - 1].values()) current_values = set(self.pin_history[round_].values()) difference = current_values - previous_round else: difference = set(self.pin_history[round_].values()) log.debug("\n") log.debug("{:=^30}".format(" Round {0} ".format(round_))) log.debug("\n") if difference: log.debug("New Packages: ") for d in difference: log.debug("{:>30}".format(format_requirement(d))) elif round_ >= 3: log.debug("Stable Pins: ") for d in current_values: log.debug("{:>30}".format(format_requirement(d))) return else: log.debug("No New Packages.") # TODO: Raise a better error. raise RuntimeError("cannot resolve after {} rounds".format(max_rounds)) def get_hashes(self): for dep in self.pinned_deps.values(): if dep.name not in self.hashes: self.hashes[dep.name] = self.get_hashes_for_one(dep) return self.hashes.copy() def get_hashes_for_one(self, ireq): if not self.finder: from .dependencies import get_finder finder_args = [] if self.allow_prereleases: finder_args.append("--pre") self.finder = get_finder(*finder_args) if ireq.editable: return set() from pipenv.vendor.pip_shims import VcsSupport vcs = VcsSupport() if ( ireq.link and ireq.link.scheme in vcs.all_schemes and "ssh" in ireq.link.scheme ): return set() if not is_pinned_requirement(ireq): raise TypeError("Expected pinned requirement, got {}".format(ireq)) matching_candidates = set() with self.allow_all_wheels(): from .dependencies import find_all_matches matching_candidates = find_all_matches( self.finder, ireq, pre=self.allow_prereleases ) return { self.hash_cache.get_hash( getattr(candidate, "location", getattr(candidate, "link", None)) ) for candidate in matching_candidates } @contextmanager def allow_all_wheels(self): """Monkey patches pip.Wheel to allow wheels from all platforms and Python versions. This also saves the candidate cache and set a new one, or else the results from the previous non-patched calls will interfere. """ def _wheel_supported(self, tags=None): # Ignore current platform. Support everything. return True def _wheel_support_index_min(self, tags=None): # All wheels are equal priority for sorting. return 0 original_wheel_supported = Wheel.supported original_support_index_min = Wheel.support_index_min Wheel.supported = _wheel_supported Wheel.support_index_min = _wheel_support_index_min try: yield finally: Wheel.supported = original_wheel_supported Wheel.support_index_min = original_support_index_min