class SystemPath(object):
global_search = attr.ib(default=True)
paths = attr.ib(
default=attr.Factory(defaultdict)
) # type: DefaultDict[str, Union[PythonFinder, PathEntry]]
_executables = attr.ib(default=attr.Factory(list)) # type: List[PathEntry]
_python_executables = attr.ib(
default=attr.Factory(dict)
) # type: Dict[str, PathEntry]
path_order = attr.ib(default=attr.Factory(list)) # type: List[str]
python_version_dict = attr.ib() # type: DefaultDict[Tuple, List[PythonVersion]]
only_python = attr.ib(default=False, type=bool)
pyenv_finder = attr.ib(default=None) # type: Optional[PythonFinder]
asdf_finder = attr.ib(default=None) # type: Optional[PythonFinder]
windows_finder = attr.ib(default=None) # type: Optional[WindowsFinder]
system = attr.ib(default=False, type=bool)
_version_dict = attr.ib(
default=attr.Factory(defaultdict)
) # type: DefaultDict[Tuple, List[PathEntry]]
ignore_unsupported = attr.ib(default=False, type=bool)
__finders = attr.ib(
default=attr.Factory(dict)
) # type: Dict[str, Union[WindowsFinder, PythonFinder]]
def _register_finder(self, finder_name, finder):
# type: (str, Union[WindowsFinder, PythonFinder]) -> "SystemPath"
if finder_name not in self.__finders:
self.__finders[finder_name] = finder
return self
def clear_caches(self):
for key in ["executables", "python_executables", "version_dict", "path_entries"]:
if key in self.__dict__:
del self.__dict__[key]
for finder in list(self.__finders.keys()):
del self.__finders[finder]
self.__finders = {}
return attr.evolve(
self,
executables=[],
python_executables={},
python_version_dict=defaultdict(list),
version_dict=defaultdict(list),
pyenv_finder=None,
windows_finder=None,
asdf_finder=None,
path_order=[],
paths=defaultdict(PathEntry),
)
def __del__(self):
for key in ["executables", "python_executables", "version_dict", "path_entries"]:
try:
del self.__dict__[key]
except KeyError:
pass
for finder in list(self.__finders.keys()):
del self.__finders[finder]
self.__finders = {}
self._python_executables = {}
self._executables = []
self.python_version_dict = defaultdict(list)
self._version_dict = defaultdict(list)
self.path_order = []
self.pyenv_finder = None
self.asdf_finder = None
self.paths = defaultdict(PathEntry)
self.__finders = {}
@property
def finders(self):
# type: () -> List[str]
return [k for k in self.__finders.keys()]
@staticmethod
def check_for_pyenv():
return PYENV_INSTALLED or os.path.exists(normalize_path(PYENV_ROOT))
@staticmethod
def check_for_asdf():
return ASDF_INSTALLED or os.path.exists(normalize_path(ASDF_DATA_DIR))
@python_version_dict.default
def create_python_version_dict(self):
# type: () -> DefaultDict[Tuple, List[PythonVersion]]
return defaultdict(list)
@cached_property
def executables(self):
# type: () -> List[PathEntry]
self.executables = [
p
for p in chain(*(child.children.values() for child in self.paths.values()))
if p.is_executable
]
return self.executables
@cached_property
def python_executables(self):
# type: () -> Dict[str, PathEntry]
python_executables = {}
for child in self.paths.values():
if child.pythons:
python_executables.update(dict(child.pythons))
for finder_name, finder in self.__finders.items():
if finder.pythons:
python_executables.update(dict(finder.pythons))
self._python_executables = python_executables
return self._python_executables
@cached_property
def version_dict(self):
# type: () -> DefaultDict[Tuple, List[PathEntry]]
self._version_dict = defaultdict(
list
) # type: DefaultDict[Tuple, List[PathEntry]]
for finder_name, finder in self.__finders.items():
for version, entry in finder.versions.items():
if finder_name == "windows":
if entry not in self._version_dict[version]:
self._version_dict[version].append(entry)
continue
if entry not in self._version_dict[version] and entry.is_python:
self._version_dict[version].append(entry)
for p, entry in self.python_executables.items():
version = entry.as_python # type: PythonVersion
if not version:
continue
if not isinstance(version, tuple):
version = version.version_tuple
if version and entry not in self._version_dict[version]:
self._version_dict[version].append(entry)
return self._version_dict
def _run_setup(self):
# type: () -> "SystemPath"
if not self.__class__ == SystemPath:
return self
new_instance = self
path_order = new_instance.path_order[:]
path_entries = self.paths.copy()
if self.global_search and "PATH" in os.environ:
path_order = path_order + os.environ["PATH"].split(os.pathsep)
path_order = list(dedup(path_order))
path_instances = [
ensure_path(p.strip('"'))
for p in path_order
if not any(
is_in_path(normalize_path(str(p)), normalize_path(shim))
for shim in SHIM_PATHS
)
]
path_entries.update(
{
p.as_posix(): PathEntry.create(
path=p.absolute(), is_root=True, only_python=self.only_python
)
for p in path_instances
if p.exists()
}
)
new_instance = attr.evolve(
new_instance,
path_order=[p.as_posix() for p in path_instances if p.exists()],
paths=path_entries,
)
if os.name == "nt" and "windows" not in self.finders:
new_instance = new_instance._setup_windows()
#: slice in pyenv
if self.check_for_pyenv() and "pyenv" not in self.finders:
new_instance = new_instance._setup_pyenv()
#: slice in asdf
if self.check_for_asdf() and "asdf" not in self.finders:
new_instance = new_instance._setup_asdf()
venv = os.environ.get("VIRTUAL_ENV")
if os.name == "nt":
bin_dir = "Scripts"
else:
bin_dir = "bin"
if venv and (new_instance.system or new_instance.global_search):
p = ensure_path(venv)
path_order = [(p / bin_dir).as_posix()] + new_instance.path_order
new_instance = attr.evolve(new_instance, path_order=path_order)
paths = new_instance.paths.copy()
paths[p] = new_instance.get_path(p.joinpath(bin_dir))
new_instance = attr.evolve(new_instance, paths=paths)
if new_instance.system:
syspath = Path(sys.executable)
syspath_bin = syspath.parent
if syspath_bin.name != bin_dir and syspath_bin.joinpath(bin_dir).exists():
syspath_bin = syspath_bin / bin_dir
path_order = [syspath_bin.as_posix()] + new_instance.path_order
paths = new_instance.paths.copy()
paths[syspath_bin] = PathEntry.create(
path=syspath_bin, is_root=True, only_python=False
)
new_instance = attr.evolve(new_instance, path_order=path_order, paths=paths)
return new_instance
def _get_last_instance(self, path):
# type: (str) -> int
reversed_paths = reversed(self.path_order)
paths = [normalize_path(p) for p in reversed_paths]
normalized_target = normalize_path(path)
last_instance = next(iter(p for p in paths if normalized_target in p), None)
if last_instance is None:
raise ValueError("No instance found on path for target: {0!s}".format(path))
path_index = self.path_order.index(last_instance)
return path_index
def _slice_in_paths(self, start_idx, paths):
# type: (int, List[Path]) -> "SystemPath"
before_path = [] # type: List[str]
after_path = [] # type: List[str]
if start_idx == 0:
after_path = self.path_order[:]
elif start_idx == -1:
before_path = self.path_order[:]
else:
before_path = self.path_order[: start_idx + 1]
after_path = self.path_order[start_idx + 2 :]
path_order = before_path + [p.as_posix() for p in paths] + after_path
if path_order == self.path_order:
return self
return attr.evolve(self, path_order=path_order)
def _remove_path(self, path):
# type: (str) -> "SystemPath"
path_copy = [p for p in reversed(self.path_order[:])]
new_order = []
target = normalize_path(path)
path_map = {normalize_path(pth): pth for pth in self.paths.keys()}
new_paths = self.paths.copy()
if target in path_map:
del new_paths[path_map[target]]
for current_path in path_copy:
normalized = normalize_path(current_path)
if normalized != target:
new_order.append(normalized)
new_order = [ensure_path(p).as_posix() for p in reversed(new_order)]
return attr.evolve(self, path_order=new_order, paths=new_paths)
def _setup_asdf(self):
# type: () -> "SystemPath"
if "asdf" in self.finders and self.asdf_finder is not None:
return self
from .python import PythonFinder
os_path = os.environ["PATH"].split(os.pathsep)
asdf_finder = PythonFinder.create(
root=ASDF_DATA_DIR,
ignore_unsupported=True,
sort_function=parse_asdf_version_order,
version_glob_path="installs/python/*",
)
asdf_index = None
try:
asdf_index = self._get_last_instance(ASDF_DATA_DIR)
except ValueError:
asdf_index = 0 if is_in_path(next(iter(os_path), ""), ASDF_DATA_DIR) else -1
if asdf_index is None:
# we are in a virtualenv without global pyenv on the path, so we should
# not write pyenv to the path here
return self
# * These are the root paths for the finder
_ = [p for p in asdf_finder.roots]
new_instance = self._slice_in_paths(asdf_index, [asdf_finder.root])
paths = self.paths.copy()
paths[asdf_finder.root] = asdf_finder
paths.update(asdf_finder.roots)
return (
attr.evolve(new_instance, paths=paths, asdf_finder=asdf_finder)
._remove_path(normalize_path(os.path.join(ASDF_DATA_DIR, "shims")))
._register_finder("asdf", asdf_finder)
)
def reload_finder(self, finder_name):
# type: (str) -> "SystemPath"
if finder_name is None:
raise TypeError("Must pass a string as the name of the target finder")
finder_attr = "{0}_finder".format(finder_name)
setup_attr = "_setup_{0}".format(finder_name)
try:
current_finder = getattr(self, finder_attr) # type: Any
except AttributeError:
raise ValueError("Must pass a valid finder to reload.")
try:
setup_fn = getattr(self, setup_attr)
except AttributeError:
raise ValueError("Finder has no valid setup function: %s" % finder_name)
if current_finder is None:
# TODO: This is called 'reload', should we load a new finder for the first
# time here? lets just skip that for now to avoid unallowed finders
pass
if (finder_name == "pyenv" and not PYENV_INSTALLED) or (
finder_name == "asdf" and not ASDF_INSTALLED
):
# Don't allow loading of finders that aren't explicitly 'installed' as it were
return self
setattr(self, finder_attr, None)
if finder_name in self.__finders:
del self.__finders[finder_name]
return setup_fn()
def _setup_pyenv(self):
# type: () -> "SystemPath"
if "pyenv" in self.finders and self.pyenv_finder is not None:
return self
from .python import PythonFinder
os_path = os.environ["PATH"].split(os.pathsep)
pyenv_finder = PythonFinder.create(
root=PYENV_ROOT,
sort_function=parse_pyenv_version_order,
version_glob_path="versions/*",
ignore_unsupported=self.ignore_unsupported,
)
pyenv_index = None
try:
pyenv_index = self._get_last_instance(PYENV_ROOT)
except ValueError:
pyenv_index = 0 if is_in_path(next(iter(os_path), ""), PYENV_ROOT) else -1
if pyenv_index is None:
# we are in a virtualenv without global pyenv on the path, so we should
# not write pyenv to the path here
return self
# * These are the root paths for the finder
_ = [p for p in pyenv_finder.roots]
new_instance = self._slice_in_paths(pyenv_index, [pyenv_finder.root])
paths = new_instance.paths.copy()
paths[pyenv_finder.root] = pyenv_finder
paths.update(pyenv_finder.roots)
return (
attr.evolve(new_instance, paths=paths, pyenv_finder=pyenv_finder)
._remove_path(os.path.join(PYENV_ROOT, "shims"))
._register_finder("pyenv", pyenv_finder)
)
def _setup_windows(self):
# type: () -> "SystemPath"
if "windows" in self.finders and self.windows_finder is not None:
return self
from .windows import WindowsFinder
windows_finder = WindowsFinder.create()
root_paths = (p for p in windows_finder.paths if p.is_root)
path_addition = [p.path.as_posix() for p in root_paths]
new_path_order = self.path_order[:] + path_addition
new_paths = self.paths.copy()
new_paths.update({p.path: p for p in root_paths})
return attr.evolve(
self,
windows_finder=windows_finder,
path_order=new_path_order,
paths=new_paths,
)._register_finder("windows", windows_finder)
def get_path(self, path):
# type: (Union[str, Path]) -> PathType
if path is None:
raise TypeError("A path must be provided in order to generate a path entry.")
path = ensure_path(path)
_path = self.paths.get(path)
if not _path:
_path = self.paths.get(path.as_posix())
if not _path and path.as_posix() in self.path_order and path.exists():
_path = PathEntry.create(
path=path.absolute(), is_root=True, only_python=self.only_python
)
self.paths[path.as_posix()] = _path
if not _path:
raise ValueError("Path not found or generated: {0!r}".format(path))
return _path
def _get_paths(self):
# type: () -> Generator[Union[PathType, WindowsFinder], None, None]
for path in self.path_order:
try:
entry = self.get_path(path)
except ValueError:
continue
else:
yield entry
@cached_property
def path_entries(self):
# type: () -> List[Union[PathType, WindowsFinder]]
paths = list(self._get_paths())
return paths
def find_all(self, executable):
# type: (str) -> List[Union[PathEntry, FinderType]]
"""
Search the path for an executable. Return all copies.
:param executable: Name of the executable
:type executable: str
:returns: List[PathEntry]
"""
sub_which = operator.methodcaller("which", executable)
filtered = (sub_which(self.get_path(k)) for k in self.path_order)
return list(filtered)
def which(self, executable):
# type: (str) -> Union[PathEntry, None]
"""
Search for an executable on the path.
:param executable: Name of the executable to be located.
:type executable: str
:returns: :class:`~pythonfinder.models.PathEntry` object.
"""
sub_which = operator.methodcaller("which", executable)
filtered = (sub_which(self.get_path(k)) for k in self.path_order)
return next(iter(f for f in filtered if f is not None), None)
def _filter_paths(self, finder):
# type: (Callable) -> Iterator
for path in self._get_paths():
if path is None:
continue
python_versions = finder(path)
if python_versions is not None:
for python in python_versions:
if python is not None:
yield python
def _get_all_pythons(self, finder):
# type: (Callable) -> Iterator
for python in self._filter_paths(finder):
if python is not None and python.is_python:
yield python
def get_pythons(self, finder):
# type: (Callable) -> Iterator
sort_key = operator.attrgetter("as_python.version_sort")
pythons = [entry for entry in self._get_all_pythons(finder)]
for python in sorted(pythons, key=sort_key, reverse=True):
if python is not None:
yield python
def find_all_python_versions(
self,
major=None, # type: Optional[Union[str, int]]
minor=None, # type: Optional[int]
patch=None, # type: Optional[int]
pre=None, # type: Optional[bool]
dev=None, # type: Optional[bool]
arch=None, # type: Optional[str]
name=None, # type: Optional[str]
):
# type (...) -> List[PathEntry]
"""Search for a specific python version on the path. Return all copies
:param major: Major python version to search for.
:type major: int
:param int minor: Minor python version to search for, defaults to None
:param int patch: Patch python version to search for, defaults to None
:param bool pre: Search for prereleases (default None) - prioritize releases if None
:param bool dev: Search for devreleases (default None) - prioritize releases if None
:param str arch: Architecture to include, e.g. '64bit', defaults to None
:param str name: The name of a python version, e.g. ``anaconda3-5.3.0``
:return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested.
:rtype: List[:class:`~pythonfinder.models.PathEntry`]
"""
sub_finder = operator.methodcaller(
"find_all_python_versions", major, minor, patch, pre, dev, arch, name
)
alternate_sub_finder = None
if major and not (minor or patch or pre or dev or arch or name):
alternate_sub_finder = operator.methodcaller(
"find_all_python_versions", None, None, None, None, None, None, major
)
if os.name == "nt" and self.windows_finder:
windows_finder_version = sub_finder(self.windows_finder)
if windows_finder_version:
return windows_finder_version
values = list(self.get_pythons(sub_finder))
if not values and alternate_sub_finder is not None:
values = list(self.get_pythons(alternate_sub_finder))
return values
def find_python_version(
self,
major=None, # type: Optional[Union[str, int]]
minor=None, # type: Optional[Union[str, int]]
patch=None, # type: Optional[Union[str, int]]
pre=None, # type: Optional[bool]
dev=None, # type: Optional[bool]
arch=None, # type: Optional[str]
name=None, # type: Optional[str]
sort_by_path=False, # type: bool
):
# type: (...) -> PathEntry
"""Search for a specific python version on the path.
:param major: Major python version to search for.
:type major: int
:param int minor: Minor python version to search for, defaults to None
:param int patch: Patch python version to search for, defaults to None
:param bool pre: Search for prereleases (default None) - prioritize releases if None
:param bool dev: Search for devreleases (default None) - prioritize releases if None
:param str arch: Architecture to include, e.g. '64bit', defaults to None
:param str name: The name of a python version, e.g. ``anaconda3-5.3.0``
:param bool sort_by_path: Whether to sort by path -- default sort is by version(default: False)
:return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested.
:rtype: :class:`~pythonfinder.models.PathEntry`
"""
major, minor, patch, name = split_version_and_name(major, minor, patch, name)
sub_finder = operator.methodcaller(
"find_python_version", major, minor, patch, pre, dev, arch, name
)
alternate_sub_finder = None
if name and not (minor or patch or pre or dev or arch or major):
alternate_sub_finder = operator.methodcaller(
"find_all_python_versions", None, None, None, None, None, None, name
)
if major and minor and patch:
_tuple_pre = pre if pre is not None else False
_tuple_dev = dev if dev is not None else False
version_tuple = (major, minor, patch, _tuple_pre, _tuple_dev)
version_tuple_pre = (major, minor, patch, True, False)
if os.name == "nt" and self.windows_finder:
windows_finder_version = sub_finder(self.windows_finder)
if windows_finder_version:
return windows_finder_version
if sort_by_path:
paths = [self.get_path(k) for k in self.path_order]
for path in paths:
found_version = sub_finder(path)
if found_version:
return found_version
if alternate_sub_finder:
for path in paths:
found_version = alternate_sub_finder(path)
if found_version:
return found_version
ver = next(iter(self.get_pythons(sub_finder)), None)
if not ver and alternate_sub_finder is not None:
ver = next(iter(self.get_pythons(alternate_sub_finder)), None)
if ver:
if ver.as_python.version_tuple[:5] in self.python_version_dict:
self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver)
else:
self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver]
return ver
@classmethod
def create(
cls,
path=None, # type: str
system=False, # type: bool
only_python=False, # type: bool
global_search=True, # type: bool
ignore_unsupported=True, # type: bool
):
# type: (...) -> SystemPath
"""Create a new :class:`pythonfinder.models.SystemPath` instance.
:param path: Search path to prepend when searching, defaults to None
:param path: str, optional
:param bool system: Whether to use the running python by default instead of searching, defaults to False
:param bool only_python: Whether to search only for python executables, defaults to False
:param bool ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True
:return: A new :class:`pythonfinder.models.SystemPath` instance.
:rtype: :class:`pythonfinder.models.SystemPath`
"""
path_entries = defaultdict(
PathEntry
) # type: DefaultDict[str, Union[PythonFinder, PathEntry]]
paths = [] # type: List[str]
if ignore_unsupported:
os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = fs_str("1")
if global_search:
if "PATH" in os.environ:
paths = os.environ["PATH"].split(os.pathsep)
path_order = [] # type: List[str]
if path:
path_order = [path]
path_instance = ensure_path(path)
path_entries.update(
{
path_instance.as_posix(): PathEntry.create(
path=path_instance.absolute(),
is_root=True,
only_python=only_python,
)
}
)
paths = [path] + paths
paths = [p for p in paths if not any(is_in_path(p, shim) for shim in SHIM_PATHS)]
_path_objects = [ensure_path(p.strip('"')) for p in paths]
path_entries.update(
{
p.as_posix(): PathEntry.create(
path=p.absolute(), is_root=True, only_python=only_python
)
for p in _path_objects
if p.exists()
}
)
instance = cls(
paths=path_entries,
path_order=path_order,
only_python=only_python,
system=system,
global_search=global_search,
ignore_unsupported=ignore_unsupported,
)
instance = instance._run_setup()
return instance
class WindowsFinder(BaseFinder):
paths = attr.ib(default=attr.Factory(list), type=list)
version_list = attr.ib(default=attr.Factory(list), type=list)
_versions = attr.ib() # type: DefaultDict[Tuple, PathEntry]
_pythons = attr.ib() # type: DefaultDict[str, PathEntry]
def find_all_python_versions(
self,
major=None, # type: Optional[Union[str, int]]
minor=None, # type: Optional[int]
patch=None, # type: Optional[int]
pre=None, # type: Optional[bool]
dev=None, # type: Optional[bool]
arch=None, # type: Optional[str]
name=None, # type: Optional[str]
):
# type (...) -> List[PathEntry]
version_matcher = operator.methodcaller("matches",
major,
minor,
patch,
pre,
dev,
arch,
python_name=name)
pythons = [py for py in self.version_list if version_matcher(py)]
version_sort = operator.attrgetter("version_sort")
return [
c.comes_from
for c in sorted(pythons, key=version_sort, reverse=True)
if c.comes_from
]
def find_python_version(
self,
major=None, # type: Optional[Union[str, int]]
minor=None, # type: Optional[int]
patch=None, # type: Optional[int]
pre=None, # type: Optional[bool]
dev=None, # type: Optional[bool]
arch=None, # type: Optional[str]
name=None, # type: Optional[str]
):
# type: (...) -> Optional[PathEntry]
return next(
iter(v for v in self.find_all_python_versions(
major=major,
minor=minor,
patch=patch,
pre=pre,
dev=dev,
arch=arch,
name=name,
)),
None,
)
@_versions.default
def get_versions(self):
# type: () -> DefaultDict[Tuple, PathEntry]
versions = defaultdict(
PathEntry) # type: DefaultDict[Tuple, PathEntry]
from pipenv.vendor.pythonfinder._vendor.pep514tools import environment as pep514env
env_versions = pep514env.findall()
path = None
for version_object in env_versions:
install_path = getattr(version_object.info, "install_path", None)
name = getattr(version_object, "tag", None)
company = getattr(version_object, "company", None)
if install_path is None:
continue
try:
path = ensure_path(install_path.__getattr__(""))
except AttributeError:
continue
if not path.exists():
continue
try:
py_version = PythonVersion.from_windows_launcher(
version_object, name=name, company=company)
except (InvalidPythonVersion, AttributeError):
continue
if py_version is None:
continue
self.version_list.append(py_version)
python_path = (py_version.comes_from.path
if py_version.comes_from else py_version.executable)
python_kwargs = {
python_path: py_version
} if python_path is not None else {}
base_dir = PathEntry.create(path,
is_root=True,
only_python=True,
pythons=python_kwargs)
versions[py_version.version_tuple[:5]] = base_dir
self.paths.append(base_dir)
return versions
@property
def versions(self):
# type: () -> DefaultDict[Tuple, PathEntry]
if not self._versions:
self._versions = self.get_versions()
return self._versions
@_pythons.default
def get_pythons(self):
# type: () -> DefaultDict[str, PathEntry]
pythons = defaultdict() # type: DefaultDict[str, PathEntry]
for version in self.version_list:
_path = ensure_path(version.comes_from.path)
pythons[_path.as_posix()] = version.comes_from
return pythons
@property
def pythons(self):
# type: () -> DefaultDict[str, PathEntry]
return self._pythons
@pythons.setter
def pythons(self, value):
# type: (DefaultDict[str, PathEntry]) -> None
self._pythons = value
@classmethod
def create(cls, *args, **kwargs):
# type: (Type[FinderType], Any, Any) -> FinderType
return cls()
class BasePath(object):
path = attr.ib(default=None) # type: Path
_children = attr.ib(default=attr.Factory(dict),
order=False) # type: Dict[str, PathEntry]
only_python = attr.ib(default=False) # type: bool
name = attr.ib(type=str)
_py_version = attr.ib(default=None,
order=False) # type: Optional[PythonVersion]
_pythons = attr.ib(default=attr.Factory(defaultdict),
order=False) # type: DefaultDict[str, PathEntry]
_is_dir = attr.ib(default=None, order=False) # type: Optional[bool]
_is_executable = attr.ib(default=None, order=False) # type: Optional[bool]
_is_python = attr.ib(default=None, order=False) # type: Optional[bool]
def __str__(self):
# type: () -> str
return fs_str("{0}".format(self.path.as_posix()))
def __lt__(self, other):
# type: ("BasePath") -> bool
return self.path.as_posix() < other.path.as_posix()
def __lte__(self, other):
# type: ("BasePath") -> bool
return self.path.as_posix() <= other.path.as_posix()
def __gt__(self, other):
# type: ("BasePath") -> bool
return self.path.as_posix() > other.path.as_posix()
def __gte__(self, other):
# type: ("BasePath") -> bool
return self.path.as_posix() >= other.path.as_posix()
def which(self, name):
# type: (str) -> Optional[PathEntry]
"""Search in this path for an executable.
:param executable: The name of an executable to search for.
:type executable: str
:returns: :class:`~pythonfinder.models.PathEntry` instance.
"""
valid_names = [name] + [
"{0}.{1}".format(name, ext).lower()
if ext else "{0}".format(name).lower() for ext in KNOWN_EXTS
]
children = self.children
found = None
if self.path is not None:
found = next(
(children[(self.path / child).as_posix()]
for child in valid_names
if (self.path / child).as_posix() in children),
None,
)
return found
def __del__(self):
for key in ["_is_dir", "_is_python", "_is_executable", "_py_version"]:
if getattr(self, key, None):
try:
delattr(self, key)
except Exception:
print("failed deleting key: {0}".format(key))
self._children = {}
for key in list(self._pythons.keys()):
del self._pythons[key]
self._pythons = None
self._py_version = None
self.path = None
@property
def children(self):
# type: () -> Dict[str, PathEntry]
if not self.is_dir:
return {}
return self._children
@property
def as_python(self):
# type: () -> PythonVersion
py_version = None
if self.py_version:
return self.py_version
if not self.is_dir and self.is_python:
try:
from .python import PythonVersion
py_version = PythonVersion.from_path( # type: ignore
path=self, name=self.name)
except (ValueError, InvalidPythonVersion):
pass
if py_version is None:
pass
self.py_version = py_version
return py_version # type: ignore
@name.default
def get_name(self):
# type: () -> Optional[str]
if self.path:
return self.path.name
return None
@property
def is_dir(self):
# type: () -> bool
if self._is_dir is None:
if not self.path:
ret_val = False
try:
ret_val = self.path.is_dir()
except OSError:
ret_val = False
self._is_dir = ret_val
return self._is_dir
@is_dir.setter
def is_dir(self, val):
# type: (bool) -> None
self._is_dir = val
@is_dir.deleter
def is_dir(self):
# type: () -> None
self._is_dir = None
@property
def is_executable(self):
# type: () -> bool
if self._is_executable is None:
if not self.path:
self._is_executable = False
else:
self._is_executable = path_is_known_executable(self.path)
return self._is_executable
@is_executable.setter
def is_executable(self, val):
# type: (bool) -> None
self._is_executable = val
@is_executable.deleter
def is_executable(self):
# type: () -> None
self._is_executable = None
@property
def is_python(self):
# type: () -> bool
if self._is_python is None:
if not self.path:
self._is_python = False
else:
self._is_python = self.is_executable and (looks_like_python(
self.path.name))
return self._is_python
@is_python.setter
def is_python(self, val):
# type: (bool) -> None
self._is_python = val
@is_python.deleter
def is_python(self):
# type: () -> None
self._is_python = None
def get_py_version(self):
# type: () -> Optional[PythonVersion]
from ..environment import IGNORE_UNSUPPORTED
if self.is_dir:
return None
if self.is_python:
py_version = None
from .python import PythonVersion
try:
py_version = PythonVersion.from_path( # type: ignore
path=self, name=self.name)
except (InvalidPythonVersion, ValueError):
py_version = None
except Exception:
if not IGNORE_UNSUPPORTED:
raise
return py_version
return None
@property
def py_version(self):
# type: () -> Optional[PythonVersion]
if not self._py_version:
py_version = self.get_py_version()
self._py_version = py_version
else:
py_version = self._py_version
return py_version
@py_version.setter
def py_version(self, val):
# type: (Optional[PythonVersion]) -> None
self._py_version = val
@py_version.deleter
def py_version(self):
# type: () -> None
self._py_version = None
def _iter_pythons(self):
# type: () -> Iterator
if self.is_dir:
for entry in self.children.values():
if entry is None:
continue
elif entry.is_dir:
for python in entry._iter_pythons():
yield python
elif entry.is_python and entry.as_python is not None:
yield entry
elif self.is_python and self.as_python is not None:
yield self # type: ignore
@property
def pythons(self):
# type: () -> DefaultDict[Union[str, Path], PathEntry]
if not self._pythons:
from .path import PathEntry
self._pythons = defaultdict(PathEntry)
for python in self._iter_pythons():
python_path = python.path.as_posix() # type: ignore
self._pythons[python_path] = python
return self._pythons
def __iter__(self):
# type: () -> Iterator
for entry in self.children.values():
yield entry
def __next__(self):
# type: () -> Generator
return next(iter(self))
def next(self):
# type: () -> Generator
return self.__next__()
def find_all_python_versions(
self,
major=None, # type: Optional[Union[str, int]]
minor=None, # type: Optional[int]
patch=None, # type: Optional[int]
pre=None, # type: Optional[bool]
dev=None, # type: Optional[bool]
arch=None, # type: Optional[str]
name=None, # type: Optional[str]
):
# type: (...) -> List[PathEntry]
"""Search for a specific python version on the path. Return all copies
:param major: Major python version to search for.
:type major: int
:param int minor: Minor python version to search for, defaults to None
:param int patch: Patch python version to search for, defaults to None
:param bool pre: Search for prereleases (default None) - prioritize releases if None
:param bool dev: Search for devreleases (default None) - prioritize releases if None
:param str arch: Architecture to include, e.g. '64bit', defaults to None
:param str name: The name of a python version, e.g. ``anaconda3-5.3.0``
:return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested.
:rtype: List[:class:`~pythonfinder.models.PathEntry`]
"""
call_method = "find_all_python_versions" if self.is_dir else "find_python_version"
sub_finder = operator.methodcaller(call_method, major, minor, patch,
pre, dev, arch, name)
if not self.is_dir:
return sub_finder(self)
unnested = [sub_finder(path) for path in expand_paths(self)]
version_sort = operator.attrgetter("as_python.version_sort")
unnested = [
p for p in unnested if p is not None and p.as_python is not None
]
paths = sorted(unnested, key=version_sort, reverse=True)
return list(paths)
def find_python_version(
self,
major=None, # type: Optional[Union[str, int]]
minor=None, # type: Optional[int]
patch=None, # type: Optional[int]
pre=None, # type: Optional[bool]
dev=None, # type: Optional[bool]
arch=None, # type: Optional[str]
name=None, # type: Optional[str]
):
# type: (...) -> Optional[PathEntry]
"""Search or self for the specified Python version and return the first match.
:param major: Major version number.
:type major: int
:param int minor: Minor python version to search for, defaults to None
:param int patch: Patch python version to search for, defaults to None
:param bool pre: Search for prereleases (default None) - prioritize releases if None
:param bool dev: Search for devreleases (default None) - prioritize releases if None
:param str arch: Architecture to include, e.g. '64bit', defaults to None
:param str name: The name of a python version, e.g. ``anaconda3-5.3.0``
:returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested.
"""
version_matcher = operator.methodcaller("matches",
major,
minor,
patch,
pre,
dev,
arch,
python_name=name)
if not self.is_dir:
if self.is_python and self.as_python and version_matcher(
self.py_version):
return self # type: ignore
matching_pythons = [
[entry, entry.as_python.version_sort]
for entry in self._iter_pythons()
if (entry is not None and entry.as_python is not None
and version_matcher(entry.py_version))
]
results = sorted(matching_pythons,
key=operator.itemgetter(1, 0),
reverse=True)
return next(iter(r[0] for r in results if r is not None), None)
class PythonFinder(BaseFinder, BasePath):
root = attr.ib(default=None, validator=optional_instance_of(Path), type=Path)
# should come before versions, because its value is used in versions's default initializer.
#: Whether to ignore any paths which raise exceptions and are not actually python
ignore_unsupported = attr.ib(default=True, type=bool)
#: Glob path for python versions off of the root directory
version_glob_path = attr.ib(default="versions/*", type=str)
#: The function to use to sort version order when returning an ordered verion set
sort_function = attr.ib(default=None) # type: Callable
#: The root locations used for discovery
roots = attr.ib(default=attr.Factory(defaultdict), type=defaultdict)
#: List of paths discovered during search
paths = attr.ib(type=list)
#: shim directory
shim_dir = attr.ib(default="shims", type=str)
#: Versions discovered in the specified paths
_versions = attr.ib(default=attr.Factory(defaultdict), type=defaultdict)
_pythons = attr.ib(default=attr.Factory(defaultdict), type=defaultdict)
def __del__(self):
# type: () -> None
self._versions = defaultdict()
self._pythons = defaultdict()
self.roots = defaultdict()
self.paths = []
@property
def expanded_paths(self):
# type: () -> Generator
return (
path for path in unnest(p for p in self.versions.values()) if path is not None
)
@property
def is_pyenv(self):
# type: () -> bool
return is_in_path(str(self.root), PYENV_ROOT)
@property
def is_asdf(self):
# type: () -> bool
return is_in_path(str(self.root), ASDF_DATA_DIR)
def get_version_order(self):
# type: () -> List[Path]
version_paths = [
p
for p in self.root.glob(self.version_glob_path)
if not (p.parent.name == "envs" or p.name == "envs")
]
versions = {v.name: v for v in version_paths}
version_order = [] # type: List[Path]
if self.is_pyenv:
version_order = [
versions[v] for v in parse_pyenv_version_order() if v in versions
]
elif self.is_asdf:
version_order = [
versions[v] for v in parse_asdf_version_order() if v in versions
]
for version in version_order:
if version in version_paths:
version_paths.remove(version)
if version_order:
version_order += version_paths
else:
version_order = version_paths
return version_order
def get_bin_dir(self, base):
# type: (Union[Path, str]) -> Path
if isinstance(base, six.string_types):
base = Path(base)
if os.name == "nt":
return base
return base / "bin"
@classmethod
def version_from_bin_dir(cls, entry):
# type: (PathEntry) -> Optional[PathEntry]
py_version = None
py_version = next(iter(entry.find_all_python_versions()), None)
return py_version
def _iter_version_bases(self):
# type: () -> Iterator[Tuple[Path, PathEntry]]
from .path import PathEntry
for p in self.get_version_order():
bin_dir = self.get_bin_dir(p)
if bin_dir.exists() and bin_dir.is_dir():
entry = PathEntry.create(
path=bin_dir.absolute(), only_python=False, name=p.name, is_root=True
)
self.roots[p] = entry
yield (p, entry)
def _iter_versions(self):
# type: () -> Iterator[Tuple[Path, PathEntry, Tuple]]
for base_path, entry in self._iter_version_bases():
version = None
version_entry = None
try:
version = PythonVersion.parse(entry.name)
except (ValueError, InvalidPythonVersion):
version_entry = next(iter(entry.find_all_python_versions()), None)
if version is None:
if not self.ignore_unsupported:
raise
continue
if version_entry is not None:
version = version_entry.py_version.as_dict()
except Exception:
if not self.ignore_unsupported:
raise
logger.warning(
"Unsupported Python version %r, ignoring...",
base_path.name,
exc_info=True,
)
continue
if version is not None:
version_tuple = (
version.get("major"),
version.get("minor"),
version.get("patch"),
version.get("is_prerelease"),
version.get("is_devrelease"),
version.get("is_debug"),
)
yield (base_path, entry, version_tuple)
@property
def versions(self):
# type: () -> DefaultDict[Tuple, PathEntry]
if not self._versions:
for _, entry, version_tuple in self._iter_versions():
self._versions[version_tuple] = entry
return self._versions
def _iter_pythons(self):
# type: () -> Iterator
for path, entry, version_tuple in self._iter_versions():
if path.as_posix() in self._pythons:
yield self._pythons[path.as_posix()]
elif version_tuple not in self.versions:
for python in entry.find_all_python_versions():
yield python
else:
yield self.versions[version_tuple]
@paths.default
def get_paths(self):
# type: () -> List[PathEntry]
_paths = [base for _, base in self._iter_version_bases()]
return _paths
@property
def pythons(self):
# type: () -> DefaultDict[str, PathEntry]
if not self._pythons:
from .path import PathEntry
self._pythons = defaultdict(PathEntry) # type: DefaultDict[str, PathEntry]
for python in self._iter_pythons():
python_path = python.path.as_posix() # type: ignore
self._pythons[python_path] = python
return self._pythons
@pythons.setter
def pythons(self, value):
# type: (DefaultDict[str, PathEntry]) -> None
self._pythons = value
def get_pythons(self):
# type: () -> DefaultDict[str, PathEntry]
return self.pythons
@overload
@classmethod
def create(cls, root, sort_function, version_glob_path=None, ignore_unsupported=True):
# type: (str, Callable, Optional[str], bool) -> PythonFinder
root = ensure_path(root)
if not version_glob_path:
version_glob_path = "versions/*"
return cls(
root=root,
path=root,
ignore_unsupported=ignore_unsupported, # type: ignore
sort_function=sort_function,
version_glob_path=version_glob_path,
)
def find_all_python_versions(
self,
major=None, # type: Optional[Union[str, int]]
minor=None, # type: Optional[int]
patch=None, # type: Optional[int]
pre=None, # type: Optional[bool]
dev=None, # type: Optional[bool]
arch=None, # type: Optional[str]
name=None, # type: Optional[str]
):
# type: (...) -> List[PathEntry]
"""Search for a specific python version on the path. Return all copies
:param major: Major python version to search for.
:type major: int
:param int minor: Minor python version to search for, defaults to None
:param int patch: Patch python version to search for, defaults to None
:param bool pre: Search for prereleases (default None) - prioritize releases if None
:param bool dev: Search for devreleases (default None) - prioritize releases if None
:param str arch: Architecture to include, e.g. '64bit', defaults to None
:param str name: The name of a python version, e.g. ``anaconda3-5.3.0``
:return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested.
:rtype: List[:class:`~pythonfinder.models.PathEntry`]
"""
call_method = "find_all_python_versions" if self.is_dir else "find_python_version"
sub_finder = operator.methodcaller(
call_method, major, minor, patch, pre, dev, arch, name
)
if not any([major, minor, patch, name]):
pythons = [
next(iter(py for py in base.find_all_python_versions()), None)
for _, base in self._iter_version_bases()
]
else:
pythons = [sub_finder(path) for path in self.paths]
pythons = expand_paths(pythons, True)
version_sort = operator.attrgetter("as_python.version_sort")
paths = [
p for p in sorted(pythons, key=version_sort, reverse=True) if p is not None
]
return paths
def find_python_version(
self,
major=None, # type: Optional[Union[str, int]]
minor=None, # type: Optional[int]
patch=None, # type: Optional[int]
pre=None, # type: Optional[bool]
dev=None, # type: Optional[bool]
arch=None, # type: Optional[str]
name=None, # type: Optional[str]
):
# type: (...) -> Optional[PathEntry]
"""Search or self for the specified Python version and return the first match.
:param major: Major version number.
:type major: int
:param int minor: Minor python version to search for, defaults to None
:param int patch: Patch python version to search for, defaults to None
:param bool pre: Search for prereleases (default None) - prioritize releases if None
:param bool dev: Search for devreleases (default None) - prioritize releases if None
:param str arch: Architecture to include, e.g. '64bit', defaults to None
:param str name: The name of a python version, e.g. ``anaconda3-5.3.0``
:returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested.
"""
sub_finder = operator.methodcaller(
"find_python_version", major, minor, patch, pre, dev, arch, name
)
version_sort = operator.attrgetter("as_python.version_sort")
unnested = [sub_finder(self.roots[path]) for path in self.roots]
unnested = [
p
for p in unnested
if p is not None and p.is_python and p.as_python is not None
]
paths = sorted(list(unnested), key=version_sort, reverse=True)
return next(iter(p for p in paths if p is not None), None)
def which(self, name):
# type: (str) -> Optional[PathEntry]
"""Search in this path for an executable.
:param executable: The name of an executable to search for.
:type executable: str
:returns: :class:`~pythonfinder.models.PathEntry` instance.
"""
matches = (p.which(name) for p in self.paths)
non_empty_match = next(iter(m for m in matches if m is not None), None)
return non_empty_match
class Lockfile(object):
path = attr.ib(validator=optional_instance_of(Path), type=Path)
_requirements = attr.ib(default=attr.Factory(list), type=list)
_dev_requirements = attr.ib(default=attr.Factory(list), type=list)
projectfile = attr.ib(validator=is_projectfile, type=ProjectFile)
_lockfile = attr.ib(validator=is_lockfile, type=plette.lockfiles.Lockfile)
newlines = attr.ib(default=DEFAULT_NEWLINES, type=str)
@path.default
def _get_path(self):
return Path(os.curdir).joinpath("Pipfile.lock").absolute()
@projectfile.default
def _get_projectfile(self):
return self.load_projectfile(self.path)
@_lockfile.default
def _get_lockfile(self):
return self.projectfile.model
@property
def lockfile(self):
return self._lockfile
@property
def section_keys(self):
return ["default", "develop"]
@property
def extended_keys(self):
return [
k for k in itertools.product(self.section_keys,
["", "vcs", "editable"])
]
def get(self, k):
return self.__getitem__(k)
def __contains__(self, k):
check_lockfile = k in self.extended_keys or self.lockfile.__contains__(
k)
if check_lockfile:
return True
return super(Lockfile, self).__contains__(k)
def __setitem__(self, k, v):
lockfile = self._lockfile
lockfile.__setitem__(k, v)
def __getitem__(self, k, *args, **kwargs):
retval = None
lockfile = self._lockfile
section = None
pkg_type = None
try:
retval = lockfile[k]
except KeyError:
if "-" in k:
section, _, pkg_type = k.rpartition("-")
vals = getattr(lockfile.get(section, {}), "_data", {})
if pkg_type == "vcs":
retval = {k: v for k, v in vals.items() if is_vcs(v)}
elif pkg_type == "editable":
retval = {k: v for k, v in vals.items() if is_editable(v)}
if retval is None:
raise
else:
retval = getattr(retval, "_data", retval)
return retval
def __getattr__(self, k, *args, **kwargs):
retval = None
lockfile = super(Lockfile, self).__getattribute__("_lockfile")
try:
return super(Lockfile, self).__getattribute__(k)
except AttributeError:
retval = getattr(lockfile, k, None)
if retval is not None:
return retval
return super(Lockfile, self).__getattribute__(k, *args, **kwargs)
def get_deps(self, dev=False, only=True):
deps = {}
if dev:
deps.update(self.develop._data)
if only:
return deps
deps = merge_items([deps, self.default._data])
return deps
@classmethod
def read_projectfile(cls, path):
"""Read the specified project file and provide an interface for
writing/updating.
:param str path: Path to the target file.
:return: A project file with the model and location for interaction
:rtype: :class:`~requirementslib.models.project.ProjectFile`
"""
pf = ProjectFile.read(path, plette.lockfiles.Lockfile, invalid_ok=True)
return pf
@classmethod
def lockfile_from_pipfile(cls, pipfile_path):
from .pipfile import Pipfile
if os.path.isfile(pipfile_path):
if not os.path.isabs(pipfile_path):
pipfile_path = os.path.abspath(pipfile_path)
pipfile = Pipfile.load(os.path.dirname(pipfile_path))
return plette.lockfiles.Lockfile.with_meta_from(pipfile._pipfile)
raise PipfileNotFound(pipfile_path)
@classmethod
def load_projectfile(cls, path, create=True, data=None):
"""Given a path, load or create the necessary lockfile.
:param str path: Path to the project root or lockfile
:param bool create: Whether to create the lockfile if not found, defaults to True
:raises OSError: Thrown if the project root directory doesn't exist
:raises FileNotFoundError: Thrown if the lockfile doesn't exist and ``create=False``
:return: A project file instance for the supplied project
:rtype: :class:`~requirementslib.models.project.ProjectFile`
"""
if not path:
path = os.curdir
path = Path(path).absolute()
project_path = path if path.is_dir() else path.parent
lockfile_path = path if path.is_file(
) else project_path / "Pipfile.lock"
if not project_path.exists():
raise OSError("Project does not exist: %s" %
project_path.as_posix())
elif not lockfile_path.exists() and not create:
raise FileNotFoundError("Lockfile does not exist: %s" %
lockfile_path.as_posix())
projectfile = cls.read_projectfile(lockfile_path.as_posix())
if not lockfile_path.exists():
if not data:
path_str = lockfile_path.as_posix()
if path_str[-5:] == ".lock":
pipfile = Path(path_str[:-5])
else:
pipfile = project_path.joinpath("Pipfile")
lf = cls.lockfile_from_pipfile(pipfile)
else:
lf = plette.lockfiles.Lockfile(data)
projectfile.model = lf
return projectfile
@classmethod
def from_data(cls, path, data, meta_from_project=True):
"""Create a new lockfile instance from a dictionary.
:param str path: Path to the project root.
:param dict data: Data to load into the lockfile.
:param bool meta_from_project: Attempt to populate the meta section from the
project root, default True.
"""
if path is None:
raise MissingParameter("path")
if data is None:
raise MissingParameter("data")
if not isinstance(data, dict):
raise TypeError("Expecting a dictionary for parameter 'data'")
path = os.path.abspath(str(path))
if os.path.isdir(path):
project_path = path
elif not os.path.isdir(path) and os.path.isdir(os.path.dirname(path)):
project_path = os.path.dirname(path)
pipfile_path = os.path.join(project_path, "Pipfile")
lockfile_path = os.path.join(project_path, "Pipfile.lock")
if meta_from_project:
lockfile = cls.lockfile_from_pipfile(pipfile_path)
lockfile.update(data)
else:
lockfile = plette.lockfiles.Lockfile(data)
projectfile = ProjectFile(line_ending=DEFAULT_NEWLINES,
location=lockfile_path,
model=lockfile)
return cls(
projectfile=projectfile,
lockfile=lockfile,
newlines=projectfile.line_ending,
path=Path(projectfile.location),
)
@classmethod
def load(cls, path, create=True):
"""Create a new lockfile instance.
:param project_path: Path to project root or lockfile
:type project_path: str or :class:`pathlib.Path`
:param str lockfile_name: Name of the lockfile in the project root directory
:param pipfile_path: Path to the project pipfile
:type pipfile_path: :class:`pathlib.Path`
:returns: A new lockfile representing the supplied project paths
:rtype: :class:`~requirementslib.models.lockfile.Lockfile`
"""
try:
projectfile = cls.load_projectfile(path, create=create)
except JSONDecodeError:
path = os.path.abspath(path)
path = Path(
os.path.join(path, "Pipfile.lock") if os.path.
isdir(path) else path)
formatted_path = path.as_posix()
backup_path = "%s.bak" % formatted_path
LockfileCorruptException.show(formatted_path,
backup_path=backup_path)
path.rename(backup_path)
cls.load(formatted_path, create=True)
lockfile_path = Path(projectfile.location)
creation_args = {
"projectfile": projectfile,
"lockfile": projectfile.model,
"newlines": projectfile.line_ending,
"path": lockfile_path,
}
return cls(**creation_args)
@classmethod
def create(cls, path, create=True):
return cls.load(path, create=create)
@property
def develop(self):
return self._lockfile.develop
@property
def default(self):
return self._lockfile.default
def get_requirements(self, dev=True, only=False):
"""Produces a generator which generates requirements from the desired
section.
:param bool dev: Indicates whether to use dev requirements, defaults to False
:return: Requirements from the relevant the relevant pipfile
:rtype: :class:`~requirementslib.models.requirements.Requirement`
"""
deps = self.get_deps(dev=dev, only=only)
for k, v in deps.items():
yield Requirement.from_pipfile(k, v)
@property
def dev_requirements(self):
if not self._dev_requirements:
self._dev_requirements = list(
self.get_requirements(dev=True, only=True))
return self._dev_requirements
@property
def requirements(self):
if not self._requirements:
self._requirements = list(
self.get_requirements(dev=False, only=True))
return self._requirements
@property
def dev_requirements_list(self):
return [{
name: entry._data
} for name, entry in self._lockfile.develop.items()]
@property
def requirements_list(self):
return [{
name: entry._data
} for name, entry in self._lockfile.default.items()]
def write(self):
self.projectfile.model = copy.deepcopy(self._lockfile)
self.projectfile.write()
def as_requirements(self, include_hashes=False, dev=False):
"""Returns a list of requirements in pip-style format."""
lines = []
section = self.dev_requirements if dev else self.requirements
for req in section:
kwargs = {"include_hashes": include_hashes}
if req.editable:
kwargs["include_markers"] = False
r = req.as_line(**kwargs)
lines.append(r.strip())
return lines
class AbstractDependency(object):
name = attr.ib() # type: STRING_TYPE
specifiers = attr.ib()
markers = attr.ib()
candidates = attr.ib()
requirement = attr.ib()
parent = attr.ib()
finder = attr.ib()
dep_dict = attr.ib(default=attr.Factory(dict))
@property
def version_set(self):
"""Return the set of versions for the candidates in this abstract
dependency.
:return: A set of matching versions
:rtype: set(str)
"""
if len(self.candidates) == 1:
return set()
return set(
packaging.version.parse(version_from_ireq(c))
for c in self.candidates)
def compatible_versions(self, other):
"""Find compatible version numbers between this abstract dependency and
another one.
:param other: An abstract dependency to compare with.
:type other: :class:`~requirementslib.models.dependency.AbstractDependency`
:return: A set of compatible version strings
:rtype: set(str)
"""
if len(self.candidates) == 1 and next(iter(self.candidates)).editable:
return self
elif len(other.candidates) == 1 and next(iter(
other.candidates)).editable:
return other
return self.version_set & other.version_set
def compatible_abstract_dep(self, other):
"""Merge this abstract dependency with another one.
Return the result of the merge as a new abstract dependency.
:param other: An abstract dependency to merge with
:type other: :class:`~requirementslib.models.dependency.AbstractDependency`
:return: A new, combined abstract dependency
:rtype: :class:`~requirementslib.models.dependency.AbstractDependency`
"""
from .requirements import Requirement
if len(self.candidates) == 1 and next(iter(self.candidates)).editable:
return self
elif len(other.candidates) == 1 and next(iter(
other.candidates)).editable:
return other
new_specifiers = self.specifiers & other.specifiers
markers = set(self.markers) if self.markers else set()
if other.markers:
markers.add(other.markers)
new_markers = None
if markers:
new_markers = packaging.markers.Marker(" or ".join(
str(m) for m in sorted(markers)))
new_ireq = copy.deepcopy(self.requirement.ireq)
new_ireq.req.specifier = new_specifiers
new_ireq.req.marker = new_markers
new_requirement = Requirement.from_line(format_requirement(new_ireq))
compatible_versions = self.compatible_versions(other)
if isinstance(compatible_versions, AbstractDependency):
return compatible_versions
candidates = [
c for c in self.candidates if packaging.version.parse(
version_from_ireq(c)) in compatible_versions
]
dep_dict = {}
candidate_strings = [format_requirement(c) for c in candidates]
for c in candidate_strings:
if c in self.dep_dict:
dep_dict[c] = self.dep_dict.get(c)
return AbstractDependency(
name=self.name,
specifiers=new_specifiers,
markers=new_markers,
candidates=candidates,
requirement=new_requirement,
parent=self.parent,
dep_dict=dep_dict,
finder=self.finder,
)
def get_deps(self, candidate):
"""Get the dependencies of the supplied candidate.
:param candidate: An installrequirement
:type candidate: :class:`~pipenv.patched.notpip._internal.req.req_install.InstallRequirement`
:return: A list of abstract dependencies
:rtype: list[:class:`~requirementslib.models.dependency.AbstractDependency`]
"""
key = format_requirement(candidate)
if key not in self.dep_dict:
from .requirements import Requirement
req = Requirement.from_line(key)
req = req.merge_markers(self.markers)
self.dep_dict[key] = req.get_abstract_dependencies()
return self.dep_dict[key]
@classmethod
def from_requirement(cls, requirement, parent=None):
"""Creates a new
:class:`~requirementslib.models.dependency.AbstractDependency` from a
:class:`~requirementslib.models.requirements.Requirement` object.
This class is used to find all candidates matching a given set of specifiers
and a given requirement.
:param requirement: A requirement for resolution
:type requirement: :class:`~requirementslib.models.requirements.Requirement` object.
"""
name = requirement.normalized_name
specifiers = requirement.ireq.specifier if not requirement.editable else ""
markers = requirement.ireq.markers
extras = requirement.ireq.extras
is_pinned = is_pinned_requirement(requirement.ireq)
is_constraint = bool(parent)
_, finder = get_finder(sources=None)
candidates = []
if not is_pinned and not requirement.editable:
for r in requirement.find_all_matches(finder=finder):
req = make_install_requirement(
name,
r.version,
extras=extras,
markers=markers,
constraint=is_constraint,
)
req.req.link = getattr(r, "location", getattr(r, "link", None))
req.parent = parent
candidates.append(req)
candidates = sorted(
set(candidates),
key=lambda k: packaging.version.parse(version_from_ireq(k)
),
)
else:
candidates = [requirement.ireq]
return cls(
name=name,
specifiers=specifiers,
markers=markers,
candidates=candidates,
requirement=requirement,
parent=parent,
finder=finder,
)
@classmethod
def from_string(cls, line, parent=None):
from .requirements import Requirement
req = Requirement.from_line(line)
abstract_dep = cls.from_requirement(req, parent=parent)
return abstract_dep
class Pipfile(object):
path = attr.ib(validator=is_path, type=Path)
projectfile = attr.ib(validator=is_projectfile, type=ProjectFile)
_pipfile = attr.ib(type=PipfileLoader)
_pyproject = attr.ib(default=attr.Factory(tomlkit.document),
type=tomlkit.toml_document.TOMLDocument)
build_system = attr.ib(default=attr.Factory(dict), type=dict)
_requirements = attr.ib(default=attr.Factory(list), type=list)
_dev_requirements = attr.ib(default=attr.Factory(list), type=list)
@path.default
def _get_path(self):
# type: () -> Path
return Path(os.curdir).absolute()
@projectfile.default
def _get_projectfile(self):
# type: () -> ProjectFile
return self.load_projectfile(os.curdir, create=False)
@_pipfile.default
def _get_pipfile(self):
# type: () -> Union[plette.pipfiles.Pipfile, PipfileLoader]
return self.projectfile.model
@property
def root(self):
return self.path.parent
@property
def extended_keys(self):
return [
k for k in itertools.product(("packages",
"dev-packages"), ("", "vcs",
"editable"))
]
@property
def pipfile(self):
# type: () -> Union[PipfileLoader, plette.pipfiles.Pipfile]
return self._pipfile
def get_deps(self, dev=False, only=True):
# type: (bool, bool) -> Dict[Text, Dict[Text, Union[List[Text], Text]]]
deps = {} # type: Dict[Text, Dict[Text, Union[List[Text], Text]]]
if dev:
deps.update(dict(self.pipfile._data.get("dev-packages", {})))
if only:
return deps
return tomlkit_value_to_python(
merge_items([deps,
dict(self.pipfile._data.get("packages", {}))]))
def get(self, k):
# type: (Text) -> Any
return self.__getitem__(k)
def __contains__(self, k):
# type: (Text) -> bool
check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k)
if check_pipfile:
return True
return False
def __getitem__(self, k, *args, **kwargs):
# type: ignore
retval = None
pipfile = self._pipfile
section = None
pkg_type = None
try:
retval = pipfile[k]
except KeyError:
if "-" in k:
section, _, pkg_type = k.rpartition("-")
vals = getattr(pipfile.get(section, {}), "_data", {})
vals = tomlkit_value_to_python(vals)
if pkg_type == "vcs":
retval = {k: v for k, v in vals.items() if is_vcs(v)}
elif pkg_type == "editable":
retval = {k: v for k, v in vals.items() if is_editable(v)}
if retval is None:
raise
else:
retval = getattr(retval, "_data", retval)
return retval
def __getattr__(self, k, *args, **kwargs):
# type: ignore
retval = None
pipfile = super(Pipfile, self).__getattribute__("_pipfile")
try:
retval = super(Pipfile, self).__getattribute__(k)
except AttributeError:
retval = getattr(pipfile, k, None)
if retval is not None:
return retval
return super(Pipfile, self).__getattribute__(k, *args, **kwargs)
@property
def requires_python(self):
# type: () -> bool
return getattr(
self._pipfile.requires,
"python_version",
getattr(self._pipfile.requires, "python_full_version", None),
)
@property
def allow_prereleases(self):
# type: () -> bool
return self._pipfile.get("pipenv", {}).get("allow_prereleases", False)
@classmethod
def read_projectfile(cls, path):
# type: (Text) -> ProjectFile
"""Read the specified project file and provide an interface for
writing/updating.
:param Text path: Path to the target file.
:return: A project file with the model and location for interaction
:rtype: :class:`~requirementslib.models.project.ProjectFile`
"""
pf = ProjectFile.read(path, PipfileLoader, invalid_ok=True)
return pf
@classmethod
def load_projectfile(cls, path, create=False):
# type: (Text, bool) -> ProjectFile
"""Given a path, load or create the necessary pipfile.
:param Text path: Path to the project root or pipfile
:param bool create: Whether to create the pipfile if not found, defaults to True
:raises OSError: Thrown if the project root directory doesn't exist
:raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False``
:return: A project file instance for the supplied project
:rtype: :class:`~requirementslib.models.project.ProjectFile`
"""
if not path:
raise RuntimeError(
"Must pass a path to classmethod 'Pipfile.load'")
if not isinstance(path, Path):
path = Path(path).absolute()
pipfile_path = path if path.is_file() else path.joinpath("Pipfile")
project_path = pipfile_path.parent
if not project_path.exists():
raise FileNotFoundError("%s is not a valid project path!" % path)
elif not pipfile_path.exists() or not pipfile_path.is_file():
if not create:
raise RequirementError("%s is not a valid Pipfile" %
pipfile_path)
return cls.read_projectfile(pipfile_path.as_posix())
@classmethod
def load(cls, path, create=False):
# type: (Text, bool) -> Pipfile
"""Given a path, load or create the necessary pipfile.
:param Text path: Path to the project root or pipfile
:param bool create: Whether to create the pipfile if not found, defaults to True
:raises OSError: Thrown if the project root directory doesn't exist
:raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False``
:return: A pipfile instance pointing at the supplied project
:rtype:: class:`~requirementslib.models.pipfile.Pipfile`
"""
projectfile = cls.load_projectfile(path, create=create)
pipfile = projectfile.model
creation_args = {
"projectfile": projectfile,
"pipfile": pipfile,
"path": Path(projectfile.location),
}
return cls(**creation_args)
def write(self):
# type: () -> None
self.projectfile.model = copy.deepcopy(self._pipfile)
self.projectfile.write()
@property
def dev_packages(self):
# type: () -> List[Requirement]
return self.dev_requirements
@property
def packages(self):
# type: () -> List[Requirement]
return self.requirements
@property
def dev_requirements(self):
# type: () -> List[Requirement]
if not self._dev_requirements:
packages = tomlkit_value_to_python(
self.pipfile.get("dev-packages", {}))
self._dev_requirements = [
Requirement.from_pipfile(k, v) for k, v in packages.items()
if v is not None
]
return self._dev_requirements
@property
def requirements(self):
# type: () -> List[Requirement]
if not self._requirements:
packages = tomlkit_value_to_python(self.pipfile.get(
"packages", {}))
self._requirements = [
Requirement.from_pipfile(k, v) for k, v in packages.items()
if v is not None
]
return self._requirements
def _read_pyproject(self):
# type: () -> None
pyproject = self.path.parent.joinpath("pyproject.toml")
if pyproject.exists():
self._pyproject = tomlkit.loads(pyproject.read_text())
build_system = self._pyproject.get("build-system", None)
if build_system and not build_system.get("build_backend"):
build_system[
"build-backend"] = "setuptools.build_meta:__legacy__"
elif not build_system or not build_system.get("requires"):
build_system = {
"requires": ["setuptools>=40.8", "wheel"],
"build-backend": "setuptools.build_meta:__legacy__",
}
self.build_system = build_system
@property
def build_requires(self):
# type: () -> List[Text]
if not self.build_system:
self._read_pyproject()
return self.build_system.get("requires", [])
@property
def build_backend(self):
# type: () -> Text
if not self.build_system:
self._read_pyproject()
return self.build_system.get("build-backend", None)
class DependencyResolver(object):
pinned_deps = attr.ib(default=attr.Factory(dict))
#: A dictionary of abstract dependencies by name
dep_dict = attr.ib(default=attr.Factory(dict))
#: A dictionary of sets of version numbers that are valid for a candidate currently
candidate_dict = attr.ib(default=attr.Factory(dict))
#: A historical record of pins
pin_history = attr.ib(default=attr.Factory(dict))
#: Whether to allow prerelease dependencies
allow_prereleases = attr.ib(default=False)
#: Stores hashes for each dependency
hashes = attr.ib(default=attr.Factory(dict))
#: A hash cache
hash_cache = attr.ib(default=attr.Factory(HashCache))
#: A finder for searching the index
finder = attr.ib(default=None)
#: Whether to include hashes even from incompatible wheels
include_incompatible_hashes = attr.ib(default=True)
#: A cache for storing available canddiates when using all wheels
_available_candidates_cache = attr.ib(default=attr.Factory(dict))
@classmethod
def create(cls, finder=None, allow_prereleases=False, get_all_hashes=True):
if not finder:
from .dependencies import get_finder
finder_args = []
if allow_prereleases:
finder_args.append("--pre")
finder = get_finder(*finder_args)
creation_kwargs = {
"allow_prereleases": allow_prereleases,
"include_incompatible_hashes": get_all_hashes,
"finder": finder,
"hash_cache": HashCache(),
}
resolver = cls(**creation_kwargs)
return resolver
@property
def dependencies(self):
return list(self.dep_dict.values())
@property
def resolution(self):
return list(self.pinned_deps.values())
def add_abstract_dep(self, dep):
"""Add an abstract dependency by either creating a new entry or merging
with an old one.
:param dep: An abstract dependency to add
:type dep: :class:`~requirementslib.models.dependency.AbstractDependency`
:raises ResolutionError: Raised when the given dependency is not compatible with
an existing abstract dependency.
"""
if dep.name in self.dep_dict:
compatible_versions = self.dep_dict[dep.name].compatible_versions(dep)
if compatible_versions:
self.candidate_dict[dep.name] = compatible_versions
self.dep_dict[dep.name] = self.dep_dict[dep.name].compatible_abstract_dep(
dep
)
else:
raise ResolutionError
else:
self.candidate_dict[dep.name] = dep.version_set
self.dep_dict[dep.name] = dep
def pin_deps(self):
"""Pins the current abstract dependencies and adds them to the history
dict.
Adds any new dependencies to the abstract dependencies already
present by merging them together to form new, compatible
abstract dependencies.
"""
for name in list(self.dep_dict.keys()):
candidates = self.dep_dict[name].candidates[:]
abs_dep = self.dep_dict[name]
while candidates:
pin = candidates.pop()
# Move on from existing pins if the new pin isn't compatible
if name in self.pinned_deps:
if self.pinned_deps[name].editable:
continue
old_version = version_from_ireq(self.pinned_deps[name])
if not pin.editable:
new_version = version_from_ireq(pin)
if (
new_version != old_version
and new_version not in self.candidate_dict[name]
):
continue
pin.parent = abs_dep.parent
pin_subdeps = self.dep_dict[name].get_deps(pin)
backup = self.dep_dict.copy(), self.candidate_dict.copy()
try:
for pin_dep in pin_subdeps:
self.add_abstract_dep(pin_dep)
except ResolutionError:
self.dep_dict, self.candidate_dict = backup
continue
else:
self.pinned_deps[name] = pin
break
def resolve(self, root_nodes, max_rounds=20):
"""Resolves dependencies using a backtracking resolver and multiple
endpoints.
Note: this resolver caches aggressively.
Runs for *max_rounds* or until any two pinning rounds yield the same outcome.
:param root_nodes: A list of the root requirements.
:type root_nodes: list[:class:`~requirementslib.models.requirements.Requirement`]
:param max_rounds: The max number of resolution rounds, defaults to 20
:param max_rounds: int, optional
:raises RuntimeError: Raised when max rounds is exceeded without a resolution.
"""
if self.dep_dict:
raise RuntimeError("Do not use the same resolver more than once")
if not self.hash_cache:
self.hash_cache = HashCache()
# Coerce input into AbstractDependency instances.
# We accept str, Requirement, and AbstractDependency as input.
from ..utils import log
from .dependencies import AbstractDependency
for dep in root_nodes:
if isinstance(dep, str):
dep = AbstractDependency.from_string(dep)
elif not isinstance(dep, AbstractDependency):
dep = AbstractDependency.from_requirement(dep)
self.add_abstract_dep(dep)
for round_ in range(max_rounds):
self.pin_deps()
self.pin_history[round_] = self.pinned_deps.copy()
if round_ > 0:
previous_round = set(self.pin_history[round_ - 1].values())
current_values = set(self.pin_history[round_].values())
difference = current_values - previous_round
else:
difference = set(self.pin_history[round_].values())
log.debug("\n")
log.debug("{:=^30}".format(" Round {0} ".format(round_)))
log.debug("\n")
if difference:
log.debug("New Packages: ")
for d in difference:
log.debug("{:>30}".format(format_requirement(d)))
elif round_ >= 3:
log.debug("Stable Pins: ")
for d in current_values:
log.debug("{:>30}".format(format_requirement(d)))
return
else:
log.debug("No New Packages.")
# TODO: Raise a better error.
raise RuntimeError("cannot resolve after {} rounds".format(max_rounds))
def get_hashes(self):
for dep in self.pinned_deps.values():
if dep.name not in self.hashes:
self.hashes[dep.name] = self.get_hashes_for_one(dep)
return self.hashes.copy()
def get_hashes_for_one(self, ireq):
if not self.finder:
from .dependencies import get_finder
finder_args = []
if self.allow_prereleases:
finder_args.append("--pre")
self.finder = get_finder(*finder_args)
if ireq.editable:
return set()
from pipenv.vendor.pip_shims import VcsSupport
vcs = VcsSupport()
if (
ireq.link
and ireq.link.scheme in vcs.all_schemes
and "ssh" in ireq.link.scheme
):
return set()
if not is_pinned_requirement(ireq):
raise TypeError("Expected pinned requirement, got {}".format(ireq))
matching_candidates = set()
with self.allow_all_wheels():
from .dependencies import find_all_matches
matching_candidates = find_all_matches(
self.finder, ireq, pre=self.allow_prereleases
)
return {
self.hash_cache.get_hash(
getattr(candidate, "location", getattr(candidate, "link", None))
)
for candidate in matching_candidates
}
@contextmanager
def allow_all_wheels(self):
"""Monkey patches pip.Wheel to allow wheels from all platforms and
Python versions.
This also saves the candidate cache and set a new one, or else
the results from the previous non-patched calls will interfere.
"""
def _wheel_supported(self, tags=None):
# Ignore current platform. Support everything.
return True
def _wheel_support_index_min(self, tags=None):
# All wheels are equal priority for sorting.
return 0
original_wheel_supported = Wheel.supported
original_support_index_min = Wheel.support_index_min
Wheel.supported = _wheel_supported
Wheel.support_index_min = _wheel_support_index_min
try:
yield
finally:
Wheel.supported = original_wheel_supported
Wheel.support_index_min = original_support_index_min
