def load_projectfile(cls, path, create=False): # type: (Text, bool) -> ProjectFile """ Given a path, load or create the necessary pipfile. :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` :return: A project file instance for the supplied project :rtype: :class:`~requirementslib.models.project.ProjectFile` """ if not path: raise RuntimeError( "Must pass a path to classmethod 'Pipfile.load'") if not isinstance(path, Path): path = Path(path).absolute() pipfile_path = path if path.is_file() else path.joinpath("Pipfile") project_path = pipfile_path.parent if not project_path.exists(): raise FileNotFoundError("%s is not a valid project path!" % path) elif not pipfile_path.exists() or not pipfile_path.is_file(): if not create: raise RequirementError("%s is not a valid Pipfile" % pipfile_path) return cls.read_projectfile(pipfile_path.as_posix())
def formatted_path(self): if self.path: path = self.path if not isinstance(path, Path): path = Path(path) return path.as_posix() return
def load_projectfile(cls, path, create=True, data=None): """Given a path, load or create the necessary lockfile. :param str path: Path to the project root or lockfile :param bool create: Whether to create the lockfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the lockfile doesn't exist and ``create=False`` :return: A project file instance for the supplied project :rtype: :class:`~requirementslib.models.project.ProjectFile` """ if not path: path = os.curdir path = Path(path).absolute() project_path = path if path.is_dir() else path.parent lockfile_path = path if path.is_file() else project_path / "Pipfile.lock" if not project_path.exists(): raise OSError("Project does not exist: %s" % project_path.as_posix()) elif not lockfile_path.exists() and not create: raise FileNotFoundError("Lockfile does not exist: %s" % lockfile_path.as_posix()) projectfile = cls.read_projectfile(lockfile_path.as_posix()) if not lockfile_path.exists(): if not data: path_str = lockfile_path.as_posix() if path_str[-5:] == ".lock": pipfile = Path(path_str[:-5]) else: pipfile = project_path.joinpath("Pipfile") lf = cls.lockfile_from_pipfile(pipfile) else: lf = plette.lockfiles.Lockfile(data) projectfile.model = lf return projectfile
def get_versions(self): versions = defaultdict(VersionPath) bin_ = sysconfig._INSTALL_SCHEMES[ sysconfig._get_default_scheme()]["scripts"] for p in self.root.glob("versions/*"): if p.parent.name == "envs": continue try: version = PythonVersion.parse(p.name) except ValueError: bin_dir = Path(bin_.format(base=p.as_posix())) if bin_dir.exists(): version = self.version_from_bin_dir(bin_dir) if not version: if not self.ignore_unsupported: raise continue except Exception: if not self.ignore_unsupported: raise logger.warning('Unsupported Python version %r, ignoring...', p.name, exc_info=True) continue if not version: continue version_tuple = (version.get("major"), version.get("minor"), version.get("patch"), version.get("is_prerelease"), version.get("is_devrelease"), version.get("is_debug")) versions[version_tuple] = VersionPath.create(path=p.resolve(), only_python=True) return versions
def filter_pythons(path): """Return all valid pythons in a given path""" if not isinstance(path, Path): path = Path(str(path)) if not path.is_dir(): return path if path_is_python(path) else None return filter(lambda x: path_is_python(x), path.iterdir())
def get_pyproject(path): from vistir.compat import Path if not path: return if not isinstance(path, Path): path = Path(path) if not path.is_dir(): path = path.parent pp_toml = path.joinpath("pyproject.toml") setup_py = path.joinpath("setup.py") if not pp_toml.exists(): if setup_py.exists(): return None else: pyproject_data = {} with io.open(pp_toml.as_posix(), encoding="utf-8") as fh: pyproject_data = tomlkit.loads(fh.read()) build_system = pyproject_data.get("build-system", None) if build_system is None: if setup_py.exists(): requires = ["setuptools", "wheel"] backend = "setuptools.build_meta" else: requires = ["setuptools>=38.2.5", "wheel"] backend = "setuptools.build_meta" build_system = {"requires": requires, "build-backend": backend} pyproject_data["build_system"] = build_system else: requires = build_system.get("requires") backend = build_system.get("build-backend") return (requires, backend)
def test_only_setup_missing( capsys, tmp_path, shared_datadir, source_pipfile_dirname, update_count ): # type: (Any, Path, Path, str, int) -> None """ test generate setup.py (when it is missing) """ pipfile_dir = shared_datadir / source_pipfile_dirname with data(source_pipfile_dirname, tmp_path): # delete the setup.py file that was copied to the tmp_path (tmp_path / "setup.py").unlink() cmd(argv=["", "sync"]) generated_setup = Path("setup.py") assert generated_setup.exists() generated_setup_text = generated_setup.read_text() expected_setup_text = Path("setup.py").read_text() for kw_arg_names in ("install_requires", "dependency_links"): assert compare_list_of_string_kw_arg( generated_setup_text, expected_setup_text, kw_arg_names, ordering_matters=False, ) captured = capsys.readouterr() assert msg_formatter.generate_success(update_count) in captured.out, captured.out
def test_update( capsys, tmp_path, shared_datadir, source_pipfile_dirname, update_count ): # type: (Any, Path, Path, str, int) -> None """ test updating setup.py (when it already exists) """ pipfile_dir = shared_datadir / source_pipfile_dirname for filename in ("Pipfile", "Pipfile.lock", "setup.py"): copy_file(pipfile_dir / filename, tmp_path) with data(source_pipfile_dirname, tmp_path): cmd(argv=["", "sync"]) generated_setup = Path("setup.py") assert generated_setup.exists() generated_setup_text = generated_setup.read_text() expected_setup_text = Path("setup.py").read_text() for kw_arg_names in ("install_requires", "dependency_links"): assert compare_list_of_string_kw_arg( generated_setup_text, expected_setup_text, kw_arg_names, ordering_matters=False, ) captured = capsys.readouterr() assert msg_formatter.update_success(update_count) in captured.out
def __init__(self, *args, **kwargs): path = kwargs.pop("path", None) self._requirements = kwargs.pop("requirements", []) self._dev_requirements = kwargs.pop("dev_requirements", []) self.path = Path(path) if path else None self.newlines = u"\n" super(Lockfile, self).__init__(*args, **kwargs)
def test_sync_pipfile_no_original(capsys, tmp_path, shared_datadir, source_pipfile_dirname, update_count): """ sync --pipfile should reference Pipfile (not Pipfile.lock) when printing results """ pipfile_dir = shared_datadir / source_pipfile_dirname for filename in ("Pipfile", "Pipfile.lock", "setup.py"): copy_file(pipfile_dir / filename, tmp_path) with data(str(pipfile_dir), tmp_path) as path: setup_file = path / "setup.py" # type: Path cmd(["", "sync", "--pipfile"]) text = setup_file.read_text() generated_setup = Path("setup.py") assert generated_setup.exists() generated_setup_text = generated_setup.read_text() expected_setup_text = Path("setup.py").read_text() for kw_arg_names in ("install_requires", "dependency_links"): assert compare_list_of_string_kw_arg( generated_setup_text, expected_setup_text, kw_arg_names, ordering_matters=False, ) captured = capsys.readouterr() assert "Pipfile.lock" not in captured.out, captured.out assert "Pipfile" in captured.out, captured.out
def test_local_editable_ref(monkeypatch): with monkeypatch.context() as m: m.setattr(pip_shims.shims, "unpack_url", mock_unpack) path = Path(ARTIFACTS_DIR) / "git/requests" req = Requirement.from_pipfile( "requests", {"editable": True, "git": path.as_uri(), "ref": "2.18.4"} ) assert req.as_line() == "-e git+{0}@2.18.4#egg=requests".format(path.as_uri())
def is_installable_dir(path): if _is_installable_dir(path): return True path = Path(path) pyproject = path.joinpath("pyproject.toml") if pyproject.exists(): pyproject_toml = tomlkit.loads(pyproject.read_text()) build_system = pyproject_toml.get("build-system", {}).get("build-backend", "") if build_system: return True return False
def load(cls, path): if not path: path = os.curdir path = Path(path).absolute() if path.is_dir(): path = path / "Pipfile.lock" elif path.name == "Pipfile": path = path.parent / "Pipfile.lock" if not path.exists(): raise OSError("Path does not exist: %s" % path) return cls.create(path.parent, lockfile_name=path.name)
def is_installable_dir(path): # type: (STRING_TYPE) -> bool if pip_shims.shims.is_installable_dir(path): return True pyproject_path = os.path.join(path, "pyproject.toml") if os.path.exists(pyproject_path): pyproject = Path(pyproject_path) pyproject_toml = tomlkit.loads(pyproject.read_text()) build_system = pyproject_toml.get("build-system", {}).get("build-backend", "") if build_system: return True return False
def test_keep_outdated_keeps_markers_not_removed(PipenvInstance): with PipenvInstance(chdir=True) as p: c = p.pipenv("install six click") assert c.ok lockfile = Path(p.lockfile_path) lockfile_content = lockfile.read_text() lockfile_json = json.loads(lockfile_content) assert "six" in lockfile_json["default"] lockfile_json["default"]["six"]["markers"] = "python_version >= '2.7'" lockfile.write_text(to_text(json.dumps(lockfile_json))) c = p.pipenv("lock --keep-outdated") assert c.ok assert p.lockfile["default"]["six"].get("markers", "") == "python_version >= '2.7'"
def ensure_setup_py(base_dir): if not base_dir: base_dir = create_tracked_tempdir(prefix="requirementslib-setup") base_dir = Path(base_dir) setup_py = base_dir.joinpath("setup.py") is_new = False if setup_py.exists() else True if not setup_py.exists(): setup_py.write_text(u"") try: yield finally: if is_new: setup_py.unlink()
def check(args): if not Path("Pipfile").exists(): fatal_error("Pipfile not found") if not Path("setup.py").exists(): fatal_error("setup.py not found") if args.lockfile: local_packages, remote_packages = lockfile_parser.get_default_packages( Path("Pipfile.lock")) else: local_packages, remote_packages = pipfile_parser.get_default_packages( Path("Pipfile")) if local_packages and not args.ignore_local: package_names = ", ".join(local_packages) fatal_error( "local package found in default dependency: %s.\nDo you mean to make it dev dependency " % package_names) with open("setup.py") as setup_file: setup_code = setup_file.read() try: ( install_requires, dependency_links, ) = setup_parser.get_install_requires_dependency_links(setup_code) except (ValueError, SyntaxError) as e: fatal_error(str(e)) # fatal_error is a NoReturn function, pycharm gets confused # noinspection PyUnboundLocalVariable checker = InconsistencyChecker(install_requires, dependency_links, remote_packages, args.strict) reports = [] checks = ( checker.check_install_requires_conflict, checker.check_dependency_links_conflict, checker.check_lacking_install_requires, checker.check_lacking_dependency_links, ) for check_item in checks: try: reports += check_item() except ValueError as e: print(e, file=stderr) fatal_error("dependency check failed") if len(reports) == 0: congratulate(msg_formatter.checked_no_problem()) else: fatal_error(reports)
def get_name(self): loc = self.path or self.uri if loc: self._uri_scheme = "path" if self.path else "uri" name = None if self.link and self.link.egg_fragment: return self.link.egg_fragment elif self.link and self.link.is_wheel: from pip_shims import Wheel return Wheel(self.link.filename).name if (self._uri_scheme != "uri" and self.path and self.setup_path and self.setup_path.exists()): from setuptools.dist import distutils old_curdir = os.path.abspath(os.getcwd()) try: os.chdir(str(self.setup_path.parent)) dist = distutils.core.run_setup(self.setup_path.as_posix()) name = dist.get_name() except (FileNotFoundError, IOError) as e: dist = None except Exception as e: from pip_shims.shims import make_abstract_dist try: if not isinstance(Path, self.path): _path = Path(self.path) else: _path = self.path if self.editable: _ireq = pip_shims.shims.install_req_from_editable( _path.as_uri()) else: _ireq = pip_shims.shims.install_req_from_line( _path.as_posix()) dist = make_abstract_dist(_ireq).get_dist() name = dist.project_name except (TypeError, ValueError, AttributeError) as e: dist = None finally: os.chdir(old_curdir) hashed_loc = hashlib.sha256(loc.encode("utf-8")).hexdigest() hashed_name = hashed_loc[-7:] if not name or name == "UNKNOWN": self._has_hashed_name = True name = hashed_name if self.link and not self._has_hashed_name: self.link = create_link("{0}#egg={1}".format(self.link.url, name)) return name
def get_versions(self): versions = defaultdict() bin_ = sysconfig._INSTALL_SCHEMES[ sysconfig._get_default_scheme()]["scripts"] for p in self.root.glob("versions/*"): if p.parent.name == "envs" or p.name == "envs": continue bin_dir = Path(bin_.format(base=p.as_posix())) version_path = None if bin_dir.exists(): version_path = PathEntry.create( path=bin_dir.absolute().as_posix(), only_python=False, name=p.name, is_root=True, ) version = None try: version = PythonVersion.parse(p.name) except ValueError: entry = next(iter(version_path.find_all_python_versions()), None) if not entry: if self.ignore_unsupported: continue raise else: version = entry.py_version.as_dict() except Exception: if not self.ignore_unsupported: raise logger.warning("Unsupported Python version %r, ignoring...", p.name, exc_info=True) continue if not version: continue version_tuple = ( version.get("major"), version.get("minor"), version.get("patch"), version.get("is_prerelease"), version.get("is_devrelease"), version.get("is_debug"), ) self.roots[p] = version_path versions[version_tuple] = version_path self.paths.append(version_path) return versions
def test_get_local_ref(tmpdir): six_dir = tmpdir.join("six") import vistir c = vistir.misc.run(["git", "clone", "https://github.com/benjaminp/six.git", six_dir.strpath], return_object=True, nospin=True) assert c.returncode == 0 r = Requirement.from_line("git+{0}#egg=six".format(Path(six_dir.strpath).as_uri())) assert r.commit_hash
def from_line(cls, line): line = line.strip('"').strip("'") link = None path = None editable = line.startswith("-e ") line = line.split(" ", 1)[1] if editable else line setup_path = None if not any( [is_installable_file(line), is_valid_url(line), is_file_url(line)]): raise RequirementError( "Supplied requirement is not installable: {0!r}".format(line)) vcs_type, prefer, relpath, path, uri, link = cls.get_link_from_line( line) setup_path = Path(path) / "setup.py" if path else None arg_dict = { "path": relpath if relpath else path, "uri": unquote(link.url_without_fragment), "link": link, "editable": editable, "setup_path": setup_path, "uri_scheme": prefer, } if link and link.is_wheel: from pip_shims import Wheel arg_dict["name"] = Wheel(link.filename).name elif link.egg_fragment: arg_dict["name"] = link.egg_fragment created = cls(**arg_dict) return created
def load(cls, path, create=False): # type: (Text, bool) -> Pipfile """ Given a path, load or create the necessary pipfile. :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` :return: A pipfile instance pointing at the supplied project :rtype:: class:`~requirementslib.models.pipfile.Pipfile` """ projectfile = cls.load_projectfile(path, create=create) pipfile = projectfile.model dev_requirements = [ Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("dev-packages", {}).items() ] requirements = [ Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("packages", {}).items() ] creation_args = { "projectfile": projectfile, "pipfile": pipfile, "dev_requirements": dev_requirements, "requirements": requirements, "path": Path(projectfile.location) } return cls(**creation_args)
def from_data(cls, path, data, meta_from_project=True): """Create a new lockfile instance from a dictionary. :param str path: Path to the project root. :param dict data: Data to load into the lockfile. :param bool meta_from_project: Attempt to populate the meta section from the project root, default True. """ if path is None: raise MissingParameter("path") if data is None: raise MissingParameter("data") if not isinstance(data, dict): raise TypeError("Expecting a dictionary for parameter 'data'") path = os.path.abspath(str(path)) if os.path.isdir(path): project_path = path elif not os.path.isdir(path) and os.path.isdir(os.path.dirname(path)): project_path = os.path.dirname(path) pipfile_path = os.path.join(project_path, "Pipfile") lockfile_path = os.path.join(project_path, "Pipfile.lock") if meta_from_project: lockfile = cls.lockfile_from_pipfile(pipfile_path) lockfile.update(data) else: lockfile = plette.lockfiles.Lockfile(data) projectfile = ProjectFile(line_ending=DEFAULT_NEWLINES, location=lockfile_path, model=lockfile) return cls( projectfile=projectfile, lockfile=lockfile, newlines=projectfile.line_ending, path=Path(projectfile.location) )
def convert_pipfile_or_lock(project, pipfile=None, hashes=False, dev=False): """Convert given Pipfile/Pipfile.lock to requirements.txt content. :param project: the project path, default to `pwd`. :param pipfile: the path of Pipfile or Pipfile.lock. If it isn't given, will try Pipfile.lock first then Pipfile. :param hashes: whether to include hashes :param dev: whether to choose the dev-dependencies section. :returns: the content of requirements.txt """ if pipfile is None: if project.joinpath("Pipfile.lock").exists(): pipfile = "Pipfile.lock" elif project.joinpath("Pipfile").exists(): pipfile = "Pipfile" if pipfile and not Path(pipfile).is_absolute(): full_path = project.joinpath(pipfile).as_posix() else: full_path = pipfile if pipfile is None or not os.path.exists(full_path): raise FileNotFoundError("No Pipfile* is found.") try: with open(full_path) as f: json.load(f) except Exception: if hashes: warnings.warn( "Pipfile is given, the hashes flag won't take effect.", UserWarning ) return _convert_pipfile(full_path, dev) else: return _convert_pipfile_lock(full_path, hashes, dev)
def __attrs_post_init__(self): #: slice in pyenv if not self.__class__ == SystemPath: return if os.name == "nt": self._setup_windows() if PYENV_INSTALLED: self._setup_pyenv() if ASDF_INSTALLED: self._setup_asdf() venv = os.environ.get("VIRTUAL_ENV") if os.name == "nt": bin_dir = "Scripts" else: bin_dir = "bin" if venv and (self.system or self.global_search): p = ensure_path(venv) self.path_order = [(p / bin_dir).as_posix()] + self.path_order self.paths[p] = PathEntry.create(path=p, is_root=True, only_python=False) if self.system: syspath = Path(sys.executable) syspath_bin = syspath.parent if syspath_bin.name != bin_dir and syspath_bin.joinpath( bin_dir).exists(): syspath_bin = syspath_bin / bin_dir self.path_order = [syspath_bin.as_posix()] + self.path_order self.paths[syspath_bin] = PathEntry.create(path=syspath_bin, is_root=True, only_python=False)
def get_versions(self): from .path import PathEntry versions = defaultdict() bin_ = "{base}/bin" for p in self.get_version_order(): bin_dir = Path(bin_.format(base=p.as_posix())) version_path = None if bin_dir.exists(): version_path = PathEntry.create( path=bin_dir.absolute().as_posix(), only_python=False, name=p.name, is_root=True, ) version = None try: version = PythonVersion.parse(p.name) except (ValueError, InvalidPythonVersion): entry = next(iter(version_path.find_all_python_versions()), None) if not entry: if self.ignore_unsupported: continue raise else: version = entry.py_version.as_dict() except Exception: if not self.ignore_unsupported: raise logger.warning("Unsupported Python version %r, ignoring...", p.name, exc_info=True) continue if not version: continue version_tuple = ( version.get("major"), version.get("minor"), version.get("patch"), version.get("is_prerelease"), version.get("is_devrelease"), version.get("is_debug"), ) self.roots[p] = version_path versions[version_tuple] = version_path self.paths.append(version_path) return versions
def ensure_path(path): """Given a path (either a string or a Path object), expand variables and return a Path object. :param path: A string or a :class:`~pathlib.Path` object. :type path: str or :class:`~pathlib.Path` :return: A fully expanded Path object. :rtype: :class:`~pathlib.Path` """ if isinstance(path, Path): path = path.as_posix() path = Path(os.path.expandvars(path)) try: path = path.resolve() except OSError: path = path.absolute() return path
def get_pyproject(path): # type: (Union[STRING_TYPE, Path]) -> Optional[Tuple[List[STRING_TYPE], STRING_TYPE]] """ Given a base path, look for the corresponding ``pyproject.toml`` file and return its build_requires and build_backend. :param AnyStr path: The root path of the project, should be a directory (will be truncated) :return: A 2 tuple of build requirements and the build backend :rtype: Optional[Tuple[List[AnyStr], AnyStr]] """ if not path: return from vistir.compat import Path if not isinstance(path, Path): path = Path(path) if not path.is_dir(): path = path.parent pp_toml = path.joinpath("pyproject.toml") setup_py = path.joinpath("setup.py") if not pp_toml.exists(): if not setup_py.exists(): return None requires = ["setuptools>=40.8", "wheel"] backend = get_default_pyproject_backend() else: pyproject_data = {} with io.open(pp_toml.as_posix(), encoding="utf-8") as fh: pyproject_data = tomlkit.loads(fh.read()) build_system = pyproject_data.get("build-system", None) if build_system is None: if setup_py.exists(): requires = ["setuptools>=40.8", "wheel"] backend = get_default_pyproject_backend() else: requires = ["setuptools>=40.8", "wheel"] backend = get_default_pyproject_backend() build_system = {"requires": requires, "build-backend": backend} pyproject_data["build_system"] = build_system else: requires = build_system.get("requires", ["setuptools>=40.8", "wheel"]) backend = build_system.get("build-backend", get_default_pyproject_backend()) return requires, backend
def egg_base(self): # type: () -> S base = None # type: Optional[STRING_TYPE] if self.setup_py.exists(): base = self.setup_py.parent elif self.pyproject.exists(): base = self.pyproject.parent elif self.setup_cfg.exists(): base = self.setup_cfg.parent if base is None: base = Path(self.base_dir) if base is None: base = Path(self.extra_kwargs["src_dir"]) egg_base = base.joinpath("reqlib-metadata") if not egg_base.exists(): atexit.register(rmtree, egg_base.as_posix()) egg_base.mkdir(parents=True, exist_ok=True) return egg_base.as_posix()
def load(cls, path): if not isinstance(path, Path): path = Path(path) pipfile_path = path / "Pipfile" if not path.exists(): raise FileNotFoundError("%s is not a valid project path!" % path) elif not pipfile_path.exists() or not pipfile_path.is_file(): raise RequirementError("%s is not a valid Pipfile" % pipfile_path) with pipfile_path.open(encoding="utf-8") as fp: pipfile = super(Pipfile, cls).load(fp) pipfile.dev_requirements = [ Requirement.from_pipfile(k, v) for k, v in pipfile.dev_packages.items() ] pipfile.requirements = [ Requirement.from_pipfile(k, v) for k, v in pipfile.packages.items() ] pipfile.path = pipfile_path return pipfile