def test_get_requirements(monkeypatch):
# Test eggs in URLs
with monkeypatch.context() as m:
# m.setattr(pip_shims.shims, "unpack_url", mock_unpack)
# m.setattr(SetupInfo, "get_info", mock_run_requires)
url_with_egg = Requirement.from_line(
'https://github.com/IndustriaTech/django-user-clipboard/archive/0.6.1.zip#egg=django-user-clipboard'
).requirement
assert url_with_egg.url == 'https://github.com/IndustriaTech/django-user-clipboard/archive/0.6.1.zip'
assert url_with_egg.name == 'django-user-clipboard'
# Test URLs without eggs pointing at installable zipfiles
url = Requirement.from_line(
'https://codeload.github.com/kennethreitz/tablib/zip/v0.12.1'
).requirement
assert url.url == 'https://codeload.github.com/kennethreitz/tablib/zip/v0.12.1'
wheel_line = "https://github.com/pypa/pipenv/raw/master/tests/test_artifacts/six-1.11.0+mkl-py2.py3-none-any.whl"
wheel = Requirement.from_line(wheel_line)
assert wheel.as_pipfile() == {
"six": {'file': 'https://github.com/pypa/pipenv/raw/master/tests/test_artifacts/six-1.11.0+mkl-py2.py3-none-any.whl'}
}
# Requirementslib inserts egg fragments as names when possible if we know the appropriate name
# this allows for custom naming
assert Requirement.from_pipfile(wheel.name, list(wheel.as_pipfile().values())[0]).as_line().split("#")[0] == wheel_line
# Test VCS urls with refs and eggnames
vcs_url = Requirement.from_line(
'git+https://github.com/kennethreitz/tablib.git@master#egg=tablib'
).requirement
assert vcs_url.vcs == 'git' and vcs_url.name == 'tablib' and vcs_url.revision == 'master'
assert vcs_url.url == 'git+https://github.com/kennethreitz/tablib.git'
# Test normal package requirement
normal = Requirement.from_line('tablib').requirement
assert normal.name == 'tablib'
# Pinned package requirement
spec = Requirement.from_line('tablib==0.12.1').requirement
assert spec.name == 'tablib' and spec.specs == [('==', '0.12.1')]
# Test complex package with both extras and markers
extras_markers = Requirement.from_line(
"requests[security]; os_name=='posix'"
).requirement
assert list(extras_markers.extras) == ['security']
assert extras_markers.name == 'requests'
assert str(extras_markers.marker) == 'os_name == "posix"'
# Test VCS uris get generated correctly, retain git+git@ if supplied that way, and are named according to egg fragment
git_reformat = Requirement.from_line(
'-e [email protected]:pypa/pipenv.git#egg=pipenv'
).requirement
assert git_reformat.url == 'git+ssh://[email protected]/pypa/pipenv.git'
assert git_reformat.name == 'pipenv'
assert git_reformat.editable
# Previously VCS uris were being treated as local files, so make sure these are not handled that way
assert not git_reformat.local_file
# Test regression where VCS uris were being handled as paths rather than VCS entries
assert git_reformat.vcs == 'git'
assert git_reformat.link.url == 'git+ssh://[email protected]/pypa/pipenv.git#egg=pipenv'
# Test VCS requirements being added with extras for constraint_line
git_extras = Requirement.from_line(
'-e git+https://github.com/requests/requests.git@master#egg=requests[security]'
)
assert git_extras.as_line() == '-e git+https://github.com/requests/requests.git@master#egg=requests[security]'
assert git_extras.constraint_line == '-e git+https://github.com/requests/requests.git@master#egg=requests[security]'
def format_remote_package(
package_name,
config,
dev=False): # type: (str, PipfileConfig, bool) -> Tuple[str, str]
"""
format and return a string that can be put into either install_requires or dependency_links or extras_require
:param package_name:
:param config:
:param dev: is package a development package
:return: Tuple[keyword_target, list_argument]
:raise ValueError: if a package config is not understood
"""
if dev:
return (
"extras_require",
Requirement.from_pipfile(package_name,
config).as_line(include_hashes=False),
)
else:
# fixme: stronger checks?
# https://setuptools.readthedocs.io/en/latest/setuptools.html#dependencies-that-aren-t-in-pypi
if "file" in config: # remote built distribution '.zip' file for example
assert isinstance(config, dict)
return "dependency_links", config["file"]
if is_pypi_package(config): # pypi package
return (
"install_requires",
Requirement.from_pipfile(package_name,
config).as_line(include_hashes=False),
)
else: # vcs
assert isinstance(config, dict)
if "git" in config:
vcs = "git"
# fixme: test cases other than git
elif "bzr" in config:
vcs = "bzr"
elif "svn" in config:
vcs = "svn"
elif "hg" in config:
vcs = "hg"
else:
raise ValueError("Can not understand config of package %s" %
package_name)
link = "{vcs}+{link}".format(vcs=vcs, link=config[vcs])
if "ref" in config:
link += "@" + config["ref"]
link += "#egg=" + package_name
return "dependency_links", link
def _parse_requirement_line(line):
if len(line) == 0:
return None
if line[:2] == '-i':
return None
return {"line": line, "requirement": Requirement.from_line(line)}
def test_get_local_ref(tmpdir):
six_dir = tmpdir.join("six")
import vistir
c = vistir.misc.run(["git", "clone", "https://github.com/benjaminp/six.git", six_dir.strpath], return_object=True, nospin=True)
assert c.returncode == 0
r = Requirement.from_line("git+{0}#egg=six".format(Path(six_dir.strpath).as_uri()))
assert r.commit_hash
def test_convert_from_pip(expected, requirement):
pkg_name = first(expected.keys())
pkg_pipfile = expected[pkg_name]
if hasattr(pkg_pipfile, 'keys'
) and 'editable' in pkg_pipfile and not pkg_pipfile['editable']:
del expected[pkg_name]['editable']
assert Requirement.from_line(requirement).as_pipfile() == expected
def test_convert_from_pip_fail_if_no_egg():
"""Parsing should fail without `#egg=`.
"""
dep = 'git+https://github.com/kennethreitz/requests.git'
with pytest.raises(ValueError) as e:
dep = Requirement.from_line(dep).as_pipfile()
assert 'pipenv requires an #egg fragment for vcs' in str(e)
def test_convert_non_installable_dir_fail(pathlib_tmpdir):
"""Convert a non-installable directory link should fail
without deleting the directory."""
dep = '-e file://{}'.format(pathlib_tmpdir.as_posix())
with pytest.raises(RequirementError):
req = Requirement.from_line(dep)
assert pathlib_tmpdir.exists()
def test_stdout_is_suppressed(capsys, tmpdir):
r = Requirement.from_line(
"git+https://github.com/sarugaku/requirementslib.git@remove-python-toml#egg=requirementslib"
)
r.req.get_vcs_repo(src_dir=tmpdir.strpath)
out, err = capsys.readouterr()
assert out.strip() == "", out
assert err.strip() == "", err
def test_convert_from_pipfile(requirement, expected):
pkg_name = first(requirement.keys())
pkg_pipfile = requirement[pkg_name]
req = Requirement.from_pipfile(pkg_name, pkg_pipfile)
if " (" in expected and expected.endswith(")"):
# To strip out plette[validation] (>=0.1.1)
expected = expected.replace(" (", "").rstrip(")")
assert req.as_line() == expected.lower(
) if '://' not in expected else expected
def test_pep_508():
r = Requirement.from_line("tablib@ https://codeload.github.com/kennethreitz/tablib/zip/v0.12.1")
assert r.specifiers == "==0.12.1"
assert r.req.link.url == "https://codeload.github.com/kennethreitz/tablib/zip/v0.12.1#egg=tablib"
assert r.req.req.name == "tablib"
assert r.req.req.url == "https://codeload.github.com/kennethreitz/tablib/zip/v0.12.1"
requires, setup_requires, build_requires = r.req.dependencies
assert all(dep in requires for dep in ["openpyxl", "odfpy", "xlrd"])
assert r.as_pipfile() == {'tablib': {'file': 'https://codeload.github.com/kennethreitz/tablib/zip/v0.12.1'}}
def test_convert_from_pip_git_uri_normalize():
"""Pip does not parse this correctly, but we can (by converting to ssh://).
"""
dep = 'git+git@host:user/repo.git#egg=myname'
dep = Requirement.from_line(dep).as_pipfile()
assert dep == {
'myname': {
'git': 'git@host:user/repo.git',
}
}
def test_convert_from_pip(monkeypatch, expected, requirement):
with monkeypatch.context() as m:
m.setattr(Requirement, "run_requires", mock_run_requires)
m.setattr(SetupInfo, "get_info", mock_run_requires)
m.setattr(pip_shims.shims, "unpack_url", mock_unpack)
pkg_name = first(expected.keys())
pkg_pipfile = expected[pkg_name]
if hasattr(pkg_pipfile, 'keys') and 'editable' in pkg_pipfile and not pkg_pipfile['editable']:
del expected[pkg_name]['editable']
assert Requirement.from_line(requirement).as_pipfile() == expected
def test_convert_from_pipfile_vcs(monkeypatch):
"""ssh VCS links should be converted correctly"""
with monkeypatch.context() as m:
m.setattr(pip_shims.shims, "unpack_url", mock_unpack)
pkg_name = "shellingham"
pkg_pipfile = {"editable": True, "git": "[email protected]:sarugaku/shellingham.git"}
req = Requirement.from_pipfile(pkg_name, pkg_pipfile)
assert (
req.req.link.url ==
"git+ssh://[email protected]/sarugaku/shellingham.git#egg=shellingham"
)
def test_convert_from_pipfile(monkeypatch, requirement, expected):
with monkeypatch.context() as m:
m.setattr(pip_shims.shims, "unpack_url", mock_unpack)
m.setattr(SetupInfo, "get_info", mock_run_requires)
m.setattr(Requirement, "run_requires", mock_run_requires)
pkg_name = first(requirement.keys())
pkg_pipfile = requirement[pkg_name]
req = Requirement.from_pipfile(pkg_name, pkg_pipfile)
if " (" in expected and expected.endswith(")"):
# To strip out plette[validation] (>=0.1.1)
expected = expected.replace(" (", "").rstrip(")")
assert req.as_line() == expected.lower() if '://' not in expected else expected
def from_pipfile(self):
if Pipfile is None:
raise ImportError(
'please, install extra requirements: install-requires[pipfile]'
)
project = Pipfile.load(self.path)
requirements = []
packages = project.get_sections()['packages'].items()
for name, version in packages:
req = Requirement.from_pipfile(name, version)
requirements.append(req.as_line())
return self._parse_requirements(requirements)
def test_convert_from_pip_git_uri_normalize(monkeypatch):
"""Pip does not parse this correctly, but we can (by converting to ssh://).
"""
with monkeypatch.context() as m:
m.setattr(Requirement, "run_requires", mock_run_requires)
m.setattr(SetupInfo, "get_info", mock_run_requires)
m.setattr(pip_shims.shims, "unpack_url", mock_unpack)
dep = 'git+git@host:user/repo.git#egg=myname'
dep = Requirement.from_line(dep).as_pipfile()
assert dep == {
'myname': {
'git': 'git@host:user/repo.git',
}
}
def freeze(project=None,
default=True,
dev=True,
include_hashes=None,
target=None):
from requirementslib import Requirement
lockfile = project.lockfile
if not lockfile:
print("Pipfile.lock is required to export.", file=sys.stderr)
return 1
section_names = []
if default:
section_names.append("default")
if dev:
section_names.append("develop")
requirements = [
Requirement.from_pipfile(key, entry._data)
for key, entry in itertools.chain.from_iterable(
lockfile.get(name, {}).items() for name in section_names)
]
if include_hashes is None:
include_hashes = all(r.is_named for r in requirements)
sources = lockfile.meta.sources._data
source_lines = list(
vistir.misc.dedup(
itertools.chain(
itertools.chain.from_iterable(
_source_as_lines(source, False) for source in sources[:1]),
itertools.chain.from_iterable(
_source_as_lines(source, True) for source in sources[1:]),
)))
requirement_lines = sorted(
vistir.misc.dedup(
_requirement_as_line(requirement, sources, include_hashes)
for requirement in requirements))
with open_for_output(target) as f:
for line in source_lines:
f.write(line)
f.write("\n")
f.write("\n")
for line in requirement_lines:
f.write(line)
f.write("\n")
def add_line_to_pipfile(self, line, develop):
from requirementslib import Requirement
requirement = Requirement.from_line(line)
section = self._get_pipfile_section(develop=develop)
key = requirement.normalized_name
entry = next(iter(requirement.as_pipfile().values()))
if isinstance(entry, dict):
# HACK: TOMLKit prefers to expand tables by default, but we
# always want inline tables here. Also tomlkit.inline_table
# does not have `update()`.
table = tomlkit.inline_table()
for k, v in entry.items():
table[k] = v
entry = table
section[key] = entry
def test_get_requirements():
# Test eggs in URLs
url_with_egg = Requirement.from_line(
'https://github.com/IndustriaTech/django-user-clipboard/archive/0.6.1.zip#egg=django-user-clipboard'
).requirement
assert url_with_egg.uri == 'https://github.com/IndustriaTech/django-user-clipboard/archive/0.6.1.zip'
assert url_with_egg.name == 'django-user-clipboard'
# Test URLs without eggs pointing at installable zipfiles
url = Requirement.from_line(
'https://github.com/kennethreitz/tablib/archive/0.12.1.zip'
).requirement
assert url.uri == 'https://github.com/kennethreitz/tablib/archive/0.12.1.zip'
# Test VCS urls with refs and eggnames
vcs_url = Requirement.from_line(
'git+https://github.com/kennethreitz/tablib.git@master#egg=tablib'
).requirement
assert vcs_url.vcs == 'git' and vcs_url.name == 'tablib' and vcs_url.revision == 'master'
assert vcs_url.uri == 'git+https://github.com/kennethreitz/tablib.git'
# Test normal package requirement
normal = Requirement.from_line('tablib').requirement
assert normal.name == 'tablib'
# Pinned package requirement
spec = Requirement.from_line('tablib==0.12.1').requirement
assert spec.name == 'tablib' and spec.specs == [('==', '0.12.1')]
# Test complex package with both extras and markers
extras_markers = Requirement.from_line(
"requests[security]; os_name=='posix'"
).requirement
assert extras_markers.extras == ['security']
assert extras_markers.name == 'requests'
assert extras_markers.markers == "os_name=='posix'"
# Test VCS uris get generated correctly, retain git+git@ if supplied that way, and are named according to egg fragment
git_reformat = Requirement.from_line(
'-e [email protected]:pypa/pipenv.git#egg=pipenv'
).requirement
assert git_reformat.uri == '[email protected]:pypa/pipenv.git'
assert git_reformat.name == 'pipenv'
assert git_reformat.editable
# Previously VCS uris were being treated as local files, so make sure these are not handled that way
assert not git_reformat.local_file
# Test regression where VCS uris were being handled as paths rather than VCS entries
assert git_reformat.vcs == 'git'
assert git_reformat.link.url == 'git+ssh://[email protected]/pypa/pipenv.git#egg=pipenv'
def _convert(file_instance, sources, section_names, hashes=False):
"""
:param file_instance: pipfile or lockfile instance
:param hashes: bool
:return: list
"""
sources_lines = _sources_as_lines(sources)
requirements = [
Requirement.from_pipfile(key, entry._data)
for key, entry in itertools.chain.from_iterable(
file_instance.get(name, {}).items() for name in section_names)
]
if hashes:
hashes = all(r.is_named for r in requirements)
requirement_lines = sorted(
dedup(
_requirement_as_line(requirement, hashes)
for requirement in requirements))
return sources_lines + [''] + requirement_lines
def test_one_way_editable_extras():
dep = '-e .[socks]'
dep = Requirement.from_line(dep).as_pipfile()
k = first(dep.keys())
assert dep[k]['extras'] == ['socks']
def test_convert_from_pipfile(requirement, expected):
pkg_name = first(requirement.keys())
pkg_pipfile = requirement[pkg_name]
req = Requirement.from_pipfile(pkg_name, pkg_pipfile)
assert req.as_line() == expected.lower(
) if '://' not in expected else expected
def test_get_ref():
r = Requirement.from_line("-e git+https://github.com/sarugaku/[email protected]#egg=shellingham")
assert r.commit_hash == "9abe7464dab5cc362fe08361619d3fb15f2e16ab"
def test_local_editable_ref(monkeypatch):
with monkeypatch.context() as m:
m.setattr(pip_shims.shims, "unpack_url", mock_unpack)
path = Path(ARTIFACTS_DIR) / 'git/requests'
req = Requirement.from_pipfile("requests", {"editable": True, "git": path.as_uri(), "ref": "2.18.4"})
assert req.as_line() == "-e git+{0}@2.18.4#egg=requests".format(path.as_uri())
def format_remote_package(
package_name,
config,
dev=False,
process_dependency_links=False
): # type: (str, PipfileConfig, bool, bool) -> Tuple[str, str]
"""
format and return a string that can be put into either install_requires or dependency_links or extras_require
:param package_name:
:param config:
:param dev: is package a development package
:param process_dependency_links: assign vcs packages to `dependency_links` keyword argument
:return: Tuple[keyword_target, list_argument]
:raise ValueError: if a package config is not understood
"""
if dev:
return (
"extras_require",
Requirement.from_pipfile(package_name,
config).as_line(include_hashes=False),
)
else:
# fixme: stronger checks?
# https://setuptools.readthedocs.io/en/latest/setuptools.html#dependencies-that-aren-t-in-pypi
if "file" in config: # remote built distribution '.zip' file for example
assert isinstance(config, dict)
return "dependency_links", config["file"]
if is_pypi_package(config): # pypi package
return (
"install_requires",
Requirement.from_pipfile(package_name,
config).as_line(include_hashes=False),
)
elif process_dependency_links: # vcs
assert isinstance(config, dict)
if "git" in config:
vcs = "git"
# fixme: test cases other than git
elif "bzr" in config:
vcs = "bzr"
elif "svn" in config:
vcs = "svn"
elif "hg" in config:
vcs = "hg"
else:
raise ValueError("Can not understand config of package %s" %
package_name)
link = "{vcs}+{link}".format(vcs=vcs, link=config[vcs])
if "ref" in config:
link += "@" + config["ref"]
link += "#egg=" + package_name
return "dependency_links", link
else: # vcs
# fixme: when editable = true, e.g. django = { git = 'https://github.com/django/django.git', ref = '1.11.4', editable = true }
# this will generate 'django @ -e git+https://github.com/django/[email protected]#egg=django'
# It is unrecognizable by pip because of the "-e"
# find out how to deal with it properly
return (
"install_requires",
package_name + " @ " +
Requirement.from_pipfile(package_name, config).as_line(
include_hashes=False).strip("-e "),
)