def test_unexisting_path(self):
with pytest.raises(InstallationError) as e:
InstallRequirement.from_line(
os.path.join('this', 'path', 'does', 'not', 'exist'))
err_msg = e.value.args[0]
assert "Invalid requirement" in err_msg
assert "It looks like a path." in err_msg
def run(self, options, args):
if not options.build_dir:
options.build_dir = build_prefix
if not options.src_dir:
options.src_dir = src_prefix
if options.download_dir:
options.no_install = True
options.ignore_installed = True
options.build_dir = os.path.abspath(options.build_dir)
options.src_dir = os.path.abspath(options.src_dir)
install_options = options.install_options or []
index_urls = [options.index_url] + options.extra_index_urls
if options.no_index:
logger.notify('Ignoring indexes: %s' % ','.join(index_urls))
index_urls = []
finder = PackageFinder(
find_links=options.find_links,
index_urls=index_urls)
requirement_set = RequirementSet(
build_dir=options.build_dir,
src_dir=options.src_dir,
download_dir=options.download_dir,
download_cache=options.download_cache,
upgrade=options.upgrade,
ignore_installed=options.ignore_installed,
ignore_dependencies=options.ignore_dependencies)
for name in args:
requirement_set.add_requirement(
InstallRequirement.from_line(name, None))
for name in options.editables:
requirement_set.add_requirement(
InstallRequirement.from_editable(name, default_vcs=options.default_vcs))
for filename in options.requirements:
for req in parse_requirements(filename, finder=finder, options=options):
requirement_set.add_requirement(req)
if not options.no_download:
requirement_set.prepare_files(finder, force_root_egg_info=self.bundle, bundle=self.bundle)
else:
requirement_set.locate_files()
if not options.no_install and not self.bundle:
requirement_set.install(install_options)
installed = ' '.join([req.name for req in
requirement_set.successfully_installed])
if installed:
logger.notify('Successfully installed %s' % installed)
elif not self.bundle:
downloaded = ' '.join([req.name for req in
requirement_set.successfully_downloaded])
if downloaded:
logger.notify('Successfully downloaded %s' % downloaded)
elif self.bundle:
requirement_set.create_bundle(self.bundle_filename)
logger.notify('Created bundle in %s' % self.bundle_filename)
# Clean up
if not options.no_install:
requirement_set.cleanup_files(bundle=self.bundle)
return requirement_set
def test_exclusive_environment_markers():
"""Make sure RequirementSet accepts several excluding env markers"""
eq26 = InstallRequirement.from_line("Django>=1.6.10,<1.7 ; python_version == '2.6'")
ne26 = InstallRequirement.from_line("Django>=1.6.10,<1.8 ; python_version != '2.6'")
req_set = RequirementSet("", "", "", session=PipSession())
req_set.add_requirement(eq26)
req_set.add_requirement(ne26)
assert req_set.has_requirement("Django")
def _find_cached_match(spec):
#if spec.is_pinned:
## If this is a pinned spec, we can take a shortcut: if it is
## found in the dependency cache, we can safely assume it has
## been downloaded before, and thus must exist. We can know
## this without every reaching out to PyPI and avoid the
## network overhead.
#name, version = spec.name, first(spec.preds)[1]
#if (name, version) in self._dep_cache:
#source = 'dependency cache'
#return version, source
version = None
overrides = self.overrides.get(spec.name)
## Try the link cache, and otherwise, try PyPI
if (spec.no_extra, overrides) in self._link_cache:
link, version = self._link_cache[(spec.no_extra, overrides)]
source = 'link cache'
else:
try:
requirement = InstallRequirement.from_line(specline)
link = self.finder.find_requirement(requirement, False)
except DistributionNotFound:
requirement = InstallRequirement.from_line(
specline, prereleases=True)
link = self.finder.find_requirement(requirement, False)
link, version = self._link_hook(overrides, spec, link)
# Hack to make pickle work
link.comes_from = None
source = 'PyPI'
if link.egg_fragment:
version = link.egg_fragment.rsplit('-', 1)[1]
link = Link(
link.url_without_fragment + "#%s=%s" % self.get_hash(link)
)
elif not version:
_, version = splitext(link.filename)[0].rsplit('-', 1)
# It's more reliable to get version from pinned spec then filename
if spec.is_pinned:
version = spec.pinned
assert version, "Version must be set!"
self._link_cache[(spec.no_extra, overrides)] = (link, version)
# Take this moment to smartly insert the pinned variant of this
# spec into the link_cache, too
pinned_spec = Spec.from_pinned(spec.name, version)
self._link_cache[pinned_spec.fullname] = (link, version)
return version, source
def test_requirement_file(self):
req_file_path = os.path.join(self.tempdir, 'test.txt')
with open(req_file_path, 'w') as req_file:
req_file.write('pip\nsetuptools')
with pytest.raises(InstallationError) as e:
InstallRequirement.from_line(req_file_path)
err_msg = e.value.args[0]
assert "Invalid requirement" in err_msg
assert "It looks like a path. It does exist." in err_msg
assert "appears to be a requirements file." in err_msg
assert "If that is the case, use the '-r' flag to install" in err_msg
def populate_requirement_set(requirement_set, args, options, finder,
session, name, wheel_cache):
"""
Marshal cmd line args into a requirement set.
"""
for filename in options.constraints:
for req in parse_requirements(
filename,
constraint=True, finder=finder, options=options,
session=session, wheel_cache=wheel_cache):
requirement_set.add_requirement(req)
for req in args:
requirement_set.add_requirement(
InstallRequirement.from_line(
req, None, isolated=options.isolated_mode,
wheel_cache=wheel_cache
)
)
for req in options.editables:
requirement_set.add_requirement(
InstallRequirement.from_editable(
req,
default_vcs=options.default_vcs,
isolated=options.isolated_mode,
wheel_cache=wheel_cache
)
)
found_req_in_file = False
for filename in options.requirements:
for req in parse_requirements(
filename,
finder=finder, options=options, session=session,
wheel_cache=wheel_cache):
found_req_in_file = True
requirement_set.add_requirement(req)
# If --require-hashes was a line in a requirements file, tell
# RequirementSet about it:
requirement_set.require_hashes = options.require_hashes
if not (args or options.editables or found_req_in_file):
opts = {'name': name}
if options.find_links:
msg = ('You must give at least one requirement to '
'%(name)s (maybe you meant "pip %(name)s '
'%(links)s"?)' %
dict(opts, links=' '.join(options.find_links)))
else:
msg = ('You must give at least one requirement '
'to %(name)s (see "pip help %(name)s")' % opts)
logger.warning(msg)
def test_markers_match(self):
# match
for markers in ('python_version >= "1.0"', "sys_platform == %r" % sys.platform):
line = "name; " + markers
req = InstallRequirement.from_line(line)
assert req.markers == markers
assert req.match_markers()
# don't match
for markers in ('python_version >= "5.0"', "sys_platform != %r" % sys.platform):
line = "name; " + markers
req = InstallRequirement.from_line(line)
assert req.markers == markers
assert not req.match_markers()
def test_markers_url(self):
# test "URL; markers" syntax
url = 'http://foo.com/?p=bar.git;a=snapshot;h=v0.1;sf=tgz'
line = '%s; python_version >= "3"' % url
req = InstallRequirement.from_line(line)
assert req.link.url == url, req.url
assert str(req.markers) == 'python_version >= "3"'
# without space, markers are part of the URL
url = 'http://foo.com/?p=bar.git;a=snapshot;h=v0.1;sf=tgz'
line = '%s;python_version >= "3"' % url
req = InstallRequirement.from_line(line)
assert req.link.url == line, req.url
assert req.markers is None
def make_install_requirement(name, version, extras, markers, constraint=False):
# If no extras are specified, the extras string is blank
extras_string = ""
if extras:
# Sort extras for stability
extras_string = "[{}]".format(",".join(sorted(extras)))
if not markers:
return InstallRequirement.from_line(
str('{}{}=={}'.format(name, extras_string, version)),
constraint=constraint)
else:
return InstallRequirement.from_line(
str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))),
constraint=constraint)
def handle(self, *args, **options):
new_app_names = []
new_package_names = []
extra_settings = []
default_templates = []
for name in args:
req = InstallRequirement.from_line(name)
if not req.check_if_exists():
result = subprocess.call(["pip", "install", name])
if not result == 0:
raise Exception("Install error")
req = InstallRequirement.from_line(name)
if not req.check_if_exists():
raise Exception("Installed package not found")
distribution = req.satisfied_by
new_package_names.append(distribution.project_name)
def _get_app_names(name):
if HOOKS['APPNAMES'].has_key(name):
return HOOKS['APPNAMES'][name]
## try real package names
packages = os.listdir(distribution.location)
candidates = difflib.get_close_matches(name, packages)
if candidates:
return [candidates[0]]
return name
new_app_names += _get_app_names(distribution.project_name)
if HOOKS['EXTRA_SETTINGS'].has_key(distribution.project_name):
extra_settings += [HOOKS['EXTRA_SETTINGS'][distribution.project_name]]
if HOOKS['TEMPLATES'].has_key(distribution.project_name):
default_templates += [HOOKS['TEMPLATES'][distribution.project_name]]
_add_to_requirements_txt(new_package_names)
## settings
for d in extra_settings:
for name, values in d.items():
_settings_add_to_list(name, values)
for d in default_templates:
for name, content in d.items():
_add_default_template(name, content)
_add_to_installed_apps(new_app_names)
def get_dependencies(self, ireq):
if ireq.editable:
return self.editables[str(ireq.link)]
name, version = as_name_version_tuple(ireq)
dependencies = self.index[name][version]
return [InstallRequirement.from_line(dep) for dep in dependencies]
def find_best_match(self, ireq, prereleases=False):
if ireq.editable:
return ireq
versions = ireq.specifier.filter(self.index[ireq.req.key], prereleases=prereleases)
best_version = max(versions, key=Version)
return InstallRequirement.from_line('{}=={}'.format(ireq.req.key, best_version))
def test_finder_installs_pre_releases(data):
"""
Test PackageFinder finds pre-releases if asked to.
"""
req = InstallRequirement.from_line("bar", None, prereleases=True)
# using a local index (that has pre & dev releases)
finder = PackageFinder([], [data.index_url("pre")])
link = finder.find_requirement(req, False)
assert link.url.endswith("bar-2.0b1.tar.gz"), link.url
# using find-links
links = ["https://foo/bar-1.0.tar.gz", "https://foo/bar-2.0b1.tar.gz"]
finder = PackageFinder(links, [])
with patch.object(finder, "_get_pages", lambda x, y: []):
link = finder.find_requirement(req, False)
assert link.url == "https://foo/bar-2.0b1.tar.gz"
links.reverse()
finder = PackageFinder(links, [])
with patch.object(finder, "_get_pages", lambda x, y: []):
link = finder.find_requirement(req, False)
assert link.url == "https://foo/bar-2.0b1.tar.gz"
def test_url_with_query():
"""InstallRequirement should strip the fragment, but not the query."""
url = 'http://foo.com/?p=bar.git;a=snapshot;h=v0.1;sf=tgz'
fragment = '#egg=bar'
req = InstallRequirement.from_line(url + fragment)
assert req.url == url, req.url
def test_pypirepo_calls_reqset_with_str_paths():
"""
Make sure that paths passed to RequirementSet init are str.
Passing unicode paths on Python 2 could make pip fail later on
unpack, if the package contains non-ASCII file names, because
non-ASCII str and unicode paths cannot be combined.
"""
with patch('piptools.repositories.pypi.RequirementSet') as mocked_init:
repo = get_pypi_repository()
ireq = InstallRequirement.from_line('ansible==2.4.0.0')
# Setup a mock object to be returned from the RequirementSet call
mocked_reqset = MagicMock()
mocked_init.return_value = mocked_reqset
# Do the call
repo.get_dependencies(ireq)
# Check that RequirementSet init is called with correct type arguments
assert mocked_init.call_count == 1
(init_call_args, init_call_kwargs) = mocked_init.call_args
assert isinstance(init_call_args[0], str)
assert isinstance(init_call_args[1], str)
assert isinstance(init_call_kwargs.get('download_dir'), str)
assert isinstance(init_call_kwargs.get('wheel_download_dir'), str)
# Check that _prepare_file is called correctly
assert mocked_reqset._prepare_file.call_count == 1
(pf_call_args, pf_call_kwargs) = mocked_reqset._prepare_file.call_args
(called_with_finder, called_with_ireq) = pf_call_args
assert isinstance(called_with_finder, PackageFinder)
assert called_with_ireq == ireq
assert not pf_call_kwargs
def test_extras_for_editable_path_requirement(self):
url = '.[ex1,ex2]'
filename = 'filename'
comes_from = '-r %s (line %s)' % (filename, 1)
req = InstallRequirement.from_editable(url, comes_from=comes_from)
assert len(req.extras) == 2
assert req.extras == set(['ex1', 'ex2'])
def test_extras_for_line_path_requirement(self):
line = 'SomeProject[ex1,ex2]'
filename = 'filename'
comes_from = '-r %s (line %s)' % (filename, 1)
req = InstallRequirement.from_line(line, comes_from=comes_from)
assert len(req.extras) == 2
assert req.extras == set(['ex1', 'ex2'])
def test_extras_for_editable_url_requirement(self):
url = 'git+https://url#egg=SomeProject[ex1,ex2]'
filename = 'filename'
comes_from = '-r %s (line %s)' % (filename, 1)
req = InstallRequirement.from_editable(url, comes_from=comes_from)
assert len(req.extras) == 2
assert req.extras == set(['ex1', 'ex2'])
def _iter_dependencies(self, ireq):
"""
Given a pinned or editable InstallRequirement, collects all the
secondary dependencies for them, either by looking them up in a local
cache, or by reaching out to the repository.
Editable requirements will never be looked up, as they may have
changed at any time.
"""
if ireq.editable:
for dependency in self.repository.get_dependencies(ireq):
yield dependency
return
elif not is_pinned_requirement(ireq):
raise TypeError("Expected pinned or editable requirement, got {}".format(ireq))
# Now, either get the dependencies from the dependency cache (for
# speed), or reach out to the external repository to
# download and inspect the package version and get dependencies
# from there
if ireq not in self.dependency_cache:
log.debug(" {} not in cache, need to check index".format(format_requirement(ireq)), fg="yellow")
dependencies = self.repository.get_dependencies(ireq)
self.dependency_cache[ireq] = sorted(str(ireq.req) for ireq in dependencies)
# Example: ['Werkzeug>=0.9', 'Jinja2>=2.4']
dependency_strings = self.dependency_cache[ireq]
log.debug(
" {:25} requires {}".format(
format_requirement(ireq), ", ".join(sorted(dependency_strings, key=lambda s: s.lower())) or "-"
)
)
for dependency_string in dependency_strings:
yield InstallRequirement.from_line(dependency_string)
def test_finder_priority_page_over_deplink():
"""Test PackageFinder prefers page links over equivalent dependency links"""
req = InstallRequirement.from_line('gmpy==1.15', None)
finder = PackageFinder([], ["http://pypi.python.org/simple"])
finder.add_dependency_links(['http://c.pypi.python.org/simple/gmpy/'])
link = finder.find_requirement(req, False)
assert link.url.startswith("http://pypi")
def install(name):
"""
Try to install the package with 'name' into folder 'libs/python27'.
"""
print "Installation directory:"
print python27_dir()
requirement_set = RequirementSet(build_dir=build_prefix,
src_dir=src_prefix,
download_dir=None)
requirement_set.add_requirement(InstallRequirement.from_line(name, None))
install_options = ["--prefix=%s" % python27_dir()]
global_options = []
finder = PackageFinder(find_links=[],
index_urls=["http://pypi.python.org/simple/"])
try:
requirement_set.prepare_files(finder,
force_root_egg_info=False,
bundle=False)
requirement_set.install(install_options, global_options)
print "\nSuccessfully installed\n=================================="
for package in requirement_set.successfully_installed:
print package.name
print "\nDone.\n"
except DistributionNotFound:
print "No package found with name: %s" % name
except Exception as e:
print "Error:", e
def test_finder_only_installs_stable_releases(data):
"""
Test PackageFinder only accepts stable versioned releases by default.
"""
req = InstallRequirement.from_line("bar", None)
# using a local index (that has pre & dev releases)
finder = PackageFinder([], [data.index_url("pre")], session=PipSession())
link = finder.find_requirement(req, False)
assert link.url.endswith("bar-1.0.tar.gz"), link.url
# using find-links
links = ["https://foo/bar-1.0.tar.gz", "https://foo/bar-2.0b1.tar.gz"]
finder = PackageFinder(links, [], session=PipSession())
with patch.object(finder, "_get_pages", lambda x, y: []):
link = finder.find_requirement(req, False)
assert link.url == "https://foo/bar-1.0.tar.gz"
links.reverse()
finder = PackageFinder(links, [], session=PipSession())
with patch.object(finder, "_get_pages", lambda x, y: []):
link = finder.find_requirement(req, False)
assert link.url == "https://foo/bar-1.0.tar.gz"
def test_finder_priority_nonegg_over_eggfragments():
"""Test PackageFinder prefers non-egg links over "#egg=" links"""
req = InstallRequirement.from_line('bar==1.0', None)
links = ['http://foo/bar.py#egg=bar-1.0', 'http://foo/bar-1.0.tar.gz']
finder = PackageFinder(links, [], session=PipSession())
with patch.object(finder, "_get_pages", lambda x, y: []):
all_versions = finder._find_all_versions(req.name)
assert all_versions[0].location.url.endswith('tar.gz')
assert all_versions[1].location.url.endswith('#egg=bar-1.0')
link = finder.find_requirement(req, False)
assert link.url.endswith('tar.gz')
links.reverse()
finder = PackageFinder(links, [], session=PipSession())
with patch.object(finder, "_get_pages", lambda x, y: []):
all_versions = finder._find_all_versions(req.name)
assert all_versions[0].location.url.endswith('tar.gz')
assert all_versions[1].location.url.endswith('#egg=bar-1.0')
link = finder.find_requirement(req, False)
assert link.url.endswith('tar.gz')
def test_no_partial_name_match():
"""Finder requires the full project name to match, not just beginning."""
finder = PackageFinder([find_links], [])
req = InstallRequirement.from_line("gmpy")
found = finder.find_requirement(req, False)
assert found.url.endswith("gmpy-1.15.tar.gz"), found
def test_no_mpkg():
"""Finder skips zipfiles with "macosx10" in the name."""
finder = PackageFinder([find_links], [])
req = InstallRequirement.from_line("pkgwithmpkg")
found = finder.find_requirement(req, False)
assert found.url.endswith("pkgwithmpkg-1.0.tar.gz"), found
def run(self, options, args):
with self._build_session(options) as session:
format_control = pip.index.FormatControl(set(), set())
wheel_cache = WheelCache(options.cache_dir, format_control)
requirement_set = RequirementSet(
build_dir=None,
src_dir=None,
download_dir=None,
isolated=options.isolated_mode,
session=session,
wheel_cache=wheel_cache,
)
for name in args:
requirement_set.add_requirement(
InstallRequirement.from_line(
name, isolated=options.isolated_mode,
wheel_cache=wheel_cache
)
)
for filename in options.requirements:
for req in parse_requirements(
filename,
options=options,
session=session,
wheel_cache=wheel_cache):
requirement_set.add_requirement(req)
if not requirement_set.has_requirements:
raise InstallationError(
'You must give at least one requirement to %(name)s (see '
'"pip help %(name)s")' % dict(name=self.name)
)
requirement_set.uninstall(auto_confirm=options.yes)
def pip_install(name, tmpdir):
build_dir = os.path.join(tmpdir, 'build')
src_dir = os.path.join(tmpdir, 'src')
download_dir = os.path.join(tmpdir, 'download')
os.mkdir(build_dir)
os.mkdir(src_dir)
os.mkdir(download_dir)
finder = PackageFinder(
find_links=[],
index_urls=['https://pypi.python.org/simple/'],
use_mirrors=False,
allow_all_external=True,
allow_all_insecure=True,
)
requirement_set = RequirementSet(
build_dir=build_dir,
src_dir=src_dir,
download_dir=download_dir,
ignore_installed=True,
ignore_dependencies=True
)
requirement_set.add_requirement(InstallRequirement.from_line(name, None))
requirement_set.prepare_files(finder)
# should be exactly one
filename = os.listdir(download_dir)[0]
path = os.path.join(download_dir, filename)
return path
def test_not_find_wheel_not_supported(self):
"""
Test not finding an unsupported wheel.
"""
req = InstallRequirement.from_line("simple.dist")
finder = PackageFinder([find_links], [], use_wheel=True)
assert_raises(DistributionNotFound, finder.find_requirement, req, True)
def getDependencies(name, requirementSet=None, finder=None):
"""Get dependencies of a python project
@param name: name of python project
@param requirements: RequirementSet
@param finder: PackageFinder
"""
if requirementSet is None:
requirementSet = RequirementSet(
build_dir=os.path.abspath(build_prefix),
src_dir=os.path.abspath(src_prefix),
download_dir=None,
download_cache=None,
upgrade=False,
ignore_installed=True,
ignore_dependencies=False)
if finder is None:
finder = PackageFinder(find_links=[],
index_urls=['http://pypi.python.org/simple'])
# lead pip download all dependencies
req = InstallRequirement.from_line(name, None)
requirementSet.add_requirement(req)
requirementSet.install_files(finder)
# trace the dependencies relationships between projects
dependencies = []
traceDependencys(req, requirementSet, dependencies)
return dependencies
def test_get_dist(self):
req = InstallRequirement.from_line('foo')
req.egg_info_path = Mock(return_value='/path/to/foo.egg-info')
dist = req.get_dist()
assert isinstance(dist, pkg_resources.Distribution)
assert dist.project_name == 'foo'
assert dist.location == '/path/to'
def get_dependencies(self, ireq):
if ireq.editable:
return self.editables[str(ireq.link)]
name, version, extras = as_tuple(ireq)
# Store non-extra dependencies under the empty string
extras += ("", )
dependencies = [
dep for extra in extras for dep in self.index[name][version][extra]
]
return [
InstallRequirement.from_line(dep, constraint=ireq.constraint)
for dep in dependencies
]
def test_wheel_over_sdist_priority(self, data):
"""
Test wheels have priority over sdists.
`test_link_sorting` also covers this at lower level
"""
req = InstallRequirement.from_line("priority")
finder = PackageFinder(
[data.find_links],
[],
use_wheel=True,
session=PipSession(),
)
found = finder.find_requirement(req, True)
assert found.url.endswith("priority-1.0-py2.py3-none-any.whl"), found
def test_finder_ignores_external_links(data):
"""
Tests that PackageFinder ignores external links, with or without hashes.
"""
req = InstallRequirement.from_line("bar", None)
# using a local index
finder = PackageFinder(
[],
[data.index_url("externals")],
session=PipSession(),
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-1.0.tar.gz"
def test_finder_finds_external_links_without_hashes_scraped_per_project():
"""
Tests that PackageFinder finds externally scraped links
"""
req = InstallRequirement.from_line("bar", None)
# using a local index
index_url = path_to_url(os.path.join(tests_data, "indexes", "externals"))
finder = PackageFinder([], [index_url],
allow_external=["bar"],
allow_insecure=["bar"],
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-4.0.tar.gz"
def run(self, options, args):
requirement_set = RequirementSet(build_dir=None,
src_dir=None,
download_dir=None)
for name in args:
requirement_set.add_requirement(InstallRequirement.from_line(name))
for filename in options.requirements:
for req in parse_requirements(filename, options=options):
requirement_set.add_requirement(req)
if not requirement_set.has_requirements:
raise InstallationError('You must give at least one requirement '
'to %(name)s (see "pip help %(name)s")' %
dict(name=self.name))
requirement_set.uninstall(auto_confirm=options.yes)
def test_finder_priority_page_over_deplink():
"""
Test PackageFinder prefers page links over equivalent dependency links
"""
req = InstallRequirement.from_line('gmpy==1.15', None)
finder = PackageFinder(
[],
["https://pypi.python.org/simple"],
process_dependency_links=True,
session=PipSession(),
)
finder.add_dependency_links(['http://c.pypi.python.org/simple/gmpy/'])
link = finder.find_requirement(req, False)
assert link.url.startswith("https://pypi"), link
def trace_requirements(requirements):
"""given an iterable of pip InstallRequirements,
return the set of required packages, given their transitive requirements.
"""
requirements = tuple(pretty_req(r) for r in requirements)
working_set = fresh_working_set()
# breadth-first traversal:
from collections import deque
queue = deque(requirements)
queued = set([req.req for req in queue])
errors = []
result = []
while queue:
req = queue.popleft()
logger.debug('tracing: %s', req)
try:
dist = working_set.find(req.req)
except pkg_resources.VersionConflict as conflict:
dist = conflict.args[0]
errors.append('Error: version conflict: %s (%s) <-> %s' %
(dist, timid_relpath(dist.location), req))
if dist is None:
errors.append('Error: unmet dependency: %s' % req)
continue
result.append(dist_to_req(dist))
for sub_req in sorted(dist.requires(), key=lambda req: req.key):
from pip.req import InstallRequirement
sub_req = InstallRequirement(sub_req, req)
if req_cycle(sub_req):
logger.warn('Circular dependency! %s', sub_req)
continue
elif sub_req.req in queued:
logger.debug('already queued: %s', sub_req)
continue
else:
logger.debug('adding sub-requirement %s', sub_req)
queue.append(sub_req)
queued.add(sub_req.req)
if errors:
raise InstallationError('\n'.join(errors))
return result
def trace_requirements(requirements):
"""given an iterable of pip InstallRequirements,
return the set of required packages, given their transitive requirements.
"""
requirements = tuple(pretty_req(r) for r in requirements)
working_set = fresh_working_set()
# breadth-first traversal:
from collections import deque
queue = deque(requirements)
queued = {_package_req_to_pkg_resources_req(req.req) for req in queue}
errors = []
result = []
while queue:
req = queue.popleft()
logger.debug('tracing: %s', req)
try:
dist = working_set.find_normalized(
_package_req_to_pkg_resources_req(req.req))
except pkg_resources.VersionConflict as conflict:
dist = conflict.args[0]
errors.append('Error: version conflict: %s (%s) <-> %s' %
(dist, timid_relpath(dist.location), req))
assert dist is not None, 'Should be unreachable in pip8+'
result.append(dist_to_req(dist))
# TODO: pip does no validation of extras. should we?
extras = [extra for extra in req.extras if extra in dist.extras]
for sub_req in sorted(dist.requires(extras=extras),
key=lambda req: req.key):
sub_req = InstallRequirement(sub_req, req)
if req_cycle(sub_req):
logger.warning('Circular dependency! %s', sub_req)
continue
elif sub_req.req in queued:
logger.debug('already queued: %s', sub_req)
continue
else:
logger.debug('adding sub-requirement %s', sub_req)
queue.append(sub_req)
queued.add(sub_req.req)
if errors:
raise InstallationError('\n'.join(errors))
return result
def test_skip_invalid_wheel_link(self, caplog, data):
"""
Test if PackageFinder skips invalid wheel filenames
"""
req = InstallRequirement.from_line("invalid")
# data.find_links contains "invalid.whl", which is an invalid wheel
finder = PackageFinder(
[data.find_links],
[],
session=PipSession(),
)
with pytest.raises(DistributionNotFound):
finder.find_requirement(req, True)
assert ("invalid.whl; invalid wheel filename" in caplog.text)
def test_finder_finds_external_links_without_hashes_scraped_per_project(data):
"""
Tests that PackageFinder finds externally scraped links
"""
req = InstallRequirement.from_line("bar", None)
# using a local index
finder = PackageFinder(
[],
[data.index_url("externals")],
allow_external=["bar"],
allow_unverified=["bar"],
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-4.0.tar.gz"
def install_egg(self, egg_name):
""" Install an egg into the egg directory """
if not os.path.exists(self.egg_directory):
os.makedirs(self.egg_directory)
self.requirement_set.add_requirement(
InstallRequirement.from_line(egg_name, None))
try:
self.requirement_set.prepare_files(self.finder,
force_root_egg_info=False,
bundle=False)
self.requirement_set.install(self.install_options,
self.global_options)
except DistributionNotFound:
self.requirement_set.requirements._keys.remove(egg_name)
raise PipException()
def test_existing_over_wheel_priority(self, data):
"""
Test existing install has priority over wheels.
`test_link_sorting` also covers this at a lower level
"""
req = InstallRequirement.from_line('priority', None)
latest_version = "1.0"
satisfied_by = Mock(location="/path",
parsed_version=parse_version(latest_version),
version=latest_version)
req.satisfied_by = satisfied_by
finder = PackageFinder([data.find_links], [], use_wheel=True)
with pytest.raises(BestVersionAlreadyInstalled):
finder.find_requirement(req, True)
def test_finder_installs_dev_releases(data):
"""
Test PackageFinder finds dev releases if asked to.
"""
req = InstallRequirement.from_line("bar", None)
# using a local index (that has dev releases)
finder = PackageFinder(
[], [data.index_url("dev")],
allow_all_prereleases=True,
session=PipSession(),
)
link = finder.find_requirement(req, False)
assert link.url.endswith("bar-2.0.dev1.tar.gz"), link.url
def test_finder_deplink():
"""
Test PackageFinder with dependency links only
"""
req = InstallRequirement.from_line('gmpy==1.15', None)
finder = PackageFinder(
[],
[],
process_dependency_links=True,
session=PipSession(),
)
finder.add_dependency_links(
['https://pypi.python.org/packages/source/g/gmpy/gmpy-1.15.zip'])
link = finder.find_requirement(req, False)
assert link.url.startswith("https://pypi"), link
def _requirement_finder(finder, req_str):
"""
First try to find the given requirement. If that fails, try several
alternatives. If they all fail, raise the first exception caught.
"""
err = None
for req_name in _get_package_name_alternatives(req_str):
req = InstallRequirement(req=req_name, comes_from=None)
try:
return finder.find_requirement(req=req, upgrade=True)
except DistributionNotFound as e:
if err is None:
err = e
raise err
def test_finder_detects_latest_already_satisfied_find_links(data):
"""Test PackageFinder detects latest already satisfied using find-links"""
req = InstallRequirement.from_line('simple', None)
# the latest simple in local pkgs is 3.0
latest_version = "3.0"
satisfied_by = Mock(
location="/path",
parsed_version=parse_version(latest_version),
version=latest_version
)
req.satisfied_by = satisfied_by
finder = PackageFinder([data.find_links], [], session=PipSession())
with pytest.raises(BestVersionAlreadyInstalled):
finder.find_requirement(req, True)
def test_finder_installs_pre_releases_with_version_spec():
"""
Test PackageFinder only accepts stable versioned releases by default.
"""
req = InstallRequirement.from_line("bar>=0.0.dev0", None)
links = ["https://foo/bar-1.0.tar.gz", "https://foo/bar-2.0b1.tar.gz"]
finder = PackageFinder(links, [])
link = finder.find_requirement(req, False)
assert link.url == "https://foo/bar-2.0b1.tar.gz"
links.reverse()
finder = PackageFinder(links, [])
link = finder.find_requirement(req, False)
assert link.url == "https://foo/bar-2.0b1.tar.gz"
def test_finder_finds_external_links_without_hashes_all_all_insecure():
"""
Tests that PackageFinder finds external links if they do not have a hash
using the all external flag
"""
req = InstallRequirement.from_line("bar==3.0", None)
# using a local index
index_url = path_to_url(os.path.join(tests_data, "indexes", "externals"))
finder = PackageFinder([], [index_url],
allow_all_external=True,
allow_all_insecure=True,
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-3.0.tar.gz"
def get_pinned_version(ireq):
"""
Get pinned version of a requirement, if it is pinned.
:type ireq: InstallRequirement|str
:type ignore_editables: bool
"""
if not isinstance(ireq, InstallRequirement):
ireq = InstallRequirement(ireq, None)
assert isinstance(ireq, InstallRequirement)
if ireq.editable:
return None
return get_ireq_version(ireq)
def test_no_reuse_existing_build_dir(self, data):
"""Test prepare_files raise exception with previous build dir"""
build_dir = os.path.join(self.tempdir, 'build', 'simple')
os.makedirs(build_dir)
open(os.path.join(build_dir, "setup.py"), 'w')
reqset = self.basic_reqset()
req = InstallRequirement.from_line('simple')
reqset.add_requirement(req)
finder = PackageFinder([data.find_links], [])
assert_raises_regexp(
PreviousBuildDirError,
"pip can't proceed with [\s\S]*%s[\s\S]*%s" %
(req, build_dir.replace('\\', '\\\\')), reqset.prepare_files,
finder)
def _iter_dependencies(self, ireq):
"""
Given a pinned or editable InstallRequirement, collects all the
secondary dependencies for them, either by looking them up in a local
cache, or by reaching out to the repository.
Editable requirements will never be looked up, as they may have
changed at any time.
"""
if ireq.editable:
for dependency in self.repository.get_dependencies(ireq):
yield dependency
return
elif ireq.markers:
for dependency in self.repository.get_dependencies(ireq):
dependency.prepared = False
yield dependency
return
elif ireq.extras:
for dependency in self.repository.get_dependencies(ireq):
dependency.prepared = False
yield dependency
return
elif not is_pinned_requirement(ireq):
raise TypeError(
'Expected pinned or editable requirement, got {}'.format(ireq))
# Now, either get the dependencies from the dependency cache (for
# speed), or reach out to the external repository to
# download and inspect the package version and get dependencies
# from there
if ireq not in self.dependency_cache:
log.debug(' {} not in cache, need to check index'.format(
format_requirement(ireq)),
fg='yellow')
dependencies = self.repository.get_dependencies(ireq)
self.dependency_cache[ireq] = sorted(
str(ireq.req) for ireq in dependencies)
# Example: ['Werkzeug>=0.9', 'Jinja2>=2.4']
dependency_strings = self.dependency_cache[ireq]
log.debug(' {:25} requires {}'.format(
format_requirement(ireq),
', '.join(sorted(dependency_strings, key=lambda s: s.lower()))
or '-'))
for dependency_string in dependency_strings:
yield InstallRequirement.from_line(dependency_string,
constraint=ireq.constraint)
def get_legacy_dependencies(self, ireq):
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
They indicate the secondary dependencies for the given requirement.
"""
if not (ireq.editable or is_pinned_requirement(ireq)):
raise TypeError(
'Expected pinned or editable InstallRequirement, got {}'.
format(ireq))
if ireq not in self._dependencies_cache:
if ireq.link and not ireq.link.is_artifact:
# No download_dir for VCS sources. This also works around pip
# using git-checkout-index, which gets rid of the .git dir.
download_dir = None
else:
download_dir = self._download_dir
if not os.path.isdir(download_dir):
os.makedirs(download_dir)
if not os.path.isdir(self._wheel_download_dir):
os.makedirs(self._wheel_download_dir)
reqset = RequirementSet(
self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
ignore_installed=True,
ignore_requires_python=True)
result = reqset._prepare_file(self.finder,
ireq,
ignore_requires_python=True)
if not result:
if reqset.requires_python:
from pip.req.req_install import InstallRequirement
marker = 'python_version=="{0}"'.format(
reqset.requires_python.replace(' ', ''))
new_req = InstallRequirement.from_line('{0}; {1}'.format(
str(ireq.req), marker))
result = [new_req]
self._dependencies_cache[ireq] = result
return set(self._dependencies_cache[ireq])
def test_finder_finds_external_links_with_hashes_all(data):
"""
Tests that PackageFinder finds external links but only if they have a hash
using the all externals flag.
"""
req = InstallRequirement.from_line("bar", None)
# using a local index
finder = PackageFinder(
[],
[data.index_url("externals")],
allow_all_external=True,
session=PipSession(),
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-2.0.tar.gz"
def test_finder_priority_file_over_page(data):
"""Test PackageFinder prefers file links over equivalent page links"""
req = InstallRequirement.from_line('gmpy==1.15', None)
finder = PackageFinder(
[data.find_links],
["http://pypi.python.org/simple"],
session=PipSession(),
)
all_versions = finder._find_all_versions(req.name)
# 1 file InstallationCandidate followed by all https ones
assert all_versions[0].location.scheme == 'file'
assert all(version.location.scheme == 'https'
for version in all_versions[1:]), all_versions
link = finder.find_requirement(req, False)
assert link.url.startswith("file://")
def test_finder_finds_external_links_without_hashes_per_project(data):
"""
Tests that PackageFinder finds external links if they do not have a hash
"""
req = InstallRequirement.from_line("bar==3.0", None)
# using a local index
finder = PackageFinder(
[],
[data.index_url("externals")],
allow_external=["bar"],
allow_unverified=["bar"],
session=PipSession(),
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-3.0.tar.gz"
def test_environment_marker_extras(self, data):
"""
Test that the environment marker extras are used with
non-wheel installs.
"""
reqset = self.basic_reqset()
req = InstallRequirement.from_editable(
data.packages.join("LocalEnvironMarker"))
reqset.add_requirement(req)
finder = PackageFinder([data.find_links], [], session=PipSession())
reqset.prepare_files(finder)
# This is hacky but does test both case in py2 and py3
if sys.version_info[:2] in ((2, 7), (3, 4)):
assert reqset.has_requirement('simple')
else:
assert not reqset.has_requirement('simple')
def test_finder_finds_external_links_without_hashes_all_all_insecure(data):
"""
Tests that PackageFinder finds external links if they do not have a hash
using the all external flag
"""
req = InstallRequirement.from_line("bar==3.0", None)
# using a local index
finder = PackageFinder(
[],
[data.index_url("externals")],
allow_all_external=True,
allow_unverified=["bar"],
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-3.0.tar.gz"
def make_install_req(ver=None):
req = Mock()
req.project_name = test_project_name
if ver:
req.url = url_template % str(ver)
req.specs = [('==', ver)]
else:
req.url = url_template.replace('@', '') % ''
req.specs = []
install_requirement = InstallRequirement(req,
None,
editable=True,
url=req.url)
return install_requirement
def trace_requirements(requirements):
"""given an iterable of pip InstallRequirements,
return the set of required packages, given their transitive requirements.
"""
from collections import deque
from pip import logger
from pip.req import InstallRequirement
from pip._vendor import pkg_resources
working_set = fresh_working_set()
# breadth-first traversal:
errors = False
queue = deque(requirements)
result = []
seen_warnings = set()
while queue:
req = queue.popleft()
if req.req is None:
# a file:/// requirement
continue
try:
dist = working_set.find(req.req)
except pkg_resources.VersionConflict as conflict:
dist = conflict.args[0]
if req.name not in seen_warnings:
logger.error("Error: version conflict: %s <-> %s" %
(dist, req))
errors = True
seen_warnings.add(req.name)
if dist is None:
logger.error('Error: unmet dependency: %s' % req)
errors = True
continue
result.append(dist_to_req(dist))
for dist_req in sorted(dist.requires(), key=lambda req: req.key):
# there really shouldn't be any circular dependencies...
queue.append(InstallRequirement(dist_req, str(req)))
if errors:
exit(1)
return result
def test_finder_finds_external_links_without_hashes_scraped_all(data):
"""
Tests that PackageFinder finds externally scraped links using the all
external flag.
"""
req = InstallRequirement.from_line("bar", None)
# using a local index
finder = PackageFinder(
[],
[data.index_url("externals")],
allow_all_external=True,
allow_unverified=["bar"],
session=PipSession(),
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-4.0.tar.gz"
