def test_unexisting_path(self):
with pytest.raises(InstallationError) as e:
InstallRequirement.from_line(
os.path.join('this', 'path', 'does', 'not', 'exist'))
err_msg = e.value.args[0]
assert "Invalid requirement" in err_msg
assert "It looks like a path." in err_msg
def test_exclusive_environment_markers():
"""Make sure RequirementSet accepts several excluding env markers"""
eq26 = InstallRequirement.from_line("Django>=1.6.10,<1.7 ; python_version == '2.6'")
ne26 = InstallRequirement.from_line("Django>=1.6.10,<1.8 ; python_version != '2.6'")
req_set = RequirementSet("", "", "", session=PipSession())
req_set.add_requirement(eq26)
req_set.add_requirement(ne26)
assert req_set.has_requirement("Django")
def _find_cached_match(spec):
#if spec.is_pinned:
## If this is a pinned spec, we can take a shortcut: if it is
## found in the dependency cache, we can safely assume it has
## been downloaded before, and thus must exist. We can know
## this without every reaching out to PyPI and avoid the
## network overhead.
#name, version = spec.name, first(spec.preds)[1]
#if (name, version) in self._dep_cache:
#source = 'dependency cache'
#return version, source
version = None
overrides = self.overrides.get(spec.name)
## Try the link cache, and otherwise, try PyPI
if (spec.no_extra, overrides) in self._link_cache:
link, version = self._link_cache[(spec.no_extra, overrides)]
source = 'link cache'
else:
try:
requirement = InstallRequirement.from_line(specline)
link = self.finder.find_requirement(requirement, False)
except DistributionNotFound:
requirement = InstallRequirement.from_line(
specline, prereleases=True)
link = self.finder.find_requirement(requirement, False)
link, version = self._link_hook(overrides, spec, link)
# Hack to make pickle work
link.comes_from = None
source = 'PyPI'
if link.egg_fragment:
version = link.egg_fragment.rsplit('-', 1)[1]
link = Link(
link.url_without_fragment + "#%s=%s" % self.get_hash(link)
)
elif not version:
_, version = splitext(link.filename)[0].rsplit('-', 1)
# It's more reliable to get version from pinned spec then filename
if spec.is_pinned:
version = spec.pinned
assert version, "Version must be set!"
self._link_cache[(spec.no_extra, overrides)] = (link, version)
# Take this moment to smartly insert the pinned variant of this
# spec into the link_cache, too
pinned_spec = Spec.from_pinned(spec.name, version)
self._link_cache[pinned_spec.fullname] = (link, version)
return version, source
def test_requirement_file(self):
req_file_path = os.path.join(self.tempdir, 'test.txt')
with open(req_file_path, 'w') as req_file:
req_file.write('pip\nsetuptools')
with pytest.raises(InstallationError) as e:
InstallRequirement.from_line(req_file_path)
err_msg = e.value.args[0]
assert "Invalid requirement" in err_msg
assert "It looks like a path. It does exist." in err_msg
assert "appears to be a requirements file." in err_msg
assert "If that is the case, use the '-r' flag to install" in err_msg
def test_markers_url(self):
# test "URL; markers" syntax
url = 'http://foo.com/?p=bar.git;a=snapshot;h=v0.1;sf=tgz'
line = '%s; python_version >= "3"' % url
req = InstallRequirement.from_line(line)
assert req.link.url == url, req.url
assert str(req.markers) == 'python_version >= "3"'
# without space, markers are part of the URL
url = 'http://foo.com/?p=bar.git;a=snapshot;h=v0.1;sf=tgz'
line = '%s;python_version >= "3"' % url
req = InstallRequirement.from_line(line)
assert req.link.url == line, req.url
assert req.markers is None
def test_markers_match(self):
# match
for markers in ('python_version >= "1.0"', "sys_platform == %r" % sys.platform):
line = "name; " + markers
req = InstallRequirement.from_line(line)
assert req.markers == markers
assert req.match_markers()
# don't match
for markers in ('python_version >= "5.0"', "sys_platform != %r" % sys.platform):
line = "name; " + markers
req = InstallRequirement.from_line(line)
assert req.markers == markers
assert not req.match_markers()
def make_install_requirement(name, version, extras, markers, constraint=False):
# If no extras are specified, the extras string is blank
extras_string = ""
if extras:
# Sort extras for stability
extras_string = "[{}]".format(",".join(sorted(extras)))
if not markers:
return InstallRequirement.from_line(
str('{}{}=={}'.format(name, extras_string, version)),
constraint=constraint)
else:
return InstallRequirement.from_line(
str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))),
constraint=constraint)
def handle(self, *args, **options):
new_app_names = []
new_package_names = []
extra_settings = []
default_templates = []
for name in args:
req = InstallRequirement.from_line(name)
if not req.check_if_exists():
result = subprocess.call(["pip", "install", name])
if not result == 0:
raise Exception("Install error")
req = InstallRequirement.from_line(name)
if not req.check_if_exists():
raise Exception("Installed package not found")
distribution = req.satisfied_by
new_package_names.append(distribution.project_name)
def _get_app_names(name):
if HOOKS['APPNAMES'].has_key(name):
return HOOKS['APPNAMES'][name]
## try real package names
packages = os.listdir(distribution.location)
candidates = difflib.get_close_matches(name, packages)
if candidates:
return [candidates[0]]
return name
new_app_names += _get_app_names(distribution.project_name)
if HOOKS['EXTRA_SETTINGS'].has_key(distribution.project_name):
extra_settings += [HOOKS['EXTRA_SETTINGS'][distribution.project_name]]
if HOOKS['TEMPLATES'].has_key(distribution.project_name):
default_templates += [HOOKS['TEMPLATES'][distribution.project_name]]
_add_to_requirements_txt(new_package_names)
## settings
for d in extra_settings:
for name, values in d.items():
_settings_add_to_list(name, values)
for d in default_templates:
for name, content in d.items():
_add_default_template(name, content)
_add_to_installed_apps(new_app_names)
def install(name):
"""
Try to install the package with 'name' into folder 'libs/python27'.
"""
print "Installation directory:"
print python27_dir()
requirement_set = RequirementSet(build_dir=build_prefix,
src_dir=src_prefix,
download_dir=None)
requirement_set.add_requirement(InstallRequirement.from_line(name, None))
install_options = ["--prefix=%s" % python27_dir()]
global_options = []
finder = PackageFinder(find_links=[],
index_urls=["http://pypi.python.org/simple/"])
try:
requirement_set.prepare_files(finder,
force_root_egg_info=False,
bundle=False)
requirement_set.install(install_options, global_options)
print "\nSuccessfully installed\n=================================="
for package in requirement_set.successfully_installed:
print package.name
print "\nDone.\n"
except DistributionNotFound:
print "No package found with name: %s" % name
except Exception as e:
print "Error:", e
def test_finder_priority_page_over_deplink():
"""Test PackageFinder prefers page links over equivalent dependency links"""
req = InstallRequirement.from_line('gmpy==1.15', None)
finder = PackageFinder([], ["http://pypi.python.org/simple"])
finder.add_dependency_links(['http://c.pypi.python.org/simple/gmpy/'])
link = finder.find_requirement(req, False)
assert link.url.startswith("http://pypi")
def test_finder_priority_nonegg_over_eggfragments():
"""Test PackageFinder prefers non-egg links over "#egg=" links"""
req = InstallRequirement.from_line('bar==1.0', None)
links = ['http://foo/bar.py#egg=bar-1.0', 'http://foo/bar-1.0.tar.gz']
finder = PackageFinder(links, [], session=PipSession())
with patch.object(finder, "_get_pages", lambda x, y: []):
all_versions = finder._find_all_versions(req.name)
assert all_versions[0].location.url.endswith('tar.gz')
assert all_versions[1].location.url.endswith('#egg=bar-1.0')
link = finder.find_requirement(req, False)
assert link.url.endswith('tar.gz')
links.reverse()
finder = PackageFinder(links, [], session=PipSession())
with patch.object(finder, "_get_pages", lambda x, y: []):
all_versions = finder._find_all_versions(req.name)
assert all_versions[0].location.url.endswith('tar.gz')
assert all_versions[1].location.url.endswith('#egg=bar-1.0')
link = finder.find_requirement(req, False)
assert link.url.endswith('tar.gz')
def test_pypirepo_calls_reqset_with_str_paths():
"""
Make sure that paths passed to RequirementSet init are str.
Passing unicode paths on Python 2 could make pip fail later on
unpack, if the package contains non-ASCII file names, because
non-ASCII str and unicode paths cannot be combined.
"""
with patch('piptools.repositories.pypi.RequirementSet') as mocked_init:
repo = get_pypi_repository()
ireq = InstallRequirement.from_line('ansible==2.4.0.0')
# Setup a mock object to be returned from the RequirementSet call
mocked_reqset = MagicMock()
mocked_init.return_value = mocked_reqset
# Do the call
repo.get_dependencies(ireq)
# Check that RequirementSet init is called with correct type arguments
assert mocked_init.call_count == 1
(init_call_args, init_call_kwargs) = mocked_init.call_args
assert isinstance(init_call_args[0], str)
assert isinstance(init_call_args[1], str)
assert isinstance(init_call_kwargs.get('download_dir'), str)
assert isinstance(init_call_kwargs.get('wheel_download_dir'), str)
# Check that _prepare_file is called correctly
assert mocked_reqset._prepare_file.call_count == 1
(pf_call_args, pf_call_kwargs) = mocked_reqset._prepare_file.call_args
(called_with_finder, called_with_ireq) = pf_call_args
assert isinstance(called_with_finder, PackageFinder)
assert called_with_ireq == ireq
assert not pf_call_kwargs
def test_finder_installs_pre_releases(data):
"""
Test PackageFinder finds pre-releases if asked to.
"""
req = InstallRequirement.from_line("bar", None, prereleases=True)
# using a local index (that has pre & dev releases)
finder = PackageFinder([], [data.index_url("pre")])
link = finder.find_requirement(req, False)
assert link.url.endswith("bar-2.0b1.tar.gz"), link.url
# using find-links
links = ["https://foo/bar-1.0.tar.gz", "https://foo/bar-2.0b1.tar.gz"]
finder = PackageFinder(links, [])
with patch.object(finder, "_get_pages", lambda x, y: []):
link = finder.find_requirement(req, False)
assert link.url == "https://foo/bar-2.0b1.tar.gz"
links.reverse()
finder = PackageFinder(links, [])
with patch.object(finder, "_get_pages", lambda x, y: []):
link = finder.find_requirement(req, False)
assert link.url == "https://foo/bar-2.0b1.tar.gz"
def test_get_dist(self):
req = InstallRequirement.from_line('foo')
req.egg_info_path = Mock(return_value='/path/to/foo.egg-info')
dist = req.get_dist()
assert isinstance(dist, pkg_resources.Distribution)
assert dist.project_name == 'foo'
assert dist.location == '/path/to'
def test_extras_for_line_url_requirement(self):
line = 'git+https://url#egg=SomeProject[ex1,ex2]'
filename = 'filename'
comes_from = '-r %s (line %s)' % (filename, 1)
req = InstallRequirement.from_line(line, comes_from=comes_from)
assert len(req.extras) == 2
assert req.extras == set(['ex1', 'ex2'])
def _iter_dependencies(self, ireq):
"""
Given a pinned or editable InstallRequirement, collects all the
secondary dependencies for them, either by looking them up in a local
cache, or by reaching out to the repository.
Editable requirements will never be looked up, as they may have
changed at any time.
"""
if ireq.editable:
for dependency in self.repository.get_dependencies(ireq):
yield dependency
return
elif not is_pinned_requirement(ireq):
raise TypeError("Expected pinned or editable requirement, got {}".format(ireq))
# Now, either get the dependencies from the dependency cache (for
# speed), or reach out to the external repository to
# download and inspect the package version and get dependencies
# from there
if ireq not in self.dependency_cache:
log.debug(" {} not in cache, need to check index".format(format_requirement(ireq)), fg="yellow")
dependencies = self.repository.get_dependencies(ireq)
self.dependency_cache[ireq] = sorted(str(ireq.req) for ireq in dependencies)
# Example: ['Werkzeug>=0.9', 'Jinja2>=2.4']
dependency_strings = self.dependency_cache[ireq]
log.debug(
" {:25} requires {}".format(
format_requirement(ireq), ", ".join(sorted(dependency_strings, key=lambda s: s.lower())) or "-"
)
)
for dependency_string in dependency_strings:
yield InstallRequirement.from_line(dependency_string)
def test_no_partial_name_match():
"""Finder requires the full project name to match, not just beginning."""
finder = PackageFinder([find_links], [])
req = InstallRequirement.from_line("gmpy")
found = finder.find_requirement(req, False)
assert found.url.endswith("gmpy-1.15.tar.gz"), found
def get_requirements(req_specs, requirement_files=None):
""" Get set of requirements from pip-like input arguments
Parameters
----------
req_specs : sequence
sequence of requirement specifiers, maybe with versions
requirement_files : None or sequence, optional
sequence of filenames or URLs with requirements
Returns
-------
requirement_set : RequiremenSet instance
Pip requirements set
"""
if requirement_files is None:
requirement_files = []
session = PipSession()
requirement_set = RequirementSet(
build_dir=None,
src_dir=None,
download_dir=None,
session=session,
)
for name in req_specs:
requirement_set.add_requirement(
InstallRequirement.from_line(name))
for filename in requirement_files:
for req in parse_requirements(filename, session=session):
requirement_set.add_requirement(req)
return requirement_set
def _transfer_to_fridge(resource, url, refill_behavior):
'''
download any kind of package to the fridge, see refill_fridge for refill_behavior
'''
targetpath = os.path.join(_fridgedir(), _frozenname(resource, url))
if os.path.exists(targetpath):
if refill_behavior == RB_ABORT:
abort("could not refill fridge with %s %s , because target already exists and if_exists_abort = True" % (resource, url))
elif refill_behavior == RB_IGNORE:
warn("did not refresh fridge with %s %s , because target already exists and reload_existing = False" % (resource, url))
return
if resource == "pypi":
req = InstallRequirement.from_line(url)
pf = PackageFinder([],['http://pypi.python.org/simple'])
url = pf.find_requirement(req,False).url
elif resource in ['https', 'http', 'ftp']:
pass
filename, headers = urllib.urlretrieve(url)
if os.path.exists(targetpath):
if refill_behavior == RB_OVERWRITE:
warn("%s %s already exist and will get overwritten because reload_behavior = %s" % (resource, url, RB_OVERWRITE))
elif refill_behavior == RB_UPDATE:
if open(filename,"rb").read() == open(targetpath,"rb").read():
print ("files %s %s identical, skipped" % (filename, targetpath))
return
warn("%s %s already exist and will get overwritten because its different to original and reload_behavior = %s" % (resource, url, RB_UPDATE))
return
shutil.move(filename, targetpath)
def format_requirements(fp, packages_groups, grouped_packages,
excluded_packages, output_index_url, ext_wheels_lines,
loose_packages=set()):
fp.write('# This file has been automatically generated, DO NOT EDIT!\n')
fp.write('\n')
if output_index_url:
fp.write('--index-url %s\n' % output_index_url, )
fp.write('\n')
seen = set()
for requirements_file, packages in packages_groups:
fp.write('# Frozen requirements for "%s"\n' % requirements_file)
fp.write('\n')
distros = [likely_distro(p) for p in packages]
for distro in sorted(distros, key=lambda d: d.key):
if (distro.key in seen or
distro.key in excluded_packages or
'%s:%s' % (requirements_file, distro.key) in excluded_packages):
continue
seen.add(distro.key)
versions = grouped_packages[distro.key]
if distro.key not in loose_packages:
line = '%s==%s' % (distro.key, versions[-1][0])
else:
line = distro.key
fp.write('%s\n' % line)
for line in ext_wheels_lines[requirements_file]:
req = InstallRequirement.from_line(line)
if req.name in loose_packages:
fp.write('%s\n' % req.name)
else:
fp.write('%s\n' % line.strip())
fp.write('\n')
def test_no_mpkg():
"""Finder skips zipfiles with "macosx10" in the name."""
finder = PackageFinder([find_links], [])
req = InstallRequirement.from_line("pkgwithmpkg")
found = finder.find_requirement(req, False)
assert found.url.endswith("pkgwithmpkg-1.0.tar.gz"), found
def _install_requirement(self, requirement):
if not self.local_options:
self.local_options = []
if self._check_requirement(requirement):
return True, None
try:
# Parse requirement
requirement = InstallRequirement.from_line(str(requirement), None)
# Build the requirement set. We're doing this one at a time so we can actually detect
# which ones fail.
requirement_set = RequirementSet(build_dir=build_prefix, src_dir=src_prefix, download_dir=None)
requirement_set.add_requirement(requirement)
# Download and build requirement
try:
requirement_set.prepare_files(self.finder, force_root_egg_info=False, bundle=False)
except PreviousBuildDirError, err:
# Remove previous build directories if they're detected.. shouldn't be an issue
# now that we've removed upstream dependencies from requirements.txt.
location = requirement.build_location(build_prefix, True)
shutil.rmtree(location)
requirement_set.prepare_files(self.finder, force_root_egg_info=False, bundle=False)
# Finally, install the requirement.
requirement_set.install(self.local_options, [])
def test_finder_only_installs_stable_releases(data):
"""
Test PackageFinder only accepts stable versioned releases by default.
"""
req = InstallRequirement.from_line("bar", None)
# using a local index (that has pre & dev releases)
finder = PackageFinder([], [data.index_url("pre")], session=PipSession())
link = finder.find_requirement(req, False)
assert link.url.endswith("bar-1.0.tar.gz"), link.url
# using find-links
links = ["https://foo/bar-1.0.tar.gz", "https://foo/bar-2.0b1.tar.gz"]
finder = PackageFinder(links, [], session=PipSession())
with patch.object(finder, "_get_pages", lambda x, y: []):
link = finder.find_requirement(req, False)
assert link.url == "https://foo/bar-1.0.tar.gz"
links.reverse()
finder = PackageFinder(links, [], session=PipSession())
with patch.object(finder, "_get_pages", lambda x, y: []):
link = finder.find_requirement(req, False)
assert link.url == "https://foo/bar-1.0.tar.gz"
def find_best_match(self, ireq, prereleases=False):
if ireq.editable:
return ireq
versions = ireq.specifier.filter(self.index[ireq.req.key], prereleases=prereleases)
best_version = max(versions, key=Version)
return InstallRequirement.from_line('{}=={}'.format(ireq.req.key, best_version))
def run(self, options, args):
with self._build_session(options) as session:
format_control = pip.index.FormatControl(set(), set())
wheel_cache = WheelCache(options.cache_dir, format_control)
requirement_set = RequirementSet(
build_dir=None,
src_dir=None,
download_dir=None,
isolated=options.isolated_mode,
session=session,
wheel_cache=wheel_cache,
)
for name in args:
requirement_set.add_requirement(
InstallRequirement.from_line(
name, isolated=options.isolated_mode,
wheel_cache=wheel_cache
)
)
for filename in options.requirements:
for req in parse_requirements(
filename,
options=options,
session=session,
wheel_cache=wheel_cache):
requirement_set.add_requirement(req)
if not requirement_set.has_requirements:
raise InstallationError(
'You must give at least one requirement to %(name)s (see '
'"pip help %(name)s")' % dict(name=self.name)
)
requirement_set.uninstall(auto_confirm=options.yes)
def get_dependencies(self, ireq):
if ireq.editable:
return self.editables[str(ireq.link)]
name, version = as_name_version_tuple(ireq)
dependencies = self.index[name][version]
return [InstallRequirement.from_line(dep) for dep in dependencies]
def pip_install(name, tmpdir):
build_dir = os.path.join(tmpdir, 'build')
src_dir = os.path.join(tmpdir, 'src')
download_dir = os.path.join(tmpdir, 'download')
os.mkdir(build_dir)
os.mkdir(src_dir)
os.mkdir(download_dir)
finder = PackageFinder(
find_links=[],
index_urls=['https://pypi.python.org/simple/'],
use_mirrors=False,
allow_all_external=True,
allow_all_insecure=True,
)
requirement_set = RequirementSet(
build_dir=build_dir,
src_dir=src_dir,
download_dir=download_dir,
ignore_installed=True,
ignore_dependencies=True
)
requirement_set.add_requirement(InstallRequirement.from_line(name, None))
requirement_set.prepare_files(finder)
# should be exactly one
filename = os.listdir(download_dir)[0]
path = os.path.join(download_dir, filename)
return path
def getDependencies(name, requirementSet=None, finder=None):
"""Get dependencies of a python project
@param name: name of python project
@param requirements: RequirementSet
@param finder: PackageFinder
"""
if requirementSet is None:
requirementSet = RequirementSet(
build_dir=os.path.abspath(build_prefix),
src_dir=os.path.abspath(src_prefix),
download_dir=None,
download_cache=None,
upgrade=False,
ignore_installed=True,
ignore_dependencies=False)
if finder is None:
finder = PackageFinder(find_links=[],
index_urls=['http://pypi.python.org/simple'])
# lead pip download all dependencies
req = InstallRequirement.from_line(name, None)
requirementSet.add_requirement(req)
requirementSet.install_files(finder)
# trace the dependencies relationships between projects
dependencies = []
traceDependencys(req, requirementSet, dependencies)
return dependencies
def test_not_find_wheel_not_supported(self):
"""
Test not finding an unsupported wheel.
"""
req = InstallRequirement.from_line("simple.dist")
finder = PackageFinder([find_links], [], use_wheel=True)
assert_raises(DistributionNotFound, finder.find_requirement, req, True)
def test_url_with_query():
"""InstallRequirement should strip the fragment, but not the query."""
url = 'http://foo.com/?p=bar.git;a=snapshot;h=v0.1;sf=tgz'
fragment = '#egg=bar'
req = InstallRequirement.from_line(url + fragment)
assert req.url == url, req.url
def populate_requirement_set(requirement_set, args, options, finder,
session, name, wheel_cache):
"""
Marshal cmd line args into a requirement set.
"""
for filename in options.constraints:
for req in parse_requirements(filename,
constraint=True,
finder=finder,
options=options,
session=session,
wheel_cache=wheel_cache):
requirement_set.add_requirement(req)
for req in args:
requirement_set.add_requirement(
InstallRequirement.from_line(req,
None,
isolated=options.isolated_mode,
wheel_cache=wheel_cache))
for req in options.editables:
requirement_set.add_requirement(
InstallRequirement.from_editable(
req,
default_vcs=options.default_vcs,
isolated=options.isolated_mode,
wheel_cache=wheel_cache))
found_req_in_file = False
for filename in options.requirements:
for req in parse_requirements(filename,
finder=finder,
options=options,
session=session,
wheel_cache=wheel_cache):
found_req_in_file = True
requirement_set.add_requirement(req)
# If --require-hashes was a line in a requirements file, tell
# RequirementSet about it:
requirement_set.require_hashes = options.require_hashes
if not (args or options.editables or found_req_in_file):
opts = {'name': name}
if options.find_links:
msg = ('You must give at least one requirement to '
'%(name)s (maybe you meant "pip %(name)s '
'%(links)s"?)' %
dict(opts, links=' '.join(options.find_links)))
else:
msg = ('You must give at least one requirement '
'to %(name)s (see "pip help %(name)s")' % opts)
logger.warning(msg)
def find_packages_latests_versions(self, options):
index_urls = [options.index_url] + options.extra_index_urls
if options.no_index:
logger.notify('Ignoring indexes: %s' % ','.join(index_urls))
index_urls = []
if options.use_mirrors:
logger.deprecated(
"1.7",
"--use-mirrors has been deprecated and will be removed"
" in the future. Explicit uses of --index-url and/or "
"--extra-index-url is suggested."
)
if options.mirrors:
logger.deprecated(
"1.7",
"--mirrors has been deprecated and will be removed in "
" the future. Explicit uses of --index-url and/or "
"--extra-index-url is suggested."
)
index_urls += options.mirrors
session = self._build_session(options)
finder = self._build_package_finder(options, index_urls, session)
installed_packages = get_installed_distributions(
local_only=options.local,
include_editables=False,
)
for dist in installed_packages:
req = InstallRequirement.from_line(dist.key, None)
try:
link = finder.find_requirement(req, True)
# If link is None, means installed version is most up-to-date
if link is None:
continue
except DistributionNotFound:
continue
except BestVersionAlreadyInstalled:
remote_version = req.installed_version
else:
# It might be a good idea that link or finder had a public
# method that returned version
remote_version = finder._link_package_versions(
link, req.name
)[0]
remote_version_raw = remote_version[2]
remote_version_parsed = remote_version[0]
yield dist, remote_version_raw, remote_version_parsed
def test_finder_detects_latest_already_satisfied_find_links(data):
"""Test PackageFinder detects latest already satisfied using find-links"""
req = InstallRequirement.from_line('simple', None)
# the latest simple in local pkgs is 3.0
latest_version = "3.0"
satisfied_by = Mock(location="/path",
parsed_version=parse_version(latest_version),
version=latest_version)
req.satisfied_by = satisfied_by
finder = PackageFinder([data.find_links], [], session=PipSession())
with pytest.raises(BestVersionAlreadyInstalled):
finder.find_requirement(req, True)
def test_finder_detects_latest_already_satisfied_pypi_links():
"""Test PackageFinder detects latest already satisified using pypi links"""
req = InstallRequirement.from_line('initools', None)
#the latest initools on pypi is 0.3.1
latest_version = "0.3.1"
satisfied_by = Mock(location="/path",
parsed_version=parse_version(latest_version),
version=latest_version)
req.satisfied_by = satisfied_by
finder = PackageFinder([], ["http://pypi.python.org/simple"])
with pytest.raises(BestVersionAlreadyInstalled):
finder.find_requirement(req, True)
def test_markers(self):
for line in (
# recommended syntax
'mock3; python_version >= "3"',
# with more spaces
'mock3 ; python_version >= "3" ',
# without spaces
'mock3;python_version >= "3"',
):
req = InstallRequirement.from_line(line)
assert req.req.name == 'mock3'
assert str(req.req.specifier) == ''
assert str(req.markers) == 'python_version >= "3"'
def install_egg(self, egg_name):
""" Install an egg into the egg directory """
if not os.path.exists(self.egg_directory):
os.makedirs(self.egg_directory)
self.requirement_set.add_requirement(
InstallRequirement.from_line(egg_name, None))
try:
self.requirement_set.prepare_files(self.finder)
self.requirement_set.install(['--prefix=' + self.egg_directory],
[])
except DistributionNotFound:
self.requirement_set.requirements._keys.remove(egg_name)
raise PipException()
def test_wheel_over_sdist_priority(self, data):
"""
Test wheels have priority over sdists.
`test_link_sorting` also covers this at lower level
"""
req = InstallRequirement.from_line("priority")
finder = PackageFinder(
[data.find_links],
[],
session=PipSession(),
)
found = finder.find_requirement(req, True).location
assert found.url.endswith("priority-1.0-py2.py3-none-any.whl"), found
def test_markers(self):
for line in (
# recommanded syntax
'mock3; python_version >= "3"',
# with more spaces
'mock3 ; python_version >= "3" ',
# without spaces
'mock3;python_version >= "3"',
):
req = InstallRequirement.from_line(line)
assert req.req.project_name == 'mock3'
assert req.req.specs == []
assert req.markers == 'python_version >= "3"'
def test_finder_priority_nonegg_over_eggfragments():
"""Test PackageFinder prefers non-egg links over "#egg=" links"""
req = InstallRequirement.from_line('bar==1.0', None)
links = ['http://foo/bar.py#egg=bar-1.0', 'http://foo/bar-1.0.tar.gz']
finder = PackageFinder(links, [])
link = finder.find_requirement(req, False)
assert link.url.endswith('tar.gz')
links.reverse()
finder = PackageFinder(links, [])
link = finder.find_requirement(req, False)
assert link.url.endswith('tar.gz')
def parse_requirement(pkgstring, comparator='=='):
ins = InstallRequirement.from_line(pkgstring)
pkg_name, specs = ins.name, str(ins.specifier)
if specs:
return pkg_name, specs
req = Requirement.parse(pkg_name)
working_set = WorkingSet()
dist = working_set.find(req)
if dist:
specs = "%s%s" % (comparator, dist.version)
return req.project_name, specs
def find_packages_latest_versions(self, options):
index_urls = [options.index_url] + options.extra_index_urls
if options.no_index:
logger.info('Ignoring indexes: %s', ','.join(index_urls))
index_urls = []
dependency_links = []
for dist in get_installed_distributions(local_only=options.local,
user_only=options.user):
if dist.has_metadata('dependency_links.txt'):
dependency_links.extend(
dist.get_metadata_lines('dependency_links.txt'), )
with self._build_session(options) as session:
finder = self._build_package_finder(options, index_urls, session)
finder.add_dependency_links(dependency_links)
installed_packages = get_installed_distributions(
local_only=options.local,
user_only=options.user,
include_editables=False,
)
format_control = FormatControl(set(), set())
wheel_cache = WheelCache(options.cache_dir, format_control)
for dist in installed_packages:
req = InstallRequirement.from_line(
dist.key,
None,
isolated=options.isolated_mode,
wheel_cache=wheel_cache)
typ = 'unknown'
try:
link = finder.find_requirement(req, True)
# If link is None, means installed version is most
# up-to-date
if link is None:
continue
except DistributionNotFound:
continue
else:
canonical_name = pkg_resources.safe_name(req.name).lower()
formats = fmt_ctl_formats(format_control, canonical_name)
search = Search(req.name, canonical_name, formats)
remote_version = finder._link_package_versions(
link, search).version
if link.is_wheel:
typ = 'wheel'
else:
typ = 'sdist'
yield dist, remote_version, typ
def gen(ireq):
if self.DEFAULT_INDEX_URL in self.finder.index_urls:
url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name)
r = self.session.get(url)
latest = list(r.json()['releases'].keys())[-1]
if str(ireq.req.specifier) == '=={0}'.format(latest):
for requires in r.json().get('info', {}).get('requires_dist', {}):
i = InstallRequirement.from_line(requires)
if 'extra' not in repr(i.markers):
yield i
def test_finder_ignores_external_links(data):
"""
Tests that PackageFinder ignores external links, with or without hashes.
"""
req = InstallRequirement.from_line("bar", None)
# using a local index
finder = PackageFinder(
[],
[data.index_url("externals")],
session=PipSession(),
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-1.0.tar.gz"
def get_dependencies(self, ireq):
if ireq.editable:
return self.editables[str(ireq.link)]
name, version, extras = as_tuple(ireq)
# Store non-extra dependencies under the empty string
extras += ("", )
dependencies = [
dep for extra in extras for dep in self.index[name][version][extra]
]
return [
InstallRequirement.from_line(dep, constraint=ireq.constraint)
for dep in dependencies
]
def test_finder_finds_external_links_without_hashes_scraped_per_project_all_insecure():
"""
Tests that PackageFinder finds externally scraped links
"""
req = InstallRequirement.from_line("bar", None)
# using a local index
index_url = path_to_url(os.path.join(tests_data, "indexes", "externals"))
finder = PackageFinder([], [index_url],
allow_external=["bar"],
allow_all_insecure=True,
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-4.0.tar.gz"
def test_finder_priority_page_over_deplink():
"""
Test PackageFinder prefers page links over equivalent dependency links
"""
req = InstallRequirement.from_line('gmpy==1.15', None)
finder = PackageFinder(
[],
["https://pypi.python.org/simple"],
process_dependency_links=True,
session=PipSession(),
)
finder.add_dependency_links(['http://c.pypi.python.org/simple/gmpy/'])
link = finder.find_requirement(req, False)
assert link.url.startswith("https://pypi"), link
def test_finder_finds_external_links_without_hashes_per_project():
"""
Tests that PackageFinder finds external links if they do not have a hash
"""
req = InstallRequirement.from_line("bar==3.0", None)
# using a local index
index_url = path_to_url(os.path.join(tests_data, "indexes", "externals"))
finder = PackageFinder([], [index_url],
allow_external=["bar"],
allow_insecure=["bar"],
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-3.0.tar.gz"
def run(self, options, args):
requirement_set = RequirementSet(build_dir=None,
src_dir=None,
download_dir=None)
for name in args:
requirement_set.add_requirement(InstallRequirement.from_line(name))
for filename in options.requirements:
for req in parse_requirements(filename, options=options):
requirement_set.add_requirement(req)
if not requirement_set.has_requirements:
raise InstallationError('You must give at least one requirement '
'to %(name)s (see "pip help %(name)s")' %
dict(name=self.name))
requirement_set.uninstall(auto_confirm=options.yes)
def test_existing_over_wheel_priority(self, data):
"""
Test existing install has priority over wheels.
`test_link_sorting` also covers this at a lower level
"""
req = InstallRequirement.from_line('priority', None)
latest_version = "1.0"
satisfied_by = Mock(location="/path",
parsed_version=parse_version(latest_version),
version=latest_version)
req.satisfied_by = satisfied_by
finder = PackageFinder([data.find_links], [], use_wheel=True)
with pytest.raises(BestVersionAlreadyInstalled):
finder.find_requirement(req, True)
def test_skip_invalid_wheel_link(self, caplog, data):
"""
Test if PackageFinder skips invalid wheel filenames
"""
req = InstallRequirement.from_line("invalid")
# data.find_links contains "invalid.whl", which is an invalid wheel
finder = PackageFinder(
[data.find_links],
[],
session=PipSession(),
)
with pytest.raises(DistributionNotFound):
finder.find_requirement(req, True)
assert ("invalid.whl; invalid wheel filename" in caplog.text)
def test_finder_finds_external_links_without_hashes_scraped_per_project(data):
"""
Tests that PackageFinder finds externally scraped links
"""
req = InstallRequirement.from_line("bar", None)
# using a local index
finder = PackageFinder(
[],
[data.index_url("externals")],
allow_external=["bar"],
allow_unverified=["bar"],
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-4.0.tar.gz"
def install_egg(self, egg_name):
""" Install an egg into the egg directory """
if not os.path.exists(self.egg_directory):
os.makedirs(self.egg_directory)
self.requirement_set.add_requirement(
InstallRequirement.from_line(egg_name, None))
try:
self.requirement_set.prepare_files(self.finder,
force_root_egg_info=False,
bundle=False)
self.requirement_set.install(self.install_options,
self.global_options)
except DistributionNotFound:
self.requirement_set.requirements._keys.remove(egg_name)
raise PipException()
def test_no_reuse_existing_build_dir(self, data):
"""Test prepare_files raise exception with previous build dir"""
build_dir = os.path.join(self.tempdir, 'build', 'simple')
os.makedirs(build_dir)
open(os.path.join(build_dir, "setup.py"), 'w')
reqset = self.basic_reqset()
req = InstallRequirement.from_line('simple')
reqset.add_requirement(req)
finder = PackageFinder([data.find_links], [])
assert_raises_regexp(
PreviousBuildDirError,
"pip can't proceed with [\s\S]*%s[\s\S]*%s" %
(req, build_dir.replace('\\', '\\\\')), reqset.prepare_files,
finder)
def test_finder_deplink():
"""
Test PackageFinder with dependency links only
"""
req = InstallRequirement.from_line('gmpy==1.15', None)
finder = PackageFinder(
[],
[],
process_dependency_links=True,
session=PipSession(),
)
finder.add_dependency_links(
['https://pypi.python.org/packages/source/g/gmpy/gmpy-1.15.zip'])
link = finder.find_requirement(req, False)
assert link.url.startswith("https://pypi"), link
def test_finder_installs_dev_releases(data):
"""
Test PackageFinder finds dev releases if asked to.
"""
req = InstallRequirement.from_line("bar", None)
# using a local index (that has dev releases)
finder = PackageFinder(
[], [data.index_url("dev")],
allow_all_prereleases=True,
session=PipSession(),
)
link = finder.find_requirement(req, False)
assert link.url.endswith("bar-2.0.dev1.tar.gz"), link.url
def test_finder_installs_pre_releases_with_version_spec():
"""
Test PackageFinder only accepts stable versioned releases by default.
"""
req = InstallRequirement.from_line("bar>=0.0.dev0", None)
links = ["https://foo/bar-1.0.tar.gz", "https://foo/bar-2.0b1.tar.gz"]
finder = PackageFinder(links, [])
link = finder.find_requirement(req, False)
assert link.url == "https://foo/bar-2.0b1.tar.gz"
links.reverse()
finder = PackageFinder(links, [])
link = finder.find_requirement(req, False)
assert link.url == "https://foo/bar-2.0b1.tar.gz"
def get_legacy_dependencies(self, ireq):
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
They indicate the secondary dependencies for the given requirement.
"""
if not (ireq.editable or is_pinned_requirement(ireq)):
raise TypeError(
'Expected pinned or editable InstallRequirement, got {}'.
format(ireq))
if ireq not in self._dependencies_cache:
if ireq.link and not ireq.link.is_artifact:
# No download_dir for VCS sources. This also works around pip
# using git-checkout-index, which gets rid of the .git dir.
download_dir = None
else:
download_dir = self._download_dir
if not os.path.isdir(download_dir):
os.makedirs(download_dir)
if not os.path.isdir(self._wheel_download_dir):
os.makedirs(self._wheel_download_dir)
reqset = RequirementSet(
self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
ignore_installed=True,
ignore_requires_python=True)
result = reqset._prepare_file(self.finder,
ireq,
ignore_requires_python=True)
if not result:
if reqset.requires_python:
from pip.req.req_install import InstallRequirement
marker = 'python_version=="{0}"'.format(
reqset.requires_python.replace(' ', ''))
new_req = InstallRequirement.from_line('{0}; {1}'.format(
str(ireq.req), marker))
result = [new_req]
self._dependencies_cache[ireq] = result
return set(self._dependencies_cache[ireq])
def _iter_dependencies(self, ireq):
"""
Given a pinned or editable InstallRequirement, collects all the
secondary dependencies for them, either by looking them up in a local
cache, or by reaching out to the repository.
Editable requirements will never be looked up, as they may have
changed at any time.
"""
if ireq.editable:
for dependency in self.repository.get_dependencies(ireq):
yield dependency
return
elif ireq.markers:
for dependency in self.repository.get_dependencies(ireq):
dependency.prepared = False
yield dependency
return
elif ireq.extras:
for dependency in self.repository.get_dependencies(ireq):
dependency.prepared = False
yield dependency
return
elif not is_pinned_requirement(ireq):
raise TypeError(
'Expected pinned or editable requirement, got {}'.format(ireq))
# Now, either get the dependencies from the dependency cache (for
# speed), or reach out to the external repository to
# download and inspect the package version and get dependencies
# from there
if ireq not in self.dependency_cache:
log.debug(' {} not in cache, need to check index'.format(
format_requirement(ireq)),
fg='yellow')
dependencies = self.repository.get_dependencies(ireq)
self.dependency_cache[ireq] = sorted(
str(ireq.req) for ireq in dependencies)
# Example: ['Werkzeug>=0.9', 'Jinja2>=2.4']
dependency_strings = self.dependency_cache[ireq]
log.debug(' {:25} requires {}'.format(
format_requirement(ireq),
', '.join(sorted(dependency_strings, key=lambda s: s.lower()))
or '-'))
for dependency_string in dependency_strings:
yield InstallRequirement.from_line(dependency_string,
constraint=ireq.constraint)
def test_finder_finds_external_links_with_hashes_all(data):
"""
Tests that PackageFinder finds external links but only if they have a hash
using the all externals flag.
"""
req = InstallRequirement.from_line("bar", None)
# using a local index
finder = PackageFinder(
[],
[data.index_url("externals")],
allow_all_external=True,
session=PipSession(),
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-2.0.tar.gz"
def test_finder_finds_external_links_without_hashes_per_project(data):
"""
Tests that PackageFinder finds external links if they do not have a hash
"""
req = InstallRequirement.from_line("bar==3.0", None)
# using a local index
finder = PackageFinder(
[],
[data.index_url("externals")],
allow_external=["bar"],
allow_unverified=["bar"],
session=PipSession(),
)
link = finder.find_requirement(req, False)
assert link.filename == "bar-3.0.tar.gz"
