def cached_wheel(self, link, package_name):
not_cached = (not self._cache_dir or not link or link.is_wheel
or not link.is_artifact or not package_name)
if not_cached:
return link
canonical_name = canonicalize_name(package_name)
formats = pip.index.fmt_ctl_formats(self._format_control,
canonical_name)
if "binary" not in formats:
return link
root = self.get_cache_path_for_link(link)
try:
wheel_names = os.listdir(root)
except OSError as err:
if err.errno in {errno.ENOENT, errno.ENOTDIR}:
return link
raise
candidates = []
for wheel_name in wheel_names:
try:
wheel = Wheel(wheel_name)
except InvalidWheelFilename:
continue
if not wheel.supported():
# Built for a different python/arch/etc
continue
candidates.append((wheel.support_index_min(), wheel_name))
if not candidates:
return link
candidates.sort()
path = os.path.join(root, candidates[0][1])
return pip.index.Link(path_to_url(path))
def faster_find_requirement(self, req, upgrade):
"""see faster_pip_packagefinder"""
from pip.index import BestVersionAlreadyInstalled
if req_is_absolute(req.req):
# if the version is pinned-down by a ==
# first try to use any installed package that satisfies the req
if req.satisfied_by:
if upgrade:
# as a matter of api, find_requirement() only raises during upgrade -- shrug
raise BestVersionAlreadyInstalled
else:
return None
# then try an optimistic search for a .whl file:
from os.path import join
from glob import glob
from pip.wheel import Wheel
from pip.index import Link
for findlink in self.find_links:
if findlink.startswith('file://'):
findlink = findlink[7:]
else:
continue
# this matches the name-munging done in pip.wheel:
reqname = req.name.replace('-', '_')
for link in glob(join(findlink, reqname + '-*.whl')):
link = Link('file://' + link)
wheel = Wheel(link.filename)
if wheel.version in req.req and wheel.supported():
return link
# otherwise, do the full network search
return self.unpatched['find_requirement'](self, req, upgrade)
def _link_sort_key(self, link_tuple):
"""
Function used to generate link sort key for link tuples.
The greater the return value, the more preferred it is.
If not finding wheels, then sorted by version only.
If finding wheels, then the sort order is by version, then:
1. existing installs
2. wheels ordered via Wheel.support_index_min()
3. source archives
Note: it was considered to embed this logic into the Link
comparison operators, but then different sdist links
with the same version, would have to be considered equal
"""
parsed_version, link, _ = link_tuple
if self.use_wheel:
support_num = len(supported_tags)
if link == INSTALLED_VERSION:
pri = 1
elif link.ext == wheel_ext:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
if not wheel.supported():
raise UnsupportedWheel(
"%s is not a supported wheel for this platform. It "
"can't be sorted." % wheel.filename
)
pri = -(wheel.support_index_min())
else: # sdist
pri = -(support_num)
return (parsed_version, pri)
else:
return parsed_version
def _candidate_sort_key(self, candidate):
"""
Function used to generate link sort key for link tuples.
The greater the return value, the more preferred it is.
If not finding wheels, then sorted by version only.
If finding wheels, then the sort order is by version, then:
1. existing installs
2. wheels ordered via Wheel.support_index_min()
3. source archives
Note: it was considered to embed this logic into the Link
comparison operators, but then different sdist links
with the same version, would have to be considered equal
"""
support_num = len(supported_tags)
if candidate.location.is_wheel:
# can raise InvalidWheelFilename
wheel = Wheel(candidate.location.filename)
if not wheel.supported():
raise UnsupportedWheel(
"%s is not a supported wheel for this platform. It "
"can't be sorted." % wheel.filename
)
pri = -(wheel.support_index_min())
else: # sdist
pri = -(support_num)
return (candidate.version, pri)
def _link_sort_key(self, link_tuple):
"""
Function used to generate link sort key for link tuples.
The greater the return value, the more preferred it is.
If not finding wheels, then sorted by version only.
If finding wheels, then the sort order is by version, then:
1. existing installs
2. wheels ordered via Wheel.support_index_min()
3. source archives
Note: it was considered to embed this logic into the Link
comparison operators, but then different sdist links
with the same version, would have to be considered equal
"""
parsed_version, link, _ = link_tuple
if self.use_wheel:
support_num = len(supported_tags)
if link == INSTALLED_VERSION:
pri = 1
elif link.ext == wheel_ext:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
if not wheel.supported():
raise UnsupportedWheel(
"%s is not a supported wheel for this platform. It can't be sorted."
% wheel.filename)
pri = -(wheel.support_index_min())
else: # sdist
pri = -(support_num)
return (parsed_version, pri)
else:
return parsed_version
def faster_find_requirement(self, req, upgrade):
"""see faster_pip_packagefinder"""
from pip.index import BestVersionAlreadyInstalled
if req_is_absolute(req.req):
# if the version is pinned-down by a ==
# first try to use any installed package that satisfies the req
if req.satisfied_by:
if upgrade:
# as a matter of api, find_requirement() only raises during upgrade -- shrug
raise BestVersionAlreadyInstalled
else:
return None
# then try an optimistic search for a .whl file:
from os.path import join
from glob import glob
from pip.wheel import Wheel
from pip.index import Link
for findlink in self.find_links:
if findlink.startswith('file://'):
findlink = findlink[7:]
else:
continue
# this matches the name-munging done in pip.wheel:
reqname = req.name.replace('-', '_')
for link in glob(join(findlink, reqname + '-*.whl')):
link = Link('file://' + link)
wheel = Wheel(link.filename)
if wheel.version in req.req and wheel.supported():
return link
# otherwise, do the full network search
return self.unpatched['find_requirement'](self, req, upgrade)
def _candidate_sort_key(self, candidate):
"""
Function used to generate link sort key for link tuples.
The greater the return value, the more preferred it is.
If not finding wheels, then sorted by version only.
If finding wheels, then the sort order is by version, then:
1. existing installs
2. wheels ordered via Wheel.support_index_min(self.valid_tags)
3. source archives
Note: it was considered to embed this logic into the Link
comparison operators, but then different sdist links
with the same version, would have to be considered equal
"""
support_num = len(self.valid_tags)
if candidate.location.is_wheel:
# can raise InvalidWheelFilename
wheel = Wheel(candidate.location.filename)
if not wheel.supported(self.valid_tags):
raise UnsupportedWheel(
"%s is not a supported wheel for this platform. It "
"can't be sorted." % wheel.filename)
pri = -(wheel.support_index_min(self.valid_tags))
else: # sdist
pri = -(support_num)
return (candidate.version, pri)
def _link_package_versions(self, link, search):
"""Return an InstallationCandidate or None"""
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
self._log_skipped_link(link, "not a file")
return
if ext not in SUPPORTED_EXTENSIONS:
self._log_skipped_link(link, "unsupported archive format: %s" % ext)
return
if "binary" not in search.formats and ext == wheel_ext:
self._log_skipped_link(link, "No binaries permitted for %s" % search.supplied)
return
if "macosx10" in link.path and ext == ".zip":
self._log_skipped_link(link, "macosx10 one")
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
self._log_skipped_link(link, "invalid wheel filename")
return
if canonicalize_name(wheel.name) != search.canonical:
self._log_skipped_link(link, "wrong project name (not %s)" % search.supplied)
return
if not wheel.supported():
self._log_skipped_link(link, "it is not compatible with this Python")
return
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != wheel_ext:
self._log_skipped_link(link, "No sources permitted for %s" % search.supplied)
return
if not version:
version = egg_info_matches(egg_info, search.supplied, link)
if version is None:
self._log_skipped_link(link, "wrong project name (not %s)" % search.supplied)
return
match = self._py_version_re.search(version)
if match:
version = version[: match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
self._log_skipped_link(link, "Python version is incorrect")
return
logger.debug("Found link %s, version: %s", link, version)
return InstallationCandidate(search.supplied, version, link)
def optimistic_wheel_search(req, index_urls):
name = req.name.replace('-', '_').lower()
for index_url in index_urls:
expected_location = os.path.join(
CACHE.wheelhouse,
index_url,
ignorecase_glob(name) + '-*.whl',
)
for link in glob.glob(expected_location):
link = Link('file:' + link)
wheel = Wheel(link.filename)
if req.specifier.contains(wheel.version) and wheel.supported():
return link
def it_doesnt_wheel_local_dirs(tmpdir):
venv = tmpdir.join('venv')
install_coverage(venv)
pip = venv.join('bin/pip').strpath
run(pip, 'install', 'venv-update==' + __version__)
run(
venv.join('bin/pip-faster').strpath,
'install',
TOP.join('tests/testing/packages/dependant_package').strpath,
)
frozen_requirements = pip_freeze(str(venv)).split('\n')
assert set(frozen_requirements) == set([
'coverage==4.0.3',
'coverage-enable-subprocess==0',
'dependant-package==1',
'implicit-dependency==1',
'many-versions-package==3',
'pure-python-package==0.2.0',
'venv-update==' + __version__,
'wheel==0.29.0',
'',
])
wheelhouse = tmpdir.join('home', '.cache', 'pip-faster', 'wheelhouse')
assert set(Wheel(f.basename).name for f in wheelhouse.listdir()) == set([
'implicit-dependency',
'many-versions-package',
'pure-python-package',
])
def it_installs_stuff_with_dash_e_without_wheeling(tmpdir):
venv = tmpdir.join('venv')
install_coverage(venv)
pip = venv.join('bin/pip').strpath
run(pip, 'install', 'venv-update==' + __version__)
# Install a package from git with no extra dependencies in editable mode.
#
# We need to install a package from VCS instead of the filesystem because
# otherwise we aren't testing that editable requirements aren't wheeled
# (and instead might just be testing that local paths aren't wheeled).
requirements(
'-e git+git://github.com/Yelp/dumb-init.git@87545be699a13d0fd31f67199b7782ebd446437e#egg=dumb-init'
) # noqa
run(str(venv.join('bin/pip-faster')), 'install', '-r', 'requirements.txt')
frozen_requirements = pip_freeze(str(venv)).split('\n')
assert set(frozen_requirements) == set([
'-e git://github.com/Yelp/dumb-init.git@87545be699a13d0fd31f67199b7782ebd446437e#egg=dumb_init-dev', # noqa
'coverage-enable-subprocess==0',
'coverage==4.0.3',
'venv-update==' + __version__,
'wheel==0.29.0',
'',
])
# we shouldn't wheel things installed editable
wheelhouse = tmpdir.join('home', '.cache', 'pip-faster', 'wheelhouse')
assert set(Wheel(f.basename).name for f in wheelhouse.listdir()) == set([])
def cached_wheels(tmpdir):
for _, _, filenames in os.walk(
tmpdir.join('home', '.cache', 'pip-faster',
'wheelhouse').strpath, ):
for filename in filenames:
assert filename.endswith('.whl'), filename
yield Wheel(filename)
def get(self, link, package_name):
candidates = []
for wheel_name in self._get_candidates(link, package_name):
try:
wheel = Wheel(wheel_name)
except InvalidWheelFilename:
continue
if not wheel.supported():
# Built for a different python/arch/etc
continue
candidates.append((wheel.support_index_min(), wheel_name))
if not candidates:
return link
return self._link_for_candidate(link, min(candidates)[1])
def get(self, link, package_name):
candidates = []
for wheel_name in self._get_candidates(link, package_name):
try:
wheel = Wheel(wheel_name)
except InvalidWheelFilename:
continue
if not wheel.supported():
# Built for a different python/arch/etc
continue
candidates.append((wheel.support_index_min(), wheel_name))
if not candidates:
return link
return self._link_for_candidate(link, min(candidates)[1])
def get_platform_tags(wheel):
# type (unicode) -> List[unicode]
"""
Returns the python platform tags indicated by the file name.
:param wheel: wheel file name (without directory)
:return:
"""
from pip.wheel import Wheel
w = Wheel(wheel)
return w.pyversions
def __init__(self, url, comes_from=None, internal=None, trusted=None):
self.url = url
self.comes_from = comes_from
self.internal = internal
self.trusted = trusted
# Set whether it's a wheel
self.wheel = None
if url != Inf and self.splitext()[1] == wheel_ext:
self.wheel = Wheel(self.filename)
def from_line(cls, name, comes_from=None, prereleases=None):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
url = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
if is_url(name):
link = Link(name)
elif (os.path.isdir(path)
and (os.path.sep in name or name.startswith('.'))):
if not is_installable_dir(path):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' not "
"found." % name
)
link = Link(path_to_url(name))
elif is_archive_file(path):
if not os.path.isfile(path):
logger.warn(
'Requirement %r looks like a filename, but the file does '
'not exist',
name
)
link = Link(path_to_url(name))
# If the line has an egg= definition, but isn't editable, pull the
# requirement out. Otherwise, assume the name is the req for the non
# URL/path/archive case.
if link and req is None:
url = link.url_without_fragment
# when fragment is None, this will become an 'unnamed' requirement
req = link.egg_fragment
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', url):
url = path_to_url(os.path.normpath(os.path.abspath(link.path)))
# fail early for invalid or unsupported wheels
if link.ext == wheel_ext:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
if not wheel.supported():
raise UnsupportedWheel(
"%s is not a supported wheel on this platform." %
wheel.filename
)
else:
req = name
return cls(req, comes_from, url=url, prereleases=prereleases)
def optimistic_wheel_search(req, find_links):
assert req_is_pinned(req), req
# this matches the name-munging done in pip.wheel:
reqname = req.project_name.replace('-', '_')
reqname = ignorecase_glob(reqname)
reqname = reqname + '-*.whl'
for findlink in find_links:
if findlink.startswith('file:'):
findlink = findlink[5:]
from os.path import join
findlink = join(findlink, reqname)
logger.debug('wheel glob: %s', findlink)
from glob import glob
for link in glob(findlink):
from pip.index import Link
link = Link('file:' + link)
from pip.wheel import Wheel
wheel = Wheel(link.filename)
logger.debug('Candidate wheel: %s', link.filename)
if wheel.version in req and wheel.supported():
return link
def make_wheels():
"""Build wheels of all installed packages that don't already have one"""
bootstrap()
from pip import get_installed_distributions
from pip.wheel import Wheel
from wheel.install import WheelFile
from wheel.util import matches_requirement
supported_wheels = []
for wheel_file in glob.glob('/tmp/wheelhouse/*.whl'):
wheelobj = Wheel(wheel_file)
if wheelobj.supported():
supported_wheels.append(WheelFile(wheel_file))
for dist in get_installed_distributions():
if dist.project_name == 'kwarter':
continue
distid = '{0.project_name}=={0.version}'.format(dist)
if not matches_requirement(distid, supported_wheels):
call(
['pip', 'wheel', '-w', '/tmp/wheelhouse', '--no-deps', distid])
def optimistic_wheel_search(req, find_links):
from os.path import join, exists
best_version = pkg_resources.parse_version('')
best_link = None
for findlink in find_links:
if findlink.startswith('file:'):
findlink = findlink[5:]
elif not exists(findlink):
assert False, 'findlink not file: coverage: %r' % findlink
continue # TODO: test coverage
# this matches the name-munging done in pip.wheel:
reqname = req.name.replace('-', '_')
reqname = ignorecase_glob(reqname)
reqname = join(findlink, reqname + '-*.whl')
logger.debug('wheel glob: %s', reqname)
from glob import glob
for link in glob(reqname):
from pip.index import Link
link = Link('file://' + link)
from pip.wheel import Wheel
wheel = Wheel(link.filename)
logger.debug('Candidate wheel: %s', link.filename)
if wheel.version not in req.req:
continue
if not wheel.supported():
continue
version = pkg_resources.parse_version(wheel.version)
if version > best_version:
best_version = version
best_link = link
return best_link
def optimistic_wheel_search(req, find_links):
from os.path import join, exists
best_version = pkg_resources.parse_version('')
best_link = None
for findlink in find_links:
if findlink.startswith('file:'):
findlink = findlink[5:]
elif not exists(findlink):
assert False, 'findlink not file: coverage: %r' % findlink
continue # TODO: test coverage
# this matches the name-munging done in pip.wheel:
reqname = req.name.replace('-', '_')
reqname = ignorecase_glob(reqname)
reqname = join(findlink, reqname + '-*.whl')
logger.debug('wheel glob: %s', reqname)
from glob import glob
for link in glob(reqname):
from pip.index import Link
link = Link('file://' + link)
from pip.wheel import Wheel
wheel = Wheel(link.filename)
logger.debug('Candidate wheel: %s', link.filename)
if wheel.version not in req.req:
continue
if not wheel.supported():
continue
version = pkg_resources.parse_version(wheel.version)
if version > best_version:
best_version = version
best_link = link
return best_link
def test_old_pip_and_setuptools(tmpdir, reqs):
"""We should be able to use pip-faster's wheel building even if we have
ancient pip and setuptools.
https://github.com/Yelp/pip-faster/issues/33
"""
tmpdir.chdir()
# 1. Create an empty virtualenv.
# 2. Install old pip/setuptools that don't support wheel building.
# 3. Install pip-faster.
# 4. Install pure-python-package and assert it was wheeled during install.
venv = tmpdir.join('venv')
run('virtualenv', venv.strpath)
# We need to add public PyPI as an extra URL since we're installing
# packages (setuptools and pip) which aren't available from our PyPI fixture.
from os import environ
environ['PIP_EXTRA_INDEX_URL'] = 'https://pypi.python.org/simple/'
try:
pip = venv.join('bin/pip').strpath
for req in reqs:
run(pip, 'install', '--', req)
# wheel needs argparse but it won't get installed
if sys.version_info < (2, 7):
run(pip, 'install', 'argparse')
run(pip, 'install', 'pip-faster==' + __version__)
finally:
del environ['PIP_EXTRA_INDEX_URL']
run(str(venv.join('bin/pip-faster')), 'install', 'pure_python_package')
# it was installed
assert 'pure-python-package==0.2.0' in pip_freeze(str(venv)).split('\n')
# it was wheeled
from pip.wheel import Wheel
wheelhouse = tmpdir.join('home', '.cache', 'pip-faster', 'wheelhouse')
assert 'pure-python-package' in [
Wheel(f.basename).name for f in wheelhouse.listdir()
]
def _link_package_versions(self, link, search):
"""Return an InstallationCandidate or None"""
platform = get_platform()
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
self._log_skipped_link(link, 'not a file')
return
if ext not in SUPPORTED_EXTENSIONS:
self._log_skipped_link(
link, 'unsupported archive format: %s' % ext)
return
if "binary" not in search.formats and ext == wheel_ext:
self._log_skipped_link(
link, 'No binaries permitted for %s' % search.supplied)
return
if "macosx10" in link.path and ext == '.zip':
self._log_skipped_link(link, 'macosx10 one')
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
self._log_skipped_link(link, 'invalid wheel filename')
return
if (pkg_resources.safe_name(wheel.name).lower() !=
search.canonical):
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if not wheel.supported():
self._log_skipped_link(
link, 'it is not compatible with this Python')
return
# This is a dirty hack to prevent installing Binary Wheels from
# PyPI unless it is a Windows or Mac Binary Wheel. This is
# paired with a change to PyPI disabling uploads for the
# same. Once we have a mechanism for enabling support for
# binary wheels on linux that deals with the inherent problems
# of binary distribution this can be removed.
comes_from = getattr(link, "comes_from", None)
if (
(
not platform.startswith('win') and not
platform.startswith('macosx') and not
platform == 'cli'
) and
comes_from is not None and
urllib_parse.urlparse(
comes_from.url
).netloc.endswith(PyPI.netloc)):
if not wheel.supported(tags=supported_tags_noarch):
self._log_skipped_link(
link,
"it is a pypi-hosted binary "
"Wheel on an unsupported platform",
)
return
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != wheel_ext:
self._log_skipped_link(
link, 'No sources permitted for %s' % search.supplied)
return
if not version:
version = egg_info_matches(egg_info, search.supplied, link)
if version is None:
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if (link.internal is not None and not
link.internal and not
normalize_name(search.supplied).lower()
in self.allow_external and not
self.allow_all_external):
# We have a link that we are sure is external, so we should skip
# it unless we are allowing externals
self._log_skipped_link(link, 'it is externally hosted')
self.need_warn_external = True
return
if (link.verifiable is not None and not
link.verifiable and not
(normalize_name(search.supplied).lower()
in self.allow_unverified)):
# We have a link that we are sure we cannot verify its integrity,
# so we should skip it unless we are allowing unsafe installs
# for this requirement.
self._log_skipped_link(
link, 'it is an insecure and unverifiable file')
self.need_warn_unverified = True
return
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
self._log_skipped_link(
link, 'Python version is incorrect')
return
logger.debug('Found link %s, version: %s', link, version)
return InstallationCandidate(search.supplied, version, link)
def add_requirement(self, install_req, parent_req_name=None,
extras_requested=None):
"""Add install_req as a requirement to install.
:param parent_req_name: The name of the requirement that needed this
added. The name is used because when multiple unnamed requirements
resolve to the same name, we could otherwise end up with dependency
links that point outside the Requirements set. parent_req must
already be added. Note that None implies that this is a user
supplied requirement, vs an inferred one.
:param extras_requested: an iterable of extras used to evaluate the
environement markers.
:return: Additional requirements to scan. That is either [] if
the requirement is not applicable, or [install_req] if the
requirement is applicable and has just been added.
"""
name = install_req.name
if not install_req.match_markers(extras_requested):
logger.warning("Ignoring %s: markers '%s' don't match your "
"environment", install_req.name,
install_req.markers)
return []
# This check has to come after we filter requirements with the
# environment markers.
if install_req.link and install_req.link.is_wheel:
wheel = Wheel(install_req.link.filename)
if not wheel.supported():
raise InstallationError(
"%s is not a supported wheel on this platform." %
wheel.filename
)
install_req.as_egg = self.as_egg
install_req.use_user_site = self.use_user_site
install_req.target_dir = self.target_dir
install_req.pycompile = self.pycompile
install_req.is_direct = (parent_req_name is None)
if not name:
# url or path requirement w/o an egg fragment
self.unnamed_requirements.append(install_req)
return [install_req]
else:
try:
existing_req = self.get_requirement(name)
except KeyError:
existing_req = None
if (parent_req_name is None and existing_req and not
existing_req.constraint and
existing_req.extras == install_req.extras and not
existing_req.req.specifier == install_req.req.specifier):
raise InstallationError(
'Double requirement given: %s (already in %s, name=%r)'
% (install_req, existing_req, name))
if not existing_req:
# Add requirement
self.requirements[name] = install_req
# FIXME: what about other normalizations? E.g., _ vs. -?
if name.lower() != name:
self.requirement_aliases[name.lower()] = name
result = [install_req]
else:
# Assume there's no need to scan, and that we've already
# encountered this for scanning.
result = []
if not install_req.constraint and existing_req.constraint:
if (install_req.link and not (existing_req.link and
install_req.link.path == existing_req.link.path)):
self.reqs_to_cleanup.append(install_req)
raise InstallationError(
"Could not satisfy constraints for '%s': "
"installation from path or url cannot be "
"constrained to a version" % name)
# If we're now installing a constraint, mark the existing
# object for real installation.
existing_req.constraint = False
existing_req.extras = tuple(
sorted(set(existing_req.extras).union(
set(install_req.extras))))
logger.debug("Setting %s extras to: %s",
existing_req, existing_req.extras)
# And now we need to scan this.
result = [existing_req]
# Canonicalise to the already-added object for the backref
# check below.
install_req = existing_req
if parent_req_name:
parent_req = self.get_requirement(parent_req_name)
self._dependencies[parent_req].append(install_req)
return result
def from_line(cls,
name,
comes_from=None,
isolated=False,
options=None,
wheel_cache=None,
constraint=False):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
from pip.index import Link
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers = name.split(marker_sep, 1)
markers = markers.strip()
if not markers:
markers = None
else:
markers = Marker(markers)
else:
markers = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras = None
if is_url(name):
link = Link(name)
else:
p, extras = _strip_extras(path)
if (os.path.isdir(p)
and (os.path.sep in name or name.startswith('.'))):
if not is_installable_dir(p):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' "
"not found." % name)
link = Link(path_to_url(p))
elif is_archive_file(p):
if not os.path.isfile(p):
logger.warning(
'Requirement %r looks like a filename, but the '
'file does not exist', name)
link = Link(path_to_url(p))
# it's a local file, dir, or url
if link:
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', link.url):
link = Link(
path_to_url(os.path.normpath(os.path.abspath(link.path))))
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
req = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req = link.egg_fragment
# a requirement specifier
else:
req = name
if extras:
extras = Requirement("placeholder" + extras.lower()).extras
else:
extras = ()
if req is not None:
try:
req = Requirement(req)
except InvalidRequirement:
if os.path.sep in req:
add_msg = "It looks like a path."
add_msg += deduce_helpful_msg(req)
elif '=' in req and not any(op in req for op in operators):
add_msg = "= is not a valid operator. Did you mean == ?"
else:
add_msg = traceback.format_exc()
raise InstallationError("Invalid requirement: '%s'\n%s" %
(req, add_msg))
return cls(
req,
comes_from,
link=link,
markers=markers,
isolated=isolated,
options=options if options else {},
wheel_cache=wheel_cache,
constraint=constraint,
extras=extras,
)
def _link_package_versions(self, link, search_name):
"""
Return an iterable of triples (pkg_resources_version_key,
link, python_version) that can be extracted from the given
link.
Meant to be overridden by subclasses, not called by clients.
"""
platform = get_platform()
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
else:
egg_info, ext = link.splitext()
if not ext:
if link not in self.logged_links:
logger.debug('Skipping link %s; not a file' % link)
self.logged_links.add(link)
return []
if egg_info.endswith('.tar'):
# Special double-extension case:
egg_info = egg_info[:-4]
ext = '.tar' + ext
if ext not in self._known_extensions():
if link not in self.logged_links:
logger.debug(
'Skipping link %s; unknown archive format: %s' %
(link, ext)
)
self.logged_links.add(link)
return []
if "macosx10" in link.path and ext == '.zip':
if link not in self.logged_links:
logger.debug('Skipping link %s; macosx10 one' % (link))
self.logged_links.add(link)
return []
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
logger.debug(
'Skipping %s because the wheel filename is invalid' %
link
)
return []
if wheel.name.lower() != search_name.lower():
logger.debug(
'Skipping link %s; wrong project name (not %s)' %
(link, search_name)
)
return []
if not wheel.supported():
logger.debug(
'Skipping %s because it is not compatible with this '
'Python' % link
)
return []
# This is a dirty hack to prevent installing Binary Wheels from
# PyPI unless it is a Windows or Mac Binary Wheel. This is
# paired with a change to PyPI disabling uploads for the
# same. Once we have a mechanism for enabling support for
# binary wheels on linux that deals with the inherent problems
# of binary distribution this can be removed.
comes_from = getattr(link, "comes_from", None)
if (
(
not platform.startswith('win')
and not platform.startswith('macosx')
)
and comes_from is not None
and urlparse.urlparse(
comes_from.url
).netloc.endswith("pypi.python.org")):
if not wheel.supported(tags=supported_tags_noarch):
logger.debug(
"Skipping %s because it is a pypi-hosted binary "
"Wheel on an unsupported platform" % link
)
return []
version = wheel.version
if not version:
version = self._egg_info_matches(egg_info, search_name, link)
if version is None:
logger.debug(
'Skipping link %s; wrong project name (not %s)' %
(link, search_name)
)
return []
if (link.internal is not None
and not link.internal
and not normalize_name(search_name).lower()
in self.allow_external
and not self.allow_all_external):
# We have a link that we are sure is external, so we should skip
# it unless we are allowing externals
logger.debug("Skipping %s because it is externally hosted." % link)
self.need_warn_external = True
return []
if (link.verifiable is not None
and not link.verifiable
and not (normalize_name(search_name).lower()
in self.allow_unverified)):
# We have a link that we are sure we cannot verify its integrity,
# so we should skip it unless we are allowing unsafe installs
# for this requirement.
logger.debug("Skipping %s because it is an insecure and "
"unverifiable file." % link)
self.need_warn_unverified = True
return []
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
logger.debug(
'Skipping %s because Python version is incorrect' % link
)
return []
logger.debug('Found link %s, version: %s' % (link, version))
return [(
pkg_resources.parse_version(version),
link,
version,
)]
def _link_package_versions(self, link, search_name):
"""
Return an iterable of triples (pkg_resources_version_key,
link, python_version) that can be extracted from the given
link.
Meant to be overridden by subclasses, not called by clients.
"""
platform = get_platform()
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
else:
egg_info, ext = link.splitext()
if not ext:
if link not in self.logged_links:
logger.debug('Skipping link %s; not a file' % link)
self.logged_links.add(link)
return []
if egg_info.endswith('.tar'):
# Special double-extension case:
egg_info = egg_info[:-4]
ext = '.tar' + ext
if ext not in self._known_extensions():
if link not in self.logged_links:
logger.debug(
'Skipping link %s; unknown archive format: %s' %
(link, ext))
self.logged_links.add(link)
return []
if "macosx10" in link.path and ext == '.zip':
if link not in self.logged_links:
logger.debug('Skipping link %s; macosx10 one' % (link))
self.logged_links.add(link)
return []
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
logger.debug(
'Skipping %s because the wheel filename is invalid' %
link)
return []
if wheel.name.lower() != search_name.lower():
logger.debug(
'Skipping link %s; wrong project name (not %s)' %
(link, search_name))
return []
if not wheel.supported():
logger.debug(
'Skipping %s because it is not compatible with this Python'
% link)
return []
# This is a dirty hack to prevent installing Binary Wheels from
# PyPI unless it is a Windows or Mac Binary Wheel. This is
# paired with a change to PyPI disabling uploads for the
# same. Once we have a mechanism for enabling support for binary
# wheels on linux that deals with the inherent problems of
# binary distribution this can be removed.
comes_from = getattr(link, "comes_from", None)
if ((not platform.startswith('win')
and not platform.startswith('macosx'))
and comes_from is not None and urlparse.urlparse(
comes_from.url).netloc.endswith("pypi.python.org")
):
if not wheel.supported(tags=supported_tags_noarch):
logger.debug(
"Skipping %s because it is a pypi-hosted binary "
"Wheel on an unsupported platform" % link)
return []
version = wheel.version
if not version:
version = self._egg_info_matches(egg_info, search_name, link)
if version is None:
logger.debug('Skipping link %s; wrong project name (not %s)' %
(link, search_name))
return []
if (link.internal is not None and not link.internal and
not normalize_name(search_name).lower() in self.allow_external
and not self.allow_all_external):
# We have a link that we are sure is external, so we should skip
# it unless we are allowing externals
logger.debug("Skipping %s because it is externally hosted." % link)
self.need_warn_external = True
return []
if (link.verifiable is not None and not link.verifiable
and not (normalize_name(search_name).lower()
in self.allow_unverified)
and not self.allow_all_unverified):
# We have a link that we are sure we cannot verify it's integrity,
# so we should skip it unless we are allowing unsafe installs
# for this requirement.
logger.debug("Skipping %s because it is an insecure and "
"unverifiable file." % link)
self.need_warn_unverified = True
return []
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
logger.debug(
'Skipping %s because Python version is incorrect' % link)
return []
logger.debug('Found link %s, version: %s' % (link, version))
return [(pkg_resources.parse_version(version), link, version)]
def _link_package_versions(self, link, search_name):
"""Return an InstallationCandidate or None"""
platform = get_platform()
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
else:
egg_info, ext = link.splitext()
if not ext:
if link not in self.logged_links:
logger.debug("Skipping link %s; not a file", link)
self.logged_links.add(link)
return
if egg_info.endswith(".tar"):
# Special double-extension case:
egg_info = egg_info[:-4]
ext = ".tar" + ext
if ext not in self._known_extensions():
if link not in self.logged_links:
logger.debug("Skipping link %s; unknown archive format: %s", link, ext)
self.logged_links.add(link)
return
if "macosx10" in link.path and ext == ".zip":
if link not in self.logged_links:
logger.debug("Skipping link %s; macosx10 one", link)
self.logged_links.add(link)
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
logger.debug("Skipping %s because the wheel filename is invalid", link)
return
if pkg_resources.safe_name(wheel.name).lower() != pkg_resources.safe_name(search_name).lower():
logger.debug("Skipping link %s; wrong project name (not %s)", link, search_name)
return
if not wheel.supported():
logger.debug("Skipping %s because it is not compatible with this " "Python", link)
return
# This is a dirty hack to prevent installing Binary Wheels from
# PyPI unless it is a Windows or Mac Binary Wheel. This is
# paired with a change to PyPI disabling uploads for the
# same. Once we have a mechanism for enabling support for
# binary wheels on linux that deals with the inherent problems
# of binary distribution this can be removed.
comes_from = getattr(link, "comes_from", None)
if (
(not platform.startswith("win") and not platform.startswith("macosx") and not platform == "cli")
and comes_from is not None
and urllib_parse.urlparse(comes_from.url).netloc.endswith(PyPI.netloc)
):
if not wheel.supported(tags=supported_tags_noarch):
logger.debug(
"Skipping %s because it is a pypi-hosted binary " "Wheel on an unsupported platform", link
)
return
version = wheel.version
if not version:
version = self._egg_info_matches(egg_info, search_name, link)
if version is None:
logger.debug("Skipping link %s; wrong project name (not %s)", link, search_name)
return
if (
link.internal is not None
and not link.internal
and not normalize_name(search_name).lower() in self.allow_external
and not self.allow_all_external
):
# We have a link that we are sure is external, so we should skip
# it unless we are allowing externals
logger.debug("Skipping %s because it is externally hosted.", link)
self.need_warn_external = True
return
if (
link.verifiable is not None
and not link.verifiable
and not (normalize_name(search_name).lower() in self.allow_unverified)
):
# We have a link that we are sure we cannot verify its integrity,
# so we should skip it unless we are allowing unsafe installs
# for this requirement.
logger.debug("Skipping %s because it is an insecure and unverifiable file.", link)
self.need_warn_unverified = True
return
match = self._py_version_re.search(version)
if match:
version = version[: match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
logger.debug("Skipping %s because Python version is incorrect", link)
return
logger.debug("Found link %s, version: %s", link, version)
return InstallationCandidate(search_name, version, link)
def from_line(cls,
name,
comes_from=None,
isolated=False,
options=None,
wheel_cache=None,
constraint=False):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
from pip.index import Link
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers = name.split(marker_sep, 1)
markers = markers.strip()
if not markers:
markers = None
else:
markers = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras = None
if is_url(name):
link = Link(name)
else:
p, extras = _strip_extras(path)
if (os.path.isdir(p)
and (os.path.sep in name or name.startswith('.'))):
if not is_installable_dir(p):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' "
"not found." % name)
link = Link(path_to_url(p))
elif is_archive_file(p):
if not os.path.isfile(p):
logger.warning(
'Requirement %r looks like a filename, but the '
'file does not exist', name)
link = Link(path_to_url(p))
# it's a local file, dir, or url
if link:
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', link.url):
link = Link(
path_to_url(os.path.normpath(os.path.abspath(link.path))))
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
if not wheel.supported():
raise UnsupportedWheel(
"%s is not a supported wheel on this platform." %
wheel.filename)
req = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req = link.egg_fragment
# a requirement specifier
else:
req = name
options = options if options else {}
res = cls(req,
comes_from,
link=link,
markers=markers,
isolated=isolated,
options=options,
wheel_cache=wheel_cache,
constraint=constraint)
if extras:
res.extras = pkg_resources.Requirement.parse('__placeholder__' +
extras).extras
return res
def _link_package_versions(self, link, search):
"""Return an InstallationCandidate or None"""
platform = get_platform()
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
self._log_skipped_link(link, 'not a file')
return
if ext not in SUPPORTED_EXTENSIONS:
self._log_skipped_link(link,
'unsupported archive format: %s' % ext)
return
if "binary" not in search.formats and ext == wheel_ext:
self._log_skipped_link(
link, 'No binaries permitted for %s' % search.supplied)
return
if "macosx10" in link.path and ext == '.zip':
self._log_skipped_link(link, 'macosx10 one')
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
self._log_skipped_link(link, 'invalid wheel filename')
return
if (pkg_resources.safe_name(wheel.name).lower() !=
search.canonical):
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if not wheel.supported():
self._log_skipped_link(
link, 'it is not compatible with this Python')
return
# This is a dirty hack to prevent installing Binary Wheels from
# PyPI unless it is a Windows or Mac Binary Wheel. This is
# paired with a change to PyPI disabling uploads for the
# same. Once we have a mechanism for enabling support for
# binary wheels on linux that deals with the inherent problems
# of binary distribution this can be removed.
comes_from = getattr(link, "comes_from", None)
if ((not platform.startswith('win')
and not platform.startswith('macosx')
and not platform == 'cli') and comes_from is not None
and urllib_parse.urlparse(
comes_from.url).netloc.endswith(PyPI.netloc)):
if not wheel.supported(tags=supported_tags_noarch):
self._log_skipped_link(
link,
"it is a pypi-hosted binary "
"Wheel on an unsupported platform",
)
return
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != wheel_ext:
self._log_skipped_link(
link, 'No sources permitted for %s' % search.supplied)
return
if not version:
version = egg_info_matches(egg_info, search.supplied, link)
if version is None:
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if (link.internal is not None and not link.internal
and not normalize_name(search.supplied).lower()
in self.allow_external and not self.allow_all_external):
# We have a link that we are sure is external, so we should skip
# it unless we are allowing externals
self._log_skipped_link(link, 'it is externally hosted')
self.need_warn_external = True
return
if (link.verifiable is not None and not link.verifiable
and not (normalize_name(search.supplied).lower()
in self.allow_unverified)):
# We have a link that we are sure we cannot verify its integrity,
# so we should skip it unless we are allowing unsafe installs
# for this requirement.
self._log_skipped_link(link,
'it is an insecure and unverifiable file')
self.need_warn_unverified = True
return
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
self._log_skipped_link(link, 'Python version is incorrect')
return
logger.debug('Found link %s, version: %s', link, version)
return InstallationCandidate(search.supplied, version, link)
def _link_package_versions(self, link, search):
"""Return an InstallationCandidate or None"""
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
self._log_skipped_link(link, 'not a file')
return
if ext not in SUPPORTED_EXTENSIONS:
self._log_skipped_link(link,
'unsupported archive format: %s' % ext)
return
if "binary" not in search.formats and ext == wheel_ext:
self._log_skipped_link(
link, 'No binaries permitted for %s' % search.supplied)
return
if "macosx10" in link.path and ext == '.zip':
self._log_skipped_link(link, 'macosx10 one')
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
self._log_skipped_link(link, 'invalid wheel filename')
return
if canonicalize_name(wheel.name) != search.canonical:
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if not wheel.supported(self.valid_tags):
self._log_skipped_link(
link, 'it is not compatible with this Python')
return
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != wheel_ext:
self._log_skipped_link(
link, 'No sources permitted for %s' % search.supplied)
return
if not version:
version = egg_info_matches(egg_info, search.supplied, link)
if version is None:
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
self._log_skipped_link(link, 'Python version is incorrect')
return
try:
support_this_python = check_requires_python(link.requires_python)
except specifiers.InvalidSpecifier:
logger.debug("Package %s has an invalid Requires-Python entry: %s",
link.filename, link.requires_python)
support_this_python = True
if not support_this_python:
logger.debug(
"The package %s is incompatible with the python"
"version in use. Acceptable python versions are:%s", link,
link.requires_python)
return
logger.debug('Found link %s, version: %s', link, version)
return InstallationCandidate(search.supplied, version, link)
def _link_package_versions(self, link, search):
"""Return an InstallationCandidate or None"""
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
self._log_skipped_link(link, 'not a file')
return
if ext not in SUPPORTED_EXTENSIONS:
self._log_skipped_link(
link, 'unsupported archive format: %s' % ext)
return
if "binary" not in search.formats and ext == wheel_ext:
self._log_skipped_link(
link, 'No binaries permitted for %s' % search.supplied)
return
if "macosx10" in link.path and ext == '.zip':
self._log_skipped_link(link, 'macosx10 one')
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
self._log_skipped_link(link, 'invalid wheel filename')
return
if canonicalize_name(wheel.name) != search.canonical:
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if not wheel.supported(self.valid_tags):
self._log_skipped_link(
link, 'it is not compatible with this Python')
return
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != wheel_ext:
self._log_skipped_link(
link, 'No sources permitted for %s' % search.supplied)
return
if not version:
version = egg_info_matches(egg_info, search.supplied, link)
if version is None:
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
self._log_skipped_link(
link, 'Python version is incorrect')
return
try:
support_this_python = check_requires_python(link.requires_python)
except specifiers.InvalidSpecifier:
logger.debug("Package %s has an invalid Requires-Python entry: %s",
link.filename, link.requires_python)
support_this_python = True
if not support_this_python:
logger.debug("The package %s is incompatible with the python"
"version in use. Acceptable python versions are:%s",
link, link.requires_python)
return
logger.debug('Found link %s, version: %s', link, version)
return InstallationCandidate(search.supplied, version, link)