class PipAccelerator(object):
"""
Accelerator for pip, the Python package manager.
The :class:`PipAccelerator` class brings together the top level logic of
pip-accel. This top level logic was previously just a collection of
functions but that became more unwieldy as the amount of internal state
increased. The :class:`PipAccelerator` class is intended to make it
(relatively) easy to build something on top of pip and pip-accel.
"""
def __init__(self, config, validate=True):
"""
Initialize the pip accelerator.
:param config: The pip-accel configuration (a :class:`.Config`
object).
:param validate: :data:`True` to run :func:`validate_environment()`,
:data:`False` otherwise.
"""
self.config = config
self.bdists = BinaryDistributionManager(self.config)
if validate:
self.validate_environment()
self.initialize_directories()
self.clean_source_index()
# Keep a list of build directories created by pip-accel.
self.build_directories = []
# We hold on to returned Requirement objects so we can remove their
# temporary sources after pip-accel has finished.
self.reported_requirements = []
def validate_environment(self):
"""
Make sure :data:`sys.prefix` matches ``$VIRTUAL_ENV`` (if defined).
This may seem like a strange requirement to dictate but it avoids hairy
issues like `documented here <https://github.com/paylogic/pip-accel/issues/5>`_.
The most sneaky thing is that ``pip`` doesn't have this problem
(de-facto) because ``virtualenv`` copies ``pip`` wherever it goes...
(``pip-accel`` on the other hand has to be installed by the user).
"""
environment = os.environ.get('VIRTUAL_ENV')
if environment:
if not same_directories(sys.prefix, environment):
raise EnvironmentMismatchError("""
You are trying to install packages in environment #1 which
is different from environment #2 where pip-accel is
installed! Please install pip-accel under environment #1 to
install packages there.
Environment #1: {environment} (defined by $VIRTUAL_ENV)
Environment #2: {prefix} (Python's installation prefix)
""", environment=environment, prefix=sys.prefix)
def initialize_directories(self):
"""Automatically create the local source distribution index directory."""
makedirs(self.config.source_index)
def clean_source_index(self):
"""
Cleanup broken symbolic links in the local source distribution index.
The purpose of this method requires some context to understand. Let me
preface this by stating that I realize I'm probably overcomplicating
things, but I like to preserve forward / backward compatibility when
possible and I don't feel like dropping everyone's locally cached
source distribution archives without a good reason to do so. With that
out of the way:
- Versions of pip-accel based on pip 1.4.x maintained a local source
distribution index based on a directory containing symbolic links
pointing directly into pip's download cache. When files were removed
from pip's download cache, broken symbolic links remained in
pip-accel's local source distribution index directory. This resulted
in very confusing error messages. To avoid this
:func:`clean_source_index()` cleaned up broken symbolic links
whenever pip-accel was about to invoke pip.
- More recent versions of pip (6.x) no longer support the same style of
download cache that contains source distribution archives that can be
re-used directly by pip-accel. To cope with the changes in pip 6.x
new versions of pip-accel tell pip to download source distribution
archives directly into the local source distribution index directory
maintained by pip-accel.
- It is very reasonable for users of pip-accel to have multiple
versions of pip-accel installed on their system (imagine a dozen
Python virtual environments that won't all be updated at the same
time; this is the situation I always find myself in :-). These
versions of pip-accel will be sharing the same local source
distribution index directory.
- All of this leads up to the local source distribution index directory
containing a mixture of symbolic links and regular files with no
obvious way to atomically and gracefully upgrade the local source
distribution index directory while avoiding fights between old and
new versions of pip-accel :-).
- I could of course switch to storing the new local source distribution
index in a differently named directory (avoiding potential conflicts
between multiple versions of pip-accel) but then I would have to
introduce a new configuration option, otherwise everyone who has
configured pip-accel to store its source index in a non-default
location could still be bitten by compatibility issues.
For now I've decided to keep using the same directory for the local
source distribution index and to keep cleaning up broken symbolic
links. This enables cooperating between old and new versions of
pip-accel and avoids trashing user's local source distribution indexes.
The main disadvantage is that pip-accel is still required to clean up
broken symbolic links...
"""
cleanup_timer = Timer()
cleanup_counter = 0
for entry in os.listdir(self.config.source_index):
pathname = os.path.join(self.config.source_index, entry)
if os.path.islink(pathname) and not os.path.exists(pathname):
logger.warn("Cleaning up broken symbolic link: %s", pathname)
os.unlink(pathname)
cleanup_counter += 1
logger.debug("Cleaned up %i broken symbolic links from source index in %s.", cleanup_counter, cleanup_timer)
def install_from_arguments(self, arguments, **kw):
"""
Download, unpack, build and install the specified requirements.
This function is a simple wrapper for :func:`get_requirements()`,
:func:`install_requirements()` and :func:`cleanup_temporary_directories()`
that implements the default behavior of the pip accelerator. If you're
extending or embedding pip-accel you may want to call the underlying
methods instead.
If the requirement set includes wheels and ``setuptools >= 0.8`` is not
yet installed, it will be added to the requirement set and installed
together with the other requirement(s) in order to enable the usage of
distributions installed from wheels (their metadata is different).
:param arguments: The command line arguments to ``pip install ..`` (a
list of strings).
:param kw: Any keyword arguments are passed on to
:func:`install_requirements()`.
:returns: The result of :func:`install_requirements()`.
"""
try:
requirements = self.get_requirements(arguments, use_wheels=self.arguments_allow_wheels(arguments))
have_wheels = any(req.is_wheel for req in requirements)
if have_wheels and not self.setuptools_supports_wheels():
logger.info("Preparing to upgrade to setuptools >= 0.8 to enable wheel support ..")
requirements.extend(self.get_requirements(['setuptools >= 0.8']))
if requirements:
return self.install_requirements(requirements, **kw)
else:
logger.info("Nothing to do! (requirements already installed)")
return 0
finally:
self.cleanup_temporary_directories()
def setuptools_supports_wheels(self):
"""
Check whether setuptools should be upgraded to ``>= 0.8`` for wheel support.
:returns: :data:`True` when setuptools needs to be upgraded, :data:`False` otherwise.
"""
# Don't use pkg_resources.Requirement.parse, to avoid the override
# in distribute, that converts `setuptools' to `distribute'.
setuptools_requirement = next(pkg_resources.parse_requirements('setuptools >= 0.8'))
try:
installed_setuptools = pkg_resources.get_distribution('setuptools')
if installed_setuptools in setuptools_requirement:
# setuptools >= 0.8 is already installed; nothing to do.
return True
except pkg_resources.DistributionNotFound:
pass
# We need to install setuptools >= 0.8.
return False
def get_requirements(self, arguments, max_retries=None, use_wheels=False):
"""
Use pip to download and unpack the requested source distribution archives.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:param max_retries: The maximum number of times that pip will be asked
to download distribution archives (this helps to
deal with intermittent failures). If this is
:data:`None` then :attr:`~.Config.max_retries` is
used.
:param use_wheels: Whether pip and pip-accel are allowed to use wheels_
(:data:`False` by default for backwards compatibility
with callers that use pip-accel as a Python API).
.. warning:: Requirements which are already installed are not included
in the result. If this breaks your use case consider using
pip's ``--ignore-installed`` option.
"""
arguments = self.decorate_arguments(arguments)
with DownloadLogFilter():
# Use a new build directory for each run of get_requirements().
self.create_build_directory()
# Check whether -U or --upgrade was given.
if any(match_option(a, '-U', '--upgrade') for a in arguments):
logger.info("Checking index(es) for new version (-U or --upgrade was given) ..")
else:
# If -U or --upgrade wasn't given and all requirements can be
# satisfied using the archives in pip-accel's local source
# index we don't need pip to connect to PyPI looking for new
# versions (that will just slow us down).
try:
return self.unpack_source_dists(arguments, use_wheels=use_wheels)
except DistributionNotFound:
logger.info("We don't have all distribution archives yet!")
# Get the maximum number of retries from the configuration if the
# caller didn't specify a preference.
if max_retries is None:
max_retries = self.config.max_retries
# If not all requirements are available locally we use pip to
# download the missing source distribution archives from PyPI (we
# retry a couple of times in case pip reports recoverable
# errors).
for i in range(max_retries):
try:
return self.download_source_dists(arguments, use_wheels=use_wheels)
except Exception as e:
if i + 1 < max_retries:
# On all but the last iteration we swallow exceptions
# during downloading.
logger.warning("pip raised exception while downloading distributions: %s", e)
else:
# On the last iteration we don't swallow exceptions
# during downloading because the error reported by pip
# is the most sensible error for us to report.
raise
logger.info("Retrying after pip failed (%i/%i) ..", i + 1, max_retries)
def decorate_arguments(self, arguments):
"""
Change pathnames of local files into ``file://`` URLs with ``#md5=...`` fragments.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:returns: A copy of the command line arguments with pathnames of local
files rewritten to ``file://`` URLs.
When pip-accel calls pip to download missing distribution archives and
the user specified the pathname of a local distribution archive on the
command line, pip will (by default) *not* copy the archive into the
download directory if an archive for the same package name and
version is already present.
This can lead to the confusing situation where the user specifies a
local distribution archive to install, a different (older) archive for
the same package and version is present in the download directory and
`pip-accel` installs the older archive instead of the newer archive.
To avoid this confusing behavior, the :func:`decorate_arguments()`
method rewrites the command line arguments given to ``pip install`` so
that pathnames of local archives are changed into ``file://`` URLs that
include a fragment with the hash of the file's contents. Here's an
example:
- Local pathname: ``/tmp/pep8-1.6.3a0.tar.gz``
- File URL: ``file:///tmp/pep8-1.6.3a0.tar.gz#md5=19cbf0b633498ead63fb3c66e5f1caf6``
When pip fills the download directory and encounters a previously
cached distribution archive it will check the hash, realize the
contents have changed and replace the archive in the download
directory.
"""
arguments = list(arguments)
for i, value in enumerate(arguments):
is_constraint_file = (i >= 1 and match_option(arguments[i - 1], '-c', '--constraint'))
is_requirement_file = (i >= 1 and match_option(arguments[i - 1], '-r', '--requirement'))
if not is_constraint_file and not is_requirement_file and os.path.isfile(value):
arguments[i] = '%s#md5=%s' % (create_file_url(value), hash_files('md5', value))
return arguments
def unpack_source_dists(self, arguments, use_wheels=False):
"""
Find and unpack local source distributions and discover their metadata.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:param use_wheels: Whether pip and pip-accel are allowed to use wheels_
(:data:`False` by default for backwards compatibility
with callers that use pip-accel as a Python API).
:returns: A list of :class:`pip_accel.req.Requirement` objects.
:raises: Any exceptions raised by pip, for example
:exc:`pip.exceptions.DistributionNotFound` when not all
requirements can be satisfied.
This function checks whether there are local source distributions
available for all requirements, unpacks the source distribution
archives and finds the names and versions of the requirements. By using
the ``pip install --download`` command we avoid reimplementing the
following pip features:
- Parsing of ``requirements.txt`` (including recursive parsing).
- Resolution of possibly conflicting pinned requirements.
- Unpacking source distributions in multiple formats.
- Finding the name & version of a given source distribution.
"""
unpack_timer = Timer()
logger.info("Unpacking distribution(s) ..")
with PatchedAttribute(pip_install_module, 'PackageFinder', CustomPackageFinder):
requirements = self.get_pip_requirement_set(arguments, use_remote_index=False, use_wheels=use_wheels)
logger.info("Finished unpacking %s in %s.", pluralize(len(requirements), "distribution"), unpack_timer)
return requirements
def download_source_dists(self, arguments, use_wheels=False):
"""
Download missing source distributions.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:param use_wheels: Whether pip and pip-accel are allowed to use wheels_
(:data:`False` by default for backwards compatibility
with callers that use pip-accel as a Python API).
:raises: Any exceptions raised by pip.
"""
download_timer = Timer()
logger.info("Downloading missing distribution(s) ..")
requirements = self.get_pip_requirement_set(arguments, use_remote_index=True, use_wheels=use_wheels)
logger.info("Finished downloading distribution(s) in %s.", download_timer)
return requirements
def get_pip_requirement_set(self, arguments, use_remote_index, use_wheels=False):
"""
Get the unpacked requirement(s) specified by the caller by running pip.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:param use_remote_index: A boolean indicating whether pip is allowed to
connect to the main package index
(http://pypi.python.org by default).
:param use_wheels: Whether pip and pip-accel are allowed to use wheels_
(:data:`False` by default for backwards compatibility
with callers that use pip-accel as a Python API).
:returns: A :class:`pip.req.RequirementSet` object created by pip.
:raises: Any exceptions raised by pip.
"""
# Compose the pip command line arguments. This is where a lot of the
# core logic of pip-accel is hidden and it uses some esoteric features
# of pip so this method is heavily commented.
command_line = []
# Use `--download' to instruct pip to download requirement(s) into
# pip-accel's local source distribution index directory. This has the
# following documented side effects (see `pip install --help'):
# 1. It disables the installation of requirements (without using the
# `--no-install' option which is deprecated and slated for removal
# in pip 7.x).
# 2. It ignores requirements that are already installed (because
# pip-accel doesn't actually need to re-install requirements that
# are already installed we will have work around this later, but
# that seems fairly simple to do).
command_line.append('--download=%s' % self.config.source_index)
# Use `--find-links' to point pip at pip-accel's local source
# distribution index directory. This ensures that source distribution
# archives are never downloaded more than once (regardless of the HTTP
# cache that was introduced in pip 6.x).
command_line.append('--find-links=%s' % create_file_url(self.config.source_index))
# Use `--no-binary=:all:' to ignore wheel distributions by default in
# order to preserve backwards compatibility with callers that expect a
# requirement set consisting only of source distributions that can be
# converted to `dumb binary distributions'.
if not use_wheels and self.arguments_allow_wheels(arguments):
command_line.append('--no-binary=:all:')
# Use `--no-index' to force pip to only consider source distribution
# archives contained in pip-accel's local source distribution index
# directory. This enables pip-accel to ask pip "Can the local source
# distribution index satisfy all requirements in the given requirement
# set?" which enables pip-accel to keep pip off the internet unless
# absolutely necessary :-).
if not use_remote_index:
command_line.append('--no-index')
# Use `--no-clean' to instruct pip to unpack the source distribution
# archives and *not* clean up the unpacked source distributions
# afterwards. This enables pip-accel to replace pip's installation
# logic with cached binary distribution archives.
command_line.append('--no-clean')
# Use `--build-directory' to instruct pip to unpack the source
# distribution archives to a temporary directory managed by pip-accel.
# We will clean up the build directory when we're done using the
# unpacked source distributions.
command_line.append('--build-directory=%s' % self.build_directory)
# Append the user's `pip install ...' arguments to the command line
# that we just assembled.
command_line.extend(arguments)
logger.info("Executing command: pip install %s", ' '.join(command_line))
# Clear the build directory to prevent PreviousBuildDirError exceptions.
self.clear_build_directory()
# During the pip 6.x upgrade pip-accel switched to using `pip install
# --download' which can produce an interactive prompt as described in
# issue 51 [1]. The documented way [2] to get rid of this interactive
# prompt is pip's --exists-action option, but due to what is most
# likely a bug in pip this doesn't actually work. The environment
# variable $PIP_EXISTS_ACTION does work however, so if the user didn't
# set it we will set a reasonable default for them.
# [1] https://github.com/paylogic/pip-accel/issues/51
# [2] https://pip.pypa.io/en/latest/reference/pip.html#exists-action-option
os.environ.setdefault('PIP_EXISTS_ACTION', 'w')
# Initialize and run the `pip install' command.
command = InstallCommand()
opts, args = command.parse_args(command_line)
if not opts.ignore_installed:
# If the user didn't supply the -I, --ignore-installed option we
# will forcefully disable the option. Refer to the documentation of
# the AttributeOverrides class for further details.
opts = AttributeOverrides(opts, ignore_installed=False)
requirement_set = command.run(opts, args)
# Make sure the output of pip and pip-accel are not intermingled.
sys.stdout.flush()
if requirement_set is None:
raise NothingToDoError("""
pip didn't generate a requirement set, most likely you
specified an empty requirements file?
""")
else:
return self.transform_pip_requirement_set(requirement_set)
def transform_pip_requirement_set(self, requirement_set):
"""
Transform pip's requirement set into one that `pip-accel` can work with.
:param requirement_set: The :class:`pip.req.RequirementSet` object
reported by pip.
:returns: A list of :class:`pip_accel.req.Requirement` objects.
This function converts the :class:`pip.req.RequirementSet` object
reported by pip into a list of :class:`pip_accel.req.Requirement`
objects.
"""
filtered_requirements = []
for requirement in requirement_set.requirements.values():
# The `satisfied_by' property is set by pip when a requirement is
# already satisfied (i.e. a version of the package that satisfies
# the requirement is already installed) and -I, --ignore-installed
# is not used. We filter out these requirements because pip never
# unpacks distributions for these requirements, so pip-accel can't
# do anything useful with such requirements.
if requirement.satisfied_by:
continue
# The `constraint' property marks requirement objects that
# constrain the acceptable version(s) of another requirement but
# don't define a requirement themselves, so we filter them out.
if requirement.constraint:
continue
# All other requirements are reported to callers.
filtered_requirements.append(requirement)
self.reported_requirements.append(requirement)
return sorted([Requirement(self.config, r) for r in filtered_requirements],
key=lambda r: r.name.lower())
def install_requirements(self, requirements, **kw):
"""
Manually install a requirement set from binary and/or wheel distributions.
:param requirements: A list of :class:`pip_accel.req.Requirement` objects.
:param kw: Any keyword arguments are passed on to
:func:`~pip_accel.bdist.BinaryDistributionManager.install_binary_dist()`.
:returns: The number of packages that were just installed (an integer).
"""
install_timer = Timer()
install_types = []
if any(not req.is_wheel for req in requirements):
install_types.append('binary')
if any(req.is_wheel for req in requirements):
install_types.append('wheel')
logger.info("Installing from %s distributions ..", concatenate(install_types))
# Track installed files by default (unless the caller specifically opted out).
kw.setdefault('track_installed_files', True)
num_installed = 0
for requirement in requirements:
# If we're upgrading over an older version, first remove the
# old version to make sure we don't leave files from old
# versions around.
if is_installed(requirement.name):
uninstall(requirement.name)
# When installing setuptools we need to uninstall distribute,
# otherwise distribute will shadow setuptools and all sorts of
# strange issues can occur (e.g. upgrading to the latest
# setuptools to gain wheel support and then having everything
# blow up because distribute doesn't know about wheels).
if requirement.name == 'setuptools' and is_installed('distribute'):
uninstall('distribute')
if requirement.is_editable:
logger.debug("Installing %s in editable form using pip.", requirement)
command = InstallCommand()
opts, args = command.parse_args(['--no-deps', '--editable', requirement.source_directory])
command.run(opts, args)
elif requirement.is_wheel:
logger.info("Installing %s wheel distribution using pip ..", requirement)
wheel_version = pip_wheel_module.wheel_version(requirement.source_directory)
pip_wheel_module.check_compatibility(wheel_version, requirement.name)
requirement.pip_requirement.move_wheel_files(requirement.source_directory)
else:
binary_distribution = self.bdists.get_binary_dist(requirement)
self.bdists.install_binary_dist(binary_distribution, **kw)
num_installed += 1
logger.info("Finished installing %s in %s.",
pluralize(num_installed, "requirement"),
install_timer)
return num_installed
def arguments_allow_wheels(self, arguments):
"""
Check whether the given command line arguments allow the use of wheels.
:param arguments: A list of strings with command line arguments.
:returns: :data:`True` if the arguments allow wheels, :data:`False` if
they disallow wheels.
Contrary to what the name of this method implies its implementation
actually checks if the user hasn't *disallowed* the use of wheels using
the ``--no-use-wheel`` option (deprecated in pip 7.x) or the
``--no-binary=:all:`` option (introduced in pip 7.x). This is because
wheels are "opt out" in recent versions of pip. I just didn't like the
method name ``arguments_dont_disallow_wheels`` ;-).
"""
return not ('--no-use-wheel' in arguments or match_option_with_value(arguments, '--no-binary', ':all:'))
def create_build_directory(self):
"""Create a new build directory for pip to unpack its archives."""
self.build_directories.append(tempfile.mkdtemp(prefix='pip-accel-build-dir-'))
def clear_build_directory(self):
"""Clear the build directory where pip unpacks the source distribution archives."""
stat = os.stat(self.build_directory)
shutil.rmtree(self.build_directory)
os.makedirs(self.build_directory, stat.st_mode)
def cleanup_temporary_directories(self):
"""Delete the build directories and any temporary directories created by pip."""
while self.build_directories:
shutil.rmtree(self.build_directories.pop())
for requirement in self.reported_requirements:
requirement.remove_temporary_source()
@property
def build_directory(self):
"""Get the pathname of the current build directory (a string)."""
if not self.build_directories:
self.create_build_directory()
return self.build_directories[-1]
class PipAccelerator(object):
"""
Accelerator for pip, the Python package manager.
The :py:class:`PipAccelerator` class brings together the top level logic of
pip-accel. This top level logic was previously just a collection of
functions but that became more unwieldy as the amount of internal state
increased. The :py:class:`PipAccelerator` class is intended to make it
(relatively) easy to build something on top of pip and pip-accel.
"""
def __init__(self, config, validate=True):
"""
Initialize the pip accelerator.
:param config: The pip-accel configuration (a :py:class:`.Config`
object).
:param validate: ``True`` to run :py:func:`validate_environment()`,
``False`` otherwise.
"""
self.config = config
self.bdists = BinaryDistributionManager(self.config)
if validate:
self.validate_environment()
self.initialize_directories()
self.clean_source_index()
# Keep a list of build directories created by pip-accel.
self.build_directories = []
# We hold on to returned Requirement objects so we can remove their
# temporary sources after pip-accel has finished.
self.reported_requirements = []
def validate_environment(self):
"""
Make sure :py:data:`sys.prefix` matches ``$VIRTUAL_ENV`` (if defined).
This may seem like a strange requirement to dictate but it avoids hairy
issues like `documented here <https://github.com/paylogic/pip-accel/issues/5>`_.
The most sneaky thing is that ``pip`` doesn't have this problem
(de-facto) because ``virtualenv`` copies ``pip`` wherever it goes...
(``pip-accel`` on the other hand has to be installed by the user).
"""
environment = os.environ.get('VIRTUAL_ENV')
if environment:
try:
# Because os.path.samefile() itself can raise exceptions, e.g.
# when $VIRTUAL_ENV points to a non-existing directory, we use
# an assertion to allow us to use a single code path :-)
assert os.path.samefile(sys.prefix, environment)
except Exception:
raise EnvironmentMismatchError("""
You are trying to install packages in environment #1 which
is different from environment #2 where pip-accel is
installed! Please install pip-accel under environment #1 to
install packages there.
Environment #1: {environment} (defined by $VIRTUAL_ENV)
Environment #2: {prefix} (Python's installation prefix)
""", environment=environment,
prefix=sys.prefix)
def initialize_directories(self):
"""Automatically create the local source distribution index directory."""
makedirs(self.config.source_index)
def clean_source_index(self):
"""
The purpose of this method requires some context to understand. Let me
preface this by stating that I realize I'm probably overcomplicating
things, but I like to preserve forward / backward compatibility when
possible and I don't feel like dropping everyone's locally cached
source distribution archives without a good reason to do so. With that
out of the way:
- Versions of pip-accel based on pip 1.4.x maintained a local source
distribution index based on a directory containing symbolic links
pointing directly into pip's download cache. When files were removed
from pip's download cache, broken symbolic links remained in
pip-accel's local source distribution index directory. This resulted
in very confusing error messages. To avoid this
:py:func:`clean_source_index()` cleaned up broken symbolic links
whenever pip-accel was about to invoke pip.
- More recent versions of pip (6.x) no longer support the same style of
download cache that contains source distribution archives that can be
re-used directly by pip-accel. To cope with the changes in pip 6.x
new versions of pip-accel tell pip to download source distribution
archives directly into the local source distribution index directory
maintained by pip-accel.
- It is very reasonable for users of pip-accel to have multiple
versions of pip-accel installed on their system (imagine a dozen
Python virtual environments that won't all be updated at the same
time; this is the situation I always find myself in :-). These
versions of pip-accel will be sharing the same local source
distribution index directory.
- All of this leads up to the local source distribution index directory
containing a mixture of symbolic links and regular files with no
obvious way to atomically and gracefully upgrade the local source
distribution index directory while avoiding fights between old and
new versions of pip-accel :-).
- I could of course switch to storing the new local source distribution
index in a differently named directory (avoiding potential conflicts
between multiple versions of pip-accel) but then I would have to
introduce a new configuration option, otherwise everyone who has
configured pip-accel to store its source index in a non-default
location could still be bitten by compatibility issues.
For now I've decided to keep using the same directory for the local
source distribution index and to keep cleaning up broken symbolic
links. This enables cooperating between old and new versions of
pip-accel and avoids trashing user's local source distribution indexes.
The main disadvantage is that pip-accel is still required to clean up
broken symbolic links...
"""
cleanup_timer = Timer()
cleanup_counter = 0
for entry in os.listdir(self.config.source_index):
pathname = os.path.join(self.config.source_index, entry)
if os.path.islink(pathname) and not os.path.exists(pathname):
logger.warn("Cleaning up broken symbolic link: %s", pathname)
os.unlink(pathname)
cleanup_counter += 1
logger.debug("Cleaned up %i broken symbolic links from source index in %s.", cleanup_counter, cleanup_timer)
def install_from_arguments(self, arguments, **kw):
"""
Download, unpack, build and install the specified requirements.
This function is a simple wrapper for :py:func:`get_requirements()`,
:py:func:`install_requirements()` and :py:func:`cleanup_temporary_directories()`
that implements the default behavior of the pip accelerator. If you're
extending or embedding pip-accel you may want to call the underlying
methods instead.
If the requirement set includes wheels and ``setuptools >= 0.8`` is not
yet installed, it will be added to the requirement set and installed
together with the other requirement(s) in order to enable the usage of
distributions installed from wheels (their metadata is different).
:param arguments: The command line arguments to ``pip install ..`` (a
list of strings).
:param kw: Any keyword arguments are passed on to
:py:func:`install_requirements()`.
:returns: The result of :py:func:`install_requirements()`.
"""
try:
use_wheels = ('--no-use-wheel' not in arguments)
requirements = self.get_requirements(arguments, use_wheels=use_wheels)
have_wheels = any(req.is_wheel for req in requirements)
if have_wheels and not self.setuptools_supports_wheels():
logger.info("Preparing to upgrade to setuptools >= 0.8 to enable wheel support ..")
requirements.extend(self.get_requirements(['setuptools >= 0.8']))
if requirements:
return self.install_requirements(requirements, **kw)
else:
logger.info("Nothing to do! (requirements already installed)")
return 0
finally:
self.cleanup_temporary_directories()
def setuptools_supports_wheels(self):
"""
Check whether setuptools should be upgraded to ``>= 0.8`` for wheel support.
:returns: ``True`` when setuptools needs to be upgraded, ``False`` otherwise.
"""
# Don't use pkg_resources.Requirement.parse, to avoid the override
# in distribute, that converts `setuptools' to `distribute'.
setuptools_requirement = next(pkg_resources.parse_requirements('setuptools >= 0.8'))
try:
installed_setuptools = pkg_resources.get_distribution('setuptools')
if installed_setuptools in setuptools_requirement:
# setuptools >= 0.8 is already installed; nothing to do.
return True
except pkg_resources.DistributionNotFound:
pass
# We need to install setuptools >= 0.8.
return False
def get_requirements(self, arguments, max_retries=None, use_wheels=False):
"""
Use pip to download and unpack the requested source distribution archives.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:param max_retries: The maximum number of times that pip will be asked
to download distribution archives (this helps to
deal with intermittent failures). If this is
``None`` then :py:attr:`~.Config.max_retries` is
used.
:param use_wheels: Whether pip and pip-accel are allowed to use wheels_
(``False`` by default for backwards compatibility
with callers that use pip-accel as a Python API).
.. warning:: Requirements which are already installed are not included
in the result. If this breaks your use case consider using
pip's ``--ignore-installed`` option.
"""
# Use a new build directory for each run of get_requirements().
self.create_build_directory()
# If all requirements can be satisfied using the archives in
# pip-accel's local source index we don't need pip to connect
# to PyPI looking for new versions (that will slow us down).
try:
return self.unpack_source_dists(arguments, use_wheels=use_wheels)
except DistributionNotFound:
logger.info("We don't have all distribution archives yet!")
# Get the maximum number of retries from the configuration if the
# caller didn't specify a preference.
if max_retries is None:
max_retries = self.config.max_retries
# If not all requirements are available locally we use pip to download
# the missing source distribution archives from PyPI (we retry a couple
# of times in case pip reports recoverable errors).
for i in range(max_retries):
try:
return self.download_source_dists(arguments, use_wheels=use_wheels)
except Exception as e:
if i + 1 < max_retries:
# On all but the last iteration we swallow exceptions
# during downloading.
logger.warning("pip raised exception while downloading distributions: %s", e)
else:
# On the last iteration we don't swallow exceptions
# during downloading because the error reported by pip
# is the most sensible error for us to report.
raise
logger.info("Retrying after pip failed (%i/%i) ..", i + 1, max_retries)
def unpack_source_dists(self, arguments, use_wheels=False):
"""
Check whether there are local source distributions available for all
requirements, unpack the source distribution archives and find the
names and versions of the requirements. By using the ``pip install
--download`` command we avoid reimplementing the following pip
features:
- Parsing of ``requirements.txt`` (including recursive parsing)
- Resolution of possibly conflicting pinned requirements
- Unpacking source distributions in multiple formats
- Finding the name & version of a given source distribution
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:param use_wheels: Whether pip and pip-accel are allowed to use wheels_
(``False`` by default for backwards compatibility
with callers that use pip-accel as a Python API).
:returns: A list of :py:class:`pip_accel.req.Requirement` objects.
:raises: Any exceptions raised by pip, for example
:py:exc:`pip.exceptions.DistributionNotFound` when not all
requirements can be satisfied.
"""
unpack_timer = Timer()
logger.info("Unpacking distribution(s) ..")
with PatchedAttribute(pip_install_module, 'PackageFinder', CustomPackageFinder):
requirements = self.get_pip_requirement_set(arguments, use_remote_index=False, use_wheels=use_wheels)
logger.info("Finished unpacking %s in %s.", pluralize(len(requirements), "distribution"), unpack_timer)
return requirements
def download_source_dists(self, arguments, use_wheels=False):
"""
Download missing source distributions.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:param use_wheels: Whether pip and pip-accel are allowed to use wheels_
(``False`` by default for backwards compatibility
with callers that use pip-accel as a Python API).
:raises: Any exceptions raised by pip.
"""
download_timer = Timer()
logger.info("Downloading missing distribution(s) ..")
requirements = self.get_pip_requirement_set(arguments, use_remote_index=True, use_wheels=use_wheels)
logger.info("Finished downloading distribution(s) in %s.", download_timer)
return requirements
def get_pip_requirement_set(self, arguments, use_remote_index, use_wheels=False):
"""
Get the unpacked requirement(s) specified by the caller by running pip.
:param arguments: The command line arguments to ``pip install ..`` (a
list of strings).
:param use_remote_index: A boolean indicating whether pip is allowed to
connect to the main package index
(http://pypi.python.org by default).
:param use_wheels: Whether pip and pip-accel are allowed to use wheels_
(``False`` by default for backwards compatibility
with callers that use pip-accel as a Python API).
:returns: A :py:class:`pip.req.RequirementSet` object created by pip.
:raises: Any exceptions raised by pip.
"""
# Compose the pip command line arguments. This is where a lot of the
# core logic of pip-accel is hidden and it uses some esoteric features
# of pip so this method is heavily commented.
command_line = []
# Use `--download' to instruct pip to download requirement(s) into
# pip-accel's local source distribution index directory. This has the
# following documented side effects (see `pip install --help'):
# 1. It disables the installation of requirements (without using the
# `--no-install' option which is deprecated and slated for removal
# in pip 7.x).
# 2. It ignores requirements that are already installed (because
# pip-accel doesn't actually need to re-install requirements that
# are already installed we will have work around this later, but
# that seems fairly simple to do).
command_line.append('--download=%s' % self.config.source_index)
# Use `--find-links' to point pip at pip-accel's local source
# distribution index directory. This ensures that source distribution
# archives are never downloaded more than once (regardless of the HTTP
# cache that was introduced in pip 6.x).
command_line.append('--find-links=file://%s' % self.config.source_index)
# Use `--no-use-wheel' to ignore wheel distributions by default in
# order to preserve backwards compatibility with callers that expect a
# requirement set consisting only of source distributions that can be
# converted to `dumb binary distributions'.
if not use_wheels and '--no-use-wheel' not in arguments:
command_line.append('--no-use-wheel')
# Use `--no-index' to force pip to only consider source distribution
# archives contained in pip-accel's local source distribution index
# directory. This enables pip-accel to ask pip "Can the local source
# distribution index satisfy all requirements in the given requirement
# set?" which enables pip-accel to keep pip off the internet unless
# absolutely necessary :-).
if not use_remote_index:
command_line.append('--no-index')
# Use `--no-clean' to instruct pip to unpack the source distribution
# archives and *not* clean up the unpacked source distributions
# afterwards. This enables pip-accel to replace pip's installation
# logic with cached binary distribution archives.
command_line.append('--no-clean')
# Use `--build-directory' to instruct pip to unpack the source
# distribution archives to a temporary directory managed by pip-accel.
# We will clean up the build directory when we're done using the
# unpacked source distributions.
command_line.append('--build-directory=%s' % self.build_directory)
# Append the user's `pip install ...' arguments to the command line
# that we just assembled.
command_line.extend(arguments)
logger.info("Executing command: pip install %s", ' '.join(command_line))
# Clear the build directory to prevent PreviousBuildDirError exceptions.
self.clear_build_directory()
# During the pip 6.x upgrade pip-accel switched to using `pip install
# --download' which can produce an interactive prompt as described in
# issue 51 [1]. The documented way [2] to get rid of this interactive
# prompt is pip's --exists-action option, but due to what is most
# likely a bug in pip this doesn't actually work. The environment
# variable $PIP_EXISTS_ACTION does work however, so if the user didn't
# set it we will set a reasonable default for them.
# [1] https://github.com/paylogic/pip-accel/issues/51
# [2] https://pip.pypa.io/en/latest/reference/pip.html#exists-action-option
os.environ.setdefault('PIP_EXISTS_ACTION', 'w')
# Initialize and run the `pip install' command.
command = InstallCommand()
opts, args = command.parse_args(command_line)
if not opts.ignore_installed:
# If the user didn't supply the -I, --ignore-installed option we
# will forcefully disable the option. Refer to the documentation of
# the AttributeOverrides class for further details.
opts = AttributeOverrides(opts, ignore_installed=False)
requirement_set = command.run(opts, args)
# Make sure the output of pip and pip-accel are not intermingled.
sys.stdout.flush()
if requirement_set is None:
raise NothingToDoError("""
pip didn't generate a requirement set, most likely you
specified an empty requirements file?
""")
else:
return self.transform_pip_requirement_set(requirement_set)
def transform_pip_requirement_set(self, requirement_set):
"""
Convert the :py:class:`pip.req.RequirementSet` object reported by pip
into a list of :py:class:`pip_accel.req.Requirement` objects.
:param requirement_set: The :py:class:`pip.req.RequirementSet` object
reported by pip.
:returns: A list of :py:class:`pip_accel.req.Requirement` objects.
"""
filtered_requirements = []
for requirement in requirement_set.requirements.values():
# The `satisfied_by' property is set by pip when a requirement is
# already satisfied (i.e. a version of the package that satisfies
# the requirement is already installed) and -I, --ignore-installed
# is not used. We filter out these requirements because pip never
# unpacks distributions for these requirements, so pip-accel can't
# do anything useful with such requirements.
if not requirement.satisfied_by:
filtered_requirements.append(requirement)
self.reported_requirements.append(requirement)
return sorted([Requirement(self.config, r) for r in filtered_requirements],
key=lambda r: r.name.lower())
def install_requirements(self, requirements, **kw):
"""
Manually install a requirement set from binary and/or wheel distributions.
:param requirements: A list of :py:class:`pip_accel.req.Requirement` objects.
:param kw: Any keyword arguments are passed on to
:py:func:`~pip_accel.bdist.BinaryDistributionManager.install_binary_dist()`.
:returns: The number of packages that were just installed (an integer).
"""
install_timer = Timer()
install_types = []
if any(not req.is_wheel for req in requirements):
install_types.append('binary')
if any(req.is_wheel for req in requirements):
install_types.append('wheel')
logger.info("Installing from %s distributions ..", concatenate(install_types))
# Track installed files by default (unless the caller specifically opted out).
kw.setdefault('track_installed_files', True)
num_installed = 0
for requirement in requirements:
# If we're upgrading over an older version, first remove the
# old version to make sure we don't leave files from old
# versions around.
if is_installed(requirement.name):
uninstall(requirement.name)
# When installing setuptools we need to uninstall distribute,
# otherwise distribute will shadow setuptools and all sorts of
# strange issues can occur (e.g. upgrading to the latest
# setuptools to gain wheel support and then having everything
# blow up because distribute doesn't know about wheels).
if requirement.name == 'setuptools' and is_installed('distribute'):
uninstall('distribute')
if requirement.is_editable:
logger.debug("Installing %s in editable form using pip.", requirement)
command = InstallCommand()
opts, args = command.parse_args(['--no-deps', '--editable', requirement.source_directory])
command.run(opts, args)
elif requirement.is_wheel:
logger.info("Installing %s wheel distribution using pip ..", requirement)
wheel_version = pip_wheel_module.wheel_version(requirement.source_directory)
pip_wheel_module.check_compatibility(wheel_version, requirement.name)
requirement.pip_requirement.move_wheel_files(requirement.source_directory)
else:
binary_distribution = self.bdists.get_binary_dist(requirement)
self.bdists.install_binary_dist(binary_distribution, **kw)
num_installed += 1
logger.info("Finished installing %s in %s.",
pluralize(num_installed, "requirement"),
install_timer)
return num_installed
def create_build_directory(self):
"""
Create a new build directory for pip to unpack its archives.
"""
self.build_directories.append(tempfile.mkdtemp(prefix='pip-accel-build-dir-'))
def clear_build_directory(self):
"""Clear the build directory where pip unpacks the source distribution archives."""
stat = os.stat(self.build_directory)
shutil.rmtree(self.build_directory)
os.makedirs(self.build_directory, stat.st_mode)
def cleanup_temporary_directories(self):
"""Delete the build directories and any temporary directories created by pip."""
while self.build_directories:
shutil.rmtree(self.build_directories.pop())
for requirement in self.reported_requirements:
requirement.remove_temporary_source()
@property
def build_directory(self):
"""Get the pathname of the current build directory (a string)."""
if not self.build_directories:
self.create_build_directory()
return self.build_directories[-1]
class PipAccelerator(object):
"""
Accelerator for pip, the Python package manager.
The :py:class:`PipAccelerator` class brings together the top level logic of
pip-accel. This top level logic was previously just a collection of
functions but that became more unwieldy as the amount of internal state
increased. The :py:class:`PipAccelerator` class is intended to make it
(relatively) easy to build something on top of pip and pip-accel.
"""
def __init__(self, config, validate=True):
"""
Initialize the pip accelerator.
:param config: The pip-accel configuration (a :py:class:`.Config`
object).
:param validate: ``True`` to run :py:func:`validate_environment()`,
``False`` otherwise.
"""
self.config = config
self.bdists = BinaryDistributionManager(self.config)
if validate:
self.validate_environment()
self.initialize_directories()
self.clean_source_index()
self.update_source_index()
# Create a temporary directory for pip to unpack its archives.
self.build_directory = tempfile.mkdtemp()
# We hold on to returned Requirement objects so we can remove their
# temporary sources after pip-accel has finished.
self.reported_requirements = []
def validate_environment(self):
"""
Make sure :py:data:`sys.prefix` matches ``$VIRTUAL_ENV`` (if defined).
This may seem like a strange requirement to dictate but it avoids hairy
issues like `documented here <https://github.com/paylogic/pip-accel/issues/5>`_.
The most sneaky thing is that ``pip`` doesn't have this problem
(de-facto) because ``virtualenv`` copies ``pip`` wherever it goes...
(``pip-accel`` on the other hand has to be installed by the user).
"""
environment = os.environ.get('VIRTUAL_ENV')
if environment:
try:
# Because os.path.samefile() itself can raise exceptions, e.g.
# when $VIRTUAL_ENV points to a non-existing directory, we use
# an assertion to allow us to use a single code path :-)
assert os.path.samefile(sys.prefix, environment)
except Exception:
raise EnvironmentMismatchError("""
You are trying to install packages in environment #1 which
is different from environment #2 where pip-accel is
installed! Please install pip-accel under environment #1 to
install packages there.
Environment #1: {environment} (defined by $VIRTUAL_ENV)
Environment #2: {prefix} (Python's installation prefix)
""", environment=environment,
prefix=sys.prefix)
def initialize_directories(self):
"""Automatically create the directories for the download cache and the source index."""
for directory in [self.config.download_cache, self.config.source_index]:
makedirs(directory)
def clean_source_index(self):
"""
When files are removed from pip's download cache, broken symbolic links
remain in pip-accel's source index directory. This results in very
confusing error messages. To avoid this we cleanup broken symbolic
links before every run.
"""
cleanup_timer = Timer()
cleanup_counter = 0
for entry in os.listdir(self.config.source_index):
pathname = os.path.join(self.config.source_index, entry)
if os.path.islink(pathname) and not os.path.exists(pathname):
logger.warn("Cleaning up broken symbolic link: %s", pathname)
os.unlink(pathname)
cleanup_counter += 1
logger.debug("Cleaned up %i broken symbolic links from source index in %s.", cleanup_counter, cleanup_timer)
def update_source_index(self):
"""
Link newly downloaded source distributions found in pip's download
cache directory into pip-accel's local source index directory using
symbolic links.
"""
update_timer = Timer()
update_counter = 0
for download_name in os.listdir(self.config.download_cache):
download_path = os.path.join(self.config.download_cache, download_name)
if os.path.isfile(download_path):
url = unquote(download_name)
if not url.endswith('.content-type'):
components = urlparse(url)
original_name = os.path.basename(components.path)
modified_name = add_archive_extension(download_path, original_name)
archive_path = os.path.join(self.config.source_index, modified_name)
if not os.path.isfile(archive_path):
logger.debug("Linking files:")
logger.debug(" - Source: %s", download_path)
logger.debug(" - Target: %s", archive_path)
os.symlink(download_path, archive_path)
logger.debug("Added %i symbolic links to source index in %s.", update_counter, update_timer)
def install_from_arguments(self, arguments, **kw):
"""
Download, unpack, build and install the specified requirements.
This function is a simple wrapper for :py:func:`get_requirements()`,
:py:func:`install_requirements()` and :py:func:`cleanup_temporary_directories()`
that implements the default behavior of the pip accelerator. If you're
extending or embedding pip-accel you may want to call the underlying
methods instead.
:param arguments: The command line arguments to ``pip install ..`` (a
list of strings).
:param kw: Any keyword arguments are passed on to
:py:func:`install_requirements()`.
"""
requirements = self.get_requirements(arguments)
self.install_requirements(requirements, **kw)
self.cleanup_temporary_directories()
def get_requirements(self, arguments, max_retries=10):
"""
Use pip to download and unpack the requested source distribution archives.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:param max_retries: The maximum number of times that pip will be asked
to download source distribution archives (this
helps to deal with intermittent failures).
"""
# If all requirements can be satisfied using the archives in
# pip-accel's local source index we don't need pip to connect
# to PyPI looking for new versions (that will slow us down).
try:
return self.unpack_source_dists(arguments)
except DistributionNotFound:
logger.info("We don't have all source distribution archives yet!")
# If not all requirements are available locally we use pip to download
# the missing source distribution archives from PyPI (we retry a couple
# of times in case pip reports recoverable errors).
for i in range(max_retries):
try:
return self.download_source_dists(arguments)
except Exception as e:
if i + 1 < max_retries:
# On all but the last iteration we swallow exceptions
# during downloading.
logger.warning("pip raised exception while downloading source distributions: %s", e)
else:
# On the last iteration we don't swallow exceptions
# during downloading because the error reported by pip
# is the most sensible error for us to report.
raise
logger.info("Retrying after pip failed (%i/%i) ..", i + 1, max_retries)
def unpack_source_dists(self, arguments):
"""
Check whether there are local source distributions available for all
requirements, unpack the source distribution archives and find the
names and versions of the requirements. By using the ``pip install
--no-install`` command we avoid reimplementing the following pip
features:
- Parsing of ``requirements.txt`` (including recursive parsing)
- Resolution of possibly conflicting pinned requirements
- Unpacking source distributions in multiple formats
- Finding the name & version of a given source distribution
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:returns: A list of :py:class:`pip_accel.req.Requirement` objects.
:raises: Any exceptions raised by pip, for example
:py:exc:`pip.exceptions.DistributionNotFound` when not all
requirements can be satisfied.
"""
unpack_timer = Timer()
logger.info("Unpacking source distribution(s) ..")
# Install our custom package finder to force --no-index behavior.
original_package_finder = pip_index_module.PackageFinder
pip_install_module.PackageFinder = CustomPackageFinder
try:
requirements = self.get_pip_requirement_set(arguments, use_remote_index=False)
logger.info("Finished unpacking %s in %s.",
pluralize(len(requirements), "source distribution"),
unpack_timer)
return requirements
finally:
# Make sure to remove our custom package finder.
pip_install_module.PackageFinder = original_package_finder
def download_source_dists(self, arguments):
"""
Download missing source distributions.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:raises: Any exceptions raised by pip.
"""
try:
download_timer = Timer()
logger.info("Downloading missing source distribution(s) ..")
requirements = self.get_pip_requirement_set(arguments, use_remote_index=True)
logger.info("Finished downloading source distribution(s) in %s.", download_timer)
return requirements
finally:
# Always update the local source index directory (even if pip
# reported errors) because we never want to download an archive
# more than once.
self.update_source_index()
def get_pip_requirement_set(self, arguments, use_remote_index):
"""
Get the unpacked requirement(s) specified by the caller by running pip.
:param arguments: The command line arguments to ``pip install ..`` (a
list of strings).
:param use_remote_index: A boolean indicating whether pip is allowed to
connect to the main package index
(http://pypi.python.org by default).
:returns: A :py:class:`pip.req.RequirementSet` object created by pip.
:raises: Any exceptions raised by pip.
"""
# Compose the pip command line arguments.
command_line = ['pip', 'install', '--no-install']
if use_remote_index:
command_line.append('--download-cache=%s' % self.config.download_cache)
else:
command_line.append('--no-index')
command_line.extend([
'--find-links=file://%s' % self.config.source_index,
'--build-directory=%s' % self.build_directory,
])
command_line.extend(arguments)
logger.info("Executing command: %s", ' '.join(command_line))
# Clear the build directory to prevent PreviousBuildDirError exceptions.
self.clear_build_directory()
# pip 1.4 has some global state in its command line parser (which we
# use) and this can causes problems when we invoke more than one
# InstallCommand in the same process. Here's a workaround.
requirements_option.default = []
# Parse the command line arguments so we can pass the resulting parser
# object to InstallCommand.
cmd_name, options, args, parser = parseopts(command_line[1:])
# Initialize our custom InstallCommand.
pip = CustomInstallCommand(parser)
# Run the `pip install ...' command.
exit_status = pip.main(args[1:], options)
# Make sure the output of pip and pip-accel are not intermingled.
sys.stdout.flush()
# If our custom install command intercepted an exception we re-raise it
# after the local source index has been updated.
if exit_status != SUCCESS:
raise pip.intercepted_exception
return self.transform_pip_requirement_set(pip.requirement_set)
def transform_pip_requirement_set(self, requirement_set):
"""
Convert the :py:class:`pip.req.RequirementSet` object reported by pip
into a list of :py:class:`pip_accel.req.Requirement` objects.
.. warning:: Requirements which are already installed are not included
in the result because pip never creates unpacked source
distribution directories for these requirements. If this
breaks your use case consider looking into pip's
``--ignore-installed`` option or file a bug report against
pip-accel to force me to find a better way.
:param requirement_set: The :py:class:`pip.req.RequirementSet` object
reported by pip.
:returns: A list of :py:class:`pip_accel.req.Requirement` objects.
"""
filtered_requirements = []
for requirement in requirement_set.requirements.values():
if requirement.satisfied_by:
logger.info("Requirement already satisfied: %s.", requirement)
else:
filtered_requirements.append(requirement)
self.reported_requirements.append(requirement)
return sorted([Requirement(r) for r in filtered_requirements],
key=lambda r: r.name.lower())
def install_requirements(self, requirements, **kw):
"""
Manually install all requirements from binary distributions.
:param requirements: A list of :py:class:`pip_accel.req.Requirement` objects.
:param kw: Any keyword arguments are passed on to
:py:func:`~pip_accel.bdist.BinaryDistributionManager.install_binary_dist()`.
"""
install_timer = Timer()
logger.info("Installing from binary distributions ..")
pip = os.path.join(sys.prefix, 'bin', 'pip')
for requirement in requirements:
if run('{pip} uninstall --yes {package} >/dev/null 2>&1', pip=pip, package=requirement.name):
logger.info("Uninstalled previously installed package %s.", requirement.name)
if requirement.is_editable:
logger.debug("Installing %s (%s) in editable form using pip.", requirement.name, requirement.version)
if not run('{pip} install --no-deps --editable {url} >/dev/null 2>&1', pip=pip, url=requirement.url):
msg = "Failed to install %s (%s) in editable form!"
raise Exception(msg % (requirement.name, requirement.version))
else:
binary_distribution = self.bdists.get_binary_dist(requirement)
self.bdists.install_binary_dist(binary_distribution, **kw)
logger.info("Finished installing %s in %s.",
pluralize(len(requirements), "requirement"),
install_timer)
def clear_build_directory(self):
"""Clear the build directory where pip unpacks the source distribution archives."""
stat = os.stat(self.build_directory)
shutil.rmtree(self.build_directory)
os.makedirs(self.build_directory, stat.st_mode)
def cleanup_temporary_directories(self):
"""Delete the build directory and any temporary directories created by pip."""
shutil.rmtree(self.build_directory)
for requirement in self.reported_requirements:
requirement.remove_temporary_source()
class PipAccelerator(object):
"""
Accelerator for pip, the Python package manager.
The :py:class:`PipAccelerator` class brings together the top level logic of
pip-accel. This top level logic was previously just a collection of
functions but that became more unwieldy as the amount of internal state
increased. The :py:class:`PipAccelerator` class is intended to make it
(relatively) easy to build something on top of pip and pip-accel.
"""
def __init__(self, config, validate=True):
"""
Initialize the pip accelerator.
:param config: The pip-accel configuration (a :py:class:`.Config`
object).
:param validate: ``True`` to run :py:func:`validate_environment()`,
``False`` otherwise.
"""
self.config = config
self.bdists = BinaryDistributionManager(self.config)
if validate:
self.validate_environment()
self.initialize_directories()
self.clean_source_index()
self.update_source_index()
# Create a temporary directory for pip to unpack its archives.
self.build_directory = tempfile.mkdtemp()
# We hold on to returned Requirement objects so we can remove their
# temporary sources after pip-accel has finished.
self.reported_requirements = []
def validate_environment(self):
"""
Make sure :py:data:`sys.prefix` matches ``$VIRTUAL_ENV`` (if defined).
This may seem like a strange requirement to dictate but it avoids hairy
issues like `documented here <https://github.com/paylogic/pip-accel/issues/5>`_.
The most sneaky thing is that ``pip`` doesn't have this problem
(de-facto) because ``virtualenv`` copies ``pip`` wherever it goes...
(``pip-accel`` on the other hand has to be installed by the user).
"""
environment = os.environ.get('VIRTUAL_ENV')
if environment:
try:
# Because os.path.samefile() itself can raise exceptions, e.g.
# when $VIRTUAL_ENV points to a non-existing directory, we use
# an assertion to allow us to use a single code path :-)
assert os.path.samefile(sys.prefix, environment)
except Exception:
raise EnvironmentMismatchError("""
You are trying to install packages in environment #1 which
is different from environment #2 where pip-accel is
installed! Please install pip-accel under environment #1 to
install packages there.
Environment #1: {environment} (defined by $VIRTUAL_ENV)
Environment #2: {prefix} (Python's installation prefix)
""", environment=environment,
prefix=sys.prefix)
def initialize_directories(self):
"""Automatically create the directories for the download cache and the source index."""
for directory in [self.config.download_cache, self.config.source_index]:
makedirs(directory)
def clean_source_index(self):
"""
When files are removed from pip's download cache, broken symbolic links
remain in pip-accel's source index directory. This results in very
confusing error messages. To avoid this we cleanup broken symbolic
links before every run.
"""
cleanup_timer = Timer()
cleanup_counter = 0
for entry in os.listdir(self.config.source_index):
pathname = os.path.join(self.config.source_index, entry)
if os.path.islink(pathname) and not os.path.exists(pathname):
logger.warn("Cleaning up broken symbolic link: %s", pathname)
os.unlink(pathname)
cleanup_counter += 1
logger.debug("Cleaned up %i broken symbolic links from source index in %s.", cleanup_counter, cleanup_timer)
def update_source_index(self):
"""
Link newly downloaded source distributions found in pip's download
cache directory into pip-accel's local source index directory using
symbolic links.
"""
update_timer = Timer()
update_counter = 0
for download_name in os.listdir(self.config.download_cache):
download_path = os.path.join(self.config.download_cache, download_name)
if os.path.isfile(download_path):
url = unquote(download_name)
if not url.endswith('.content-type'):
components = urlparse(url)
original_name = os.path.basename(components.path)
modified_name = add_archive_extension(download_path, original_name)
archive_path = os.path.join(self.config.source_index, modified_name)
if not os.path.isfile(archive_path):
logger.debug("Linking files:")
logger.debug(" - Source: %s", download_path)
logger.debug(" - Target: %s", archive_path)
os.symlink(download_path, archive_path)
logger.debug("Added %i symbolic links to source index in %s.", update_counter, update_timer)
def install_from_arguments(self, arguments, **kw):
"""
Download, unpack, build and install the specified requirements.
This function is a simple wrapper for :py:func:`get_requirements()`,
:py:func:`install_requirements()` and :py:func:`cleanup_temporary_directories()`
that implements the default behavior of the pip accelerator. If you're
extending or embedding pip-accel you may want to call the underlying
methods instead.
:param arguments: The command line arguments to ``pip install ..`` (a
list of strings).
:param kw: Any keyword arguments are passed on to
:py:func:`install_requirements()`.
"""
requirements = self.get_requirements(arguments, self.config.max_retries)
self.install_requirements(requirements, **kw)
self.cleanup_temporary_directories()
def get_requirements(self, arguments, max_retries=10):
"""
Use pip to download and unpack the requested source distribution archives.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:param max_retries: The maximum number of times that pip will be asked
to download source distribution archives (this
helps to deal with intermittent failures).
"""
# If all requirements can be satisfied using the archives in
# pip-accel's local source index we don't need pip to connect
# to PyPI looking for new versions (that will slow us down).
try:
return self.unpack_source_dists(arguments)
except DistributionNotFound:
logger.info("We don't have all source distribution archives yet!")
# If not all requirements are available locally we use pip to download
# the missing source distribution archives from PyPI (we retry a couple
# of times in case pip reports recoverable errors).
for i in range(max_retries):
try:
return self.download_source_dists(arguments)
except Exception as e:
if i + 1 < max_retries:
# On all but the last iteration we swallow exceptions
# during downloading.
logger.warning("pip raised exception while downloading source distributions: %s", e)
else:
# On the last iteration we don't swallow exceptions
# during downloading because the error reported by pip
# is the most sensible error for us to report.
raise
logger.info("Retrying after pip failed (%i/%i) ..", i + 1, max_retries)
def unpack_source_dists(self, arguments):
"""
Check whether there are local source distributions available for all
requirements, unpack the source distribution archives and find the
names and versions of the requirements. By using the ``pip install
--no-install`` command we avoid reimplementing the following pip
features:
- Parsing of ``requirements.txt`` (including recursive parsing)
- Resolution of possibly conflicting pinned requirements
- Unpacking source distributions in multiple formats
- Finding the name & version of a given source distribution
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:returns: A list of :py:class:`pip_accel.req.Requirement` objects.
:raises: Any exceptions raised by pip, for example
:py:exc:`pip.exceptions.DistributionNotFound` when not all
requirements can be satisfied.
"""
unpack_timer = Timer()
logger.info("Unpacking source distribution(s) ..")
# Install our custom package finder to force --no-index behavior.
original_package_finder = pip_index_module.PackageFinder
pip_install_module.PackageFinder = CustomPackageFinder
try:
requirements = self.get_pip_requirement_set(arguments, use_remote_index=False)
logger.info("Finished unpacking %s in %s.",
pluralize(len(requirements), "source distribution"),
unpack_timer)
return requirements
finally:
# Make sure to remove our custom package finder.
pip_install_module.PackageFinder = original_package_finder
def download_source_dists(self, arguments):
"""
Download missing source distributions.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:raises: Any exceptions raised by pip.
"""
try:
download_timer = Timer()
logger.info("Downloading missing source distribution(s) ..")
requirements = self.get_pip_requirement_set(arguments, use_remote_index=True)
logger.info("Finished downloading source distribution(s) in %s.", download_timer)
return requirements
finally:
# Always update the local source index directory (even if pip
# reported errors) because we never want to download an archive
# more than once.
self.update_source_index()
def get_pip_requirement_set(self, arguments, use_remote_index):
"""
Get the unpacked requirement(s) specified by the caller by running pip.
:param arguments: The command line arguments to ``pip install ..`` (a
list of strings).
:param use_remote_index: A boolean indicating whether pip is allowed to
connect to the main package index
(http://pypi.python.org by default).
:returns: A :py:class:`pip.req.RequirementSet` object created by pip.
:raises: Any exceptions raised by pip.
"""
# Compose the pip command line arguments.
command_line = ['pip', 'install', '--no-install']
if use_remote_index:
command_line.append('--download-cache=%s' % self.config.download_cache)
else:
command_line.append('--no-index')
command_line.extend([
'--find-links=file://%s' % self.config.source_index,
'--build-directory=%s' % self.build_directory,
])
command_line.extend(arguments)
logger.info("Executing command: %s", ' '.join(command_line))
# Clear the build directory to prevent PreviousBuildDirError exceptions.
self.clear_build_directory()
# pip 1.4 has some global state in its command line parser (which we
# use) and this can causes problems when we invoke more than one
# InstallCommand in the same process. Here's a workaround.
requirements_option.default = []
# Parse the command line arguments so we can pass the resulting parser
# object to InstallCommand.
cmd_name, options, args, parser = parseopts(command_line[1:])
# Initialize our custom InstallCommand.
pip = CustomInstallCommand(parser)
# Run the `pip install ...' command.
exit_status = pip.main(args[1:], options)
# Make sure the output of pip and pip-accel are not intermingled.
sys.stdout.flush()
# If our custom install command intercepted an exception we re-raise it
# after the local source index has been updated.
if exit_status != SUCCESS:
raise pip.intercepted_exception
elif pip.requirement_set is None:
raise NothingToDoError("""
pip didn't generate a requirement set, most likely you
specified an empty requirements file?
""")
else:
return self.transform_pip_requirement_set(pip.requirement_set)
def transform_pip_requirement_set(self, requirement_set):
"""
Convert the :py:class:`pip.req.RequirementSet` object reported by pip
into a list of :py:class:`pip_accel.req.Requirement` objects.
.. warning:: Requirements which are already installed are not included
in the result because pip never creates unpacked source
distribution directories for these requirements. If this
breaks your use case consider looking into pip's
``--ignore-installed`` option or file a bug report against
pip-accel to force me to find a better way.
:param requirement_set: The :py:class:`pip.req.RequirementSet` object
reported by pip.
:returns: A list of :py:class:`pip_accel.req.Requirement` objects.
"""
filtered_requirements = []
for requirement in requirement_set.requirements.values():
if requirement.satisfied_by:
logger.info("Requirement already satisfied: %s.", requirement)
else:
filtered_requirements.append(requirement)
self.reported_requirements.append(requirement)
return sorted([Requirement(r) for r in filtered_requirements],
key=lambda r: r.name.lower())
def install_requirements(self, requirements, **kw):
"""
Manually install all requirements from binary distributions.
:param requirements: A list of :py:class:`pip_accel.req.Requirement` objects.
:param kw: Any keyword arguments are passed on to
:py:func:`~pip_accel.bdist.BinaryDistributionManager.install_binary_dist()`.
"""
install_timer = Timer()
logger.info("Installing from binary distributions ..")
for requirement in requirements:
if run('pip uninstall --yes {package}', package=requirement.name):
logger.info("Uninstalled previously installed package %s.", requirement.name)
if requirement.is_editable:
logger.debug("Installing %s (%s) in editable form using pip.", requirement.name, requirement.version)
if not run('pip install --no-deps --editable {url}', url=requirement.url):
msg = "Failed to install %s (%s) in editable form!"
raise Exception(msg % (requirement.name, requirement.version))
else:
binary_distribution = self.bdists.get_binary_dist(requirement)
self.bdists.install_binary_dist(binary_distribution, **kw)
logger.info("Finished installing %s in %s.",
pluralize(len(requirements), "requirement"),
install_timer)
def clear_build_directory(self):
"""Clear the build directory where pip unpacks the source distribution archives."""
stat = os.stat(self.build_directory)
shutil.rmtree(self.build_directory)
os.makedirs(self.build_directory, stat.st_mode)
def cleanup_temporary_directories(self):
"""Delete the build directory and any temporary directories created by pip."""
shutil.rmtree(self.build_directory)
for requirement in self.reported_requirements:
requirement.remove_temporary_source()