def main():
"""The command line interface for the ``pip-accel`` program."""
arguments = sys.argv[1:]
# If no arguments are given, the help text of pip-accel is printed.
if not arguments:
usage()
sys.exit(0)
# If no install subcommand is given we pass the command line straight
# to pip without any changes and exit immediately afterwards.
if 'install' not in arguments:
# This will not return.
os.execvp('pip', ['pip'] + arguments)
else:
arguments = [arg for arg in arguments if arg != 'install']
# Initialize logging output.
coloredlogs.install()
# Adjust verbosity based on -v, -q, --verbose, --quiet options.
for argument in list(arguments):
if match_option(argument, '-v', '--verbose'):
coloredlogs.increase_verbosity()
elif match_option(argument, '-q', '--quiet'):
coloredlogs.decrease_verbosity()
# Perform the requested action(s).
try:
accelerator = PipAccelerator(Config())
accelerator.install_from_arguments(arguments)
except NothingToDoError as e:
# Don't print a traceback for this (it's not very user friendly) and
# exit with status zero to stay compatible with pip. For more details
# please refer to https://github.com/paylogic/pip-accel/issues/47.
logger.warning("%s", e)
sys.exit(0)
except Exception:
logger.exception("Caught unhandled exception!")
sys.exit(1)
def main():
"""The command line interface for the ``pip-accel`` program."""
arguments = sys.argv[1:]
# If no arguments are given, the help text of pip-accel is printed.
if not arguments:
usage()
sys.exit(0)
# If no install subcommand is given we pass the command line straight
# to pip without any changes and exit immediately afterwards.
if 'install' not in arguments:
# This will not return.
os.execvp('pip', ['pip'] + arguments)
else:
arguments = [arg for arg in arguments if arg != 'install']
# Initialize logging output.
coloredlogs.install()
# Adjust verbosity based on -v, -q, --verbose, --quiet options.
for argument in list(arguments):
if match_option(argument, '-v', '--verbose'):
coloredlogs.increase_verbosity()
elif match_option(argument, '-q', '--quiet'):
coloredlogs.decrease_verbosity()
# Perform the requested action(s).
try:
accelerator = PipAccelerator(Config())
accelerator.install_from_arguments(arguments)
except NothingToDoError as e:
# Don't print a traceback for this (it's not very user friendly) and
# exit with status zero to stay compatible with pip. For more details
# please refer to https://github.com/paylogic/pip-accel/issues/47.
logger.warning("%s", e)
sys.exit(0)
except Exception:
logger.exception("Caught unhandled exception!")
sys.exit(1)
def get_requirements(self, arguments, max_retries=None, use_wheels=False):
"""
Use pip to download and unpack the requested source distribution archives.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:param max_retries: The maximum number of times that pip will be asked
to download distribution archives (this helps to
deal with intermittent failures). If this is
:data:`None` then :attr:`~.Config.max_retries` is
used.
:param use_wheels: Whether pip and pip-accel are allowed to use wheels_
(:data:`False` by default for backwards compatibility
with callers that use pip-accel as a Python API).
.. warning:: Requirements which are already installed are not included
in the result. If this breaks your use case consider using
pip's ``--ignore-installed`` option.
"""
arguments = self.decorate_arguments(arguments)
# Demote hash sum mismatch log messages from CRITICAL to DEBUG (hiding
# implementation details from users unless they want to see them).
with DownloadLogFilter():
with SetupRequiresPatch(self.config, self.eggs_links):
# Use a new build directory for each run of get_requirements().
self.create_build_directory()
# Check whether -U or --upgrade was given.
if any(match_option(a, '-U', '--upgrade') for a in arguments):
logger.info("Checking index(es) for new version (-U or --upgrade was given) ..")
else:
# If -U or --upgrade wasn't given and all requirements can be
# satisfied using the archives in pip-accel's local source
# index we don't need pip to connect to PyPI looking for new
# versions (that will just slow us down).
try:
return self.unpack_source_dists(arguments, use_wheels=use_wheels)
except DistributionNotFound:
logger.info("We don't have all distribution archives yet!")
# Get the maximum number of retries from the configuration if the
# caller didn't specify a preference.
if max_retries is None:
max_retries = self.config.max_retries
# If not all requirements are available locally we use pip to
# download the missing source distribution archives from PyPI (we
# retry a couple of times in case pip reports recoverable
# errors).
for i in range(max_retries):
try:
return self.download_source_dists(arguments, use_wheels=use_wheels)
except Exception as e:
if i + 1 < max_retries:
# On all but the last iteration we swallow exceptions
# during downloading.
logger.warning("pip raised exception while downloading distributions: %s", e)
else:
# On the last iteration we don't swallow exceptions
# during downloading because the error reported by pip
# is the most sensible error for us to report.
raise
logger.info("Retrying after pip failed (%i/%i) ..", i + 1, max_retries)
def get_requirements(self, arguments, max_retries=None, use_wheels=False):
"""
Use pip to download and unpack the requested source distribution archives.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:param max_retries: The maximum number of times that pip will be asked
to download distribution archives (this helps to
deal with intermittent failures). If this is
:data:`None` then :attr:`~.Config.max_retries` is
used.
:param use_wheels: Whether pip and pip-accel are allowed to use wheels_
(:data:`False` by default for backwards compatibility
with callers that use pip-accel as a Python API).
.. warning:: Requirements which are already installed are not included
in the result. If this breaks your use case consider using
pip's ``--ignore-installed`` option.
"""
arguments = self.decorate_arguments(arguments)
# Demote hash sum mismatch log messages from CRITICAL to DEBUG (hiding
# implementation details from users unless they want to see them).
with DownloadLogFilter():
with SetupRequiresPatch(self.config, self.eggs_links):
# Use a new build directory for each run of get_requirements().
self.create_build_directory()
# Check whether -U or --upgrade was given.
if any(match_option(a, '-U', '--upgrade') for a in arguments):
logger.info("Checking index(es) for new version (-U or --upgrade was given) ..")
else:
# If -U or --upgrade wasn't given and all requirements can be
# satisfied using the archives in pip-accel's local source
# index we don't need pip to connect to PyPI looking for new
# versions (that will just slow us down).
try:
return self.unpack_source_dists(arguments, use_wheels=use_wheels)
except DistributionNotFound:
logger.info("We don't have all distribution archives yet!")
# Get the maximum number of retries from the configuration if the
# caller didn't specify a preference.
if max_retries is None:
max_retries = self.config.max_retries
# If not all requirements are available locally we use pip to
# download the missing source distribution archives from PyPI (we
# retry a couple of times in case pip reports recoverable
# errors).
for i in range(max_retries):
try:
return self.download_source_dists(arguments, use_wheels=use_wheels)
except Exception as e:
if i + 1 < max_retries:
# On all but the last iteration we swallow exceptions
# during downloading.
logger.warning("pip raised exception while downloading distributions: %s", e)
else:
# On the last iteration we don't swallow exceptions
# during downloading because the error reported by pip
# is the most sensible error for us to report.
raise
logger.info("Retrying after pip failed (%i/%i) ..", i + 1, max_retries)
def decorate_arguments(self, arguments):
"""
Change pathnames of local files into ``file://`` URLs with ``#md5=...`` fragments.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:returns: A copy of the command line arguments with pathnames of local
files rewritten to ``file://`` URLs.
When pip-accel calls pip to download missing distribution archives and
the user specified the pathname of a local distribution archive on the
command line, pip will (by default) *not* copy the archive into the
download directory if an archive for the same package name and
version is already present.
This can lead to the confusing situation where the user specifies a
local distribution archive to install, a different (older) archive for
the same package and version is present in the download directory and
`pip-accel` installs the older archive instead of the newer archive.
To avoid this confusing behavior, the :func:`decorate_arguments()`
method rewrites the command line arguments given to ``pip install`` so
that pathnames of local archives are changed into ``file://`` URLs that
include a fragment with the hash of the file's contents. Here's an
example:
- Local pathname: ``/tmp/pep8-1.6.3a0.tar.gz``
- File URL: ``file:///tmp/pep8-1.6.3a0.tar.gz#md5=19cbf0b633498ead63fb3c66e5f1caf6``
When pip fills the download directory and encounters a previously
cached distribution archive it will check the hash, realize the
contents have changed and replace the archive in the download
directory.
"""
arguments = list(arguments)
for i, value in enumerate(arguments):
is_constraint_file = (i >= 1 and match_option(
arguments[i - 1], '-c', '--constraint'))
is_requirement_file = (i >= 1 and match_option(
arguments[i - 1], '-r', '--requirement'))
if not is_constraint_file and not is_requirement_file and os.path.isfile(
value):
arguments[i] = '%s#md5=%s' % (create_file_url(value),
hash_files('md5', value))
return arguments
def decorate_arguments(self, arguments):
"""
Change pathnames of local files into ``file://`` URLs with ``#md5=...`` fragments.
:param arguments: The command line arguments to ``pip install ...`` (a
list of strings).
:returns: A copy of the command line arguments with pathnames of local
files rewritten to ``file://`` URLs.
When pip-accel calls pip to download missing distribution archives and
the user specified the pathname of a local distribution archive on the
command line, pip will (by default) *not* copy the archive into the
download directory if an archive for the same package name and
version is already present.
This can lead to the confusing situation where the user specifies a
local distribution archive to install, a different (older) archive for
the same package and version is present in the download directory and
`pip-accel` installs the older archive instead of the newer archive.
To avoid this confusing behavior, the :func:`decorate_arguments()`
method rewrites the command line arguments given to ``pip install`` so
that pathnames of local archives are changed into ``file://`` URLs that
include a fragment with the hash of the file's contents. Here's an
example:
- Local pathname: ``/tmp/pep8-1.6.3a0.tar.gz``
- File URL: ``file:///tmp/pep8-1.6.3a0.tar.gz#md5=19cbf0b633498ead63fb3c66e5f1caf6``
When pip fills the download directory and encounters a previously
cached distribution archive it will check the hash, realize the
contents have changed and replace the archive in the download
directory.
"""
arguments = list(arguments)
for i, value in enumerate(arguments):
is_constraint_file = (i >= 1 and match_option(arguments[i - 1], '-c', '--constraint'))
is_requirement_file = (i >= 1 and match_option(arguments[i - 1], '-r', '--requirement'))
if not is_constraint_file and not is_requirement_file and os.path.isfile(value):
arguments[i] = '%s#md5=%s' % (create_file_url(value), hash_files('md5', value))
return arguments
