def build_missing_binary_dists(requirements):
"""
Convert source distributions to binary distributions.
:param requirements: A list of tuples in the format of the return value of
the :py:func:`unpack_source_dists()` function.
:returns: ``True`` if it succeeds in building a binary distribution,
``False`` otherwise (probably because of missing binary
dependencies like system libraries).
"""
existing_binary_dists = find_cached_binary_dists()
logger.info("Building binary distributions ..")
pyversion = get_python_version()
for name, version, directory in requirements:
# Check if a binary distribution already exists.
filename = existing_binary_dists.get((name.lower(), version, pyversion))
if filename:
logger.debug("Existing binary distribution for %s (%s) found at %s.", name, version, filename)
continue
# Make sure the source distribution contains a setup script.
setup_script = os.path.join(directory, 'setup.py')
if not os.path.isfile(setup_script):
logger.warn("Package %s (%s) is not a source distribution.", name, version)
continue
# Try to build the binary distribution.
if not build_binary_dist(name, version, directory, pyversion):
sanity_check_dependencies(name)
if not build_binary_dist(name, version, directory, pyversion):
return False
logger.info("Finished building binary distributions.")
return True
def install_requirements(requirements, install_prefix=ENVIRONMENT):
"""
Manually install all requirements from binary distributions.
:param requirements: A list of tuples in the format of the return value of
:py:func:`unpack_source_dists()`.
:param install_prefix: The "prefix" under which the requirements should be
installed. This will be a pathname like ``/usr``,
``/usr/local`` or the pathname of a virtual
environment.
:returns: ``True`` if it succeeds in installing all requirements from
binary distribution archives, ``False`` otherwise.
"""
install_timer = Timer()
existing_binary_dists = find_cached_binary_dists()
pyversion = get_python_version()
logger.info("Installing from binary distributions ..")
for name, version, directory in requirements:
filename = existing_binary_dists.get((name.lower(), version, pyversion))
if not filename:
logger.error("No binary distribution of %s (%s) available!", name, version)
return False
install_binary_dist(name, filename, install_prefix=install_prefix)
logger.info("Finished installing all requirements in %s.", install_timer)
return True
def download_source_dists(arguments):
"""
Download missing source distributions.
:param arguments: A list with the arguments intended for ``pip``.
"""
download_timer = Timer()
logger.info("Downloading source distributions ..")
# Execute pip to download missing source distributions.
try:
run_pip(arguments + ['--no-install'], use_remote_index=True)
logger.info("Finished downloading source distributions in %s.", download_timer)
except Exception, e:
logger.warn("pip raised an exception while downloading source distributions: %s.", e)
def download_source_dists(arguments, build_directory):
"""
Download missing source distributions.
:param arguments: A list with the arguments intended for ``pip``.
"""
download_timer = Timer()
logger.info("Downloading source distributions ..")
# Execute pip to download missing source distributions.
try:
run_pip(arguments + ['--no-install'], use_remote_index=True, build_directory=build_directory)
logger.info("Finished downloading source distributions in %s.", download_timer)
except PreviousBuildDirError:
# Shouldn't be swallowed.
raise
except Exception, e:
logger.warn("pip raised an exception while downloading source distributions: %s.", e)
def main():
"""
Main logic of the ``pip-accel`` command.
"""
arguments = sys.argv[1:]
# If no arguments are given, the help text of pip-accel is printed.
if not arguments:
print_usage()
sys.exit(0)
# If no install subcommand is given we pass the command line straight
# to pip without any changes and exit immediately afterwards.
elif 'install' not in arguments:
sys.exit(os.spawnvp(os.P_WAIT, 'pip', ['pip'] + arguments))
# Initialize logging output.
coloredlogs.install()
# Increase verbosity based on -v, --verbose options.
for argument in arguments:
if argument == '--verbose' or (len(argument) >= 2 and argument[0] ==
'-' and argument[1] != '-' and 'v' in argument):
coloredlogs.increase_verbosity()
# Make sure the prefix is the same as the environment.
if not os.path.samefile(sys.prefix, ENVIRONMENT):
logger.error("You are trying to install packages in environment #1 which is different from environment #2 where pip-accel is installed! Please install pip-accel under environment #1 to install packages there.")
logger.info("Environment #1: %s ($VIRTUAL_ENV)", ENVIRONMENT)
logger.info("Environment #2: %s (installation prefix)", sys.prefix)
sys.exit(1)
main_timer = Timer()
initialize_directories()
build_directory = tempfile.mkdtemp()
# Execute "pip install" in a loop in order to retry after intermittent
# error responses from servers (which can happen quite frequently).
try:
for i in xrange(1, MAX_RETRIES):
try:
requirements = unpack_source_dists(arguments, build_directory)
except DistributionNotFound:
logger.warn("We don't have all source distributions yet!")
download_source_dists(arguments, build_directory)
else:
if not requirements:
logger.info("No unsatisfied requirements found, probably there's nothing to do.")
else:
install_requirements(requirements)
logger.info("Done! Took %s to install %i package%s.", main_timer, len(requirements), '' if len(requirements) == 1 else 's')
return
logger.warn("pip failed, retrying (%i/%i) ..", i + 1, MAX_RETRIES)
except InstallationError:
# Abort early when pip reports installation errors.
logger.fatal("pip reported unrecoverable installation errors. Please fix and rerun!")
sys.exit(1)
finally:
# Always cleanup temporary build directory.
shutil.rmtree(build_directory)
# Abort when after N retries we still failed to download source distributions.
logger.fatal("External command failed %i times, aborting!" % MAX_RETRIES)
sys.exit(1)
def build_binary_dist(name, version, directory, pyversion):
"""
Convert a single, unpacked source distribution to a binary distribution.
:param name: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:returns: ``True`` if we succeed in building a binary distribution,
``False`` otherwise (probably because of missing binary
dependencies like system libraries).
"""
# Cleanup previously generated distributions.
dist_directory = os.path.join(directory, 'dist')
if os.path.isdir(dist_directory):
logger.info("Cleaning up previously generated distributions in %s ..", dist_directory)
shutil.rmtree(dist_directory)
# Let the user know what's going on.
build_text = "Building binary distribution of %s (%s) .." % (name, version)
logger.info("%s", build_text)
# Compose the command line needed to build the binary distribution.
command_line = '"%s/bin/python" setup.py bdist_dumb --format=gztar' % ENVIRONMENT
logger.debug("Executing external command: %s", command_line)
# Redirect all output of the build to a temporary file.
fd, temporary_file = tempfile.mkstemp()
command_line = '%s > "%s" 2>&1' % (command_line, temporary_file)
# Start the build.
build = subprocess.Popen(['sh', '-c', command_line], cwd=directory)
# Wait for the build to finish.
if INTERACTIVE:
# Provide feedback to the user in the mean time.
spinner = Spinner(build_text)
while build.poll() is None:
spinner.step()
time.sleep(0.1)
spinner.clear()
else:
build.wait()
# Make sure the build succeeded.
if build.returncode != 0:
logger.error("Failed to build binary distribution of %s! (version: %s)", name, version)
with open(temporary_file) as handle:
logger.info("Build output (will probably provide a hint as to what went wrong):\n%s", handle.read())
return False
# Move the generated distribution to the binary index.
filenames = os.listdir(dist_directory)
if len(filenames) != 1:
logger.error("Build process did not result in one binary distribution! (matches: %s)", filenames)
return False
cache_file = '%s:%s:%s.tar.gz' % (name, version, pyversion)
logger.info("Copying binary distribution %s to cache as %s.", filenames[0], cache_file)
cache_binary_dist(os.path.join(directory, 'dist', filenames[0]),
os.path.join(binary_index, cache_file))
return True
def run_pip(arguments, use_remote_index, build_directory=None):
"""
Execute a modified ``pip install`` command. This function assumes that the
arguments concern a ``pip install`` command (:py:func:`main()` makes sure
of this).
:param arguments: A list of strings containing the arguments that will be
passed to ``pip``.
:param use_remote_index: A boolean indicating whether ``pip`` is allowed to
contact http://pypi.python.org.
:returns: A ``RequirementSet`` object created by ``pip``, unless an
exception is raised by ``pip`` (in which case the exception will
bubble up).
"""
command_line = []
for i, arg in enumerate(arguments):
if arg == 'install':
command_line += ['pip'] + arguments[:i + 1] + [
'--download-cache=%s' % download_cache,
'--find-links=file://%s' % source_index
]
if build_directory:
command_line += ['--build-directory=%s' % build_directory]
if not use_remote_index:
command_line += ['--no-index']
command_line += arguments[i + 1:]
break
else:
command_line = ['pip'] + arguments
logger.info("Executing command: %s", ' '.join(command_line))
# XXX Nasty hack required for pip 1.4 compatibility (workaround for global state).
requirements_option.default = []
cmd_name, options, args, parser = parseopts(command_line[1:])
pip = CustomInstallCommand(parser)
exit_status = pip.main(args[1:], options)
# Make sure the output of pip and pip-accel are not intermingled.
sys.stdout.flush()
update_source_dists_index()
if exit_status == SUCCESS:
return pip.requirement_set
else:
raise pip.intercepted_exception
def download_source_dists(arguments, build_directory):
"""
Download missing source distributions.
:param arguments: A list with the arguments intended for ``pip``.
"""
download_timer = Timer()
logger.info("Downloading source distributions ..")
clear_build_directory(build_directory)
# Execute pip to download missing source distributions.
try:
run_pip(arguments + ['--no-install'],
use_remote_index=True,
build_directory=build_directory)
logger.info("Finished downloading source distributions in %s.",
download_timer)
except Exception, e:
logger.warn(
"pip raised an exception while downloading source distributions: %s.",
e)
def run_pip(arguments, use_remote_index, build_directory=None):
"""
Execute a modified ``pip install`` command. This function assumes that the
arguments concern a ``pip install`` command (:py:func:`main()` makes sure
of this).
:param arguments: A list of strings containing the arguments that will be
passed to ``pip``.
:param use_remote_index: A boolean indicating whether ``pip`` is allowed to
contact http://pypi.python.org.
:returns: A ``RequirementSet`` object created by ``pip``, unless an
exception is raised by ``pip`` (in which case the exception will
bubble up).
"""
command_line = []
for i, arg in enumerate(arguments):
if arg == 'install':
command_line += ['pip'] + arguments[:i+1] + [
'--download-cache=%s' % download_cache,
'--find-links=file://%s' % source_index]
if build_directory:
command_line += ['--build-directory=%s' % build_directory]
if not use_remote_index:
command_line += ['--no-index']
command_line += arguments[i+1:]
break
else:
command_line = ['pip'] + arguments
logger.info("Executing command: %s", ' '.join(command_line))
# XXX Nasty hack required for pip 1.4 compatibility (workaround for global state).
requirements_option.default = []
cmd_name, options, args, parser = parseopts(command_line[1:])
pip = CustomInstallCommand(parser)
exit_status = pip.main(args[1:], options)
# Make sure the output of pip and pip-accel are not intermingled.
sys.stdout.flush()
update_source_dists_index()
if exit_status == SUCCESS:
return pip.requirement_set
else:
raise pip.intercepted_exception
def confirm_installation(project_name, packages, command_line):
"""
Notify the user that there are missing dependencies and how they can be
installed. Then ask the user whether we are allowed to install the
dependencies.
:param packages: A list of strings with the names of the packages that are
missing.
:param command_line: A list of strings with the command line needed to
install the packages.
Raises :py:class:`RefusedAutomaticInstallation` when the user refuses to
let pip-accel install any missing dependencies.
"""
logger.info("%s: You seem to be missing %i dependenc%s: %s", project_name,
len(packages), len(packages) == 1 and 'y' or 'ies', " ".join(packages))
logger.info("%s: I can install %s for you with this command: %s",
project_name, len(packages) == 1 and 'it' or 'them',
" ".join(command_line))
try:
prompt = "Do you want me to install %s dependenc%s? [y/N] "
choice = raw_input(prompt % (len(packages) == 1 and 'this' or 'these',
len(packages) == 1 and 'y' or 'ies'))
if choice.lower().strip() == 'y':
logger.info("Got permission to install missing dependenc%s.",
len(packages) == 1 and 'y' or 'ies')
return
except:
pass
logger.error("%s: Refused installation of missing dependenc%s!",
project_name, len(packages) == 1 and 'y' or 'ies')
msg = "%s: User canceled automatic installation of missing dependenc%s!"
raise RefusedAutomaticInstallation, msg % (project_name, 'y' if len(packages) == 1 else 'ies')
def install_requirements(requirements, install_prefix=ENVIRONMENT):
"""
Manually install all requirements from binary distributions.
:param requirements: A list of tuples in the format of the return value of
:py:func:`unpack_source_dists()`.
:param install_prefix: The "prefix" under which the requirements should be
installed. This will be a pathname like ``/usr``,
``/usr/local`` or the pathname of a virtual
environment.
:returns: ``True`` if it succeeds in installing all requirements from
binary distribution archives, ``False`` otherwise.
"""
install_timer = Timer()
logger.info("Installing from binary distributions ..")
python = os.path.join(install_prefix, 'bin', 'python')
pip = os.path.join(install_prefix, 'bin', 'pip')
for name, version, directory in requirements:
if os.system('%s uninstall --yes %s >/dev/null 2>&1' %
(pipes.quote(pip), pipes.quote(name))) == 0:
logger.info("Uninstalled previously installed package %s.", name)
members = get_binary_dist(name,
version,
directory,
prefix=install_prefix,
python=python)
install_binary_dist(members, prefix=install_prefix, python=python)
logger.info("Finished installing all requirements in %s.", install_timer)
return True
def find_cached_binary_dists():
"""
Find all previously cached binary distributions.
:returns: A dictionary with (package-name, package-version, python-version)
tuples as keys and pathnames of binary archives as values.
"""
logger.info("Scanning binary distribution index ..")
distributions = {}
for filename in sorted(os.listdir(binary_index), key=str.lower):
if filename.endswith('.tar.gz'):
basename = re.sub('\.tar.gz$', '', filename)
parts = basename.split(':')
if len(parts) == 3:
key = (parts[0].lower(), parts[1], parts[2])
logger.debug("Matched %s in %s.", key, filename)
distributions[key] = os.path.join(binary_index, filename)
continue
logger.debug("Failed to match filename: %s.", filename)
logger.info("Found %i existing binary distribution%s.",
len(distributions), '' if len(distributions) == 1 else 's')
for (name, version, pyversion), filename in distributions.iteritems():
logger.debug(" - %s (%s, %s) in %s.", name, version, pyversion, filename)
return distributions
def update_source_dists_index():
"""
Link newly downloaded source distributions into the local index directory
using symbolic links.
"""
for download_name in os.listdir(download_cache):
download_path = os.path.join(download_cache, download_name)
url = urllib.unquote(download_name)
if not url.endswith('.content-type'):
components = urlparse.urlparse(url)
archive_name = os.path.basename(components.path)
archive_path = os.path.join(source_index, add_extension(download_path, archive_name))
if not os.path.isfile(archive_path):
logger.info("Linking files:")
logger.info(" - Source: %s", download_path)
logger.info(" - Target: %s", archive_path)
os.symlink(download_path, archive_path)
def update_source_dists_index():
"""
Link newly downloaded source distributions into the local index directory
using symbolic links.
"""
for download_name in os.listdir(download_cache):
download_path = os.path.join(download_cache, download_name)
url = urllib.unquote(download_name)
if not url.endswith('.content-type'):
components = urlparse.urlparse(url)
archive_name = os.path.basename(components.path)
archive_path = os.path.join(
source_index, add_extension(download_path, archive_name))
if not os.path.isfile(archive_path):
logger.info("Linking files:")
logger.info(" - Source: %s", download_path)
logger.info(" - Target: %s", archive_path)
os.symlink(download_path, archive_path)
def install_binary_dist(package, filename, install_prefix=ENVIRONMENT):
"""
Install a binary distribution created with ``python setup.py bdist`` into
the given prefix (a directory like ``/usr``, ``/usr/local`` or a virtual
environment).
:param package: The name of the package to install.
:param filename: The pathname of the tar archive.
:param install_prefix: The "prefix" under which the requirements should be
installed. This will be a pathname like ``/usr``,
``/usr/local`` or the pathname of a virtual
environment.
"""
# TODO This is quite slow for modules like Django. Speed it up! Two choices:
# 1. Run the external tar program to unpack the archive. This will
# slightly complicate the fixing up of hashbangs.
# 2. Using links? The plan: We can maintain a "seed" environment under
# $PIP_ACCEL_CACHE and use symbolic and/or hard links to populate other
# places based on the "seed" environment.
install_timer = Timer()
python = os.path.join(install_prefix, 'bin', 'python')
pip = os.path.join(install_prefix, 'bin', 'pip')
if os.system('"%s" uninstall --yes "%s" >/dev/null 2>&1' % (pip, package)) == 0:
logger.info("Uninstalled previously installed package %s.", package)
logger.info("Installing package %s from binary distribution %s to %s ..", package, filename, install_prefix)
archive = tarfile.open(filename, 'r:gz')
for member in archive.getmembers():
install_path = os.path.join(install_prefix, member.name)
directory = os.path.dirname(install_path)
if not os.path.isdir(directory):
logger.debug("Creating directory: %s ..", directory)
os.makedirs(directory)
logger.debug("Writing file: %s ..", install_path)
file_handle = archive.extractfile(member)
with open(install_path, 'w') as handle:
contents = file_handle.read()
if contents.startswith('#!/'):
contents = fix_hashbang(python, contents)
handle.write(contents)
os.chmod(install_path, member.mode)
archive.close()
logger.info("Finished installing binary distribution in %s.", install_timer)
def unpack_source_dists(arguments, build_directory):
"""
Check whether there are local source distributions available for all
requirements, unpack the source distribution archives and find the names
and versions of the requirements. By using the ``pip install --no-install``
command we avoid reimplementing the following pip features:
- Parsing of ``requirements.txt`` (including recursive parsing)
- Resolution of possibly conflicting pinned requirements
- Unpacking source distributions in multiple formats
- Finding the name & version of a given source distribution
:param arguments: A list of strings with the command line arguments to be
passed to the ``pip`` command.
:returns: A list of tuples with three strings each: The name of a
requirement (package), its version number and the directory where
the unpacked source distribution is located. If ``pip`` fails, an
exception will be raised by ``pip``.
"""
unpack_timer = Timer()
logger.info("Unpacking local source distributions ..")
clear_build_directory(build_directory)
# Execute pip to unpack the source distributions.
requirement_set = run_pip(arguments + ['--no-install'],
use_remote_index=False,
build_directory=build_directory)
logger.info("Unpacked local source distributions in %s.", unpack_timer)
requirements = []
for install_requirement in sorted_requirements(requirement_set):
if install_requirement.satisfied_by:
logger.info("Requirement already satisfied: %s.",
install_requirement)
else:
req = ensure_parsed_requirement(install_requirement)
requirements.append(
(req.project_name, install_requirement.installed_version,
install_requirement.source_dir))
return requirements
def unpack_source_dists(arguments, build_directory):
"""
Check whether there are local source distributions available for all
requirements, unpack the source distribution archives and find the names
and versions of the requirements. By using the ``pip install --no-install``
command we avoid reimplementing the following pip features:
- Parsing of ``requirements.txt`` (including recursive parsing)
- Resolution of possibly conflicting pinned requirements
- Unpacking source distributions in multiple formats
- Finding the name & version of a given source distribution
:param arguments: A list of strings with the command line arguments to be
passed to the ``pip`` command.
:returns: A list of tuples with three strings each: The name of a
requirement (package), its version number and the directory where
the unpacked source distribution is located. If ``pip`` fails, an
exception will be raised by ``pip``.
"""
unpack_timer = Timer()
logger.info("Unpacking local source distributions ..")
clear_build_directory(build_directory)
# Execute pip to unpack the source distributions.
requirement_set = run_pip(arguments + ['--no-install'],
use_remote_index=False,
build_directory=build_directory)
logger.info("Unpacked local source distributions in %s.", unpack_timer)
requirements = []
for install_requirement in sorted_requirements(requirement_set):
if install_requirement.satisfied_by:
logger.info("Requirement already satisfied: %s.", install_requirement)
else:
req = ensure_parsed_requirement(install_requirement)
requirements.append((req.project_name,
install_requirement.installed_version,
install_requirement.source_dir))
return requirements
def install_requirements(requirements, install_prefix=ENVIRONMENT):
"""
Manually install all requirements from binary distributions.
:param requirements: A list of tuples in the format of the return value of
:py:func:`unpack_source_dists()`.
:param install_prefix: The "prefix" under which the requirements should be
installed. This will be a pathname like ``/usr``,
``/usr/local`` or the pathname of a virtual
environment.
:returns: ``True`` if it succeeds in installing all requirements from
binary distribution archives, ``False`` otherwise.
"""
install_timer = Timer()
logger.info("Installing from binary distributions ..")
python = os.path.join(install_prefix, 'bin', 'python')
pip = os.path.join(install_prefix, 'bin', 'pip')
for name, version, directory in requirements:
if os.system('%s uninstall --yes %s >/dev/null 2>&1' % (pipes.quote(pip), pipes.quote(name))) == 0:
logger.info("Uninstalled previously installed package %s.", name)
members = get_binary_dist(name, version, directory, prefix=install_prefix, python=python)
install_binary_dist(members, prefix=install_prefix, python=python)
logger.info("Finished installing all requirements in %s.", install_timer)
return True
def confirm_installation(project_name, packages, command_line):
"""
Notify the user that there are missing dependencies and how they can be
installed. Then ask the user whether we are allowed to install the
dependencies.
:param packages: A list of strings with the names of the packages that are
missing.
:param command_line: A list of strings with the command line needed to
install the packages.
Raises :py:class:`RefusedAutomaticInstallation` when the user refuses to
let pip-accel install any missing dependencies.
"""
logger.info("%s: You seem to be missing %i dependenc%s: %s", project_name,
len(packages),
len(packages) == 1 and 'y' or 'ies', " ".join(packages))
logger.info("%s: I can install %s for you with this command: %s",
project_name,
len(packages) == 1 and 'it' or 'them', " ".join(command_line))
try:
prompt = "Do you want me to install %s dependenc%s? [y/N] "
choice = raw_input(prompt % (len(packages) == 1 and 'this' or 'these',
len(packages) == 1 and 'y' or 'ies'))
if choice.lower().strip() == 'y':
logger.info("Got permission to install missing dependenc%s.",
len(packages) == 1 and 'y' or 'ies')
return
except:
pass
logger.error("%s: Refused installation of missing dependenc%s!",
project_name,
len(packages) == 1 and 'y' or 'ies')
msg = "%s: User canceled automatic installation of missing dependenc%s!"
raise RefusedAutomaticInstallation, msg % (project_name, 'y' if
len(packages) == 1 else 'ies')
def sanity_check_dependencies(project_name, auto_install=None):
"""
If ``pip-accel`` fails to build a binary distribution, it will call this
function as a last chance to install missing dependencies. If this function
does not raise an exception, ``pip-accel`` will retry the build once.
:param project_name: The project name of a requirement as found on PyPI.
:param auto_install: ``True`` if dependencies on system packages may be
automatically installed, ``False`` if missing system
packages should raise an error, ``None`` if the
decision should be based on the environment variable
``PIP_ACCEL_AUTO_INSTALL``.
If missing system packages are found, this function will try to install
them. If anything "goes wrong" an exception is raised:
- If ``auto_install=False`` then :py:class:`DependencyCheckFailed` is
raised
- If installation of missing packages fails
:py:class:`DependencyInstallationFailed` is raised
- If the user refuses to let pip-accel install missing packages
:py:class:`RefusedAutomaticInstallation` is raised.
If all goes well nothing is raised, nor is anything returned.
"""
# Has the caller forbidden us from automatic installation?
auto_install_forbidden = (auto_install == False)
if auto_install is not None:
# If the caller made an explicit choice, we'll respect that.
auto_install_allowed = auto_install
else:
# Otherwise we check the environment variable.
auto_install_allowed = 'PIP_ACCEL_AUTO_INSTALL' in os.environ
logger.info("%s: Checking for missing dependencies ..", project_name)
known_dependencies = current_platform.find_dependencies(project_name.lower())
if not known_dependencies:
logger.info("%s: No known dependencies... Maybe you have a suggestion?", project_name)
else:
installed_packages = set(current_platform.find_installed())
missing_packages = [p for p in known_dependencies if p not in installed_packages]
if not missing_packages:
logger.info("%s: All known dependencies are already installed.", project_name)
elif auto_install_forbidden:
msg = "Missing %i system package%s (%s) required by %s but automatic installation is disabled!"
raise DependencyCheckFailed, msg % (len(missing_packages), '' if len(missing_packages) == 1 else 's',
', '.join(missing_packages), project_name)
else:
command_line = ['sudo'] + current_platform.install_command(missing_packages)
if not auto_install_allowed:
confirm_installation(project_name, missing_packages, command_line)
logger.info("%s: Missing %i dependenc%s: %s",
project_name, len(missing_packages),
'y' if len(missing_packages) == 1 else 'ies',
" ".join(missing_packages))
exit_code = os.spawnvp(os.P_WAIT, 'sudo', command_line)
if exit_code == 0:
logger.info("%s: Successfully installed %i missing dependenc%s.",
project_name, len(missing_packages),
len(missing_packages) == 1 and 'y' or 'ies')
else:
logger.error("%s: Failed to install %i missing dependenc%s!",
project_name, len(missing_packages),
len(missing_packages) == 1 and 'y' or 'ies')
msg = "Failed to install %i system package%s required by %s! (command failed: %s)"
raise DependencyInstallationFailed, msg % (len(missing_packages), '' if len(missing_packages) == 1 else 's',
project_name, command_line)
def sanity_check_dependencies(project_name, auto_install=None):
"""
If ``pip-accel`` fails to build a binary distribution, it will call this
function as a last chance to install missing dependencies. If this function
does not raise an exception, ``pip-accel`` will retry the build once.
:param project_name: The project name of a requirement as found on PyPI.
:param auto_install: ``True`` if dependencies on system packages may be
automatically installed, ``False`` if missing system
packages should raise an error, ``None`` if the
decision should be based on the environment variable
``PIP_ACCEL_AUTO_INSTALL``.
If missing system packages are found, this function will try to install
them. If anything "goes wrong" an exception is raised:
- If ``auto_install=False`` then :py:class:`DependencyCheckFailed` is
raised
- If installation of missing packages fails
:py:class:`DependencyInstallationFailed` is raised
- If the user refuses to let pip-accel install missing packages
:py:class:`RefusedAutomaticInstallation` is raised.
If all goes well nothing is raised, nor is anything returned.
"""
# Has the caller forbidden us from automatic installation?
auto_install_forbidden = (auto_install == False)
if auto_install is not None:
# If the caller made an explicit choice, we'll respect that.
auto_install_allowed = auto_install
else:
# Otherwise we check the environment variable.
auto_install_allowed = 'PIP_ACCEL_AUTO_INSTALL' in os.environ
logger.info("%s: Checking for missing dependencies ..", project_name)
known_dependencies = current_platform.find_dependencies(
project_name.lower())
if not known_dependencies:
logger.info(
"%s: No known dependencies... Maybe you have a suggestion?",
project_name)
else:
installed_packages = set(current_platform.find_installed())
missing_packages = [
p for p in known_dependencies if p not in installed_packages
]
if not missing_packages:
logger.info("%s: All known dependencies are already installed.",
project_name)
elif auto_install_forbidden:
msg = "Missing %i system package%s (%s) required by %s but automatic installation is disabled!"
raise DependencyCheckFailed, msg % (
len(missing_packages), '' if len(missing_packages) == 1 else
's', ', '.join(missing_packages), project_name)
else:
command_line = [
'sudo'
] + current_platform.install_command(missing_packages)
if not auto_install_allowed:
confirm_installation(project_name, missing_packages,
command_line)
logger.info("%s: Missing %i dependenc%s: %s", project_name,
len(missing_packages),
'y' if len(missing_packages) == 1 else 'ies',
" ".join(missing_packages))
exit_code = os.spawnvp(os.P_WAIT, 'sudo', command_line)
if exit_code == 0:
logger.info(
"%s: Successfully installed %i missing dependenc%s.",
project_name, len(missing_packages),
len(missing_packages) == 1 and 'y' or 'ies')
else:
logger.error("%s: Failed to install %i missing dependenc%s!",
project_name, len(missing_packages),
len(missing_packages) == 1 and 'y' or 'ies')
msg = "Failed to install %i system package%s required by %s! (command failed: %s)"
raise DependencyInstallationFailed, msg % (
len(missing_packages), '' if len(missing_packages) == 1
else 's', project_name, command_line)
def main():
"""
Main logic of the ``pip-accel`` command.
"""
arguments = sys.argv[1:]
# If no arguments are given, the help text of pip-accel is printed.
if not arguments:
print_usage()
sys.exit(0)
# If no install subcommand is given we pass the command line straight
# to pip without any changes and exit immediately afterwards.
elif 'install' not in arguments:
sys.exit(os.spawnvp(os.P_WAIT, 'pip', ['pip'] + arguments))
# Initialize logging output.
coloredlogs.install()
# Increase verbosity based on -v, --verbose options.
for argument in arguments:
if argument == '--verbose' or (len(argument) >= 2
and argument[0] == '-'
and argument[1] != '-'
and 'v' in argument):
coloredlogs.increase_verbosity()
# Make sure the prefix is the same as the environment.
if not os.path.samefile(sys.prefix, ENVIRONMENT):
logger.error(
"You are trying to install packages in environment #1 which is different from environment #2 where pip-accel is installed! Please install pip-accel under environment #1 to install packages there."
)
logger.info("Environment #1: %s ($VIRTUAL_ENV)", ENVIRONMENT)
logger.info("Environment #2: %s (installation prefix)", sys.prefix)
sys.exit(1)
main_timer = Timer()
initialize_directories()
build_directory = tempfile.mkdtemp()
# Execute "pip install" in a loop in order to retry after intermittent
# error responses from servers (which can happen quite frequently).
try:
for i in xrange(1, MAX_RETRIES):
try:
requirements = unpack_source_dists(arguments, build_directory)
except DistributionNotFound:
logger.warn("We don't have all source distributions yet!")
download_source_dists(arguments, build_directory)
else:
if not requirements:
logger.info(
"No unsatisfied requirements found, probably there's nothing to do."
)
else:
install_requirements(requirements)
logger.info("Done! Took %s to install %i package%s.",
main_timer, len(requirements),
'' if len(requirements) == 1 else 's')
return
logger.warn("pip failed, retrying (%i/%i) ..", i + 1, MAX_RETRIES)
except InstallationError:
# Abort early when pip reports installation errors.
logger.fatal(
"pip reported unrecoverable installation errors. Please fix and rerun!"
)
sys.exit(1)
finally:
# Always cleanup temporary build directory.
shutil.rmtree(build_directory)
# Abort when after N retries we still failed to download source distributions.
logger.fatal("External command failed %i times, aborting!" % MAX_RETRIES)
sys.exit(1)
