def install_requirements(requirements, install_prefix=ENVIRONMENT):
"""
Manually install all requirements from binary distributions.
:param requirements: A list of tuples in the format of the return value of
:py:func:`unpack_source_dists()`.
:param install_prefix: The "prefix" under which the requirements should be
installed. This will be a pathname like ``/usr``,
``/usr/local`` or the pathname of a virtual
environment.
:returns: ``True`` if it succeeds in installing all requirements from
binary distribution archives, ``False`` otherwise.
"""
install_timer = Timer()
existing_binary_dists = find_cached_binary_dists()
pyversion = get_python_version()
logger.info("Installing from binary distributions ..")
for name, version, directory in requirements:
filename = existing_binary_dists.get((name.lower(), version, pyversion))
if not filename:
logger.error("No binary distribution of %s (%s) available!", name, version)
return False
install_binary_dist(name, filename, install_prefix=install_prefix)
logger.info("Finished installing all requirements in %s.", install_timer)
return True
def confirm_installation(project_name, packages, command_line):
"""
Notify the user that there are missing dependencies and how they can be
installed. Then ask the user whether we are allowed to install the
dependencies.
:param packages: A list of strings with the names of the packages that are
missing.
:param command_line: A list of strings with the command line needed to
install the packages.
Raises :py:class:`RefusedAutomaticInstallation` when the user refuses to
let pip-accel install any missing dependencies.
"""
logger.info("%s: You seem to be missing %i dependenc%s: %s", project_name,
len(packages), len(packages) == 1 and 'y' or 'ies', " ".join(packages))
logger.info("%s: I can install %s for you with this command: %s",
project_name, len(packages) == 1 and 'it' or 'them',
" ".join(command_line))
try:
prompt = "Do you want me to install %s dependenc%s? [y/N] "
choice = raw_input(prompt % (len(packages) == 1 and 'this' or 'these',
len(packages) == 1 and 'y' or 'ies'))
if choice.lower().strip() == 'y':
logger.info("Got permission to install missing dependenc%s.",
len(packages) == 1 and 'y' or 'ies')
return
except:
pass
logger.error("%s: Refused installation of missing dependenc%s!",
project_name, len(packages) == 1 and 'y' or 'ies')
msg = "%s: User canceled automatic installation of missing dependenc%s!"
raise RefusedAutomaticInstallation, msg % (project_name, 'y' if len(packages) == 1 else 'ies')
def main():
"""
Main logic of the ``pip-accel`` command.
"""
arguments = sys.argv[1:]
# If no arguments are given, the help text of pip-accel is printed.
if not arguments:
print_usage()
sys.exit(0)
# If no install subcommand is given we pass the command line straight
# to pip without any changes and exit immediately afterwards.
elif 'install' not in arguments:
sys.exit(os.spawnvp(os.P_WAIT, 'pip', ['pip'] + arguments))
# Initialize logging output.
coloredlogs.install()
# Increase verbosity based on -v, --verbose options.
for argument in arguments:
if argument == '--verbose' or (len(argument) >= 2 and argument[0] ==
'-' and argument[1] != '-' and 'v' in argument):
coloredlogs.increase_verbosity()
# Make sure the prefix is the same as the environment.
if not os.path.samefile(sys.prefix, ENVIRONMENT):
logger.error("You are trying to install packages in environment #1 which is different from environment #2 where pip-accel is installed! Please install pip-accel under environment #1 to install packages there.")
logger.info("Environment #1: %s ($VIRTUAL_ENV)", ENVIRONMENT)
logger.info("Environment #2: %s (installation prefix)", sys.prefix)
sys.exit(1)
main_timer = Timer()
initialize_directories()
build_directory = tempfile.mkdtemp()
# Execute "pip install" in a loop in order to retry after intermittent
# error responses from servers (which can happen quite frequently).
try:
for i in xrange(1, MAX_RETRIES):
try:
requirements = unpack_source_dists(arguments, build_directory)
except DistributionNotFound:
logger.warn("We don't have all source distributions yet!")
download_source_dists(arguments, build_directory)
else:
if not requirements:
logger.info("No unsatisfied requirements found, probably there's nothing to do.")
else:
install_requirements(requirements)
logger.info("Done! Took %s to install %i package%s.", main_timer, len(requirements), '' if len(requirements) == 1 else 's')
return
logger.warn("pip failed, retrying (%i/%i) ..", i + 1, MAX_RETRIES)
except InstallationError:
# Abort early when pip reports installation errors.
logger.fatal("pip reported unrecoverable installation errors. Please fix and rerun!")
sys.exit(1)
finally:
# Always cleanup temporary build directory.
shutil.rmtree(build_directory)
# Abort when after N retries we still failed to download source distributions.
logger.fatal("External command failed %i times, aborting!" % MAX_RETRIES)
sys.exit(1)
def build_binary_dist(name, version, directory, pyversion):
"""
Convert a single, unpacked source distribution to a binary distribution.
:param name: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:returns: ``True`` if we succeed in building a binary distribution,
``False`` otherwise (probably because of missing binary
dependencies like system libraries).
"""
# Cleanup previously generated distributions.
dist_directory = os.path.join(directory, 'dist')
if os.path.isdir(dist_directory):
logger.info("Cleaning up previously generated distributions in %s ..", dist_directory)
shutil.rmtree(dist_directory)
# Let the user know what's going on.
build_text = "Building binary distribution of %s (%s) .." % (name, version)
logger.info("%s", build_text)
# Compose the command line needed to build the binary distribution.
command_line = '"%s/bin/python" setup.py bdist_dumb --format=gztar' % ENVIRONMENT
logger.debug("Executing external command: %s", command_line)
# Redirect all output of the build to a temporary file.
fd, temporary_file = tempfile.mkstemp()
command_line = '%s > "%s" 2>&1' % (command_line, temporary_file)
# Start the build.
build = subprocess.Popen(['sh', '-c', command_line], cwd=directory)
# Wait for the build to finish.
if INTERACTIVE:
# Provide feedback to the user in the mean time.
spinner = Spinner(build_text)
while build.poll() is None:
spinner.step()
time.sleep(0.1)
spinner.clear()
else:
build.wait()
# Make sure the build succeeded.
if build.returncode != 0:
logger.error("Failed to build binary distribution of %s! (version: %s)", name, version)
with open(temporary_file) as handle:
logger.info("Build output (will probably provide a hint as to what went wrong):\n%s", handle.read())
return False
# Move the generated distribution to the binary index.
filenames = os.listdir(dist_directory)
if len(filenames) != 1:
logger.error("Build process did not result in one binary distribution! (matches: %s)", filenames)
return False
cache_file = '%s:%s:%s.tar.gz' % (name, version, pyversion)
logger.info("Copying binary distribution %s to cache as %s.", filenames[0], cache_file)
cache_binary_dist(os.path.join(directory, 'dist', filenames[0]),
os.path.join(binary_index, cache_file))
return True
def confirm_installation(project_name, packages, command_line):
"""
Notify the user that there are missing dependencies and how they can be
installed. Then ask the user whether we are allowed to install the
dependencies.
:param packages: A list of strings with the names of the packages that are
missing.
:param command_line: A list of strings with the command line needed to
install the packages.
Raises :py:class:`RefusedAutomaticInstallation` when the user refuses to
let pip-accel install any missing dependencies.
"""
logger.info("%s: You seem to be missing %i dependenc%s: %s", project_name,
len(packages),
len(packages) == 1 and 'y' or 'ies', " ".join(packages))
logger.info("%s: I can install %s for you with this command: %s",
project_name,
len(packages) == 1 and 'it' or 'them', " ".join(command_line))
try:
prompt = "Do you want me to install %s dependenc%s? [y/N] "
choice = raw_input(prompt % (len(packages) == 1 and 'this' or 'these',
len(packages) == 1 and 'y' or 'ies'))
if choice.lower().strip() == 'y':
logger.info("Got permission to install missing dependenc%s.",
len(packages) == 1 and 'y' or 'ies')
return
except:
pass
logger.error("%s: Refused installation of missing dependenc%s!",
project_name,
len(packages) == 1 and 'y' or 'ies')
msg = "%s: User canceled automatic installation of missing dependenc%s!"
raise RefusedAutomaticInstallation, msg % (project_name, 'y' if
len(packages) == 1 else 'ies')
def main():
"""
Main logic of the ``pip-accel`` command.
"""
arguments = sys.argv[1:]
# If no arguments are given, the help text of pip-accel is printed.
if not arguments:
print_usage()
sys.exit(0)
# If no install subcommand is given we pass the command line straight
# to pip without any changes and exit immediately afterwards.
elif 'install' not in arguments:
sys.exit(os.spawnvp(os.P_WAIT, 'pip', ['pip'] + arguments))
# Initialize logging output.
coloredlogs.install()
# Increase verbosity based on -v, --verbose options.
for argument in arguments:
if argument == '--verbose' or (len(argument) >= 2
and argument[0] == '-'
and argument[1] != '-'
and 'v' in argument):
coloredlogs.increase_verbosity()
# Make sure the prefix is the same as the environment.
if not os.path.samefile(sys.prefix, ENVIRONMENT):
logger.error(
"You are trying to install packages in environment #1 which is different from environment #2 where pip-accel is installed! Please install pip-accel under environment #1 to install packages there."
)
logger.info("Environment #1: %s ($VIRTUAL_ENV)", ENVIRONMENT)
logger.info("Environment #2: %s (installation prefix)", sys.prefix)
sys.exit(1)
main_timer = Timer()
initialize_directories()
build_directory = tempfile.mkdtemp()
# Execute "pip install" in a loop in order to retry after intermittent
# error responses from servers (which can happen quite frequently).
try:
for i in xrange(1, MAX_RETRIES):
try:
requirements = unpack_source_dists(arguments, build_directory)
except DistributionNotFound:
logger.warn("We don't have all source distributions yet!")
download_source_dists(arguments, build_directory)
else:
if not requirements:
logger.info(
"No unsatisfied requirements found, probably there's nothing to do."
)
else:
install_requirements(requirements)
logger.info("Done! Took %s to install %i package%s.",
main_timer, len(requirements),
'' if len(requirements) == 1 else 's')
return
logger.warn("pip failed, retrying (%i/%i) ..", i + 1, MAX_RETRIES)
except InstallationError:
# Abort early when pip reports installation errors.
logger.fatal(
"pip reported unrecoverable installation errors. Please fix and rerun!"
)
sys.exit(1)
finally:
# Always cleanup temporary build directory.
shutil.rmtree(build_directory)
# Abort when after N retries we still failed to download source distributions.
logger.fatal("External command failed %i times, aborting!" % MAX_RETRIES)
sys.exit(1)
def sanity_check_dependencies(project_name, auto_install=None):
"""
If ``pip-accel`` fails to build a binary distribution, it will call this
function as a last chance to install missing dependencies. If this function
does not raise an exception, ``pip-accel`` will retry the build once.
:param project_name: The project name of a requirement as found on PyPI.
:param auto_install: ``True`` if dependencies on system packages may be
automatically installed, ``False`` if missing system
packages should raise an error, ``None`` if the
decision should be based on the environment variable
``PIP_ACCEL_AUTO_INSTALL``.
If missing system packages are found, this function will try to install
them. If anything "goes wrong" an exception is raised:
- If ``auto_install=False`` then :py:class:`DependencyCheckFailed` is
raised
- If installation of missing packages fails
:py:class:`DependencyInstallationFailed` is raised
- If the user refuses to let pip-accel install missing packages
:py:class:`RefusedAutomaticInstallation` is raised.
If all goes well nothing is raised, nor is anything returned.
"""
# Has the caller forbidden us from automatic installation?
auto_install_forbidden = (auto_install == False)
if auto_install is not None:
# If the caller made an explicit choice, we'll respect that.
auto_install_allowed = auto_install
else:
# Otherwise we check the environment variable.
auto_install_allowed = 'PIP_ACCEL_AUTO_INSTALL' in os.environ
logger.info("%s: Checking for missing dependencies ..", project_name)
known_dependencies = current_platform.find_dependencies(
project_name.lower())
if not known_dependencies:
logger.info(
"%s: No known dependencies... Maybe you have a suggestion?",
project_name)
else:
installed_packages = set(current_platform.find_installed())
missing_packages = [
p for p in known_dependencies if p not in installed_packages
]
if not missing_packages:
logger.info("%s: All known dependencies are already installed.",
project_name)
elif auto_install_forbidden:
msg = "Missing %i system package%s (%s) required by %s but automatic installation is disabled!"
raise DependencyCheckFailed, msg % (
len(missing_packages), '' if len(missing_packages) == 1 else
's', ', '.join(missing_packages), project_name)
else:
command_line = [
'sudo'
] + current_platform.install_command(missing_packages)
if not auto_install_allowed:
confirm_installation(project_name, missing_packages,
command_line)
logger.info("%s: Missing %i dependenc%s: %s", project_name,
len(missing_packages),
'y' if len(missing_packages) == 1 else 'ies',
" ".join(missing_packages))
exit_code = os.spawnvp(os.P_WAIT, 'sudo', command_line)
if exit_code == 0:
logger.info(
"%s: Successfully installed %i missing dependenc%s.",
project_name, len(missing_packages),
len(missing_packages) == 1 and 'y' or 'ies')
else:
logger.error("%s: Failed to install %i missing dependenc%s!",
project_name, len(missing_packages),
len(missing_packages) == 1 and 'y' or 'ies')
msg = "Failed to install %i system package%s required by %s! (command failed: %s)"
raise DependencyInstallationFailed, msg % (
len(missing_packages), '' if len(missing_packages) == 1
else 's', project_name, command_line)
def sanity_check_dependencies(project_name, auto_install=None):
"""
If ``pip-accel`` fails to build a binary distribution, it will call this
function as a last chance to install missing dependencies. If this function
does not raise an exception, ``pip-accel`` will retry the build once.
:param project_name: The project name of a requirement as found on PyPI.
:param auto_install: ``True`` if dependencies on system packages may be
automatically installed, ``False`` if missing system
packages should raise an error, ``None`` if the
decision should be based on the environment variable
``PIP_ACCEL_AUTO_INSTALL``.
If missing system packages are found, this function will try to install
them. If anything "goes wrong" an exception is raised:
- If ``auto_install=False`` then :py:class:`DependencyCheckFailed` is
raised
- If installation of missing packages fails
:py:class:`DependencyInstallationFailed` is raised
- If the user refuses to let pip-accel install missing packages
:py:class:`RefusedAutomaticInstallation` is raised.
If all goes well nothing is raised, nor is anything returned.
"""
# Has the caller forbidden us from automatic installation?
auto_install_forbidden = (auto_install == False)
if auto_install is not None:
# If the caller made an explicit choice, we'll respect that.
auto_install_allowed = auto_install
else:
# Otherwise we check the environment variable.
auto_install_allowed = 'PIP_ACCEL_AUTO_INSTALL' in os.environ
logger.info("%s: Checking for missing dependencies ..", project_name)
known_dependencies = current_platform.find_dependencies(project_name.lower())
if not known_dependencies:
logger.info("%s: No known dependencies... Maybe you have a suggestion?", project_name)
else:
installed_packages = set(current_platform.find_installed())
missing_packages = [p for p in known_dependencies if p not in installed_packages]
if not missing_packages:
logger.info("%s: All known dependencies are already installed.", project_name)
elif auto_install_forbidden:
msg = "Missing %i system package%s (%s) required by %s but automatic installation is disabled!"
raise DependencyCheckFailed, msg % (len(missing_packages), '' if len(missing_packages) == 1 else 's',
', '.join(missing_packages), project_name)
else:
command_line = ['sudo'] + current_platform.install_command(missing_packages)
if not auto_install_allowed:
confirm_installation(project_name, missing_packages, command_line)
logger.info("%s: Missing %i dependenc%s: %s",
project_name, len(missing_packages),
'y' if len(missing_packages) == 1 else 'ies',
" ".join(missing_packages))
exit_code = os.spawnvp(os.P_WAIT, 'sudo', command_line)
if exit_code == 0:
logger.info("%s: Successfully installed %i missing dependenc%s.",
project_name, len(missing_packages),
len(missing_packages) == 1 and 'y' or 'ies')
else:
logger.error("%s: Failed to install %i missing dependenc%s!",
project_name, len(missing_packages),
len(missing_packages) == 1 and 'y' or 'ies')
msg = "Failed to install %i system package%s required by %s! (command failed: %s)"
raise DependencyInstallationFailed, msg % (len(missing_packages), '' if len(missing_packages) == 1 else 's',
project_name, command_line)