def build_missing_binary_dists(requirements): """ Convert source distributions to binary distributions. :param requirements: A list of tuples in the format of the return value of the :py:func:`unpack_source_dists()` function. :returns: ``True`` if it succeeds in building a binary distribution, ``False`` otherwise (probably because of missing binary dependencies like system libraries). """ existing_binary_dists = find_cached_binary_dists() logger.info("Building binary distributions ..") pyversion = get_python_version() for name, version, directory in requirements: # Check if a binary distribution already exists. filename = existing_binary_dists.get((name.lower(), version, pyversion)) if filename: logger.debug("Existing binary distribution for %s (%s) found at %s.", name, version, filename) continue # Make sure the source distribution contains a setup script. setup_script = os.path.join(directory, 'setup.py') if not os.path.isfile(setup_script): logger.warn("Package %s (%s) is not a source distribution.", name, version) continue # Try to build the binary distribution. if not build_binary_dist(name, version, directory, pyversion): sanity_check_dependencies(name) if not build_binary_dist(name, version, directory, pyversion): return False logger.info("Finished building binary distributions.") return True
def install_requirements(requirements, install_prefix=ENVIRONMENT): """ Manually install all requirements from binary distributions. :param requirements: A list of tuples in the format of the return value of :py:func:`unpack_source_dists()`. :param install_prefix: The "prefix" under which the requirements should be installed. This will be a pathname like ``/usr``, ``/usr/local`` or the pathname of a virtual environment. :returns: ``True`` if it succeeds in installing all requirements from binary distribution archives, ``False`` otherwise. """ install_timer = Timer() existing_binary_dists = find_cached_binary_dists() pyversion = get_python_version() logger.info("Installing from binary distributions ..") for name, version, directory in requirements: filename = existing_binary_dists.get((name.lower(), version, pyversion)) if not filename: logger.error("No binary distribution of %s (%s) available!", name, version) return False install_binary_dist(name, filename, install_prefix=install_prefix) logger.info("Finished installing all requirements in %s.", install_timer) return True
def download_source_dists(arguments): """ Download missing source distributions. :param arguments: A list with the arguments intended for ``pip``. """ download_timer = Timer() logger.info("Downloading source distributions ..") # Execute pip to download missing source distributions. try: run_pip(arguments + ['--no-install'], use_remote_index=True) logger.info("Finished downloading source distributions in %s.", download_timer) except Exception, e: logger.warn("pip raised an exception while downloading source distributions: %s.", e)
def download_source_dists(arguments, build_directory): """ Download missing source distributions. :param arguments: A list with the arguments intended for ``pip``. """ download_timer = Timer() logger.info("Downloading source distributions ..") # Execute pip to download missing source distributions. try: run_pip(arguments + ['--no-install'], use_remote_index=True, build_directory=build_directory) logger.info("Finished downloading source distributions in %s.", download_timer) except PreviousBuildDirError: # Shouldn't be swallowed. raise except Exception, e: logger.warn("pip raised an exception while downloading source distributions: %s.", e)
def main(): """ Main logic of the ``pip-accel`` command. """ arguments = sys.argv[1:] # If no arguments are given, the help text of pip-accel is printed. if not arguments: print_usage() sys.exit(0) # If no install subcommand is given we pass the command line straight # to pip without any changes and exit immediately afterwards. elif 'install' not in arguments: sys.exit(os.spawnvp(os.P_WAIT, 'pip', ['pip'] + arguments)) # Initialize logging output. coloredlogs.install() # Increase verbosity based on -v, --verbose options. for argument in arguments: if argument == '--verbose' or (len(argument) >= 2 and argument[0] == '-' and argument[1] != '-' and 'v' in argument): coloredlogs.increase_verbosity() # Make sure the prefix is the same as the environment. if not os.path.samefile(sys.prefix, ENVIRONMENT): logger.error("You are trying to install packages in environment #1 which is different from environment #2 where pip-accel is installed! Please install pip-accel under environment #1 to install packages there.") logger.info("Environment #1: %s ($VIRTUAL_ENV)", ENVIRONMENT) logger.info("Environment #2: %s (installation prefix)", sys.prefix) sys.exit(1) main_timer = Timer() initialize_directories() build_directory = tempfile.mkdtemp() # Execute "pip install" in a loop in order to retry after intermittent # error responses from servers (which can happen quite frequently). try: for i in xrange(1, MAX_RETRIES): try: requirements = unpack_source_dists(arguments, build_directory) except DistributionNotFound: logger.warn("We don't have all source distributions yet!") download_source_dists(arguments, build_directory) else: if not requirements: logger.info("No unsatisfied requirements found, probably there's nothing to do.") else: install_requirements(requirements) logger.info("Done! Took %s to install %i package%s.", main_timer, len(requirements), '' if len(requirements) == 1 else 's') return logger.warn("pip failed, retrying (%i/%i) ..", i + 1, MAX_RETRIES) except InstallationError: # Abort early when pip reports installation errors. logger.fatal("pip reported unrecoverable installation errors. Please fix and rerun!") sys.exit(1) finally: # Always cleanup temporary build directory. shutil.rmtree(build_directory) # Abort when after N retries we still failed to download source distributions. logger.fatal("External command failed %i times, aborting!" % MAX_RETRIES) sys.exit(1)
def build_binary_dist(name, version, directory, pyversion): """ Convert a single, unpacked source distribution to a binary distribution. :param name: The name of the requirement to build. :param version: The version of the requirement to build. :param directory: The directory where the unpacked sources of the requirement are available. :returns: ``True`` if we succeed in building a binary distribution, ``False`` otherwise (probably because of missing binary dependencies like system libraries). """ # Cleanup previously generated distributions. dist_directory = os.path.join(directory, 'dist') if os.path.isdir(dist_directory): logger.info("Cleaning up previously generated distributions in %s ..", dist_directory) shutil.rmtree(dist_directory) # Let the user know what's going on. build_text = "Building binary distribution of %s (%s) .." % (name, version) logger.info("%s", build_text) # Compose the command line needed to build the binary distribution. command_line = '"%s/bin/python" setup.py bdist_dumb --format=gztar' % ENVIRONMENT logger.debug("Executing external command: %s", command_line) # Redirect all output of the build to a temporary file. fd, temporary_file = tempfile.mkstemp() command_line = '%s > "%s" 2>&1' % (command_line, temporary_file) # Start the build. build = subprocess.Popen(['sh', '-c', command_line], cwd=directory) # Wait for the build to finish. if INTERACTIVE: # Provide feedback to the user in the mean time. spinner = Spinner(build_text) while build.poll() is None: spinner.step() time.sleep(0.1) spinner.clear() else: build.wait() # Make sure the build succeeded. if build.returncode != 0: logger.error("Failed to build binary distribution of %s! (version: %s)", name, version) with open(temporary_file) as handle: logger.info("Build output (will probably provide a hint as to what went wrong):\n%s", handle.read()) return False # Move the generated distribution to the binary index. filenames = os.listdir(dist_directory) if len(filenames) != 1: logger.error("Build process did not result in one binary distribution! (matches: %s)", filenames) return False cache_file = '%s:%s:%s.tar.gz' % (name, version, pyversion) logger.info("Copying binary distribution %s to cache as %s.", filenames[0], cache_file) cache_binary_dist(os.path.join(directory, 'dist', filenames[0]), os.path.join(binary_index, cache_file)) return True
def run_pip(arguments, use_remote_index, build_directory=None): """ Execute a modified ``pip install`` command. This function assumes that the arguments concern a ``pip install`` command (:py:func:`main()` makes sure of this). :param arguments: A list of strings containing the arguments that will be passed to ``pip``. :param use_remote_index: A boolean indicating whether ``pip`` is allowed to contact http://pypi.python.org. :returns: A ``RequirementSet`` object created by ``pip``, unless an exception is raised by ``pip`` (in which case the exception will bubble up). """ command_line = [] for i, arg in enumerate(arguments): if arg == 'install': command_line += ['pip'] + arguments[:i + 1] + [ '--download-cache=%s' % download_cache, '--find-links=file://%s' % source_index ] if build_directory: command_line += ['--build-directory=%s' % build_directory] if not use_remote_index: command_line += ['--no-index'] command_line += arguments[i + 1:] break else: command_line = ['pip'] + arguments logger.info("Executing command: %s", ' '.join(command_line)) # XXX Nasty hack required for pip 1.4 compatibility (workaround for global state). requirements_option.default = [] cmd_name, options, args, parser = parseopts(command_line[1:]) pip = CustomInstallCommand(parser) exit_status = pip.main(args[1:], options) # Make sure the output of pip and pip-accel are not intermingled. sys.stdout.flush() update_source_dists_index() if exit_status == SUCCESS: return pip.requirement_set else: raise pip.intercepted_exception
def download_source_dists(arguments, build_directory): """ Download missing source distributions. :param arguments: A list with the arguments intended for ``pip``. """ download_timer = Timer() logger.info("Downloading source distributions ..") clear_build_directory(build_directory) # Execute pip to download missing source distributions. try: run_pip(arguments + ['--no-install'], use_remote_index=True, build_directory=build_directory) logger.info("Finished downloading source distributions in %s.", download_timer) except Exception, e: logger.warn( "pip raised an exception while downloading source distributions: %s.", e)
def run_pip(arguments, use_remote_index, build_directory=None): """ Execute a modified ``pip install`` command. This function assumes that the arguments concern a ``pip install`` command (:py:func:`main()` makes sure of this). :param arguments: A list of strings containing the arguments that will be passed to ``pip``. :param use_remote_index: A boolean indicating whether ``pip`` is allowed to contact http://pypi.python.org. :returns: A ``RequirementSet`` object created by ``pip``, unless an exception is raised by ``pip`` (in which case the exception will bubble up). """ command_line = [] for i, arg in enumerate(arguments): if arg == 'install': command_line += ['pip'] + arguments[:i+1] + [ '--download-cache=%s' % download_cache, '--find-links=file://%s' % source_index] if build_directory: command_line += ['--build-directory=%s' % build_directory] if not use_remote_index: command_line += ['--no-index'] command_line += arguments[i+1:] break else: command_line = ['pip'] + arguments logger.info("Executing command: %s", ' '.join(command_line)) # XXX Nasty hack required for pip 1.4 compatibility (workaround for global state). requirements_option.default = [] cmd_name, options, args, parser = parseopts(command_line[1:]) pip = CustomInstallCommand(parser) exit_status = pip.main(args[1:], options) # Make sure the output of pip and pip-accel are not intermingled. sys.stdout.flush() update_source_dists_index() if exit_status == SUCCESS: return pip.requirement_set else: raise pip.intercepted_exception
def confirm_installation(project_name, packages, command_line): """ Notify the user that there are missing dependencies and how they can be installed. Then ask the user whether we are allowed to install the dependencies. :param packages: A list of strings with the names of the packages that are missing. :param command_line: A list of strings with the command line needed to install the packages. Raises :py:class:`RefusedAutomaticInstallation` when the user refuses to let pip-accel install any missing dependencies. """ logger.info("%s: You seem to be missing %i dependenc%s: %s", project_name, len(packages), len(packages) == 1 and 'y' or 'ies', " ".join(packages)) logger.info("%s: I can install %s for you with this command: %s", project_name, len(packages) == 1 and 'it' or 'them', " ".join(command_line)) try: prompt = "Do you want me to install %s dependenc%s? [y/N] " choice = raw_input(prompt % (len(packages) == 1 and 'this' or 'these', len(packages) == 1 and 'y' or 'ies')) if choice.lower().strip() == 'y': logger.info("Got permission to install missing dependenc%s.", len(packages) == 1 and 'y' or 'ies') return except: pass logger.error("%s: Refused installation of missing dependenc%s!", project_name, len(packages) == 1 and 'y' or 'ies') msg = "%s: User canceled automatic installation of missing dependenc%s!" raise RefusedAutomaticInstallation, msg % (project_name, 'y' if len(packages) == 1 else 'ies')
def install_requirements(requirements, install_prefix=ENVIRONMENT): """ Manually install all requirements from binary distributions. :param requirements: A list of tuples in the format of the return value of :py:func:`unpack_source_dists()`. :param install_prefix: The "prefix" under which the requirements should be installed. This will be a pathname like ``/usr``, ``/usr/local`` or the pathname of a virtual environment. :returns: ``True`` if it succeeds in installing all requirements from binary distribution archives, ``False`` otherwise. """ install_timer = Timer() logger.info("Installing from binary distributions ..") python = os.path.join(install_prefix, 'bin', 'python') pip = os.path.join(install_prefix, 'bin', 'pip') for name, version, directory in requirements: if os.system('%s uninstall --yes %s >/dev/null 2>&1' % (pipes.quote(pip), pipes.quote(name))) == 0: logger.info("Uninstalled previously installed package %s.", name) members = get_binary_dist(name, version, directory, prefix=install_prefix, python=python) install_binary_dist(members, prefix=install_prefix, python=python) logger.info("Finished installing all requirements in %s.", install_timer) return True
def find_cached_binary_dists(): """ Find all previously cached binary distributions. :returns: A dictionary with (package-name, package-version, python-version) tuples as keys and pathnames of binary archives as values. """ logger.info("Scanning binary distribution index ..") distributions = {} for filename in sorted(os.listdir(binary_index), key=str.lower): if filename.endswith('.tar.gz'): basename = re.sub('\.tar.gz$', '', filename) parts = basename.split(':') if len(parts) == 3: key = (parts[0].lower(), parts[1], parts[2]) logger.debug("Matched %s in %s.", key, filename) distributions[key] = os.path.join(binary_index, filename) continue logger.debug("Failed to match filename: %s.", filename) logger.info("Found %i existing binary distribution%s.", len(distributions), '' if len(distributions) == 1 else 's') for (name, version, pyversion), filename in distributions.iteritems(): logger.debug(" - %s (%s, %s) in %s.", name, version, pyversion, filename) return distributions
def update_source_dists_index(): """ Link newly downloaded source distributions into the local index directory using symbolic links. """ for download_name in os.listdir(download_cache): download_path = os.path.join(download_cache, download_name) url = urllib.unquote(download_name) if not url.endswith('.content-type'): components = urlparse.urlparse(url) archive_name = os.path.basename(components.path) archive_path = os.path.join(source_index, add_extension(download_path, archive_name)) if not os.path.isfile(archive_path): logger.info("Linking files:") logger.info(" - Source: %s", download_path) logger.info(" - Target: %s", archive_path) os.symlink(download_path, archive_path)
def update_source_dists_index(): """ Link newly downloaded source distributions into the local index directory using symbolic links. """ for download_name in os.listdir(download_cache): download_path = os.path.join(download_cache, download_name) url = urllib.unquote(download_name) if not url.endswith('.content-type'): components = urlparse.urlparse(url) archive_name = os.path.basename(components.path) archive_path = os.path.join( source_index, add_extension(download_path, archive_name)) if not os.path.isfile(archive_path): logger.info("Linking files:") logger.info(" - Source: %s", download_path) logger.info(" - Target: %s", archive_path) os.symlink(download_path, archive_path)
def install_binary_dist(package, filename, install_prefix=ENVIRONMENT): """ Install a binary distribution created with ``python setup.py bdist`` into the given prefix (a directory like ``/usr``, ``/usr/local`` or a virtual environment). :param package: The name of the package to install. :param filename: The pathname of the tar archive. :param install_prefix: The "prefix" under which the requirements should be installed. This will be a pathname like ``/usr``, ``/usr/local`` or the pathname of a virtual environment. """ # TODO This is quite slow for modules like Django. Speed it up! Two choices: # 1. Run the external tar program to unpack the archive. This will # slightly complicate the fixing up of hashbangs. # 2. Using links? The plan: We can maintain a "seed" environment under # $PIP_ACCEL_CACHE and use symbolic and/or hard links to populate other # places based on the "seed" environment. install_timer = Timer() python = os.path.join(install_prefix, 'bin', 'python') pip = os.path.join(install_prefix, 'bin', 'pip') if os.system('"%s" uninstall --yes "%s" >/dev/null 2>&1' % (pip, package)) == 0: logger.info("Uninstalled previously installed package %s.", package) logger.info("Installing package %s from binary distribution %s to %s ..", package, filename, install_prefix) archive = tarfile.open(filename, 'r:gz') for member in archive.getmembers(): install_path = os.path.join(install_prefix, member.name) directory = os.path.dirname(install_path) if not os.path.isdir(directory): logger.debug("Creating directory: %s ..", directory) os.makedirs(directory) logger.debug("Writing file: %s ..", install_path) file_handle = archive.extractfile(member) with open(install_path, 'w') as handle: contents = file_handle.read() if contents.startswith('#!/'): contents = fix_hashbang(python, contents) handle.write(contents) os.chmod(install_path, member.mode) archive.close() logger.info("Finished installing binary distribution in %s.", install_timer)
def unpack_source_dists(arguments, build_directory): """ Check whether there are local source distributions available for all requirements, unpack the source distribution archives and find the names and versions of the requirements. By using the ``pip install --no-install`` command we avoid reimplementing the following pip features: - Parsing of ``requirements.txt`` (including recursive parsing) - Resolution of possibly conflicting pinned requirements - Unpacking source distributions in multiple formats - Finding the name & version of a given source distribution :param arguments: A list of strings with the command line arguments to be passed to the ``pip`` command. :returns: A list of tuples with three strings each: The name of a requirement (package), its version number and the directory where the unpacked source distribution is located. If ``pip`` fails, an exception will be raised by ``pip``. """ unpack_timer = Timer() logger.info("Unpacking local source distributions ..") clear_build_directory(build_directory) # Execute pip to unpack the source distributions. requirement_set = run_pip(arguments + ['--no-install'], use_remote_index=False, build_directory=build_directory) logger.info("Unpacked local source distributions in %s.", unpack_timer) requirements = [] for install_requirement in sorted_requirements(requirement_set): if install_requirement.satisfied_by: logger.info("Requirement already satisfied: %s.", install_requirement) else: req = ensure_parsed_requirement(install_requirement) requirements.append( (req.project_name, install_requirement.installed_version, install_requirement.source_dir)) return requirements
def unpack_source_dists(arguments, build_directory): """ Check whether there are local source distributions available for all requirements, unpack the source distribution archives and find the names and versions of the requirements. By using the ``pip install --no-install`` command we avoid reimplementing the following pip features: - Parsing of ``requirements.txt`` (including recursive parsing) - Resolution of possibly conflicting pinned requirements - Unpacking source distributions in multiple formats - Finding the name & version of a given source distribution :param arguments: A list of strings with the command line arguments to be passed to the ``pip`` command. :returns: A list of tuples with three strings each: The name of a requirement (package), its version number and the directory where the unpacked source distribution is located. If ``pip`` fails, an exception will be raised by ``pip``. """ unpack_timer = Timer() logger.info("Unpacking local source distributions ..") clear_build_directory(build_directory) # Execute pip to unpack the source distributions. requirement_set = run_pip(arguments + ['--no-install'], use_remote_index=False, build_directory=build_directory) logger.info("Unpacked local source distributions in %s.", unpack_timer) requirements = [] for install_requirement in sorted_requirements(requirement_set): if install_requirement.satisfied_by: logger.info("Requirement already satisfied: %s.", install_requirement) else: req = ensure_parsed_requirement(install_requirement) requirements.append((req.project_name, install_requirement.installed_version, install_requirement.source_dir)) return requirements
def sanity_check_dependencies(project_name, auto_install=None): """ If ``pip-accel`` fails to build a binary distribution, it will call this function as a last chance to install missing dependencies. If this function does not raise an exception, ``pip-accel`` will retry the build once. :param project_name: The project name of a requirement as found on PyPI. :param auto_install: ``True`` if dependencies on system packages may be automatically installed, ``False`` if missing system packages should raise an error, ``None`` if the decision should be based on the environment variable ``PIP_ACCEL_AUTO_INSTALL``. If missing system packages are found, this function will try to install them. If anything "goes wrong" an exception is raised: - If ``auto_install=False`` then :py:class:`DependencyCheckFailed` is raised - If installation of missing packages fails :py:class:`DependencyInstallationFailed` is raised - If the user refuses to let pip-accel install missing packages :py:class:`RefusedAutomaticInstallation` is raised. If all goes well nothing is raised, nor is anything returned. """ # Has the caller forbidden us from automatic installation? auto_install_forbidden = (auto_install == False) if auto_install is not None: # If the caller made an explicit choice, we'll respect that. auto_install_allowed = auto_install else: # Otherwise we check the environment variable. auto_install_allowed = 'PIP_ACCEL_AUTO_INSTALL' in os.environ logger.info("%s: Checking for missing dependencies ..", project_name) known_dependencies = current_platform.find_dependencies(project_name.lower()) if not known_dependencies: logger.info("%s: No known dependencies... Maybe you have a suggestion?", project_name) else: installed_packages = set(current_platform.find_installed()) missing_packages = [p for p in known_dependencies if p not in installed_packages] if not missing_packages: logger.info("%s: All known dependencies are already installed.", project_name) elif auto_install_forbidden: msg = "Missing %i system package%s (%s) required by %s but automatic installation is disabled!" raise DependencyCheckFailed, msg % (len(missing_packages), '' if len(missing_packages) == 1 else 's', ', '.join(missing_packages), project_name) else: command_line = ['sudo'] + current_platform.install_command(missing_packages) if not auto_install_allowed: confirm_installation(project_name, missing_packages, command_line) logger.info("%s: Missing %i dependenc%s: %s", project_name, len(missing_packages), 'y' if len(missing_packages) == 1 else 'ies', " ".join(missing_packages)) exit_code = os.spawnvp(os.P_WAIT, 'sudo', command_line) if exit_code == 0: logger.info("%s: Successfully installed %i missing dependenc%s.", project_name, len(missing_packages), len(missing_packages) == 1 and 'y' or 'ies') else: logger.error("%s: Failed to install %i missing dependenc%s!", project_name, len(missing_packages), len(missing_packages) == 1 and 'y' or 'ies') msg = "Failed to install %i system package%s required by %s! (command failed: %s)" raise DependencyInstallationFailed, msg % (len(missing_packages), '' if len(missing_packages) == 1 else 's', project_name, command_line)
def sanity_check_dependencies(project_name, auto_install=None): """ If ``pip-accel`` fails to build a binary distribution, it will call this function as a last chance to install missing dependencies. If this function does not raise an exception, ``pip-accel`` will retry the build once. :param project_name: The project name of a requirement as found on PyPI. :param auto_install: ``True`` if dependencies on system packages may be automatically installed, ``False`` if missing system packages should raise an error, ``None`` if the decision should be based on the environment variable ``PIP_ACCEL_AUTO_INSTALL``. If missing system packages are found, this function will try to install them. If anything "goes wrong" an exception is raised: - If ``auto_install=False`` then :py:class:`DependencyCheckFailed` is raised - If installation of missing packages fails :py:class:`DependencyInstallationFailed` is raised - If the user refuses to let pip-accel install missing packages :py:class:`RefusedAutomaticInstallation` is raised. If all goes well nothing is raised, nor is anything returned. """ # Has the caller forbidden us from automatic installation? auto_install_forbidden = (auto_install == False) if auto_install is not None: # If the caller made an explicit choice, we'll respect that. auto_install_allowed = auto_install else: # Otherwise we check the environment variable. auto_install_allowed = 'PIP_ACCEL_AUTO_INSTALL' in os.environ logger.info("%s: Checking for missing dependencies ..", project_name) known_dependencies = current_platform.find_dependencies( project_name.lower()) if not known_dependencies: logger.info( "%s: No known dependencies... Maybe you have a suggestion?", project_name) else: installed_packages = set(current_platform.find_installed()) missing_packages = [ p for p in known_dependencies if p not in installed_packages ] if not missing_packages: logger.info("%s: All known dependencies are already installed.", project_name) elif auto_install_forbidden: msg = "Missing %i system package%s (%s) required by %s but automatic installation is disabled!" raise DependencyCheckFailed, msg % ( len(missing_packages), '' if len(missing_packages) == 1 else 's', ', '.join(missing_packages), project_name) else: command_line = [ 'sudo' ] + current_platform.install_command(missing_packages) if not auto_install_allowed: confirm_installation(project_name, missing_packages, command_line) logger.info("%s: Missing %i dependenc%s: %s", project_name, len(missing_packages), 'y' if len(missing_packages) == 1 else 'ies', " ".join(missing_packages)) exit_code = os.spawnvp(os.P_WAIT, 'sudo', command_line) if exit_code == 0: logger.info( "%s: Successfully installed %i missing dependenc%s.", project_name, len(missing_packages), len(missing_packages) == 1 and 'y' or 'ies') else: logger.error("%s: Failed to install %i missing dependenc%s!", project_name, len(missing_packages), len(missing_packages) == 1 and 'y' or 'ies') msg = "Failed to install %i system package%s required by %s! (command failed: %s)" raise DependencyInstallationFailed, msg % ( len(missing_packages), '' if len(missing_packages) == 1 else 's', project_name, command_line)
def main(): """ Main logic of the ``pip-accel`` command. """ arguments = sys.argv[1:] # If no arguments are given, the help text of pip-accel is printed. if not arguments: print_usage() sys.exit(0) # If no install subcommand is given we pass the command line straight # to pip without any changes and exit immediately afterwards. elif 'install' not in arguments: sys.exit(os.spawnvp(os.P_WAIT, 'pip', ['pip'] + arguments)) # Initialize logging output. coloredlogs.install() # Increase verbosity based on -v, --verbose options. for argument in arguments: if argument == '--verbose' or (len(argument) >= 2 and argument[0] == '-' and argument[1] != '-' and 'v' in argument): coloredlogs.increase_verbosity() # Make sure the prefix is the same as the environment. if not os.path.samefile(sys.prefix, ENVIRONMENT): logger.error( "You are trying to install packages in environment #1 which is different from environment #2 where pip-accel is installed! Please install pip-accel under environment #1 to install packages there." ) logger.info("Environment #1: %s ($VIRTUAL_ENV)", ENVIRONMENT) logger.info("Environment #2: %s (installation prefix)", sys.prefix) sys.exit(1) main_timer = Timer() initialize_directories() build_directory = tempfile.mkdtemp() # Execute "pip install" in a loop in order to retry after intermittent # error responses from servers (which can happen quite frequently). try: for i in xrange(1, MAX_RETRIES): try: requirements = unpack_source_dists(arguments, build_directory) except DistributionNotFound: logger.warn("We don't have all source distributions yet!") download_source_dists(arguments, build_directory) else: if not requirements: logger.info( "No unsatisfied requirements found, probably there's nothing to do." ) else: install_requirements(requirements) logger.info("Done! Took %s to install %i package%s.", main_timer, len(requirements), '' if len(requirements) == 1 else 's') return logger.warn("pip failed, retrying (%i/%i) ..", i + 1, MAX_RETRIES) except InstallationError: # Abort early when pip reports installation errors. logger.fatal( "pip reported unrecoverable installation errors. Please fix and rerun!" ) sys.exit(1) finally: # Always cleanup temporary build directory. shutil.rmtree(build_directory) # Abort when after N retries we still failed to download source distributions. logger.fatal("External command failed %i times, aborting!" % MAX_RETRIES) sys.exit(1)