def build_missing_binary_dists(requirements): """ Convert source distributions to binary distributions. :param requirements: A list of tuples in the format of the return value of the :py:func:`unpack_source_dists()` function. :returns: ``True`` if it succeeds in building a binary distribution, ``False`` otherwise (probably because of missing binary dependencies like system libraries). """ existing_binary_dists = find_cached_binary_dists() logger.info("Building binary distributions ..") pyversion = get_python_version() for name, version, directory in requirements: # Check if a binary distribution already exists. filename = existing_binary_dists.get((name.lower(), version, pyversion)) if filename: logger.debug("Existing binary distribution for %s (%s) found at %s.", name, version, filename) continue # Make sure the source distribution contains a setup script. setup_script = os.path.join(directory, 'setup.py') if not os.path.isfile(setup_script): logger.warn("Package %s (%s) is not a source distribution.", name, version) continue # Try to build the binary distribution. if not build_binary_dist(name, version, directory, pyversion): sanity_check_dependencies(name) if not build_binary_dist(name, version, directory, pyversion): return False logger.info("Finished building binary distributions.") return True
def main(): """ Main logic of the ``pip-accel`` command. """ arguments = sys.argv[1:] # If no arguments are given, the help text of pip-accel is printed. if not arguments: print_usage() sys.exit(0) # If no install subcommand is given we pass the command line straight # to pip without any changes and exit immediately afterwards. elif 'install' not in arguments: sys.exit(os.spawnvp(os.P_WAIT, 'pip', ['pip'] + arguments)) # Initialize logging output. coloredlogs.install() # Increase verbosity based on -v, --verbose options. for argument in arguments: if argument == '--verbose' or (len(argument) >= 2 and argument[0] == '-' and argument[1] != '-' and 'v' in argument): coloredlogs.increase_verbosity() # Make sure the prefix is the same as the environment. if not os.path.samefile(sys.prefix, ENVIRONMENT): logger.error("You are trying to install packages in environment #1 which is different from environment #2 where pip-accel is installed! Please install pip-accel under environment #1 to install packages there.") logger.info("Environment #1: %s ($VIRTUAL_ENV)", ENVIRONMENT) logger.info("Environment #2: %s (installation prefix)", sys.prefix) sys.exit(1) main_timer = Timer() initialize_directories() build_directory = tempfile.mkdtemp() # Execute "pip install" in a loop in order to retry after intermittent # error responses from servers (which can happen quite frequently). try: for i in xrange(1, MAX_RETRIES): try: requirements = unpack_source_dists(arguments, build_directory) except DistributionNotFound: logger.warn("We don't have all source distributions yet!") download_source_dists(arguments, build_directory) else: if not requirements: logger.info("No unsatisfied requirements found, probably there's nothing to do.") else: install_requirements(requirements) logger.info("Done! Took %s to install %i package%s.", main_timer, len(requirements), '' if len(requirements) == 1 else 's') return logger.warn("pip failed, retrying (%i/%i) ..", i + 1, MAX_RETRIES) except InstallationError: # Abort early when pip reports installation errors. logger.fatal("pip reported unrecoverable installation errors. Please fix and rerun!") sys.exit(1) finally: # Always cleanup temporary build directory. shutil.rmtree(build_directory) # Abort when after N retries we still failed to download source distributions. logger.fatal("External command failed %i times, aborting!" % MAX_RETRIES) sys.exit(1)
def download_source_dists(arguments): """ Download missing source distributions. :param arguments: A list with the arguments intended for ``pip``. """ download_timer = Timer() logger.info("Downloading source distributions ..") # Execute pip to download missing source distributions. try: run_pip(arguments + ['--no-install'], use_remote_index=True) logger.info("Finished downloading source distributions in %s.", download_timer) except Exception, e: logger.warn("pip raised an exception while downloading source distributions: %s.", e)
def cache_binary_dist(input_path, output_path): """ Transform a binary distribution archive created with ``python setup.py bdist_dumb --format=gztar`` into a form that can be cached for future use. This comes down to making the pathnames inside the archive relative to the `prefix` that the binary distribution was built for. :param input_path: The pathname of the original binary distribution archive :param output_path: The pathname of the binary distribution in the cache directory. """ # Copy the tar archive file by file so we can rewrite their pathnames. logger.debug("Expected prefix in binary distribution archive: %s.", ENVIRONMENT) input_bdist = tarfile.open(input_path, 'r:gz') output_bdist = tarfile.open(output_path, 'w:gz') for member in input_bdist.getmembers(): # In my testing the `dumb' tar files created with the `python setup.py # bdist' command contain pathnames that are relative to `/' which is # kind of awkward: I would like to use os.path.relpath() on them but # that won't give the correct result without preprocessing... original_pathname = member.name absolute_pathname = re.sub(r'^\./', '/', original_pathname) if member.isdev(): logger.debug("Warning: Ignoring device file: %s.", absolute_pathname) elif not member.isdir(): modified_pathname = os.path.relpath(absolute_pathname, ENVIRONMENT) if os.path.isabs(modified_pathname): logger.warn("Failed to transform pathname in binary distribution to relative path! (original: %r, modified: %r)", original_pathname, modified_pathname) else: # Some binary distributions include C header files (see for # example the greenlet package) however the subdirectory of # include/ in a virtual environment is a symbolic link to a # subdirectory of /usr/include/ so we should never try to # install C header files inside the directory pointed to by the # symbolic link. Instead we implement the same workaround that # pip uses to avoid this problem. modified_pathname = re.sub('^include/', 'include/site/', modified_pathname) logger.debug("Transformed %r -> %r.", original_pathname, modified_pathname) # Get the file data from the input archive. file_data = input_bdist.extractfile(original_pathname) # Use all metadata from the input archive but modify the filename. member.name = modified_pathname # Copy modified metadata + original file data to output archive. output_bdist.addfile(member, file_data) input_bdist.close() output_bdist.close()
def download_source_dists(arguments, build_directory): """ Download missing source distributions. :param arguments: A list with the arguments intended for ``pip``. """ download_timer = Timer() logger.info("Downloading source distributions ..") # Execute pip to download missing source distributions. try: run_pip(arguments + ['--no-install'], use_remote_index=True, build_directory=build_directory) logger.info("Finished downloading source distributions in %s.", download_timer) except PreviousBuildDirError: # Shouldn't be swallowed. raise except Exception, e: logger.warn("pip raised an exception while downloading source distributions: %s.", e)
def download_source_dists(arguments, build_directory): """ Download missing source distributions. :param arguments: A list with the arguments intended for ``pip``. """ download_timer = Timer() logger.info("Downloading source distributions ..") clear_build_directory(build_directory) # Execute pip to download missing source distributions. try: run_pip(arguments + ['--no-install'], use_remote_index=True, build_directory=build_directory) logger.info("Finished downloading source distributions in %s.", download_timer) except Exception, e: logger.warn( "pip raised an exception while downloading source distributions: %s.", e)
def previous_build_workaround(partial_function, build_directory): """ Since pip 1.4, pip refuses to touch existing build directories, even if pip itself created those build directories! This function implements a brute force workaround until I find a better way. :param partial_function: The function to apply the workaround to. :param build_directory: The pathname of the build directory. :returns: The return value of the partial function. """ try: return partial_function() except PreviousBuildDirError: logger.warn("Caught a 'previous build directory' error, applying workaround ..") # This sucks but I don't see any way around it. Shouldn't pip try to # prevent this situation from happening? We always start from a clean # build directory anyway :-s. shutil.rmtree(build_directory) os.makedirs(build_directory) return partial_function()
def main(): """ Main logic of the ``pip-accel`` command. """ arguments = sys.argv[1:] # If no arguments are given, the help text of pip-accel is printed. if not arguments: print_usage() sys.exit(0) # If no install subcommand is given we pass the command line straight # to pip without any changes and exit immediately afterwards. elif 'install' not in arguments: sys.exit(os.spawnvp(os.P_WAIT, 'pip', ['pip'] + arguments)) # Initialize logging output. coloredlogs.install() # Increase verbosity based on -v, --verbose options. for argument in arguments: if argument == '--verbose' or (len(argument) >= 2 and argument[0] == '-' and argument[1] != '-' and 'v' in argument): coloredlogs.increase_verbosity() # Make sure the prefix is the same as the environment. if not os.path.samefile(sys.prefix, ENVIRONMENT): logger.error( "You are trying to install packages in environment #1 which is different from environment #2 where pip-accel is installed! Please install pip-accel under environment #1 to install packages there." ) logger.info("Environment #1: %s ($VIRTUAL_ENV)", ENVIRONMENT) logger.info("Environment #2: %s (installation prefix)", sys.prefix) sys.exit(1) main_timer = Timer() initialize_directories() build_directory = tempfile.mkdtemp() # Execute "pip install" in a loop in order to retry after intermittent # error responses from servers (which can happen quite frequently). try: for i in xrange(1, MAX_RETRIES): try: requirements = unpack_source_dists(arguments, build_directory) except DistributionNotFound: logger.warn("We don't have all source distributions yet!") download_source_dists(arguments, build_directory) else: if not requirements: logger.info( "No unsatisfied requirements found, probably there's nothing to do." ) else: install_requirements(requirements) logger.info("Done! Took %s to install %i package%s.", main_timer, len(requirements), '' if len(requirements) == 1 else 's') return logger.warn("pip failed, retrying (%i/%i) ..", i + 1, MAX_RETRIES) except InstallationError: # Abort early when pip reports installation errors. logger.fatal( "pip reported unrecoverable installation errors. Please fix and rerun!" ) sys.exit(1) finally: # Always cleanup temporary build directory. shutil.rmtree(build_directory) # Abort when after N retries we still failed to download source distributions. logger.fatal("External command failed %i times, aborting!" % MAX_RETRIES) sys.exit(1)