def build_missing_binary_dists(requirements):
"""
Convert source distributions to binary distributions.
:param requirements: A list of tuples in the format of the return value of
the :py:func:`unpack_source_dists()` function.
:returns: ``True`` if it succeeds in building a binary distribution,
``False`` otherwise (probably because of missing binary
dependencies like system libraries).
"""
existing_binary_dists = find_cached_binary_dists()
logger.info("Building binary distributions ..")
pyversion = get_python_version()
for name, version, directory in requirements:
# Check if a binary distribution already exists.
filename = existing_binary_dists.get((name.lower(), version, pyversion))
if filename:
logger.debug("Existing binary distribution for %s (%s) found at %s.", name, version, filename)
continue
# Make sure the source distribution contains a setup script.
setup_script = os.path.join(directory, 'setup.py')
if not os.path.isfile(setup_script):
logger.warn("Package %s (%s) is not a source distribution.", name, version)
continue
# Try to build the binary distribution.
if not build_binary_dist(name, version, directory, pyversion):
sanity_check_dependencies(name)
if not build_binary_dist(name, version, directory, pyversion):
return False
logger.info("Finished building binary distributions.")
return True
def main():
"""
Main logic of the ``pip-accel`` command.
"""
arguments = sys.argv[1:]
# If no arguments are given, the help text of pip-accel is printed.
if not arguments:
print_usage()
sys.exit(0)
# If no install subcommand is given we pass the command line straight
# to pip without any changes and exit immediately afterwards.
elif 'install' not in arguments:
sys.exit(os.spawnvp(os.P_WAIT, 'pip', ['pip'] + arguments))
# Initialize logging output.
coloredlogs.install()
# Increase verbosity based on -v, --verbose options.
for argument in arguments:
if argument == '--verbose' or (len(argument) >= 2 and argument[0] ==
'-' and argument[1] != '-' and 'v' in argument):
coloredlogs.increase_verbosity()
# Make sure the prefix is the same as the environment.
if not os.path.samefile(sys.prefix, ENVIRONMENT):
logger.error("You are trying to install packages in environment #1 which is different from environment #2 where pip-accel is installed! Please install pip-accel under environment #1 to install packages there.")
logger.info("Environment #1: %s ($VIRTUAL_ENV)", ENVIRONMENT)
logger.info("Environment #2: %s (installation prefix)", sys.prefix)
sys.exit(1)
main_timer = Timer()
initialize_directories()
build_directory = tempfile.mkdtemp()
# Execute "pip install" in a loop in order to retry after intermittent
# error responses from servers (which can happen quite frequently).
try:
for i in xrange(1, MAX_RETRIES):
try:
requirements = unpack_source_dists(arguments, build_directory)
except DistributionNotFound:
logger.warn("We don't have all source distributions yet!")
download_source_dists(arguments, build_directory)
else:
if not requirements:
logger.info("No unsatisfied requirements found, probably there's nothing to do.")
else:
install_requirements(requirements)
logger.info("Done! Took %s to install %i package%s.", main_timer, len(requirements), '' if len(requirements) == 1 else 's')
return
logger.warn("pip failed, retrying (%i/%i) ..", i + 1, MAX_RETRIES)
except InstallationError:
# Abort early when pip reports installation errors.
logger.fatal("pip reported unrecoverable installation errors. Please fix and rerun!")
sys.exit(1)
finally:
# Always cleanup temporary build directory.
shutil.rmtree(build_directory)
# Abort when after N retries we still failed to download source distributions.
logger.fatal("External command failed %i times, aborting!" % MAX_RETRIES)
sys.exit(1)
def download_source_dists(arguments):
"""
Download missing source distributions.
:param arguments: A list with the arguments intended for ``pip``.
"""
download_timer = Timer()
logger.info("Downloading source distributions ..")
# Execute pip to download missing source distributions.
try:
run_pip(arguments + ['--no-install'], use_remote_index=True)
logger.info("Finished downloading source distributions in %s.", download_timer)
except Exception, e:
logger.warn("pip raised an exception while downloading source distributions: %s.", e)
def cache_binary_dist(input_path, output_path):
"""
Transform a binary distribution archive created with ``python setup.py
bdist_dumb --format=gztar`` into a form that can be cached for future use.
This comes down to making the pathnames inside the archive relative to the
`prefix` that the binary distribution was built for.
:param input_path: The pathname of the original binary distribution archive
:param output_path: The pathname of the binary distribution in the cache
directory.
"""
# Copy the tar archive file by file so we can rewrite their pathnames.
logger.debug("Expected prefix in binary distribution archive: %s.", ENVIRONMENT)
input_bdist = tarfile.open(input_path, 'r:gz')
output_bdist = tarfile.open(output_path, 'w:gz')
for member in input_bdist.getmembers():
# In my testing the `dumb' tar files created with the `python setup.py
# bdist' command contain pathnames that are relative to `/' which is
# kind of awkward: I would like to use os.path.relpath() on them but
# that won't give the correct result without preprocessing...
original_pathname = member.name
absolute_pathname = re.sub(r'^\./', '/', original_pathname)
if member.isdev():
logger.debug("Warning: Ignoring device file: %s.", absolute_pathname)
elif not member.isdir():
modified_pathname = os.path.relpath(absolute_pathname, ENVIRONMENT)
if os.path.isabs(modified_pathname):
logger.warn("Failed to transform pathname in binary distribution to relative path! (original: %r, modified: %r)",
original_pathname, modified_pathname)
else:
# Some binary distributions include C header files (see for
# example the greenlet package) however the subdirectory of
# include/ in a virtual environment is a symbolic link to a
# subdirectory of /usr/include/ so we should never try to
# install C header files inside the directory pointed to by the
# symbolic link. Instead we implement the same workaround that
# pip uses to avoid this problem.
modified_pathname = re.sub('^include/', 'include/site/', modified_pathname)
logger.debug("Transformed %r -> %r.", original_pathname, modified_pathname)
# Get the file data from the input archive.
file_data = input_bdist.extractfile(original_pathname)
# Use all metadata from the input archive but modify the filename.
member.name = modified_pathname
# Copy modified metadata + original file data to output archive.
output_bdist.addfile(member, file_data)
input_bdist.close()
output_bdist.close()
def download_source_dists(arguments, build_directory):
"""
Download missing source distributions.
:param arguments: A list with the arguments intended for ``pip``.
"""
download_timer = Timer()
logger.info("Downloading source distributions ..")
# Execute pip to download missing source distributions.
try:
run_pip(arguments + ['--no-install'], use_remote_index=True, build_directory=build_directory)
logger.info("Finished downloading source distributions in %s.", download_timer)
except PreviousBuildDirError:
# Shouldn't be swallowed.
raise
except Exception, e:
logger.warn("pip raised an exception while downloading source distributions: %s.", e)
def download_source_dists(arguments, build_directory):
"""
Download missing source distributions.
:param arguments: A list with the arguments intended for ``pip``.
"""
download_timer = Timer()
logger.info("Downloading source distributions ..")
clear_build_directory(build_directory)
# Execute pip to download missing source distributions.
try:
run_pip(arguments + ['--no-install'],
use_remote_index=True,
build_directory=build_directory)
logger.info("Finished downloading source distributions in %s.",
download_timer)
except Exception, e:
logger.warn(
"pip raised an exception while downloading source distributions: %s.",
e)
def previous_build_workaround(partial_function, build_directory):
"""
Since pip 1.4, pip refuses to touch existing build directories, even if pip
itself created those build directories! This function implements a brute
force workaround until I find a better way.
:param partial_function: The function to apply the workaround to.
:param build_directory: The pathname of the build directory.
:returns: The return value of the partial function.
"""
try:
return partial_function()
except PreviousBuildDirError:
logger.warn("Caught a 'previous build directory' error, applying workaround ..")
# This sucks but I don't see any way around it. Shouldn't pip try to
# prevent this situation from happening? We always start from a clean
# build directory anyway :-s.
shutil.rmtree(build_directory)
os.makedirs(build_directory)
return partial_function()
def main():
"""
Main logic of the ``pip-accel`` command.
"""
arguments = sys.argv[1:]
# If no arguments are given, the help text of pip-accel is printed.
if not arguments:
print_usage()
sys.exit(0)
# If no install subcommand is given we pass the command line straight
# to pip without any changes and exit immediately afterwards.
elif 'install' not in arguments:
sys.exit(os.spawnvp(os.P_WAIT, 'pip', ['pip'] + arguments))
# Initialize logging output.
coloredlogs.install()
# Increase verbosity based on -v, --verbose options.
for argument in arguments:
if argument == '--verbose' or (len(argument) >= 2
and argument[0] == '-'
and argument[1] != '-'
and 'v' in argument):
coloredlogs.increase_verbosity()
# Make sure the prefix is the same as the environment.
if not os.path.samefile(sys.prefix, ENVIRONMENT):
logger.error(
"You are trying to install packages in environment #1 which is different from environment #2 where pip-accel is installed! Please install pip-accel under environment #1 to install packages there."
)
logger.info("Environment #1: %s ($VIRTUAL_ENV)", ENVIRONMENT)
logger.info("Environment #2: %s (installation prefix)", sys.prefix)
sys.exit(1)
main_timer = Timer()
initialize_directories()
build_directory = tempfile.mkdtemp()
# Execute "pip install" in a loop in order to retry after intermittent
# error responses from servers (which can happen quite frequently).
try:
for i in xrange(1, MAX_RETRIES):
try:
requirements = unpack_source_dists(arguments, build_directory)
except DistributionNotFound:
logger.warn("We don't have all source distributions yet!")
download_source_dists(arguments, build_directory)
else:
if not requirements:
logger.info(
"No unsatisfied requirements found, probably there's nothing to do."
)
else:
install_requirements(requirements)
logger.info("Done! Took %s to install %i package%s.",
main_timer, len(requirements),
'' if len(requirements) == 1 else 's')
return
logger.warn("pip failed, retrying (%i/%i) ..", i + 1, MAX_RETRIES)
except InstallationError:
# Abort early when pip reports installation errors.
logger.fatal(
"pip reported unrecoverable installation errors. Please fix and rerun!"
)
sys.exit(1)
finally:
# Always cleanup temporary build directory.
shutil.rmtree(build_directory)
# Abort when after N retries we still failed to download source distributions.
logger.fatal("External command failed %i times, aborting!" % MAX_RETRIES)
sys.exit(1)
