def is_supported():
"""
Check whether the current system supports the implemented package
manager.
"""
logger.debug("Falling back to dummy package manager interface ..")
return True
def fix_hashbang(python, contents):
"""
Rewrite the hashbang in an executable script so that the Python program
inside the virtual environment is used instead of a system wide Python.
:param python: The absolute pathname of the Python program inside the
virtual environment.
:param contents: A string with the contents of the script whose hashbang
should be fixed.
:returns: The modified contents of the script as a string.
"""
# Separate the first line in the file from the remainder of the contents
# while preserving the end of line sequence (CR+LF or just an LF) and
# without having to split all lines in the file (there's no point).
parts = re.split(r'(\r?\n)', contents, 1)
hashbang = parts[0]
# Get the base name of the command in the hashbang and deal with hashbangs
# like `#!/usr/bin/env python'.
modified_name = re.sub('^env ', '', os.path.basename(hashbang))
# Only rewrite hashbangs that actually involve Python.
if re.match(r'^python(\d+(\.\d+)*)?$', modified_name):
logger.debug("Hashbang %r looks like a Python hashbang! We'll rewrite it!", hashbang)
parts[0] = '#!%s' % python
contents = ''.join(parts)
else:
logger.debug("Warning: Failed to match hashbang: %r.", hashbang)
return contents
def build_missing_binary_dists(requirements):
"""
Convert source distributions to binary distributions.
:param requirements: A list of tuples in the format of the return value of
the :py:func:`unpack_source_dists()` function.
:returns: ``True`` if it succeeds in building a binary distribution,
``False`` otherwise (probably because of missing binary
dependencies like system libraries).
"""
existing_binary_dists = find_cached_binary_dists()
logger.info("Building binary distributions ..")
pyversion = get_python_version()
for name, version, directory in requirements:
# Check if a binary distribution already exists.
filename = existing_binary_dists.get((name.lower(), version, pyversion))
if filename:
logger.debug("Existing binary distribution for %s (%s) found at %s.", name, version, filename)
continue
# Make sure the source distribution contains a setup script.
setup_script = os.path.join(directory, 'setup.py')
if not os.path.isfile(setup_script):
logger.warn("Package %s (%s) is not a source distribution.", name, version)
continue
# Try to build the binary distribution.
if not build_binary_dist(name, version, directory, pyversion):
sanity_check_dependencies(name)
if not build_binary_dist(name, version, directory, pyversion):
return False
logger.info("Finished building binary distributions.")
return True
def is_supported():
"""
Check whether the current system supports the implemented package
manager.
"""
logger.debug("Falling back to dummy package manager interface ..")
return True
def build_binary_dist(name, version, directory, pyversion):
"""
Convert a single, unpacked source distribution to a binary distribution.
:param name: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:returns: ``True`` if we succeed in building a binary distribution,
``False`` otherwise (probably because of missing binary
dependencies like system libraries).
"""
# Cleanup previously generated distributions.
dist_directory = os.path.join(directory, 'dist')
if os.path.isdir(dist_directory):
logger.info("Cleaning up previously generated distributions in %s ..", dist_directory)
shutil.rmtree(dist_directory)
# Let the user know what's going on.
build_text = "Building binary distribution of %s (%s) .." % (name, version)
logger.info("%s", build_text)
# Compose the command line needed to build the binary distribution.
command_line = '"%s/bin/python" setup.py bdist_dumb --format=gztar' % ENVIRONMENT
logger.debug("Executing external command: %s", command_line)
# Redirect all output of the build to a temporary file.
fd, temporary_file = tempfile.mkstemp()
command_line = '%s > "%s" 2>&1' % (command_line, temporary_file)
# Start the build.
build = subprocess.Popen(['sh', '-c', command_line], cwd=directory)
# Wait for the build to finish.
if INTERACTIVE:
# Provide feedback to the user in the mean time.
spinner = Spinner(build_text)
while build.poll() is None:
spinner.step()
time.sleep(0.1)
spinner.clear()
else:
build.wait()
# Make sure the build succeeded.
if build.returncode != 0:
logger.error("Failed to build binary distribution of %s! (version: %s)", name, version)
with open(temporary_file) as handle:
logger.info("Build output (will probably provide a hint as to what went wrong):\n%s", handle.read())
return False
# Move the generated distribution to the binary index.
filenames = os.listdir(dist_directory)
if len(filenames) != 1:
logger.error("Build process did not result in one binary distribution! (matches: %s)", filenames)
return False
cache_file = '%s:%s:%s.tar.gz' % (name, version, pyversion)
logger.info("Copying binary distribution %s to cache as %s.", filenames[0], cache_file)
cache_binary_dist(os.path.join(directory, 'dist', filenames[0]),
os.path.join(binary_index, cache_file))
return True
def clear_build_directory(directory):
"""
Since pip 1.4 there is a new exception that's raised by pip:
``PreviousBuildDirError``. Unfortunately pip-accel apparently triggers the
worst possible side effect of this new "feature" and the only way to avoid
it is to start with an empty build directory in every step of pip-accel's
process (possibly very inefficient).
:param directory: The build directory to clear.
"""
logger.debug("Clearing build directory ..")
if os.path.isdir(directory):
shutil.rmtree(directory)
os.makedirs(directory)
def clear_build_directory(directory):
"""
Since pip 1.4 there is a new exception that's raised by pip:
``PreviousBuildDirError``. Unfortunately pip-accel apparently triggers the
worst possible side effect of this new "feature" and the only way to avoid
it is to start with an empty build directory in every step of pip-accel's
process (possibly very inefficient).
:param directory: The build directory to clear.
"""
logger.debug("Clearing build directory ..")
if os.path.isdir(directory):
shutil.rmtree(directory)
os.makedirs(directory)
def __init__(self):
"""
Load any predefined dependencies for the platform from a configuration
file bundled with the pip accelerator.
"""
config_name = getattr(self, 'config', '')
if config_name:
script = os.path.abspath(__file__)
directory = os.path.dirname(script)
config_path = os.path.join(directory, '%s.ini' % config_name)
if os.path.isfile(config_path):
logger.debug("Loading system package configuration from %s ..", config_path)
parser = ConfigParser.RawConfigParser()
parser.read(config_path)
if parser.has_section('dependencies'):
self.dependencies = dict((n.lower(), v.split()) for n, v in parser.items('dependencies'))
def __init__(self):
"""
Load any predefined dependencies for the platform from a configuration
file bundled with the pip accelerator.
"""
config_name = getattr(self, 'config', '')
if config_name:
script = os.path.abspath(__file__)
directory = os.path.dirname(script)
config_path = os.path.join(directory, '%s.ini' % config_name)
if os.path.isfile(config_path):
logger.debug("Loading system package configuration from %s ..",
config_path)
parser = ConfigParser.RawConfigParser()
parser.read(config_path)
if parser.has_section('dependencies'):
self.dependencies = dict(
(n.lower(), v.split())
for n, v in parser.items('dependencies'))
def install_binary_dist(package, filename, install_prefix=ENVIRONMENT):
"""
Install a binary distribution created with ``python setup.py bdist`` into
the given prefix (a directory like ``/usr``, ``/usr/local`` or a virtual
environment).
:param package: The name of the package to install.
:param filename: The pathname of the tar archive.
:param install_prefix: The "prefix" under which the requirements should be
installed. This will be a pathname like ``/usr``,
``/usr/local`` or the pathname of a virtual
environment.
"""
# TODO This is quite slow for modules like Django. Speed it up! Two choices:
# 1. Run the external tar program to unpack the archive. This will
# slightly complicate the fixing up of hashbangs.
# 2. Using links? The plan: We can maintain a "seed" environment under
# $PIP_ACCEL_CACHE and use symbolic and/or hard links to populate other
# places based on the "seed" environment.
install_timer = Timer()
python = os.path.join(install_prefix, 'bin', 'python')
pip = os.path.join(install_prefix, 'bin', 'pip')
if os.system('"%s" uninstall --yes "%s" >/dev/null 2>&1' % (pip, package)) == 0:
logger.info("Uninstalled previously installed package %s.", package)
logger.info("Installing package %s from binary distribution %s to %s ..", package, filename, install_prefix)
archive = tarfile.open(filename, 'r:gz')
for member in archive.getmembers():
install_path = os.path.join(install_prefix, member.name)
directory = os.path.dirname(install_path)
if not os.path.isdir(directory):
logger.debug("Creating directory: %s ..", directory)
os.makedirs(directory)
logger.debug("Writing file: %s ..", install_path)
file_handle = archive.extractfile(member)
with open(install_path, 'w') as handle:
contents = file_handle.read()
if contents.startswith('#!/'):
contents = fix_hashbang(python, contents)
handle.write(contents)
os.chmod(install_path, member.mode)
archive.close()
logger.info("Finished installing binary distribution in %s.", install_timer)
def initialize_directories():
"""
Create the directories for the download cache, the source index and the
binary index if any of them don't exist yet and reset the binary index
when its format changes.
"""
# Create all required directories on the fly.
for directory in [download_cache, source_index, binary_index]:
if not os.path.isdir(directory):
os.makedirs(directory)
# Invalidate the binary distribution cache when the
# format is changed in backwards incompatible ways.
if os.path.isfile(index_version_file):
with open(index_version_file) as handle:
if int(handle.read()) == CACHE_FORMAT_REVISION:
logger.debug("Binary distribution cache format is compatible.")
return
logger.debug(
"Binary distribution cache format is incompatible; clearing cache ..")
for entry in os.listdir(binary_index):
pathname = os.path.join(binary_index, entry)
logger.debug(" - Deleting %s.", pathname)
os.unlink(pathname)
with open(index_version_file, 'w') as handle:
handle.write("%i\n" % CACHE_FORMAT_REVISION)
def is_supported():
"""
Use the ``lsb_release`` program to check whether we are on a Debian
Linux system (or a derivative distribution of Debian like Ubuntu).
:returns: ``True`` if we are, ``False`` if we're not.
"""
logger.debug("Checking if we're on a Debian (derived) system ..")
handle = os.popen('lsb_release -si 2>/dev/null')
output = handle.read()
handle.close()
distributor_id = output.strip()
logger.debug("Distributor ID: %s", distributor_id)
if distributor_id.lower() in ('debian', 'ubuntu'):
logger.debug("Using Debian package manager interface ..")
return True
def is_supported():
"""
Use the ``lsb_release`` program to check whether we are on a Debian
Linux system (or a derivative distribution of Debian like Ubuntu).
:returns: ``True`` if we are, ``False`` if we're not.
"""
logger.debug("Checking if we're on a Debian (derived) system ..")
handle = os.popen('lsb_release -si 2>/dev/null')
output = handle.read()
handle.close()
distributor_id = output.strip()
logger.debug("Distributor ID: %s", distributor_id)
if distributor_id.lower() in ('debian', 'ubuntu'):
logger.debug("Using Debian package manager interface ..")
return True
def cache_binary_dist(input_path, output_path):
"""
Transform a binary distribution archive created with ``python setup.py
bdist_dumb --format=gztar`` into a form that can be cached for future use.
This comes down to making the pathnames inside the archive relative to the
`prefix` that the binary distribution was built for.
:param input_path: The pathname of the original binary distribution archive
:param output_path: The pathname of the binary distribution in the cache
directory.
"""
# Copy the tar archive file by file so we can rewrite their pathnames.
logger.debug("Expected prefix in binary distribution archive: %s.", ENVIRONMENT)
input_bdist = tarfile.open(input_path, 'r:gz')
output_bdist = tarfile.open(output_path, 'w:gz')
for member in input_bdist.getmembers():
# In my testing the `dumb' tar files created with the `python setup.py
# bdist' command contain pathnames that are relative to `/' which is
# kind of awkward: I would like to use os.path.relpath() on them but
# that won't give the correct result without preprocessing...
original_pathname = member.name
absolute_pathname = re.sub(r'^\./', '/', original_pathname)
if member.isdev():
logger.debug("Warning: Ignoring device file: %s.", absolute_pathname)
elif not member.isdir():
modified_pathname = os.path.relpath(absolute_pathname, ENVIRONMENT)
if os.path.isabs(modified_pathname):
logger.warn("Failed to transform pathname in binary distribution to relative path! (original: %r, modified: %r)",
original_pathname, modified_pathname)
else:
# Some binary distributions include C header files (see for
# example the greenlet package) however the subdirectory of
# include/ in a virtual environment is a symbolic link to a
# subdirectory of /usr/include/ so we should never try to
# install C header files inside the directory pointed to by the
# symbolic link. Instead we implement the same workaround that
# pip uses to avoid this problem.
modified_pathname = re.sub('^include/', 'include/site/', modified_pathname)
logger.debug("Transformed %r -> %r.", original_pathname, modified_pathname)
# Get the file data from the input archive.
file_data = input_bdist.extractfile(original_pathname)
# Use all metadata from the input archive but modify the filename.
member.name = modified_pathname
# Copy modified metadata + original file data to output archive.
output_bdist.addfile(member, file_data)
input_bdist.close()
output_bdist.close()
def initialize_directories():
"""
Create the directories for the download cache, the source index and the
binary index if any of them don't exist yet and reset the binary index
when its format changes.
"""
# Create all required directories on the fly.
for directory in [download_cache, source_index, binary_index]:
if not os.path.isdir(directory):
os.makedirs(directory)
# Invalidate the binary distribution cache when the
# format is changed in backwards incompatible ways.
if os.path.isfile(index_version_file):
with open(index_version_file) as handle:
if int(handle.read()) == CACHE_FORMAT_REVISION:
logger.debug("Binary distribution cache format is compatible.")
return
logger.debug("Binary distribution cache format is incompatible; clearing cache ..")
for entry in os.listdir(binary_index):
pathname = os.path.join(binary_index, entry)
logger.debug(" - Deleting %s.", pathname)
os.unlink(pathname)
with open(index_version_file, 'w') as handle:
handle.write("%i\n" % CACHE_FORMAT_REVISION)
def find_cached_binary_dists():
"""
Find all previously cached binary distributions.
:returns: A dictionary with (package-name, package-version, python-version)
tuples as keys and pathnames of binary archives as values.
"""
logger.info("Scanning binary distribution index ..")
distributions = {}
for filename in sorted(os.listdir(binary_index), key=str.lower):
if filename.endswith('.tar.gz'):
basename = re.sub('\.tar.gz$', '', filename)
parts = basename.split(':')
if len(parts) == 3:
key = (parts[0].lower(), parts[1], parts[2])
logger.debug("Matched %s in %s.", key, filename)
distributions[key] = os.path.join(binary_index, filename)
continue
logger.debug("Failed to match filename: %s.", filename)
logger.info("Found %i existing binary distribution%s.",
len(distributions), '' if len(distributions) == 1 else 's')
for (name, version, pyversion), filename in distributions.iteritems():
logger.debug(" - %s (%s, %s) in %s.", name, version, pyversion, filename)
return distributions
