def __init__(self, text, **kw):
"""
Initialize a :class:`PipAcceleratorError` object.
Accepts the same arguments as :func:`.compact()`.
"""
super(PipAcceleratorError, self).__init__(compact(text, **kw))
def __init__(self, *args, **kw):
"""Takes the same arguments as :py:func:`pip_accel.utils.compact()`."""
super(CacheBackendError, self).__init__(compact(*args, **kw))
def __init__(self, text, **kw):
"""Accepts the same arguments as :py:func:`.compact()`."""
super(PipAcceleratorError, self).__init__(compact(text, **kw))
def __init__(self, *args, **kw):
"""Takes the same arguments as :py:func:`pip_accel.utils.compact()`."""
super(CacheBackendError, self).__init__(compact(*args, **kw))
def build_binary_dist_helper(self, requirement, setup_command):
"""
Convert an unpacked source distribution to a binary distribution.
:param requirement: A :class:`.Requirement` object.
:param setup_command: A list of strings with the arguments to
``setup.py``.
:returns: The pathname of the resulting binary distribution (a string).
:raises: :exc:`.BuildFailed` when the build reports an error (e.g.
because of missing binary dependencies like system
libraries).
:raises: :exc:`.NoBuildOutput` when the build does not produce the
expected binary distribution archive.
"""
build_timer = Timer()
# Make sure the source distribution contains a setup script.
setup_script = os.path.join(requirement.source_directory, 'setup.py')
if not os.path.isfile(setup_script):
msg = "Directory %s (%s %s) doesn't contain a source distribution!"
raise InvalidSourceDistribution(msg % (requirement.source_directory, requirement.name, requirement.version))
# Let the user know what's going on.
build_text = "Building %s binary distribution" % requirement
logger.info("%s ..", build_text)
# Cleanup previously generated distributions.
dist_directory = os.path.join(requirement.source_directory, 'dist')
if os.path.isdir(dist_directory):
logger.debug("Cleaning up previously generated distributions in %s ..", dist_directory)
shutil.rmtree(dist_directory)
# Let the user know (approximately) which command is being executed
# (I don't think it's necessary to show them the nasty details :-).
logger.debug("Executing external command: %s",
' '.join(map(pipes.quote, [self.config.python_executable, 'setup.py'] + setup_command)))
# Compose the command line needed to build the binary distribution.
# This nasty command line forces the use of setuptools (instead of
# distutils) just like pip does. This will cause the `*.egg-info'
# metadata to be written to a directory instead of a file, which
# (amongst other things) enables tracking of installed files.
command_line = [
self.config.python_executable, '-c',
';'.join([
'import setuptools',
'__file__=%r' % setup_script,
r"exec(compile(open(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))",
])
] + setup_command
# Redirect all output of the build to a temporary file.
fd, temporary_file = tempfile.mkstemp()
try:
# Start the build.
build = subprocess.Popen(command_line, cwd=requirement.source_directory, stdout=fd, stderr=fd)
# Wait for the build to finish and provide feedback to the user in the mean time.
spinner = Spinner(label=build_text, timer=build_timer)
while build.poll() is None:
spinner.step()
# Don't tax the CPU too much.
time.sleep(0.2)
spinner.clear()
# Make sure the build succeeded and produced a binary distribution archive.
try:
# If the build reported an error we'll try to provide the user with
# some hints about what went wrong.
if build.returncode != 0:
raise BuildFailed("Failed to build {name} ({version}) binary distribution!",
name=requirement.name, version=requirement.version)
# Check if the build created the `dist' directory (the os.listdir()
# call below will raise an exception if we don't check for this).
if not os.path.isdir(dist_directory):
raise NoBuildOutput("Build of {name} ({version}) did not produce a binary distribution archive!",
name=requirement.name, version=requirement.version)
# Check if we can find the binary distribution archive.
filenames = os.listdir(dist_directory)
if len(filenames) != 1:
variables = dict(name=requirement.name,
version=requirement.version,
filenames=concatenate(sorted(filenames)))
raise NoBuildOutput("""
Build of {name} ({version}) produced more than one
distribution archive! (matches: {filenames})
""", **variables)
except Exception as e:
# Decorate the exception with the output of the failed build.
with open(temporary_file) as handle:
build_output = handle.read()
enhanced_message = compact("""
{message}
Please check the build output because it will probably
provide a hint about what went wrong.
Build output:
{output}
""", message=e.args[0], output=build_output.strip())
e.args = (enhanced_message,)
raise
logger.info("Finished building %s in %s.", requirement.name, build_timer)
return os.path.join(dist_directory, filenames[0])
finally:
# Close file descriptor before removing the temporary file.
# Without closing Windows is complaining that the file cannot
# be removed because it is used by another process.
os.close(fd)
os.unlink(temporary_file)
def build_binary_dist_helper(self, requirement, setup_command):
"""
Convert a single, unpacked source distribution to a binary
distribution. Raises an exception if it fails to create the binary
distribution (probably because of missing binary dependencies like
system libraries).
:param requirement: A :py:class:`.Requirement` object.
:param setup_command: A list of strings with the arguments to
``setup.py``.
:returns: The pathname of the resulting binary distribution (a string).
:raises: :py:exc:`.BuildFailed` when the build reports an error.
:raises: :py:exc:`.NoBuildOutput` when the build does not produce the
expected binary distribution archive.
"""
build_timer = Timer()
# Make sure the source distribution contains a setup script.
setup_script = os.path.join(requirement.source_directory, 'setup.py')
if not os.path.isfile(setup_script):
msg = "Directory %s (%s %s) doesn't contain a source distribution!"
raise InvalidSourceDistribution(msg % (requirement.source_directory, requirement.name, requirement.version))
# Let the user know what's going on.
build_text = "Building %s (%s) binary distribution" % (requirement.name, requirement.version)
logger.info("%s ..", build_text)
# Cleanup previously generated distributions.
dist_directory = os.path.join(requirement.source_directory, 'dist')
if os.path.isdir(dist_directory):
logger.debug("Cleaning up previously generated distributions in %s ..", dist_directory)
shutil.rmtree(dist_directory)
# Compose the command line needed to build the binary distribution.
command_line = ' '.join(pipes.quote(t) for t in [self.config.python_executable, 'setup.py'] + setup_command)
logger.debug("Executing external command: %s", command_line)
# Redirect all output of the build to a temporary file.
fd, temporary_file = tempfile.mkstemp()
command_line = '%s > "%s" 2>&1' % (command_line, temporary_file)
try:
# Start the build.
build = subprocess.Popen(['sh', '-c', command_line], cwd=requirement.source_directory)
# Wait for the build to finish and provide feedback to the user in the mean time.
spinner = Spinner(label=build_text, timer=build_timer)
while build.poll() is None:
spinner.step()
# Don't tax the CPU too much.
time.sleep(0.2)
spinner.clear()
# Make sure the build succeeded and produced a binary distribution archive.
try:
# If the build reported an error we'll try to provide the user with
# some hints about what went wrong.
if build.returncode != 0:
raise BuildFailed("Failed to build {name} ({version}) binary distribution!",
name=requirement.name, version=requirement.version)
# Check if the build created the `dist' directory (the os.listdir()
# call below will raise an exception if we don't check for this).
if not os.path.isdir(dist_directory):
raise NoBuildOutput("Build of {name} ({version}) did not produce a binary distribution archive!",
name=requirement.name, version=requirement.version)
# Check if we can find the binary distribution archive.
filenames = os.listdir(dist_directory)
if len(filenames) != 1:
raise NoBuildOutput("Build of {name} ({version}) produced more than one distribution archive! (matches: {filenames})",
name=requirement.name, version=requirement.version, filenames=concatenate(sorted(filenames)))
except Exception as e:
# Decorate the exception with the output of the failed build.
with open(temporary_file) as handle:
build_output = handle.read()
enhanced_message = compact("""
{message}
Please check the build output because it will probably
provide a hint about what went wrong.
Build output:
{output}
""", message=e.args[0],
output=build_output.strip())
e.args = (enhanced_message,)
raise
logger.info("Finished building %s (%s) in %s.", requirement.name, requirement.version, build_timer)
return os.path.join(dist_directory, filenames[0])
finally:
os.unlink(temporary_file)
def build_binary_dist_helper(self, requirement, setup_command):
"""
Convert an unpacked source distribution to a binary distribution.
:param requirement: A :class:`.Requirement` object.
:param setup_command: A list of strings with the arguments to
``setup.py``.
:returns: The pathname of the resulting binary distribution (a string).
:raises: :exc:`.BuildFailed` when the build reports an error (e.g.
because of missing binary dependencies like system
libraries).
:raises: :exc:`.NoBuildOutput` when the build does not produce the
expected binary distribution archive.
"""
build_timer = Timer()
# Make sure the source distribution contains a setup script.
setup_script = os.path.join(requirement.source_directory, 'setup.py')
if not os.path.isfile(setup_script):
msg = "Directory %s (%s %s) doesn't contain a source distribution!"
raise InvalidSourceDistribution(
msg % (requirement.source_directory, requirement.name,
requirement.version))
# Let the user know what's going on.
build_text = "Building %s binary distribution" % requirement
logger.info("%s ..", build_text)
# Cleanup previously generated distributions.
dist_directory = os.path.join(requirement.source_directory, 'dist')
if os.path.isdir(dist_directory):
logger.debug(
"Cleaning up previously generated distributions in %s ..",
dist_directory)
shutil.rmtree(dist_directory)
# Let the user know (approximately) which command is being executed
# (I don't think it's necessary to show them the nasty details :-).
logger.debug(
"Executing external command: %s", ' '.join(
map(pipes.quote, [self.config.python_executable, 'setup.py'] +
setup_command)))
# Compose the command line needed to build the binary distribution.
# This nasty command line forces the use of setuptools (instead of
# distutils) just like pip does. This will cause the `*.egg-info'
# metadata to be written to a directory instead of a file, which
# (amongst other things) enables tracking of installed files.
command_line = [
self.config.python_executable, '-c', ';'.join([
'import setuptools',
'__file__=%r' % setup_script,
r"exec(compile(open(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))",
])
] + setup_command
# Redirect all output of the build to a temporary file.
fd, temporary_file = tempfile.mkstemp()
try:
# Start the build.
build = subprocess.Popen(command_line,
cwd=requirement.source_directory,
stdout=fd,
stderr=fd)
# Wait for the build to finish and provide feedback to the user in the mean time.
spinner = Spinner(label=build_text, timer=build_timer)
while build.poll() is None:
spinner.step()
# Don't tax the CPU too much.
time.sleep(0.2)
spinner.clear()
# Make sure the build succeeded and produced a binary distribution archive.
try:
# If the build reported an error we'll try to provide the user with
# some hints about what went wrong.
if build.returncode != 0:
raise BuildFailed(
"Failed to build {name} ({version}) binary distribution!",
name=requirement.name,
version=requirement.version)
# Check if the build created the `dist' directory (the os.listdir()
# call below will raise an exception if we don't check for this).
if not os.path.isdir(dist_directory):
raise NoBuildOutput(
"Build of {name} ({version}) did not produce a binary distribution archive!",
name=requirement.name,
version=requirement.version)
# Check if we can find the binary distribution archive.
filenames = os.listdir(dist_directory)
if len(filenames) != 1:
variables = dict(name=requirement.name,
version=requirement.version,
filenames=concatenate(sorted(filenames)))
raise NoBuildOutput(
"""
Build of {name} ({version}) produced more than one
distribution archive! (matches: {filenames})
""", **variables)
except Exception as e:
# Decorate the exception with the output of the failed build.
with open(temporary_file) as handle:
build_output = handle.read()
enhanced_message = compact("""
{message}
Please check the build output because it will probably
provide a hint about what went wrong.
Build output:
{output}
""",
message=e.args[0],
output=build_output.strip())
e.args = (enhanced_message, )
raise
logger.info("Finished building %s in %s.", requirement.name,
build_timer)
return os.path.join(dist_directory, filenames[0])
finally:
# Close file descriptor before removing the temporary file.
# Without closing Windows is complaining that the file cannot
# be removed because it is used by another process.
os.close(fd)
os.unlink(temporary_file)
def install_binary_dist(self, members, virtualenv_compatible=True, prefix=None, python=None):
"""
Install a binary distribution created by :py:class:`build_binary_dist()`
into the given prefix (a directory like ``/usr``, ``/usr/local`` or a
virtual environment).
:param members: An iterable of tuples with two values each:
1. A :py:class:`tarfile.TarInfo` object.
2. A file-like object.
:param prefix: The "prefix" under which the requirements should be
installed. This will be a pathname like ``/usr``,
``/usr/local`` or the pathname of a virtual environment.
Defaults to :py:attr:`.Config.install_prefix`.
:param python: The pathname of the Python executable to use in the shebang
line of all executable Python scripts inside the binary
distribution. Defaults to :py:attr:`.Config.python_executable`.
:param virtualenv_compatible: Whether to enable workarounds to make the
resulting filenames compatible with
virtual environments (defaults to
``True``).
:raises: :py:exc:`~exceptions.ValueError` when the Python executable is
not located inside the installation prefix.
"""
# TODO This is quite slow for modules like Django. Speed it up! Two choices:
# 1. Run the external tar program to unpack the archive. This will
# slightly complicate the fixing up of hashbangs.
# 2. Using links? The plan: We can maintain a "seed" environment under
# $PIP_ACCEL_CACHE and use symbolic and/or hard links to populate other
# places based on the "seed" environment.
prefix = os.path.normpath(prefix or self.config.install_prefix)
python = os.path.normpath(python or self.config.python_executable)
# Make sure the installation prefix and Python executable match.
if os.path.commonprefix([prefix, python]) != prefix:
raise ValueError(compact("""
The configured Python executable ({executable}) is not located
inside the installation prefix ({prefix}). I don't think this
can work, so please make sure that if you override one
configuration option you also set the other one!
""", executable=python, prefix=prefix))
module_search_path = set(map(os.path.normpath, sys.path))
for member, from_handle in members:
pathname = member.name
if virtualenv_compatible:
# Some binary distributions include C header files (see for example
# the greenlet package) however the subdirectory of include/ in a
# virtual environment is a symbolic link to a subdirectory of
# /usr/include/ so we should never try to install C header files
# inside the directory pointed to by the symbolic link. Instead we
# implement the same workaround that pip uses to avoid this
# problem.
pathname = re.sub('^include/', 'include/site/', pathname)
if self.config.on_debian and '/site-packages/' in pathname:
# On Debian based system wide Python installs the /site-packages/
# directory is not in Python's module search path while
# /dist-packages/ is. We try to be compatible with this.
match = re.match('^(.+?)/site-packages', pathname)
if match:
site_packages = os.path.normpath(os.path.join(prefix, match.group(0)))
dist_packages = os.path.normpath(os.path.join(prefix, match.group(1), 'dist-packages'))
if dist_packages in module_search_path and site_packages not in module_search_path:
pathname = pathname.replace('/site-packages/', '/dist-packages/')
pathname = os.path.join(prefix, pathname)
directory = os.path.dirname(pathname)
if not os.path.isdir(directory):
logger.debug("Creating directory: %s ..", directory)
makedirs(directory)
logger.debug("Creating file: %s ..", pathname)
with open(pathname, 'wb') as to_handle:
contents = from_handle.read()
if contents.startswith(b'#!/'):
contents = self.fix_hashbang(contents, python)
to_handle.write(contents)
os.chmod(pathname, member.mode)
def build_binary_dist_helper(self, requirement):
"""
Convert a single, unpacked source distribution to a binary
distribution. Raises an exception if it fails to create the binary
distribution (probably because of missing binary dependencies like
system libraries).
:param requirement: A :py:class:`.Requirement` object.
:returns: The pathname of the resulting binary distribution (a string).
:raises: :py:exc:`.BuildFailed` when the build reports an error.
:raises: :py:exc:`.NoBuildOutput` when the build does not produce the
expected binary distribution archive.
"""
build_timer = Timer()
# Make sure the source distribution contains a setup script.
setup_script = os.path.join(requirement.source_directory, 'setup.py')
if not os.path.isfile(setup_script):
msg = "Directory %s (%s %s) doesn't contain a source distribution!"
raise InvalidSourceDistribution(
msg % (requirement.source_directory, requirement.name,
requirement.version))
# Let the user know what's going on.
build_text = "Building %s (%s) binary distribution" % (
requirement.name, requirement.version)
logger.info("%s ..", build_text)
# Cleanup previously generated distributions.
dist_directory = os.path.join(requirement.source_directory, 'dist')
if os.path.isdir(dist_directory):
logger.debug(
"Cleaning up previously generated distributions in %s ..",
dist_directory)
shutil.rmtree(dist_directory)
makedirs(dist_directory)
# Create a temporary directory for pip installing into, and set up the
# install_lib directory structure inside it. We do this so that we can
# pip install into this as our target.
temporary_dir = tempfile.mkdtemp()
distutils_inst = install(Distribution())
distutils_inst.prefix = '' # This will be changed if we're in a virtualenv.
distutils_inst.finalize_options()
pip_target = os.path.normpath(temporary_dir +
distutils_inst.install_lib)
# Compose the command line needed to build the binary distribution.
command_line = ' '.join(
pipes.quote(t)
for t in ['pip', 'install', '.', '--target', pip_target])
logger.debug("Executing external command: %s", command_line)
# Redirect all output of the build to a temporary file.
fd, temporary_file = tempfile.mkstemp()
command_line = '%s > "%s" 2>&1' % (command_line, temporary_file)
try:
# Start the build.
build = subprocess.Popen(['sh', '-c', command_line],
cwd=requirement.source_directory)
# Wait for the build to finish and provide feedback to the user in the mean time.
spinner = Spinner(label=build_text, timer=build_timer)
while build.poll() is None:
spinner.step()
# Don't tax the CPU too much.
time.sleep(0.2)
spinner.clear()
# At this point, we may have a number of dependencies in the directory we want
# to tar up that should not be part of the package distribution. For instance,
# s3 will also wrap up the concurrent, futures, and requests packages. We fix
# this by reading {name}-{version}-{py-version}.egg-info/installed-files.txt
# and removing any files or directories that are not in it.
# 1. Find the appropriate .egg-info/ directory.
egg_info_dir = None
egg_info_start = '-'.join([requirement.name, requirement.version])
for egg_info_root, dirs, _ in os.walk(temporary_dir):
for d in dirs:
if d.startswith(egg_info_start) and d.endswith(
'.egg-info'):
egg_info_dir = d
break
if egg_info_dir is not None:
break
# 2. If we have a .egg-info/, try to read the installed-files.txt contents.
inst_files = set()
if egg_info_dir is not None:
egg_info_path = os.path.join(egg_info_root, egg_info_dir)
inst_files_path = os.path.join(egg_info_path,
'installed-files.txt')
try:
with open(inst_files_path) as f:
for line in f:
abs_path = os.path.abspath(
os.path.join(egg_info_path, line.strip()))
inst_files.add(abs_path)
inst_files.add(os.path.dirname(abs_path))
except IOError, ioe:
loger.warn('Unable to open %s: %s' %
(inst_files_path, ioe))
# 3. If we were able to get a set of files and directories that belong in the
# distribution, then we can delete everything else before archiving it.
if inst_files:
dirs, files = next(os.walk(egg_info_root))[1:]
for d in dirs:
d = os.path.abspath(os.path.join(egg_info_root, d))
if d not in inst_files:
logger.info('Removing %s (not part of the package)' %
d)
shutil.rmtree(d)
for f in files:
f = os.path.abspath(os.path.join(egg_info_root, f))
if f not in inst_files:
logger.info('Removing %s (not part of the package)' %
f)
os.unlink(f)
# Tar up the contents of temporary_dir into the correct file name and put it in the dist dir.
tarball_path = os.path.join(temporary_dir, requirement.name)
path = archive_util.make_archive(tarball_path,
'gztar',
root_dir=temporary_dir)
shutil.copy(path, dist_directory)
# Make sure the build succeeded and produced a binary distribution archive.
try:
# If the build reported an error we'll try to provide the user with
# some hints about what went wrong.
if build.returncode != 0:
raise BuildFailed(
"Failed to build {name} ({version}) binary distribution!",
name=requirement.name,
version=requirement.version)
# Check if the build created the `dist' directory (the os.listdir()
# call below will raise an exception if we don't check for this).
if not os.path.isdir(dist_directory):
raise NoBuildOutput(
"Build of {name} ({version}) did not produce a binary distribution archive!",
name=requirement.name,
version=requirement.version)
# Check if we can find the binary distribution archive.
filenames = os.listdir(dist_directory)
if len(filenames) != 1:
raise NoBuildOutput(
"Build of {name} ({version}) produced more than one distribution archive! (matches: {filenames})",
name=requirement.name,
version=requirement.version,
filenames=concatenate(sorted(filenames)))
except Exception as e:
# Decorate the exception with the output of the failed build.
with open(temporary_file) as handle:
build_output = handle.read()
enhanced_message = compact("""
{message}
Please check the build output because it will probably
provide a hint about what went wrong.
Build output:
{output}
""",
message=e.args[0],
output=build_output.strip())
e.args = (enhanced_message, )
raise
logger.info("Finished building %s (%s) in %s.", requirement.name,
requirement.version, build_timer)
return os.path.join(dist_directory, filenames[0])
def __init__(self, text, **kw):
"""Accepts the same arguments as :py:func:`.compact()`."""
super(PipAcceleratorError, self).__init__(compact(text, **kw))
