def __init__(self, text, **kw): """ Initialize a :class:`PipAcceleratorError` object. Accepts the same arguments as :func:`.compact()`. """ super(PipAcceleratorError, self).__init__(compact(text, **kw))
def __init__(self, *args, **kw): """Takes the same arguments as :py:func:`pip_accel.utils.compact()`.""" super(CacheBackendError, self).__init__(compact(*args, **kw))
def __init__(self, text, **kw): """Accepts the same arguments as :py:func:`.compact()`.""" super(PipAcceleratorError, self).__init__(compact(text, **kw))
def build_binary_dist_helper(self, requirement, setup_command): """ Convert an unpacked source distribution to a binary distribution. :param requirement: A :class:`.Requirement` object. :param setup_command: A list of strings with the arguments to ``setup.py``. :returns: The pathname of the resulting binary distribution (a string). :raises: :exc:`.BuildFailed` when the build reports an error (e.g. because of missing binary dependencies like system libraries). :raises: :exc:`.NoBuildOutput` when the build does not produce the expected binary distribution archive. """ build_timer = Timer() # Make sure the source distribution contains a setup script. setup_script = os.path.join(requirement.source_directory, 'setup.py') if not os.path.isfile(setup_script): msg = "Directory %s (%s %s) doesn't contain a source distribution!" raise InvalidSourceDistribution(msg % (requirement.source_directory, requirement.name, requirement.version)) # Let the user know what's going on. build_text = "Building %s binary distribution" % requirement logger.info("%s ..", build_text) # Cleanup previously generated distributions. dist_directory = os.path.join(requirement.source_directory, 'dist') if os.path.isdir(dist_directory): logger.debug("Cleaning up previously generated distributions in %s ..", dist_directory) shutil.rmtree(dist_directory) # Let the user know (approximately) which command is being executed # (I don't think it's necessary to show them the nasty details :-). logger.debug("Executing external command: %s", ' '.join(map(pipes.quote, [self.config.python_executable, 'setup.py'] + setup_command))) # Compose the command line needed to build the binary distribution. # This nasty command line forces the use of setuptools (instead of # distutils) just like pip does. This will cause the `*.egg-info' # metadata to be written to a directory instead of a file, which # (amongst other things) enables tracking of installed files. command_line = [ self.config.python_executable, '-c', ';'.join([ 'import setuptools', '__file__=%r' % setup_script, r"exec(compile(open(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))", ]) ] + setup_command # Redirect all output of the build to a temporary file. fd, temporary_file = tempfile.mkstemp() try: # Start the build. build = subprocess.Popen(command_line, cwd=requirement.source_directory, stdout=fd, stderr=fd) # Wait for the build to finish and provide feedback to the user in the mean time. spinner = Spinner(label=build_text, timer=build_timer) while build.poll() is None: spinner.step() # Don't tax the CPU too much. time.sleep(0.2) spinner.clear() # Make sure the build succeeded and produced a binary distribution archive. try: # If the build reported an error we'll try to provide the user with # some hints about what went wrong. if build.returncode != 0: raise BuildFailed("Failed to build {name} ({version}) binary distribution!", name=requirement.name, version=requirement.version) # Check if the build created the `dist' directory (the os.listdir() # call below will raise an exception if we don't check for this). if not os.path.isdir(dist_directory): raise NoBuildOutput("Build of {name} ({version}) did not produce a binary distribution archive!", name=requirement.name, version=requirement.version) # Check if we can find the binary distribution archive. filenames = os.listdir(dist_directory) if len(filenames) != 1: variables = dict(name=requirement.name, version=requirement.version, filenames=concatenate(sorted(filenames))) raise NoBuildOutput(""" Build of {name} ({version}) produced more than one distribution archive! (matches: {filenames}) """, **variables) except Exception as e: # Decorate the exception with the output of the failed build. with open(temporary_file) as handle: build_output = handle.read() enhanced_message = compact(""" {message} Please check the build output because it will probably provide a hint about what went wrong. Build output: {output} """, message=e.args[0], output=build_output.strip()) e.args = (enhanced_message,) raise logger.info("Finished building %s in %s.", requirement.name, build_timer) return os.path.join(dist_directory, filenames[0]) finally: # Close file descriptor before removing the temporary file. # Without closing Windows is complaining that the file cannot # be removed because it is used by another process. os.close(fd) os.unlink(temporary_file)
def build_binary_dist_helper(self, requirement, setup_command): """ Convert a single, unpacked source distribution to a binary distribution. Raises an exception if it fails to create the binary distribution (probably because of missing binary dependencies like system libraries). :param requirement: A :py:class:`.Requirement` object. :param setup_command: A list of strings with the arguments to ``setup.py``. :returns: The pathname of the resulting binary distribution (a string). :raises: :py:exc:`.BuildFailed` when the build reports an error. :raises: :py:exc:`.NoBuildOutput` when the build does not produce the expected binary distribution archive. """ build_timer = Timer() # Make sure the source distribution contains a setup script. setup_script = os.path.join(requirement.source_directory, 'setup.py') if not os.path.isfile(setup_script): msg = "Directory %s (%s %s) doesn't contain a source distribution!" raise InvalidSourceDistribution(msg % (requirement.source_directory, requirement.name, requirement.version)) # Let the user know what's going on. build_text = "Building %s (%s) binary distribution" % (requirement.name, requirement.version) logger.info("%s ..", build_text) # Cleanup previously generated distributions. dist_directory = os.path.join(requirement.source_directory, 'dist') if os.path.isdir(dist_directory): logger.debug("Cleaning up previously generated distributions in %s ..", dist_directory) shutil.rmtree(dist_directory) # Compose the command line needed to build the binary distribution. command_line = ' '.join(pipes.quote(t) for t in [self.config.python_executable, 'setup.py'] + setup_command) logger.debug("Executing external command: %s", command_line) # Redirect all output of the build to a temporary file. fd, temporary_file = tempfile.mkstemp() command_line = '%s > "%s" 2>&1' % (command_line, temporary_file) try: # Start the build. build = subprocess.Popen(['sh', '-c', command_line], cwd=requirement.source_directory) # Wait for the build to finish and provide feedback to the user in the mean time. spinner = Spinner(label=build_text, timer=build_timer) while build.poll() is None: spinner.step() # Don't tax the CPU too much. time.sleep(0.2) spinner.clear() # Make sure the build succeeded and produced a binary distribution archive. try: # If the build reported an error we'll try to provide the user with # some hints about what went wrong. if build.returncode != 0: raise BuildFailed("Failed to build {name} ({version}) binary distribution!", name=requirement.name, version=requirement.version) # Check if the build created the `dist' directory (the os.listdir() # call below will raise an exception if we don't check for this). if not os.path.isdir(dist_directory): raise NoBuildOutput("Build of {name} ({version}) did not produce a binary distribution archive!", name=requirement.name, version=requirement.version) # Check if we can find the binary distribution archive. filenames = os.listdir(dist_directory) if len(filenames) != 1: raise NoBuildOutput("Build of {name} ({version}) produced more than one distribution archive! (matches: {filenames})", name=requirement.name, version=requirement.version, filenames=concatenate(sorted(filenames))) except Exception as e: # Decorate the exception with the output of the failed build. with open(temporary_file) as handle: build_output = handle.read() enhanced_message = compact(""" {message} Please check the build output because it will probably provide a hint about what went wrong. Build output: {output} """, message=e.args[0], output=build_output.strip()) e.args = (enhanced_message,) raise logger.info("Finished building %s (%s) in %s.", requirement.name, requirement.version, build_timer) return os.path.join(dist_directory, filenames[0]) finally: os.unlink(temporary_file)
def build_binary_dist_helper(self, requirement, setup_command): """ Convert an unpacked source distribution to a binary distribution. :param requirement: A :class:`.Requirement` object. :param setup_command: A list of strings with the arguments to ``setup.py``. :returns: The pathname of the resulting binary distribution (a string). :raises: :exc:`.BuildFailed` when the build reports an error (e.g. because of missing binary dependencies like system libraries). :raises: :exc:`.NoBuildOutput` when the build does not produce the expected binary distribution archive. """ build_timer = Timer() # Make sure the source distribution contains a setup script. setup_script = os.path.join(requirement.source_directory, 'setup.py') if not os.path.isfile(setup_script): msg = "Directory %s (%s %s) doesn't contain a source distribution!" raise InvalidSourceDistribution( msg % (requirement.source_directory, requirement.name, requirement.version)) # Let the user know what's going on. build_text = "Building %s binary distribution" % requirement logger.info("%s ..", build_text) # Cleanup previously generated distributions. dist_directory = os.path.join(requirement.source_directory, 'dist') if os.path.isdir(dist_directory): logger.debug( "Cleaning up previously generated distributions in %s ..", dist_directory) shutil.rmtree(dist_directory) # Let the user know (approximately) which command is being executed # (I don't think it's necessary to show them the nasty details :-). logger.debug( "Executing external command: %s", ' '.join( map(pipes.quote, [self.config.python_executable, 'setup.py'] + setup_command))) # Compose the command line needed to build the binary distribution. # This nasty command line forces the use of setuptools (instead of # distutils) just like pip does. This will cause the `*.egg-info' # metadata to be written to a directory instead of a file, which # (amongst other things) enables tracking of installed files. command_line = [ self.config.python_executable, '-c', ';'.join([ 'import setuptools', '__file__=%r' % setup_script, r"exec(compile(open(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))", ]) ] + setup_command # Redirect all output of the build to a temporary file. fd, temporary_file = tempfile.mkstemp() try: # Start the build. build = subprocess.Popen(command_line, cwd=requirement.source_directory, stdout=fd, stderr=fd) # Wait for the build to finish and provide feedback to the user in the mean time. spinner = Spinner(label=build_text, timer=build_timer) while build.poll() is None: spinner.step() # Don't tax the CPU too much. time.sleep(0.2) spinner.clear() # Make sure the build succeeded and produced a binary distribution archive. try: # If the build reported an error we'll try to provide the user with # some hints about what went wrong. if build.returncode != 0: raise BuildFailed( "Failed to build {name} ({version}) binary distribution!", name=requirement.name, version=requirement.version) # Check if the build created the `dist' directory (the os.listdir() # call below will raise an exception if we don't check for this). if not os.path.isdir(dist_directory): raise NoBuildOutput( "Build of {name} ({version}) did not produce a binary distribution archive!", name=requirement.name, version=requirement.version) # Check if we can find the binary distribution archive. filenames = os.listdir(dist_directory) if len(filenames) != 1: variables = dict(name=requirement.name, version=requirement.version, filenames=concatenate(sorted(filenames))) raise NoBuildOutput( """ Build of {name} ({version}) produced more than one distribution archive! (matches: {filenames}) """, **variables) except Exception as e: # Decorate the exception with the output of the failed build. with open(temporary_file) as handle: build_output = handle.read() enhanced_message = compact(""" {message} Please check the build output because it will probably provide a hint about what went wrong. Build output: {output} """, message=e.args[0], output=build_output.strip()) e.args = (enhanced_message, ) raise logger.info("Finished building %s in %s.", requirement.name, build_timer) return os.path.join(dist_directory, filenames[0]) finally: # Close file descriptor before removing the temporary file. # Without closing Windows is complaining that the file cannot # be removed because it is used by another process. os.close(fd) os.unlink(temporary_file)
def install_binary_dist(self, members, virtualenv_compatible=True, prefix=None, python=None): """ Install a binary distribution created by :py:class:`build_binary_dist()` into the given prefix (a directory like ``/usr``, ``/usr/local`` or a virtual environment). :param members: An iterable of tuples with two values each: 1. A :py:class:`tarfile.TarInfo` object. 2. A file-like object. :param prefix: The "prefix" under which the requirements should be installed. This will be a pathname like ``/usr``, ``/usr/local`` or the pathname of a virtual environment. Defaults to :py:attr:`.Config.install_prefix`. :param python: The pathname of the Python executable to use in the shebang line of all executable Python scripts inside the binary distribution. Defaults to :py:attr:`.Config.python_executable`. :param virtualenv_compatible: Whether to enable workarounds to make the resulting filenames compatible with virtual environments (defaults to ``True``). :raises: :py:exc:`~exceptions.ValueError` when the Python executable is not located inside the installation prefix. """ # TODO This is quite slow for modules like Django. Speed it up! Two choices: # 1. Run the external tar program to unpack the archive. This will # slightly complicate the fixing up of hashbangs. # 2. Using links? The plan: We can maintain a "seed" environment under # $PIP_ACCEL_CACHE and use symbolic and/or hard links to populate other # places based on the "seed" environment. prefix = os.path.normpath(prefix or self.config.install_prefix) python = os.path.normpath(python or self.config.python_executable) # Make sure the installation prefix and Python executable match. if os.path.commonprefix([prefix, python]) != prefix: raise ValueError(compact(""" The configured Python executable ({executable}) is not located inside the installation prefix ({prefix}). I don't think this can work, so please make sure that if you override one configuration option you also set the other one! """, executable=python, prefix=prefix)) module_search_path = set(map(os.path.normpath, sys.path)) for member, from_handle in members: pathname = member.name if virtualenv_compatible: # Some binary distributions include C header files (see for example # the greenlet package) however the subdirectory of include/ in a # virtual environment is a symbolic link to a subdirectory of # /usr/include/ so we should never try to install C header files # inside the directory pointed to by the symbolic link. Instead we # implement the same workaround that pip uses to avoid this # problem. pathname = re.sub('^include/', 'include/site/', pathname) if self.config.on_debian and '/site-packages/' in pathname: # On Debian based system wide Python installs the /site-packages/ # directory is not in Python's module search path while # /dist-packages/ is. We try to be compatible with this. match = re.match('^(.+?)/site-packages', pathname) if match: site_packages = os.path.normpath(os.path.join(prefix, match.group(0))) dist_packages = os.path.normpath(os.path.join(prefix, match.group(1), 'dist-packages')) if dist_packages in module_search_path and site_packages not in module_search_path: pathname = pathname.replace('/site-packages/', '/dist-packages/') pathname = os.path.join(prefix, pathname) directory = os.path.dirname(pathname) if not os.path.isdir(directory): logger.debug("Creating directory: %s ..", directory) makedirs(directory) logger.debug("Creating file: %s ..", pathname) with open(pathname, 'wb') as to_handle: contents = from_handle.read() if contents.startswith(b'#!/'): contents = self.fix_hashbang(contents, python) to_handle.write(contents) os.chmod(pathname, member.mode)
def build_binary_dist_helper(self, requirement): """ Convert a single, unpacked source distribution to a binary distribution. Raises an exception if it fails to create the binary distribution (probably because of missing binary dependencies like system libraries). :param requirement: A :py:class:`.Requirement` object. :returns: The pathname of the resulting binary distribution (a string). :raises: :py:exc:`.BuildFailed` when the build reports an error. :raises: :py:exc:`.NoBuildOutput` when the build does not produce the expected binary distribution archive. """ build_timer = Timer() # Make sure the source distribution contains a setup script. setup_script = os.path.join(requirement.source_directory, 'setup.py') if not os.path.isfile(setup_script): msg = "Directory %s (%s %s) doesn't contain a source distribution!" raise InvalidSourceDistribution( msg % (requirement.source_directory, requirement.name, requirement.version)) # Let the user know what's going on. build_text = "Building %s (%s) binary distribution" % ( requirement.name, requirement.version) logger.info("%s ..", build_text) # Cleanup previously generated distributions. dist_directory = os.path.join(requirement.source_directory, 'dist') if os.path.isdir(dist_directory): logger.debug( "Cleaning up previously generated distributions in %s ..", dist_directory) shutil.rmtree(dist_directory) makedirs(dist_directory) # Create a temporary directory for pip installing into, and set up the # install_lib directory structure inside it. We do this so that we can # pip install into this as our target. temporary_dir = tempfile.mkdtemp() distutils_inst = install(Distribution()) distutils_inst.prefix = '' # This will be changed if we're in a virtualenv. distutils_inst.finalize_options() pip_target = os.path.normpath(temporary_dir + distutils_inst.install_lib) # Compose the command line needed to build the binary distribution. command_line = ' '.join( pipes.quote(t) for t in ['pip', 'install', '.', '--target', pip_target]) logger.debug("Executing external command: %s", command_line) # Redirect all output of the build to a temporary file. fd, temporary_file = tempfile.mkstemp() command_line = '%s > "%s" 2>&1' % (command_line, temporary_file) try: # Start the build. build = subprocess.Popen(['sh', '-c', command_line], cwd=requirement.source_directory) # Wait for the build to finish and provide feedback to the user in the mean time. spinner = Spinner(label=build_text, timer=build_timer) while build.poll() is None: spinner.step() # Don't tax the CPU too much. time.sleep(0.2) spinner.clear() # At this point, we may have a number of dependencies in the directory we want # to tar up that should not be part of the package distribution. For instance, # s3 will also wrap up the concurrent, futures, and requests packages. We fix # this by reading {name}-{version}-{py-version}.egg-info/installed-files.txt # and removing any files or directories that are not in it. # 1. Find the appropriate .egg-info/ directory. egg_info_dir = None egg_info_start = '-'.join([requirement.name, requirement.version]) for egg_info_root, dirs, _ in os.walk(temporary_dir): for d in dirs: if d.startswith(egg_info_start) and d.endswith( '.egg-info'): egg_info_dir = d break if egg_info_dir is not None: break # 2. If we have a .egg-info/, try to read the installed-files.txt contents. inst_files = set() if egg_info_dir is not None: egg_info_path = os.path.join(egg_info_root, egg_info_dir) inst_files_path = os.path.join(egg_info_path, 'installed-files.txt') try: with open(inst_files_path) as f: for line in f: abs_path = os.path.abspath( os.path.join(egg_info_path, line.strip())) inst_files.add(abs_path) inst_files.add(os.path.dirname(abs_path)) except IOError, ioe: loger.warn('Unable to open %s: %s' % (inst_files_path, ioe)) # 3. If we were able to get a set of files and directories that belong in the # distribution, then we can delete everything else before archiving it. if inst_files: dirs, files = next(os.walk(egg_info_root))[1:] for d in dirs: d = os.path.abspath(os.path.join(egg_info_root, d)) if d not in inst_files: logger.info('Removing %s (not part of the package)' % d) shutil.rmtree(d) for f in files: f = os.path.abspath(os.path.join(egg_info_root, f)) if f not in inst_files: logger.info('Removing %s (not part of the package)' % f) os.unlink(f) # Tar up the contents of temporary_dir into the correct file name and put it in the dist dir. tarball_path = os.path.join(temporary_dir, requirement.name) path = archive_util.make_archive(tarball_path, 'gztar', root_dir=temporary_dir) shutil.copy(path, dist_directory) # Make sure the build succeeded and produced a binary distribution archive. try: # If the build reported an error we'll try to provide the user with # some hints about what went wrong. if build.returncode != 0: raise BuildFailed( "Failed to build {name} ({version}) binary distribution!", name=requirement.name, version=requirement.version) # Check if the build created the `dist' directory (the os.listdir() # call below will raise an exception if we don't check for this). if not os.path.isdir(dist_directory): raise NoBuildOutput( "Build of {name} ({version}) did not produce a binary distribution archive!", name=requirement.name, version=requirement.version) # Check if we can find the binary distribution archive. filenames = os.listdir(dist_directory) if len(filenames) != 1: raise NoBuildOutput( "Build of {name} ({version}) produced more than one distribution archive! (matches: {filenames})", name=requirement.name, version=requirement.version, filenames=concatenate(sorted(filenames))) except Exception as e: # Decorate the exception with the output of the failed build. with open(temporary_file) as handle: build_output = handle.read() enhanced_message = compact(""" {message} Please check the build output because it will probably provide a hint about what went wrong. Build output: {output} """, message=e.args[0], output=build_output.strip()) e.args = (enhanced_message, ) raise logger.info("Finished building %s (%s) in %s.", requirement.name, requirement.version, build_timer) return os.path.join(dist_directory, filenames[0])