def get_binary_dist(package, version, directory, url=None, python='/usr/bin/python', prefix='/usr'): """ Get the cached binary distribution archive that was previously built for the given package (name, version) (and optionally URL). If no archive has been cached yet, a new binary distribution archive is created and added to the cache. :param package: The name of the requirement to build. :param version: The version of the requirement to build. :param directory: The directory where the unpacked sources of the requirement are available. :param url: The URL of the requirement, optional. When given this is used to generate the filename of the cached binary distribution. :param python: The pathname of the Python executable to use to run ``setup.py`` (obviously this should point to a working Python installation). :param prefix: The prefix that the original binary distribution was created for. :returns: An iterable of tuples with two values each: A :py:class:`tarfile.TarInfo` object and a file-like object. """ tag = hashlib.sha1(str(version + url).encode()).hexdigest() if url else version cache_file = os.path.join( binary_index, '%s:%s:%s.tar.gz' % (package, tag, get_python_version())) if not os.path.isfile(cache_file): logger.debug("%s (%s) hasn't been cached yet, doing so now.", package, version) # Build the binary distribution. try: raw_file = build_binary_dist(package, version, directory, python=python) except BuildFailed: sanity_check_dependencies(package) raw_file = build_binary_dist(package, version, directory, python=python) # Transform the binary distribution archive into a form that we can re-use. transformed_file = '%s.tmp-%i' % (cache_file, os.getpid()) archive = tarfile.open(transformed_file, 'w:gz') for member, from_handle in transform_binary_dist(raw_file, prefix=prefix): archive.addfile(member, from_handle) archive.close() # Try to avoid race conditions between multiple processes by atomically # moving the transformed binary distribution into its final place. os.rename(transformed_file, cache_file) logger.debug("%s (%s) cached as %s.", package, version, cache_file) archive = tarfile.open(cache_file, 'r:gz') for member in archive.getmembers(): yield member, archive.extractfile(member.name) archive.close()
def generate_filename(self, requirement): """ Generate a distribution archive filename for a package. :param requirement: A :py:class:`.Requirement` object. :returns: The filename of the distribution archive (a string) including a single leading directory component to indicate the cache format revision. """ return 'v%i/%s:%s:%s.tar.gz' % (self.config.cache_format_revision, requirement.name, requirement.version, get_python_version())
def generate_filename(self, package, version, url=None): """ Generate a distribution archive filename for a package. :param package: The name of the package (a string). :param version: The version of the package (a string). :param url: The URL of the requirement (a string or ``None``). :returns: The filename (without directory components) of the distribution archive. """ if url and url.startswith('file://'): url = None tag = hashlib.sha1(str(version + url).encode()).hexdigest() if url else version return '%s:%s:%s.tar.gz' % (package, tag, get_python_version())
def get_binary_dist(package, version, directory, python='/usr/bin/python', prefix='/usr'): """ Get the cached binary distribution archive that was previously built for the given package (name, version). If no archive has been cached yet, a new binary distribution archive is created and added to the cache. :param package: The name of the requirement to build. :param version: The version of the requirement to build. :param directory: The directory where the unpacked sources of the requirement are available. :param python: The pathname of the Python executable to use to run ``setup.py`` (obviously this should point to a working Python installation). :param prefix: The prefix that the original binary distribution was created for. :returns: An iterable of tuples with two values each: A :py:class:`tarfile.TarInfo` object and a file-like object. """ cache_file = os.path.join( binary_index, '%s:%s:%s.tar.gz' % (package, version, get_python_version())) if not os.path.isfile(cache_file): logger.debug("%s (%s) hasn't been cached yet, doing so now.", package, version) # Build the binary distribution. try: raw_file = build_binary_dist(package, version, directory, python=python) except BuildFailed: sanity_check_dependencies(package) raw_file = build_binary_dist(package, version, directory, python=python) # Transform the binary distribution archive into a form that we can re-use. archive = tarfile.open(cache_file, 'w:gz') for member, from_handle in transform_binary_dist(raw_file, prefix=prefix): archive.addfile(member, from_handle) archive.close() logger.debug("%s (%s) cached as %s.", package, version, cache_file) archive = tarfile.open(cache_file, 'r:gz') for member in archive.getmembers(): yield member, archive.extractfile(member.name) archive.close()
def get_binary_dist(package, version, directory, url=None, python='/usr/bin/python', prefix='/usr'): """ Get the cached binary distribution archive that was previously built for the given package (name, version) (and optionally URL). If no archive has been cached yet, a new binary distribution archive is created and added to the cache. :param package: The name of the requirement to build. :param version: The version of the requirement to build. :param directory: The directory where the unpacked sources of the requirement are available. :param url: The URL of the requirement, optional. When given this is used to generate the filename of the cached binary distribution. :param python: The pathname of the Python executable to use to run ``setup.py`` (obviously this should point to a working Python installation). :param prefix: The prefix that the original binary distribution was created for. :returns: An iterable of tuples with two values each: A :py:class:`tarfile.TarInfo` object and a file-like object. """ tag = hashlib.sha1(str(version + url).encode()).hexdigest() if url else version cache_file = os.path.join(binary_index, '%s:%s:%s.tar.gz' % (package, tag, get_python_version())) if not os.path.isfile(cache_file): logger.debug("%s (%s) hasn't been cached yet, doing so now.", package, version) # Build the binary distribution. try: raw_file = build_binary_dist(package, version, directory, python=python) except BuildFailed: sanity_check_dependencies(package) raw_file = build_binary_dist(package, version, directory, python=python) # Transform the binary distribution archive into a form that we can re-use. transformed_file = '%s.tmp-%i' % (cache_file, os.getpid()) archive = tarfile.open(transformed_file, 'w:gz') for member, from_handle in transform_binary_dist(raw_file, prefix=prefix): archive.addfile(member, from_handle) archive.close() # Try to avoid race conditions between multiple processes by atomically # moving the transformed binary distribution into its final place. os.rename(transformed_file, cache_file) logger.debug("%s (%s) cached as %s.", package, version, cache_file) archive = tarfile.open(cache_file, 'r:gz') for member in archive.getmembers(): yield member, archive.extractfile(member.name) archive.close()
def generate_filename(self, requirement): """ Generate a distribution archive filename for a package. :param requirement: A :py:class:`.Requirement` object. :returns: The filename of the distribution archive (a string) including a single leading directory component to indicate the cache format revision. """ url = requirement.url if url and url.startswith('file://'): # Ignore the URL if it is a file:// URL because those frequently # point to temporary directories whose pathnames change with every # invocation of pip-accel. If we would include the file:// URL in # the cache key we would be generating a unique cache key on # every run (not good for performance ;-). url = None tag = sha1(requirement.version + url) if url else requirement.version return 'v%i/%s-%s-%s.tar.gz' % (self.config.cache_format_revision, requirement.name, tag, get_python_version())
def generate_filename(self, requirement): """ Generate a distribution archive filename for a package. :param requirement: A :py:class:`.Requirement` object. :returns: The filename of the distribution archive (a string) including a single leading directory component to indicate the cache format revision. """ url = requirement.url if url and url.startswith('file://'): # Ignore the URL if it is a file:// URL because those frequently # point to temporary directories whose pathnames change with every # invocation of pip-accel. If we would include the file:// URL in # the cache key we would be generating a unique cache key on # every run (not good for performance ;-). url = None tag = sha1(requirement.version + url) if url else requirement.version return 'v%i/%s:%s:%s.tar.gz' % (self.config.cache_format_revision, requirement.name, tag, get_python_version())
def get_binary_dist(package, version, directory, python='/usr/bin/python', prefix='/usr'): """ Get the cached binary distribution archive that was previously built for the given package (name, version). If no archive has been cached yet, a new binary distribution archive is created and added to the cache. :param package: The name of the requirement to build. :param version: The version of the requirement to build. :param directory: The directory where the unpacked sources of the requirement are available. :param python: The pathname of the Python executable to use to run ``setup.py`` (obviously this should point to a working Python installation). :param prefix: The prefix that the original binary distribution was created for. :returns: An iterable of tuples with two values each: A :py:class:`tarfile.TarInfo` object and a file-like object. """ cache_file = os.path.join(binary_index, '%s:%s:%s.tar.gz' % (package, version, get_python_version())) if not os.path.isfile(cache_file): logger.debug("%s (%s) hasn't been cached yet, doing so now.", package, version) # Build the binary distribution. try: raw_file = build_binary_dist(package, version, directory, python=python) except BuildFailed: sanity_check_dependencies(package) raw_file = build_binary_dist(package, version, directory, python=python) # Transform the binary distribution archive into a form that we can re-use. archive = tarfile.open(cache_file, 'w:gz') for member, from_handle in transform_binary_dist(raw_file, prefix=prefix): archive.addfile(member, from_handle) archive.close() logger.debug("%s (%s) cached as %s.", package, version, cache_file) archive = tarfile.open(cache_file, 'r:gz') for member in archive.getmembers(): yield member, archive.extractfile(member.name) archive.close()
def get_binary_dist(package, version, directory, url=None, python='/usr/bin/python', prefix='/usr'): """ Get the cached binary distribution archive that was previously built for the given package (name, version) (and optionally URL). If no archive has been cached yet, a new binary distribution archive is created and added to the cache. :param package: The name of the requirement to build. :param version: The version of the requirement to build. :param directory: The directory where the unpacked sources of the requirement are available. :param url: The URL of the requirement (optional). When given this is used to generate the filename of the cached binary distribution. :param python: The pathname of the Python executable to use to run ``setup.py`` (obviously this should point to a working Python installation). :param prefix: The prefix that the original binary distribution was created for. :returns: An iterable of tuples with two values each: A :py:class:`tarfile.TarInfo` object and a file-like object. .. note:: The ``url`` parameter is ignored if it contains a ``file://`` URL. The reason for this is as follows: - When pip is passed the pathname of a directory containing an unpacked source distribution it will set the URL of the requirement to a ``file://`` URL pointing to the directory. - Exporting source distributions from a VCS repository to a temporary directory and installing them with pip-accel is a very reasonable use case. - The two previous points combined mean the "URL" of the package would change with every run of pip-accel, triggering a time consuming rebuild of the binary distribution. """ if url and url.startswith('file://'): url = None tag = hashlib.sha1(str(version + url).encode()).hexdigest() if url else version cache_file = os.path.join(binary_index, '%s:%s:%s.tar.gz' % (package, tag, get_python_version())) if not os.path.isfile(cache_file): logger.debug("%s (%s) hasn't been cached yet, doing so now.", package, version) # Build the binary distribution. try: raw_file = build_binary_dist(package, version, directory, python=python) except BuildFailed: sanity_check_dependencies(package) raw_file = build_binary_dist(package, version, directory, python=python) # Transform the binary distribution archive into a form that we can re-use. transformed_file = '%s.tmp-%i' % (cache_file, os.getpid()) archive = tarfile.open(transformed_file, 'w:gz') for member, from_handle in transform_binary_dist(raw_file, prefix=prefix): archive.addfile(member, from_handle) archive.close() # Try to avoid race conditions between multiple processes by atomically # moving the transformed binary distribution into its final place. os.rename(transformed_file, cache_file) logger.debug("%s (%s) cached as %s.", package, version, cache_file) archive = tarfile.open(cache_file, 'r:gz') for member in archive.getmembers(): yield member, archive.extractfile(member.name) archive.close()