def build_missing_binary_dists(requirements):
"""
Convert source distributions to binary distributions.
:param requirements: A list of tuples in the format of the return value of
the :py:func:`unpack_source_dists()` function.
:returns: ``True`` if it succeeds in building a binary distribution,
``False`` otherwise (probably because of missing binary
dependencies like system libraries).
"""
existing_binary_dists = find_cached_binary_dists()
logger.info("Building binary distributions ..")
pyversion = get_python_version()
for name, version, directory in requirements:
# Check if a binary distribution already exists.
filename = existing_binary_dists.get((name.lower(), version, pyversion))
if filename:
logger.debug("Existing binary distribution for %s (%s) found at %s.", name, version, filename)
continue
# Make sure the source distribution contains a setup script.
setup_script = os.path.join(directory, 'setup.py')
if not os.path.isfile(setup_script):
logger.warn("Package %s (%s) is not a source distribution.", name, version)
continue
# Try to build the binary distribution.
if not build_binary_dist(name, version, directory, pyversion):
sanity_check_dependencies(name)
if not build_binary_dist(name, version, directory, pyversion):
return False
logger.info("Finished building binary distributions.")
return True
def get_binary_dist(package,
version,
directory,
url=None,
python='/usr/bin/python',
prefix='/usr'):
"""
Get the cached binary distribution archive that was previously built for
the given package (name, version) (and optionally URL). If no archive has
been cached yet, a new binary distribution archive is created and added to
the cache.
:param package: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:param url: The URL of the requirement, optional. When given this is used
to generate the filename of the cached binary distribution.
:param python: The pathname of the Python executable to use to run
``setup.py`` (obviously this should point to a working
Python installation).
:param prefix: The prefix that the original binary distribution was created for.
:returns: An iterable of tuples with two values each: A
:py:class:`tarfile.TarInfo` object and a file-like object.
"""
tag = hashlib.sha1(str(version +
url).encode()).hexdigest() if url else version
cache_file = os.path.join(
binary_index, '%s:%s:%s.tar.gz' % (package, tag, get_python_version()))
if not os.path.isfile(cache_file):
logger.debug("%s (%s) hasn't been cached yet, doing so now.", package,
version)
# Build the binary distribution.
try:
raw_file = build_binary_dist(package,
version,
directory,
python=python)
except BuildFailed:
sanity_check_dependencies(package)
raw_file = build_binary_dist(package,
version,
directory,
python=python)
# Transform the binary distribution archive into a form that we can re-use.
transformed_file = '%s.tmp-%i' % (cache_file, os.getpid())
archive = tarfile.open(transformed_file, 'w:gz')
for member, from_handle in transform_binary_dist(raw_file,
prefix=prefix):
archive.addfile(member, from_handle)
archive.close()
# Try to avoid race conditions between multiple processes by atomically
# moving the transformed binary distribution into its final place.
os.rename(transformed_file, cache_file)
logger.debug("%s (%s) cached as %s.", package, version, cache_file)
archive = tarfile.open(cache_file, 'r:gz')
for member in archive.getmembers():
yield member, archive.extractfile(member.name)
archive.close()
def get_binary_dist(package,
version,
directory,
python='/usr/bin/python',
prefix='/usr'):
"""
Get the cached binary distribution archive that was previously built for
the given package (name, version). If no archive has been cached yet, a
new binary distribution archive is created and added to the cache.
:param package: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:param python: The pathname of the Python executable to use to run
``setup.py`` (obviously this should point to a working
Python installation).
:param prefix: The prefix that the original binary distribution was created for.
:returns: An iterable of tuples with two values each: A
:py:class:`tarfile.TarInfo` object and a file-like object.
"""
cache_file = os.path.join(
binary_index,
'%s:%s:%s.tar.gz' % (package, version, get_python_version()))
if not os.path.isfile(cache_file):
logger.debug("%s (%s) hasn't been cached yet, doing so now.", package,
version)
# Build the binary distribution.
try:
raw_file = build_binary_dist(package,
version,
directory,
python=python)
except BuildFailed:
sanity_check_dependencies(package)
raw_file = build_binary_dist(package,
version,
directory,
python=python)
# Transform the binary distribution archive into a form that we can re-use.
archive = tarfile.open(cache_file, 'w:gz')
for member, from_handle in transform_binary_dist(raw_file,
prefix=prefix):
archive.addfile(member, from_handle)
archive.close()
logger.debug("%s (%s) cached as %s.", package, version, cache_file)
archive = tarfile.open(cache_file, 'r:gz')
for member in archive.getmembers():
yield member, archive.extractfile(member.name)
archive.close()
def get_binary_dist(package, version, directory, url=None, python='/usr/bin/python', prefix='/usr'):
"""
Get the cached binary distribution archive that was previously built for
the given package (name, version) (and optionally URL). If no archive has
been cached yet, a new binary distribution archive is created and added to
the cache.
:param package: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:param url: The URL of the requirement, optional. When given this is used
to generate the filename of the cached binary distribution.
:param python: The pathname of the Python executable to use to run
``setup.py`` (obviously this should point to a working
Python installation).
:param prefix: The prefix that the original binary distribution was created for.
:returns: An iterable of tuples with two values each: A
:py:class:`tarfile.TarInfo` object and a file-like object.
"""
tag = hashlib.sha1(str(version + url).encode()).hexdigest() if url else version
cache_file = os.path.join(binary_index, '%s:%s:%s.tar.gz' % (package, tag, get_python_version()))
if not os.path.isfile(cache_file):
logger.debug("%s (%s) hasn't been cached yet, doing so now.", package, version)
# Build the binary distribution.
try:
raw_file = build_binary_dist(package, version, directory, python=python)
except BuildFailed:
sanity_check_dependencies(package)
raw_file = build_binary_dist(package, version, directory, python=python)
# Transform the binary distribution archive into a form that we can re-use.
transformed_file = '%s.tmp-%i' % (cache_file, os.getpid())
archive = tarfile.open(transformed_file, 'w:gz')
for member, from_handle in transform_binary_dist(raw_file, prefix=prefix):
archive.addfile(member, from_handle)
archive.close()
# Try to avoid race conditions between multiple processes by atomically
# moving the transformed binary distribution into its final place.
os.rename(transformed_file, cache_file)
logger.debug("%s (%s) cached as %s.", package, version, cache_file)
archive = tarfile.open(cache_file, 'r:gz')
for member in archive.getmembers():
yield member, archive.extractfile(member.name)
archive.close()
def get_binary_dist(package, version, directory, python='/usr/bin/python', prefix='/usr'):
"""
Get the cached binary distribution archive that was previously built for
the given package (name, version). If no archive has been cached yet, a
new binary distribution archive is created and added to the cache.
:param package: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:param python: The pathname of the Python executable to use to run
``setup.py`` (obviously this should point to a working
Python installation).
:param prefix: The prefix that the original binary distribution was created for.
:returns: An iterable of tuples with two values each: A
:py:class:`tarfile.TarInfo` object and a file-like object.
"""
cache_file = os.path.join(binary_index, '%s:%s:%s.tar.gz' % (package, version, get_python_version()))
if not os.path.isfile(cache_file):
logger.debug("%s (%s) hasn't been cached yet, doing so now.", package, version)
# Build the binary distribution.
try:
raw_file = build_binary_dist(package, version, directory, python=python)
except BuildFailed:
sanity_check_dependencies(package)
raw_file = build_binary_dist(package, version, directory, python=python)
# Transform the binary distribution archive into a form that we can re-use.
archive = tarfile.open(cache_file, 'w:gz')
for member, from_handle in transform_binary_dist(raw_file, prefix=prefix):
archive.addfile(member, from_handle)
archive.close()
logger.debug("%s (%s) cached as %s.", package, version, cache_file)
archive = tarfile.open(cache_file, 'r:gz')
for member in archive.getmembers():
yield member, archive.extractfile(member.name)
archive.close()
def get_binary_dist(package, version, directory, url=None, python='/usr/bin/python', prefix='/usr'):
"""
Get the cached binary distribution archive that was previously built for
the given package (name, version) (and optionally URL). If no archive has
been cached yet, a new binary distribution archive is created and added to
the cache.
:param package: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:param url: The URL of the requirement (optional). When given this is used
to generate the filename of the cached binary distribution.
:param python: The pathname of the Python executable to use to run
``setup.py`` (obviously this should point to a working
Python installation).
:param prefix: The prefix that the original binary distribution was created for.
:returns: An iterable of tuples with two values each: A
:py:class:`tarfile.TarInfo` object and a file-like object.
.. note:: The ``url`` parameter is ignored if it contains a ``file://``
URL. The reason for this is as follows:
- When pip is passed the pathname of a directory containing an
unpacked source distribution it will set the URL of the
requirement to a ``file://`` URL pointing to the directory.
- Exporting source distributions from a VCS repository to a
temporary directory and installing them with pip-accel is a
very reasonable use case.
- The two previous points combined mean the "URL" of the package
would change with every run of pip-accel, triggering a time
consuming rebuild of the binary distribution.
"""
if url and url.startswith('file://'):
url = None
tag = hashlib.sha1(str(version + url).encode()).hexdigest() if url else version
cache_file = os.path.join(binary_index, '%s:%s:%s.tar.gz' % (package, tag, get_python_version()))
if not os.path.isfile(cache_file):
logger.debug("%s (%s) hasn't been cached yet, doing so now.", package, version)
# Build the binary distribution.
try:
raw_file = build_binary_dist(package, version, directory, python=python)
except BuildFailed:
sanity_check_dependencies(package)
raw_file = build_binary_dist(package, version, directory, python=python)
# Transform the binary distribution archive into a form that we can re-use.
transformed_file = '%s.tmp-%i' % (cache_file, os.getpid())
archive = tarfile.open(transformed_file, 'w:gz')
for member, from_handle in transform_binary_dist(raw_file, prefix=prefix):
archive.addfile(member, from_handle)
archive.close()
# Try to avoid race conditions between multiple processes by atomically
# moving the transformed binary distribution into its final place.
os.rename(transformed_file, cache_file)
logger.debug("%s (%s) cached as %s.", package, version, cache_file)
archive = tarfile.open(cache_file, 'r:gz')
for member in archive.getmembers():
yield member, archive.extractfile(member.name)
archive.close()
def get_binary_dist(package, version, directory, url, cache, python='/usr/bin/python', prefix='/usr'):
"""
Get the cached binary distribution archive that was previously built for
the given package (name, version) (and optionally URL). If no archive has
been cached yet, a new binary distribution archive is created and added to
the cache.
:param package: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:param url: The URL of the requirement (may be ``None``). This is used to
generate the filename of the cached binary distribution.
:param cache: A :py:class:`.CacheManager` object.
:param python: The pathname of the Python executable to use to run
``setup.py`` (obviously this should point to a working
Python installation).
:param prefix: The prefix that the original binary distribution was created for.
:returns: An iterable of tuples with two values each: A
:py:class:`tarfile.TarInfo` object and a file-like object.
.. note:: The ``url`` parameter is ignored if it contains a ``file://``
URL. The reason for this is as follows:
- When pip is passed the pathname of a directory containing an
unpacked source distribution it will set the URL of the
requirement to a ``file://`` URL pointing to the directory.
- Exporting source distributions from a VCS repository to a
temporary directory and installing them with pip-accel is a
very reasonable use case.
- The two previous points combined mean the "URL" of the package
would change with every run of pip-accel, triggering a time
consuming rebuild of the binary distribution.
"""
cache_file = cache.get(package, version, url)
if not cache_file:
logger.debug("%s (%s) hasn't been cached yet, doing so now.", package, version)
# Build the binary distribution.
try:
raw_file = build_binary_dist(package, version, directory, python=python)
except BuildFailed:
sanity_check_dependencies(package)
raw_file = build_binary_dist(package, version, directory, python=python)
# Transform the binary distribution archive into a form that we can re-use.
transformed_file = os.path.join(tempfile.gettempdir(), os.path.basename(raw_file))
archive = tarfile.open(transformed_file, 'w:gz')
for member, from_handle in transform_binary_dist(raw_file, prefix=prefix):
archive.addfile(member, from_handle)
archive.close()
# Push the binary distribution archive to all available backends.
with open(transformed_file, 'rb') as handle:
cache.put(package, version, url, handle)
# Cleanup the temporary file.
os.remove(transformed_file)
# Get the absolute pathname of the file in the local cache.
cache_file = cache.get(package, version, url)
archive = tarfile.open(cache_file, 'r:gz')
for member in archive.getmembers():
yield member, archive.extractfile(member.name)
archive.close()
def get_binary_dist(package,
version,
directory,
url,
cache,
python='/usr/bin/python',
prefix='/usr'):
"""
Get the cached binary distribution archive that was previously built for
the given package (name, version) (and optionally URL). If no archive has
been cached yet, a new binary distribution archive is created and added to
the cache.
:param package: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:param url: The URL of the requirement (may be ``None``). This is used to
generate the filename of the cached binary distribution.
:param cache: A :py:class:`.CacheManager` object.
:param python: The pathname of the Python executable to use to run
``setup.py`` (obviously this should point to a working
Python installation).
:param prefix: The prefix that the original binary distribution was created for.
:returns: An iterable of tuples with two values each: A
:py:class:`tarfile.TarInfo` object and a file-like object.
.. note:: The ``url`` parameter is ignored if it contains a ``file://``
URL. The reason for this is as follows:
- When pip is passed the pathname of a directory containing an
unpacked source distribution it will set the URL of the
requirement to a ``file://`` URL pointing to the directory.
- Exporting source distributions from a VCS repository to a
temporary directory and installing them with pip-accel is a
very reasonable use case.
- The two previous points combined mean the "URL" of the package
would change with every run of pip-accel, triggering a time
consuming rebuild of the binary distribution.
"""
cache_file = cache.get(package, version, url)
if not cache_file:
logger.debug("%s (%s) hasn't been cached yet, doing so now.", package,
version)
# Build the binary distribution.
try:
raw_file = build_binary_dist(package,
version,
directory,
python=python)
except BuildFailed:
sanity_check_dependencies(package)
raw_file = build_binary_dist(package,
version,
directory,
python=python)
# Transform the binary distribution archive into a form that we can re-use.
transformed_file = os.path.join(tempfile.gettempdir(),
os.path.basename(raw_file))
archive = tarfile.open(transformed_file, 'w:gz')
for member, from_handle in transform_binary_dist(raw_file,
prefix=prefix):
archive.addfile(member, from_handle)
archive.close()
# Push the binary distribution archive to all available backends.
with open(transformed_file, 'rb') as handle:
cache.put(package, version, url, handle)
# Cleanup the temporary file.
os.remove(transformed_file)
# Get the absolute pathname of the file in the local cache.
cache_file = cache.get(package, version, url)
archive = tarfile.open(cache_file, 'r:gz')
for member in archive.getmembers():
yield member, archive.extractfile(member.name)
archive.close()