def get_binary_dist(package,
version,
directory,
url=None,
python='/usr/bin/python',
prefix='/usr'):
"""
Get the cached binary distribution archive that was previously built for
the given package (name, version) (and optionally URL). If no archive has
been cached yet, a new binary distribution archive is created and added to
the cache.
:param package: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:param url: The URL of the requirement, optional. When given this is used
to generate the filename of the cached binary distribution.
:param python: The pathname of the Python executable to use to run
``setup.py`` (obviously this should point to a working
Python installation).
:param prefix: The prefix that the original binary distribution was created for.
:returns: An iterable of tuples with two values each: A
:py:class:`tarfile.TarInfo` object and a file-like object.
"""
tag = hashlib.sha1(str(version +
url).encode()).hexdigest() if url else version
cache_file = os.path.join(
binary_index, '%s:%s:%s.tar.gz' % (package, tag, get_python_version()))
if not os.path.isfile(cache_file):
logger.debug("%s (%s) hasn't been cached yet, doing so now.", package,
version)
# Build the binary distribution.
try:
raw_file = build_binary_dist(package,
version,
directory,
python=python)
except BuildFailed:
sanity_check_dependencies(package)
raw_file = build_binary_dist(package,
version,
directory,
python=python)
# Transform the binary distribution archive into a form that we can re-use.
transformed_file = '%s.tmp-%i' % (cache_file, os.getpid())
archive = tarfile.open(transformed_file, 'w:gz')
for member, from_handle in transform_binary_dist(raw_file,
prefix=prefix):
archive.addfile(member, from_handle)
archive.close()
# Try to avoid race conditions between multiple processes by atomically
# moving the transformed binary distribution into its final place.
os.rename(transformed_file, cache_file)
logger.debug("%s (%s) cached as %s.", package, version, cache_file)
archive = tarfile.open(cache_file, 'r:gz')
for member in archive.getmembers():
yield member, archive.extractfile(member.name)
archive.close()
def generate_filename(self, requirement):
"""
Generate a distribution archive filename for a package.
:param requirement: A :py:class:`.Requirement` object.
:returns: The filename of the distribution archive (a string)
including a single leading directory component to indicate
the cache format revision.
"""
return 'v%i/%s:%s:%s.tar.gz' % (self.config.cache_format_revision,
requirement.name, requirement.version, get_python_version())
def generate_filename(self, requirement):
"""
Generate a distribution archive filename for a package.
:param requirement: A :py:class:`.Requirement` object.
:returns: The filename of the distribution archive (a string)
including a single leading directory component to indicate
the cache format revision.
"""
return 'v%i/%s:%s:%s.tar.gz' % (self.config.cache_format_revision,
requirement.name, requirement.version,
get_python_version())
def generate_filename(self, package, version, url=None):
"""
Generate a distribution archive filename for a package.
:param package: The name of the package (a string).
:param version: The version of the package (a string).
:param url: The URL of the requirement (a string or ``None``).
:returns: The filename (without directory components) of the
distribution archive.
"""
if url and url.startswith('file://'):
url = None
tag = hashlib.sha1(str(version + url).encode()).hexdigest() if url else version
return '%s:%s:%s.tar.gz' % (package, tag, get_python_version())
def get_binary_dist(package,
version,
directory,
python='/usr/bin/python',
prefix='/usr'):
"""
Get the cached binary distribution archive that was previously built for
the given package (name, version). If no archive has been cached yet, a
new binary distribution archive is created and added to the cache.
:param package: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:param python: The pathname of the Python executable to use to run
``setup.py`` (obviously this should point to a working
Python installation).
:param prefix: The prefix that the original binary distribution was created for.
:returns: An iterable of tuples with two values each: A
:py:class:`tarfile.TarInfo` object and a file-like object.
"""
cache_file = os.path.join(
binary_index,
'%s:%s:%s.tar.gz' % (package, version, get_python_version()))
if not os.path.isfile(cache_file):
logger.debug("%s (%s) hasn't been cached yet, doing so now.", package,
version)
# Build the binary distribution.
try:
raw_file = build_binary_dist(package,
version,
directory,
python=python)
except BuildFailed:
sanity_check_dependencies(package)
raw_file = build_binary_dist(package,
version,
directory,
python=python)
# Transform the binary distribution archive into a form that we can re-use.
archive = tarfile.open(cache_file, 'w:gz')
for member, from_handle in transform_binary_dist(raw_file,
prefix=prefix):
archive.addfile(member, from_handle)
archive.close()
logger.debug("%s (%s) cached as %s.", package, version, cache_file)
archive = tarfile.open(cache_file, 'r:gz')
for member in archive.getmembers():
yield member, archive.extractfile(member.name)
archive.close()
def generate_filename(self, package, version, url=None):
"""
Generate a distribution archive filename for a package.
:param package: The name of the package (a string).
:param version: The version of the package (a string).
:param url: The URL of the requirement (a string or ``None``).
:returns: The filename (without directory components) of the
distribution archive.
"""
if url and url.startswith('file://'):
url = None
tag = hashlib.sha1(str(version +
url).encode()).hexdigest() if url else version
return '%s:%s:%s.tar.gz' % (package, tag, get_python_version())
def get_binary_dist(package, version, directory, url=None, python='/usr/bin/python', prefix='/usr'):
"""
Get the cached binary distribution archive that was previously built for
the given package (name, version) (and optionally URL). If no archive has
been cached yet, a new binary distribution archive is created and added to
the cache.
:param package: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:param url: The URL of the requirement, optional. When given this is used
to generate the filename of the cached binary distribution.
:param python: The pathname of the Python executable to use to run
``setup.py`` (obviously this should point to a working
Python installation).
:param prefix: The prefix that the original binary distribution was created for.
:returns: An iterable of tuples with two values each: A
:py:class:`tarfile.TarInfo` object and a file-like object.
"""
tag = hashlib.sha1(str(version + url).encode()).hexdigest() if url else version
cache_file = os.path.join(binary_index, '%s:%s:%s.tar.gz' % (package, tag, get_python_version()))
if not os.path.isfile(cache_file):
logger.debug("%s (%s) hasn't been cached yet, doing so now.", package, version)
# Build the binary distribution.
try:
raw_file = build_binary_dist(package, version, directory, python=python)
except BuildFailed:
sanity_check_dependencies(package)
raw_file = build_binary_dist(package, version, directory, python=python)
# Transform the binary distribution archive into a form that we can re-use.
transformed_file = '%s.tmp-%i' % (cache_file, os.getpid())
archive = tarfile.open(transformed_file, 'w:gz')
for member, from_handle in transform_binary_dist(raw_file, prefix=prefix):
archive.addfile(member, from_handle)
archive.close()
# Try to avoid race conditions between multiple processes by atomically
# moving the transformed binary distribution into its final place.
os.rename(transformed_file, cache_file)
logger.debug("%s (%s) cached as %s.", package, version, cache_file)
archive = tarfile.open(cache_file, 'r:gz')
for member in archive.getmembers():
yield member, archive.extractfile(member.name)
archive.close()
def generate_filename(self, requirement):
"""
Generate a distribution archive filename for a package.
:param requirement: A :py:class:`.Requirement` object.
:returns: The filename of the distribution archive (a string)
including a single leading directory component to indicate
the cache format revision.
"""
url = requirement.url
if url and url.startswith('file://'):
# Ignore the URL if it is a file:// URL because those frequently
# point to temporary directories whose pathnames change with every
# invocation of pip-accel. If we would include the file:// URL in
# the cache key we would be generating a unique cache key on
# every run (not good for performance ;-).
url = None
tag = sha1(requirement.version + url) if url else requirement.version
return 'v%i/%s-%s-%s.tar.gz' % (self.config.cache_format_revision,
requirement.name, tag, get_python_version())
def generate_filename(self, requirement):
"""
Generate a distribution archive filename for a package.
:param requirement: A :py:class:`.Requirement` object.
:returns: The filename of the distribution archive (a string)
including a single leading directory component to indicate
the cache format revision.
"""
url = requirement.url
if url and url.startswith('file://'):
# Ignore the URL if it is a file:// URL because those frequently
# point to temporary directories whose pathnames change with every
# invocation of pip-accel. If we would include the file:// URL in
# the cache key we would be generating a unique cache key on
# every run (not good for performance ;-).
url = None
tag = sha1(requirement.version + url) if url else requirement.version
return 'v%i/%s:%s:%s.tar.gz' % (self.config.cache_format_revision,
requirement.name, tag, get_python_version())
def get_binary_dist(package, version, directory, python='/usr/bin/python', prefix='/usr'):
"""
Get the cached binary distribution archive that was previously built for
the given package (name, version). If no archive has been cached yet, a
new binary distribution archive is created and added to the cache.
:param package: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:param python: The pathname of the Python executable to use to run
``setup.py`` (obviously this should point to a working
Python installation).
:param prefix: The prefix that the original binary distribution was created for.
:returns: An iterable of tuples with two values each: A
:py:class:`tarfile.TarInfo` object and a file-like object.
"""
cache_file = os.path.join(binary_index, '%s:%s:%s.tar.gz' % (package, version, get_python_version()))
if not os.path.isfile(cache_file):
logger.debug("%s (%s) hasn't been cached yet, doing so now.", package, version)
# Build the binary distribution.
try:
raw_file = build_binary_dist(package, version, directory, python=python)
except BuildFailed:
sanity_check_dependencies(package)
raw_file = build_binary_dist(package, version, directory, python=python)
# Transform the binary distribution archive into a form that we can re-use.
archive = tarfile.open(cache_file, 'w:gz')
for member, from_handle in transform_binary_dist(raw_file, prefix=prefix):
archive.addfile(member, from_handle)
archive.close()
logger.debug("%s (%s) cached as %s.", package, version, cache_file)
archive = tarfile.open(cache_file, 'r:gz')
for member in archive.getmembers():
yield member, archive.extractfile(member.name)
archive.close()
def get_binary_dist(package, version, directory, url=None, python='/usr/bin/python', prefix='/usr'):
"""
Get the cached binary distribution archive that was previously built for
the given package (name, version) (and optionally URL). If no archive has
been cached yet, a new binary distribution archive is created and added to
the cache.
:param package: The name of the requirement to build.
:param version: The version of the requirement to build.
:param directory: The directory where the unpacked sources of the
requirement are available.
:param url: The URL of the requirement (optional). When given this is used
to generate the filename of the cached binary distribution.
:param python: The pathname of the Python executable to use to run
``setup.py`` (obviously this should point to a working
Python installation).
:param prefix: The prefix that the original binary distribution was created for.
:returns: An iterable of tuples with two values each: A
:py:class:`tarfile.TarInfo` object and a file-like object.
.. note:: The ``url`` parameter is ignored if it contains a ``file://``
URL. The reason for this is as follows:
- When pip is passed the pathname of a directory containing an
unpacked source distribution it will set the URL of the
requirement to a ``file://`` URL pointing to the directory.
- Exporting source distributions from a VCS repository to a
temporary directory and installing them with pip-accel is a
very reasonable use case.
- The two previous points combined mean the "URL" of the package
would change with every run of pip-accel, triggering a time
consuming rebuild of the binary distribution.
"""
if url and url.startswith('file://'):
url = None
tag = hashlib.sha1(str(version + url).encode()).hexdigest() if url else version
cache_file = os.path.join(binary_index, '%s:%s:%s.tar.gz' % (package, tag, get_python_version()))
if not os.path.isfile(cache_file):
logger.debug("%s (%s) hasn't been cached yet, doing so now.", package, version)
# Build the binary distribution.
try:
raw_file = build_binary_dist(package, version, directory, python=python)
except BuildFailed:
sanity_check_dependencies(package)
raw_file = build_binary_dist(package, version, directory, python=python)
# Transform the binary distribution archive into a form that we can re-use.
transformed_file = '%s.tmp-%i' % (cache_file, os.getpid())
archive = tarfile.open(transformed_file, 'w:gz')
for member, from_handle in transform_binary_dist(raw_file, prefix=prefix):
archive.addfile(member, from_handle)
archive.close()
# Try to avoid race conditions between multiple processes by atomically
# moving the transformed binary distribution into its final place.
os.rename(transformed_file, cache_file)
logger.debug("%s (%s) cached as %s.", package, version, cache_file)
archive = tarfile.open(cache_file, 'r:gz')
for member in archive.getmembers():
yield member, archive.extractfile(member.name)
archive.close()
