def get_all_wheels(session, wheel_dir):
from belt import models
class Whl(object):
def __init__(self, matchdict, wheel_dir, wheel):
self.__dict__.update(matchdict)
self.wheel_dir = wheel_dir
self.wheel = wheel
@property
def path(self):
if not hasattr(self, '_path'):
self._path = os.path.abspath(os.path.join(self.wheel_dir, self.wheel))
return self._path
wheels = (session.query(models.File)
.filter(models.File.filename.like("%whl")).all())
wheels = [wheel.filename for wheel in wheels if
os.path.exists(wheel.fullpath)]
wheels = set(os.listdir(wheel_dir)).difference(wheels)
for wheel in wheels:
match = WHEEL_INFO_RE(wheel)
if match is None:
continue
yield Whl(match.groupdict(), wheel_dir, wheel)
def test_extension_filenames(self):
for ext_name, exts in self.index['extensions'].items():
self.assertEqual(
ext_name.find('_'), -1,
"Extension names should not contain underscores. "
"Found {}".format(ext_name))
for item in exts:
self.assertTrue(
item['filename'].endswith('.whl'),
"Filename {} must end with .whl".format(item['filename']))
self.assertEqual(
ext_name, item['metadata']['name'],
"Extension name mismatch in extensions['{}']. "
"Found an extension in the list with name "
"{}".format(ext_name, item['metadata']['name']))
parsed_filename = WHEEL_INFO_RE(item['filename'])
p = parsed_filename.groupdict()
self.assertTrue(
p.get('name'),
"Can't get name for {}".format(item['filename']))
universal_wheel = p.get('pyver') == 'py2.py3' and p.get(
'abi') == 'none' and p.get('plat') == 'any'
self.assertTrue(
universal_wheel,
"{} of {} not universal (platform independent) wheel. "
"It should end in py2.py3-none-any.whl".format(
item['filename'], ext_name))
def get_metadata(self):
from wheel.install import WHEEL_INFO_RE
from glob import glob
if not extension_exists(self.name):
return None
metadata = {}
ext_dir = self.path or get_extension_path(self.name)
info_dirs = glob(os.path.join(ext_dir, '*.*-info'))
azext_metadata = WheelExtension.get_azext_metadata(ext_dir)
if azext_metadata:
metadata.update(azext_metadata)
for dist_info_dirname in info_dirs:
parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname)
if parsed_dist_info_dir:
parsed_dist_info_dir = parsed_dist_info_dir.groupdict().get(
'name')
if os.path.split(parsed_dist_info_dir)[-1] == self.name.replace(
'-', '_'):
whl_metadata_filepath = os.path.join(dist_info_dirname,
WHL_METADATA_FILENAME)
if os.path.isfile(whl_metadata_filepath):
with open(whl_metadata_filepath) as f:
metadata.update(json.loads(f.read()))
return metadata
def build_wheels(local_pypi, wheel_dir):
from belt.utils import get_search_names
built_wheels = []
for wheel in os.listdir(wheel_dir):
match = WHEEL_INFO_RE(wheel)
if match is None:
continue
tags = match.groupdict()
# path_to_wheel = os.path.abspath(os.path.join(wheel_dir, wheel))
built_wheels.append(tags['namever'].lower())
for package_dir in os.listdir(local_pypi):
dir_ = os.path.join(local_pypi, package_dir)
for pkg in os.listdir(dir_):
name = os.path.join(dir_, pkg)
_, ext = os.path.splitext(name)
if ext in ['.md5', '.whl']:
continue
package_name, version = split_package_name(pkg)
package_name = '{}-{}'.format(package_name, version)
# FIXME why are names inconsistent forcing us to check both
# variations?
options = get_search_names(package_name)
for opt in options:
if opt.lower() in built_wheels:
break
else:
args = 'pip wheel --wheel-dir {} {}'.format(wheel_dir, name)
subprocess.call(args, shell=True)
def _is_not_platform_specific(item):
parsed_filename = WHEEL_INFO_RE(item['filename'])
p = parsed_filename.groupdict()
if p.get('pyver') == 'py2.py3' and p.get('abi') == 'none' and p.get('plat') == 'any':
return True
logger.debug("Skipping '%s' as not universal wheel."
"We do not currently support platform specific extension detection. "
"They can be installed with the full URL %s", item['filename'], item.get('downloadUrl'))
return False
def _is_not_platform_specific(item):
parsed_filename = WHEEL_INFO_RE(item['filename'])
p = parsed_filename.groupdict()
if p.get('pyver') == 'py2.py3' and p.get('abi') == 'none' and p.get('plat') == 'any':
return True
logger.debug("Skipping '%s' as not universal wheel."
"We do not currently support platform specific extension detection. "
"They can be installed with the full URL %s", item['filename'], item.get('downloadUrl'))
return False
def split_package_name(name):
wheel_match = WHEEL_INFO_RE(name)
if wheel_match:
pkg_name = wheel_match.group('name')
version = wheel_match.group('ver')
else:
pkg_name = re.split(r'-\d+', name, 1)[0]
version = name[len(pkg_name) + 1:]
version = ARCHIVE_SUFFIX.sub('', version)
return pkg_name, version
def _add_whl_ext(source): # pylint: disable=too-many-statements
url_parse_result = urlparse(source)
is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https')
logger.debug('Extension source is url? %s', is_url)
whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source)
parsed_filename = WHEEL_INFO_RE(whl_filename)
extension_name = parsed_filename.groupdict().get('name') if parsed_filename else None
if not extension_name:
raise CLIError('Unable to determine extension name from {}. Is the file name correct?'.format(source))
if extension_exists(extension_name):
raise CLIError('The extension {} already exists.'.format(extension_name))
ext_file = None
if is_url:
# Download from URL
tmp_dir = tempfile.mkdtemp()
ext_file = os.path.join(tmp_dir, whl_filename)
logger.debug('Downloading %s to %s', source, ext_file)
try:
_whl_download_from_url(url_parse_result, ext_file)
except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err)))
logger.debug('Downloaded to %s', ext_file)
else:
# Get file path
ext_file = os.path.realpath(os.path.expanduser(source))
if not os.path.isfile(ext_file):
raise CLIError("File {} not found.".format(source))
# Validate the extension
logger.debug('Validating the extension {}'.format(ext_file))
try:
_validate_whl_extension(ext_file)
except AssertionError:
logger.debug(traceback.format_exc())
raise CLIError('The extension is invalid. Use --debug for more information.')
except CLIError as e:
raise e
logger.debug('Validation successful on {}'.format(ext_file))
# Install with pip
extension_path = get_extension_path(extension_name)
pip_args = ['install', '--target', extension_path, ext_file]
logger.debug('Executing pip with args: %s', pip_args)
with HomebrewPipPatch():
pip_status_code = _run_pip(pip_args)
if pip_status_code > 0:
logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path)
shutil.rmtree(extension_path, ignore_errors=True)
raise CLIError('An error occurred. Pip failed with status code {}. '
'Use --debug for more information.'.format(pip_status_code))
# Save the whl we used to install the extension in the extension dir.
dst = os.path.join(extension_path, whl_filename)
shutil.copyfile(ext_file, dst)
logger.debug('Saved the whl to %s', dst)
def _add_whl_ext(source): # pylint: disable=too-many-statements
import pip
url_parse_result = urlparse(source)
is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https')
logger.debug('Extension source is url? %s', is_url)
whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source)
parsed_filename = WHEEL_INFO_RE(whl_filename)
extension_name = parsed_filename.groupdict().get('name') if parsed_filename else None
if not extension_name:
raise CLIError('Unable to determine extension name from {}. Is the file name correct?'.format(source))
if extension_exists(extension_name):
raise CLIError('The extension {} already exists.'.format(extension_name))
ext_file = None
if is_url:
# Download from URL
tmp_dir = tempfile.mkdtemp()
ext_file = os.path.join(tmp_dir, whl_filename)
logger.debug('Downloading %s to %s', source, ext_file)
try:
_whl_download_from_url(url_parse_result, ext_file)
except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err)))
logger.debug('Downloaded to %s', ext_file)
else:
# Get file path
ext_file = os.path.realpath(os.path.expanduser(source))
if not os.path.isfile(ext_file):
raise CLIError("File {} not found.".format(source))
# Validate the extension
logger.debug('Validating the extension {}'.format(ext_file))
try:
_validate_whl_extension(ext_file)
except AssertionError:
logger.debug(traceback.format_exc())
raise CLIError('The extension is invalid. Use --debug for more information.')
except CLIError as e:
raise e
logger.debug('Validation successful on {}'.format(ext_file))
# Install with pip
extension_path = get_extension_path(extension_name)
pip_args = ['install', '--target', extension_path, ext_file]
logger.debug('Executing pip with args: %s', pip_args)
pip_status_code = _run_pip(pip, pip_args)
if pip_status_code > 0:
logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path)
shutil.rmtree(extension_path, ignore_errors=True)
raise CLIError('An error occurred. Pip failed with status code {}. '
'Use --debug for more information.'.format(pip_status_code))
# Save the whl we used to install the extension in the extension dir.
dst = os.path.join(extension_path, whl_filename)
shutil.copyfile(ext_file, dst)
logger.debug('Saved the whl to %s', dst)
def add_platforms(wheel_ctx, platforms):
"""Add platform tags `platforms` to a wheel
Add any platform tags in `platforms` that are missing
to wheel_ctx's filename and ``WHEEL`` file.
Parameters
----------
wheel_ctx : InWheelCtx
An open wheel context
platforms : iterable
platform tags to add to wheel filename and WHEEL tags - e.g.
``('macosx_10_9_intel', 'macosx_10_9_x86_64')
"""
info_fname = pjoin(_dist_info_dir(wheel_ctx.path), 'WHEEL')
info = read_pkg_info(info_fname)
if info['Root-Is-Purelib'] == 'true':
raise WheelToolsError('Cannot add platforms to pure wheel')
# Check what tags we have
if wheel_ctx.out_wheel is None:
in_wheel = wheel_ctx.in_wheel
else:
raise NotImplementedError()
parsed_fname = WHEEL_INFO_RE(basename(in_wheel))
in_fname_tags = parsed_fname.groupdict()['plat'].split('.')
extra_fname_tags = [tag for tag in platforms if tag not in in_fname_tags]
in_wheel_base, ext = splitext(basename(in_wheel))
out_wheel_base = '.'.join([in_wheel_base] + list(extra_fname_tags))
out_wheel = out_wheel_base + ext
in_info_tags = [tag for name, tag in info.items() if name == 'Tag']
# Python version, C-API version combinations
pyc_apis = ['-'.join(tag.split('-')[:2]) for tag in in_info_tags]
# unique Python version, C-API version combinations
pyc_apis = unique_by_index(pyc_apis)
# Add new platform tags for each Python version, C-API combination
required_tags = ['-'.join(tup) for tup in product(pyc_apis, platforms)]
needs_write = False
for req_tag in required_tags:
if req_tag in in_info_tags:
continue
needs_write = True
info.add_header('Tag', req_tag)
if needs_write:
write_pkg_info(info_fname, info)
# Tell context manager to write wheel on exit by setting filename
wheel_ctx.out_wheel = out_wheel
return wheel_ctx.out_wheel
def add_platforms(wheel_ctx, platforms):
"""Add platform tags `platforms` to a wheel
Add any platform tags in `platforms` that are missing
to wheel_ctx's filename and ``WHEEL`` file.
Parameters
----------
wheel_ctx : InWheelCtx
An open wheel context
platforms : iterable
platform tags to add to wheel filename and WHEEL tags - e.g.
``('macosx_10_9_intel', 'macosx_10_9_x86_64')
"""
info_fname = pjoin(_dist_info_dir(wheel_ctx.path), 'WHEEL')
info = read_pkg_info(info_fname)
if info['Root-Is-Purelib'] == 'true':
raise WheelToolsError('Cannot add platforms to pure wheel')
# Check what tags we have
if wheel_ctx.out_wheel is None:
in_wheel = wheel_ctx.in_wheel
else:
raise NotImplementedError()
parsed_fname = WHEEL_INFO_RE(basename(in_wheel))
in_fname_tags = parsed_fname.groupdict()['plat'].split('.')
extra_fname_tags = [tag for tag in platforms if tag not in in_fname_tags]
in_wheel_base, ext = splitext(basename(in_wheel))
out_wheel_base = '.'.join([in_wheel_base] + list(extra_fname_tags))
out_wheel = out_wheel_base + ext
in_info_tags = [tag for name, tag in info.items() if name == 'Tag']
# Python version, C-API version combinations
pyc_apis = ['-'.join(tag.split('-')[:2]) for tag in in_info_tags]
# unique Python version, C-API version combinations
pyc_apis = unique_by_index(pyc_apis)
# Add new platform tags for each Python version, C-API combination
required_tags = ['-'.join(tup) for tup in product(pyc_apis, platforms)]
needs_write = False
for req_tag in required_tags:
if req_tag in in_info_tags:
continue
needs_write = True
info.add_header('Tag', req_tag)
if needs_write:
write_pkg_info(info_fname, info)
# Tell context manager to write wheel on exit by setting filename
wheel_ctx.out_wheel = out_wheel
return wheel_ctx.out_wheel
def copy_wheels_to_pypi(wheel_dir, local_pypi):
for wheel in os.listdir(wheel_dir):
match = WHEEL_INFO_RE(wheel)
if match is None:
continue
tags = match.groupdict()
path_to_wheel = os.path.abspath(os.path.join(wheel_dir, wheel))
wheel_destination = WheelDestination(local_pypi, tags['name'], tags['ver'])
local_wheel = os.path.join(wheel_destination.path, os.path.basename(path_to_wheel))
if os.path.exists(local_wheel):
continue
mkdir_p(wheel_destination.path)
shutil.copyfile(path_to_wheel, local_wheel)
def test_source_wheels(self):
# Test we can build all sources into wheels and that metadata from the wheel is valid
built_whl_dir = tempfile.mkdtemp()
source_extensions = [
os.path.join(SRC_PATH, n) for n in os.listdir(SRC_PATH)
if os.path.isdir(os.path.join(SRC_PATH, n))
]
for s in source_extensions:
if not os.path.isfile(os.path.join(s, 'setup.py')):
continue
try:
check_call([
'python', 'setup.py', 'bdist_wheel', '-q', '-d',
built_whl_dir
],
cwd=s,
stdout=PIPE,
stderr=PIPE)
except CalledProcessError as err:
self.fail("Unable to build extension {} : {}".format(s, err))
for filename in os.listdir(built_whl_dir):
ext_file = os.path.join(built_whl_dir, filename)
ext_dir = tempfile.mkdtemp(dir=built_whl_dir)
ext_name = WHEEL_INFO_RE(filename).groupdict().get('name')
metadata = get_ext_metadata(ext_dir, ext_file, ext_name)
run_requires = metadata.get('run_requires')
if run_requires and ext_name not in SKIP_DEP_CHECK:
deps = run_requires[0]['requires']
self.assertTrue(
all(not dep.startswith('azure-') for dep in deps),
"Dependencies of {} use disallowed extension dependencies. "
"Remove these dependencies: {}".format(filename, deps))
shutil.rmtree(built_whl_dir)
def get_metadata(self):
from wheel.install import WHEEL_INFO_RE
if not extension_exists(self.name):
return None
metadata = {}
ext_dir = get_extension_path(self.name)
dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')]
azext_metadata = WheelExtension.get_azext_metadata(ext_dir)
if azext_metadata:
metadata.update(azext_metadata)
for dist_info_dirname in dist_info_dirs:
parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname)
if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == self.name.replace('-', '_'):
whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME)
if os.path.isfile(whl_metadata_filepath):
with open(whl_metadata_filepath) as f:
metadata.update(json.load(f))
return metadata
def test_extension_filenames(self):
for ext_name, exts in self.index['extensions'].items():
self.assertEqual(ext_name.find('_'), -1, "Extension names should not contain underscores. "
"Found {}".format(ext_name))
for item in exts:
self.assertTrue(item['filename'].endswith('.whl'),
"Filename {} must end with .whl".format(item['filename']))
self.assertEqual(ext_name, item['metadata']['name'],
"Extension name mismatch in extensions['{}']. "
"Found an extension in the list with name "
"{}".format(ext_name, item['metadata']['name']))
parsed_filename = WHEEL_INFO_RE(item['filename'])
p = parsed_filename.groupdict()
self.assertTrue(p.get('name'), "Can't get name for {}".format(item['filename']))
universal_wheel = p.get('pyver') == 'py2.py3' and p.get('abi') == 'none' and p.get('plat') == 'any'
self.assertTrue(universal_wheel,
"{} of {} not universal (platform independent) wheel. "
"It should end in py2.py3-none-any.whl".format(item['filename'], ext_name))
def get_ext_metadata(ext_dir, ext_file, ext_name):
# Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L89
WHL_METADATA_FILENAME = 'metadata.json'
zip_ref = zipfile.ZipFile(ext_file, 'r')
zip_ref.extractall(ext_dir)
zip_ref.close()
metadata = {}
dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')]
azext_metadata = _get_azext_metadata(ext_dir)
if azext_metadata:
metadata.update(azext_metadata)
for dist_info_dirname in dist_info_dirs:
parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname)
if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'):
whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME)
if os.path.isfile(whl_metadata_filepath):
with open(whl_metadata_filepath) as f:
metadata.update(json.load(f))
return metadata
def execute(args, p):
import os
import sys
from wheel.install import WHEEL_INFO_RE # type: ignore
from .wheeltools import InWheelCtx, add_platforms, WheelToolsError
from .wheel_abi import analyze_wheel_abi
wheel_abi = analyze_wheel_abi(args.WHEEL_FILE)
parsed_fname = WHEEL_INFO_RE(basename(args.WHEEL_FILE))
in_fname_tags = parsed_fname.groupdict()['plat'].split('.')
print('%s recieves the following tag: "%s".' %
(basename(args.WHEEL_FILE), wheel_abi.overall_tag))
print('Use ``auditwheel show`` for more details')
if wheel_abi.overall_tag in in_fname_tags:
print('No tags to be added. Exiting.')
return 1
# todo: move more of this logic to separate file
if not exists(args.WHEEL_DIR):
os.makedirs(args.WHEEL_DIR)
with InWheelCtx(args.WHEEL_FILE) as ctx:
try:
out_wheel = add_platforms(ctx, [wheel_abi.overall_tag])
except WheelToolsError as e:
print('\n%s.' % str(e), file=sys.stderr)
return 1
if out_wheel:
# tell context manager to write wheel on exit with
# the proper output directory
ctx.out_wheel = join(args.WHEEL_DIR, basename(out_wheel))
return 0
def execute(args, p):
import os
import sys
from wheel.install import WHEEL_INFO_RE # type: ignore
from .wheeltools import InWheelCtx, add_platforms, WheelToolsError
from .wheel_abi import analyze_wheel_abi
wheel_abi = analyze_wheel_abi(args.WHEEL_FILE)
parsed_fname = WHEEL_INFO_RE(basename(args.WHEEL_FILE))
in_fname_tags = parsed_fname.groupdict()['plat'].split('.')
print('%s recieves the following tag: "%s".' % (basename(args.WHEEL_FILE),
wheel_abi.overall_tag))
print('Use ``auditwheel show`` for more details')
if wheel_abi.overall_tag in in_fname_tags:
print('No tags to be added. Exiting.')
return 1
# todo: move more of this logic to separate file
if not exists(args.WHEEL_DIR):
os.makedirs(args.WHEEL_DIR)
with InWheelCtx(args.WHEEL_FILE) as ctx:
try:
out_wheel = add_platforms(ctx, [wheel_abi.overall_tag])
except WheelToolsError as e:
print('\n%s.' % str(e), file=sys.stderr)
return 1
if out_wheel:
# tell context manager to write wheel on exit with
# the proper output directory
ctx.out_wheel = join(args.WHEEL_DIR, basename(out_wheel))
return 0
def get_metadata(self):
from wheel.install import WHEEL_INFO_RE
from glob import glob
if not extension_exists(self.name):
return None
metadata = {}
ext_dir = self.path or get_extension_path(self.name)
info_dirs = glob(os.path.join(ext_dir, '*.*-info'))
azext_metadata = WheelExtension.get_azext_metadata(ext_dir)
if azext_metadata:
metadata.update(azext_metadata)
for dist_info_dirname in info_dirs:
parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname)
if parsed_dist_info_dir:
parsed_dist_info_dir = parsed_dist_info_dir.groupdict().get('name')
if os.path.split(parsed_dist_info_dir)[-1] == self.name.replace('-', '_'):
whl_metadata_filepath = os.path.join(dist_info_dirname, WHL_METADATA_FILENAME)
if os.path.isfile(whl_metadata_filepath):
with open(whl_metadata_filepath) as f:
metadata.update(json.loads(f.read()))
return metadata
def _add_whl_ext(source, ext_sha256=None): # pylint: disable=too-many-statements
if not source.endswith('.whl'):
raise ValueError(
'Unknown extension type. Only Python wheels are supported.')
url_parse_result = urlparse(source)
is_url = (url_parse_result.scheme == 'http'
or url_parse_result.scheme == 'https')
logger.debug('Extension source is url? %s', is_url)
whl_filename = os.path.basename(
url_parse_result.path) if is_url else os.path.basename(source)
parsed_filename = WHEEL_INFO_RE(whl_filename)
# Extension names can have - but .whl format changes it to _ (PEP 0427). Undo this.
extension_name = parsed_filename.groupdict().get('name').replace(
'_', '-') if parsed_filename else None
if not extension_name:
raise CLIError(
'Unable to determine extension name from {}. Is the file name correct?'
.format(source))
if extension_exists(extension_name):
raise CLIError(
'The extension {} already exists.'.format(extension_name))
ext_file = None
if is_url:
# Download from URL
tmp_dir = tempfile.mkdtemp()
ext_file = os.path.join(tmp_dir, whl_filename)
logger.debug('Downloading %s to %s', source, ext_file)
try:
_whl_download_from_url(url_parse_result, ext_file)
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError) as err:
raise CLIError(
'Please ensure you have network connection. Error detail: {}'.
format(str(err)))
logger.debug('Downloaded to %s', ext_file)
else:
# Get file path
ext_file = os.path.realpath(os.path.expanduser(source))
if not os.path.isfile(ext_file):
raise CLIError("File {} not found.".format(source))
# Validate the extension
logger.debug('Validating the extension %s', ext_file)
if ext_sha256:
valid_checksum, computed_checksum = is_valid_sha256sum(
ext_file, ext_sha256)
if valid_checksum:
logger.debug("Checksum of %s is OK", ext_file)
else:
logger.debug(
"Invalid checksum for %s. Expected '%s', computed '%s'.",
ext_file, ext_sha256, computed_checksum)
raise CLIError(
"The checksum of the extension does not match the expected value. "
"Use --debug for more information.")
try:
_validate_whl_extension(ext_file)
except AssertionError:
logger.debug(traceback.format_exc())
raise CLIError(
'The extension is invalid. Use --debug for more information.')
except CLIError as e:
raise e
logger.debug('Validation successful on %s', ext_file)
# Install with pip
extension_path = get_extension_path(extension_name)
pip_args = ['install', '--target', extension_path, ext_file]
logger.debug('Executing pip with args: %s', pip_args)
with HomebrewPipPatch():
pip_status_code = _run_pip(pip_args)
if pip_status_code > 0:
logger.debug(
'Pip failed so deleting anything we might have installed at %s',
extension_path)
shutil.rmtree(extension_path, ignore_errors=True)
raise CLIError(
'An error occurred. Pip failed with status code {}. '
'Use --debug for more information.'.format(pip_status_code))
# Save the whl we used to install the extension in the extension dir.
dst = os.path.join(extension_path, whl_filename)
shutil.copyfile(ext_file, dst)
logger.debug('Saved the whl to %s', dst)
def add_platforms(wheel_ctx, platforms, remove_platforms=()):
"""Add platform tags `platforms` to a wheel
Add any platform tags in `platforms` that are missing
to wheel_ctx's filename and ``WHEEL`` file.
Parameters
----------
wheel_ctx : InWheelCtx
An open wheel context
platforms : iterable
platform tags to add to wheel filename and WHEEL tags - e.g.
``('macosx_10_9_intel', 'macosx_10_9_x86_64')
remove_platforms : iterable
platform tags to remove to the wheel filename and WHEEL tags, e.g.
``('linux_x86_64',)`` when ``('manylinux_x86_64')`` is added
"""
definitely_not_purelib = False
info_fname = pjoin(_dist_info_dir(wheel_ctx.path), 'WHEEL')
info = read_pkg_info(info_fname)
# Check what tags we have
if wheel_ctx.out_wheel is not None:
out_dir = dirname(wheel_ctx.out_wheel)
wheel_fname = basename(wheel_ctx.out_wheel)
else:
out_dir = '.'
wheel_fname = basename(wheel_ctx.in_wheel)
parsed_fname = WHEEL_INFO_RE(wheel_fname)
fparts = parsed_fname.groupdict()
original_fname_tags = fparts['plat'].split('.')
logger.info('Previous filename tags: %s', ', '.join(original_fname_tags))
fname_tags = {
tag
for tag in original_fname_tags if tag not in remove_platforms
}
fname_tags |= set(platforms)
# Can't be 'any' and another platform
if 'any' in fname_tags and len(fname_tags) > 1:
fname_tags.remove('any')
remove_platforms.append('any')
definitely_not_purelib = True
if fname_tags != original_fname_tags:
logger.info('New filename tags: %s', ', '.join(fname_tags))
else:
logger.info('No filename tags change needed.')
_, ext = splitext(wheel_fname)
fparts['plat'] = '.'.join(fname_tags)
fparts['ext'] = ext
out_wheel_fname = "{namever}-{pyver}-{abi}-{plat}{ext}".format(**fparts)
out_wheel = pjoin(out_dir, out_wheel_fname)
in_info_tags = [tag for name, tag in info.items() if name == 'Tag']
logger.info('Previous WHEEL info tags: %s', ', '.join(in_info_tags))
# Python version, C-API version combinations
pyc_apis = ['-'.join(tag.split('-')[:2]) for tag in in_info_tags]
# unique Python version, C-API version combinations
pyc_apis = unique_by_index(pyc_apis)
# Add new platform tags for each Python version, C-API combination
wanted_tags = ['-'.join(tup) for tup in product(pyc_apis, platforms)]
new_tags = [tag for tag in wanted_tags if tag not in in_info_tags]
unwanted_tags = [
'-'.join(tup) for tup in product(pyc_apis, remove_platforms)
]
updated_tags = [tag for tag in in_info_tags if tag not in unwanted_tags]
updated_tags += new_tags
if updated_tags != in_info_tags:
del info['Tag']
for tag in updated_tags:
info.add_header('Tag', tag)
if definitely_not_purelib:
info['Root-Is-Purelib'] = 'False'
logger.info('Changed wheel type to Platlib')
logger.info('New WHEEL info tags: %s', ', '.join(info.get_all('Tag')))
write_pkg_info(info_fname, info)
else:
logger.info('No WHEEL info change needed.')
return out_wheel
def _add_whl_ext(source, ext_sha256=None, pip_extra_index_urls=None, pip_proxy=None): # pylint: disable=too-many-statements
if not source.endswith('.whl'):
raise ValueError('Unknown extension type. Only Python wheels are supported.')
url_parse_result = urlparse(source)
is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https')
logger.debug('Extension source is url? %s', is_url)
whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source)
parsed_filename = WHEEL_INFO_RE(whl_filename)
# Extension names can have - but .whl format changes it to _ (PEP 0427). Undo this.
extension_name = parsed_filename.groupdict().get('name').replace('_', '-') if parsed_filename else None
if not extension_name:
raise CLIError('Unable to determine extension name from {}. Is the file name correct?'.format(source))
if extension_exists(extension_name):
raise CLIError('The extension {} already exists.'.format(extension_name))
ext_file = None
if is_url:
# Download from URL
tmp_dir = tempfile.mkdtemp()
ext_file = os.path.join(tmp_dir, whl_filename)
logger.debug('Downloading %s to %s', source, ext_file)
try:
_whl_download_from_url(url_parse_result, ext_file)
except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err)))
logger.debug('Downloaded to %s', ext_file)
else:
# Get file path
ext_file = os.path.realpath(os.path.expanduser(source))
if not os.path.isfile(ext_file):
raise CLIError("File {} not found.".format(source))
# Validate the extension
logger.debug('Validating the extension %s', ext_file)
if ext_sha256:
valid_checksum, computed_checksum = is_valid_sha256sum(ext_file, ext_sha256)
if valid_checksum:
logger.debug("Checksum of %s is OK", ext_file)
else:
logger.debug("Invalid checksum for %s. Expected '%s', computed '%s'.",
ext_file, ext_sha256, computed_checksum)
raise CLIError("The checksum of the extension does not match the expected value. "
"Use --debug for more information.")
try:
_validate_whl_extension(ext_file)
except AssertionError:
logger.debug(traceback.format_exc())
raise CLIError('The extension is invalid. Use --debug for more information.')
except CLIError as e:
raise e
logger.debug('Validation successful on %s', ext_file)
# Check for distro consistency
check_distro_consistency()
# Install with pip
extension_path = get_extension_path(extension_name)
pip_args = ['install', '--target', extension_path, ext_file]
if pip_proxy:
pip_args = pip_args + ['--proxy', pip_proxy]
if pip_extra_index_urls:
for extra_index_url in pip_extra_index_urls:
pip_args = pip_args + ['--extra-index-url', extra_index_url]
logger.debug('Executing pip with args: %s', pip_args)
with HomebrewPipPatch():
pip_status_code = _run_pip(pip_args)
if pip_status_code > 0:
logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path)
shutil.rmtree(extension_path, ignore_errors=True)
raise CLIError('An error occurred. Pip failed with status code {}. '
'Use --debug for more information.'.format(pip_status_code))
# Save the whl we used to install the extension in the extension dir.
dst = os.path.join(extension_path, whl_filename)
shutil.copyfile(ext_file, dst)
logger.debug('Saved the whl to %s', dst)
def add_platforms(wheel_ctx, platforms, remove_platforms=()):
"""Add platform tags `platforms` to a wheel
Add any platform tags in `platforms` that are missing
to wheel_ctx's filename and ``WHEEL`` file.
Parameters
----------
wheel_ctx : InWheelCtx
An open wheel context
platforms : iterable
platform tags to add to wheel filename and WHEEL tags - e.g.
``('macosx_10_9_intel', 'macosx_10_9_x86_64')
remove_platforms : iterable
platform tags to remove to the wheel filename and WHEEL tags, e.g.
``('linux_x86_64',)`` when ``('manylinux_x86_64')`` is added
"""
info_fname = pjoin(_dist_info_dir(wheel_ctx.path), 'WHEEL')
info = read_pkg_info(info_fname)
if info['Root-Is-Purelib'] == 'true':
print('No need to add platforms to pure wheel - Skipping {}'.format(wheel_ctx.in_wheel))
return
# Check what tags we have
if wheel_ctx.out_wheel is not None:
out_dir = dirname(wheel_ctx.out_wheel)
wheel_fname = basename(wheel_ctx.out_wheel)
else:
out_dir = '.'
wheel_fname = basename(wheel_ctx.in_wheel)
parsed_fname = WHEEL_INFO_RE(wheel_fname)
fparts = parsed_fname.groupdict()
original_fname_tags = fparts['plat'].split('.')
print('Previous filename tags:', ', '.join(original_fname_tags))
fname_tags = [tag for tag in original_fname_tags
if tag not in remove_platforms]
for platform in platforms:
if platform not in fname_tags:
fname_tags.append(platform)
if fname_tags != original_fname_tags:
print('New filename tags:', ', '.join(fname_tags))
else:
print('No filename tags change needed.')
wheel_base, ext = splitext(wheel_fname)
fparts['plat'] = '.'.join(fname_tags)
fparts['ext'] = ext
out_wheel_fname = "{namever}-{pyver}-{abi}-{plat}{ext}".format(**fparts)
out_wheel = pjoin(out_dir, out_wheel_fname)
in_info_tags = [tag for name, tag in info.items() if name == 'Tag']
print('Previous WHEEL info tags:', ', '.join(in_info_tags))
# Python version, C-API version combinations
pyc_apis = ['-'.join(tag.split('-')[:2]) for tag in in_info_tags]
# unique Python version, C-API version combinations
pyc_apis = unique_by_index(pyc_apis)
# Add new platform tags for each Python version, C-API combination
wanted_tags = ['-'.join(tup) for tup in product(pyc_apis, platforms)]
new_tags = [tag for tag in wanted_tags if tag not in in_info_tags]
unwanted_tags = ['-'.join(tup)
for tup in product(pyc_apis, remove_platforms)]
updated_tags = [tag for tag in in_info_tags if tag not in unwanted_tags]
updated_tags += new_tags
needs_write = updated_tags != in_info_tags
if needs_write:
del info['Tag']
for tag in updated_tags:
info.add_header('Tag', tag)
print('New WHEEL info tags:', ', '.join(info.get_all('Tag')))
write_pkg_info(info_fname, info)
else:
print('No WHEEL info change needed.')
return out_wheel
