Ejemplo n.º 1
0
Archivo: axle.py Proyecto: rob-b/belt
def get_all_wheels(session, wheel_dir):
    from belt import models

    class Whl(object):

        def __init__(self, matchdict, wheel_dir, wheel):
            self.__dict__.update(matchdict)
            self.wheel_dir = wheel_dir
            self.wheel = wheel

        @property
        def path(self):
            if not hasattr(self, '_path'):
                self._path = os.path.abspath(os.path.join(self.wheel_dir, self.wheel))
            return self._path
    wheels = (session.query(models.File)
              .filter(models.File.filename.like("%whl")).all())
    wheels = [wheel.filename for wheel in wheels if
              os.path.exists(wheel.fullpath)]
    wheels = set(os.listdir(wheel_dir)).difference(wheels)
    for wheel in wheels:
        match = WHEEL_INFO_RE(wheel)
        if match is None:
            continue
        yield Whl(match.groupdict(), wheel_dir, wheel)
Ejemplo n.º 2
0
 def test_extension_filenames(self):
     for ext_name, exts in self.index['extensions'].items():
         self.assertEqual(
             ext_name.find('_'), -1,
             "Extension names should not contain underscores. "
             "Found {}".format(ext_name))
         for item in exts:
             self.assertTrue(
                 item['filename'].endswith('.whl'),
                 "Filename {} must end with .whl".format(item['filename']))
             self.assertEqual(
                 ext_name, item['metadata']['name'],
                 "Extension name mismatch in extensions['{}']. "
                 "Found an extension in the list with name "
                 "{}".format(ext_name, item['metadata']['name']))
             parsed_filename = WHEEL_INFO_RE(item['filename'])
             p = parsed_filename.groupdict()
             self.assertTrue(
                 p.get('name'),
                 "Can't get name for {}".format(item['filename']))
             universal_wheel = p.get('pyver') == 'py2.py3' and p.get(
                 'abi') == 'none' and p.get('plat') == 'any'
             self.assertTrue(
                 universal_wheel,
                 "{} of {} not universal (platform independent) wheel. "
                 "It should end in py2.py3-none-any.whl".format(
                     item['filename'], ext_name))
Ejemplo n.º 3
0
    def get_metadata(self):
        from wheel.install import WHEEL_INFO_RE
        from glob import glob
        if not extension_exists(self.name):
            return None
        metadata = {}
        ext_dir = self.path or get_extension_path(self.name)
        info_dirs = glob(os.path.join(ext_dir, '*.*-info'))
        azext_metadata = WheelExtension.get_azext_metadata(ext_dir)
        if azext_metadata:
            metadata.update(azext_metadata)

        for dist_info_dirname in info_dirs:
            parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname)
            if parsed_dist_info_dir:
                parsed_dist_info_dir = parsed_dist_info_dir.groupdict().get(
                    'name')

            if os.path.split(parsed_dist_info_dir)[-1] == self.name.replace(
                    '-', '_'):
                whl_metadata_filepath = os.path.join(dist_info_dirname,
                                                     WHL_METADATA_FILENAME)
                if os.path.isfile(whl_metadata_filepath):
                    with open(whl_metadata_filepath) as f:
                        metadata.update(json.loads(f.read()))

        return metadata
Ejemplo n.º 4
0
Archivo: axle.py Proyecto: rob-b/belt
def build_wheels(local_pypi, wheel_dir):
    from belt.utils import get_search_names

    built_wheels = []
    for wheel in os.listdir(wheel_dir):
        match = WHEEL_INFO_RE(wheel)
        if match is None:
            continue
        tags = match.groupdict()
        # path_to_wheel = os.path.abspath(os.path.join(wheel_dir, wheel))
        built_wheels.append(tags['namever'].lower())

    for package_dir in os.listdir(local_pypi):
        dir_ = os.path.join(local_pypi, package_dir)
        for pkg in os.listdir(dir_):
            name = os.path.join(dir_, pkg)
            _, ext = os.path.splitext(name)
            if ext in ['.md5', '.whl']:
                continue
            package_name, version = split_package_name(pkg)
            package_name = '{}-{}'.format(package_name, version)

            # FIXME why are names inconsistent forcing us to check both
            # variations?
            options = get_search_names(package_name)
            for opt in options:
                if opt.lower() in built_wheels:
                    break
            else:
                args = 'pip wheel --wheel-dir {} {}'.format(wheel_dir, name)
                subprocess.call(args, shell=True)
Ejemplo n.º 5
0
def _is_not_platform_specific(item):
    parsed_filename = WHEEL_INFO_RE(item['filename'])
    p = parsed_filename.groupdict()
    if p.get('pyver') == 'py2.py3' and p.get('abi') == 'none' and p.get('plat') == 'any':
        return True
    logger.debug("Skipping '%s' as not universal wheel."
                 "We do not currently support platform specific extension detection. "
                 "They can be installed with the full URL %s", item['filename'], item.get('downloadUrl'))
    return False
Ejemplo n.º 6
0
def _is_not_platform_specific(item):
    parsed_filename = WHEEL_INFO_RE(item['filename'])
    p = parsed_filename.groupdict()
    if p.get('pyver') == 'py2.py3' and p.get('abi') == 'none' and p.get('plat') == 'any':
        return True
    logger.debug("Skipping '%s' as not universal wheel."
                 "We do not currently support platform specific extension detection. "
                 "They can be installed with the full URL %s", item['filename'], item.get('downloadUrl'))
    return False
Ejemplo n.º 7
0
Archivo: axle.py Proyecto: rob-b/belt
def split_package_name(name):

    wheel_match = WHEEL_INFO_RE(name)
    if wheel_match:
        pkg_name = wheel_match.group('name')
        version = wheel_match.group('ver')
    else:
        pkg_name = re.split(r'-\d+', name, 1)[0]
        version = name[len(pkg_name) + 1:]
        version = ARCHIVE_SUFFIX.sub('', version)
    return pkg_name, version
Ejemplo n.º 8
0
def _add_whl_ext(source):  # pylint: disable=too-many-statements
    url_parse_result = urlparse(source)
    is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https')
    logger.debug('Extension source is url? %s', is_url)
    whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source)
    parsed_filename = WHEEL_INFO_RE(whl_filename)
    extension_name = parsed_filename.groupdict().get('name') if parsed_filename else None
    if not extension_name:
        raise CLIError('Unable to determine extension name from {}. Is the file name correct?'.format(source))
    if extension_exists(extension_name):
        raise CLIError('The extension {} already exists.'.format(extension_name))
    ext_file = None
    if is_url:
        # Download from URL
        tmp_dir = tempfile.mkdtemp()
        ext_file = os.path.join(tmp_dir, whl_filename)
        logger.debug('Downloading %s to %s', source, ext_file)
        try:
            _whl_download_from_url(url_parse_result, ext_file)
        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
            raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err)))
        logger.debug('Downloaded to %s', ext_file)
    else:
        # Get file path
        ext_file = os.path.realpath(os.path.expanduser(source))
        if not os.path.isfile(ext_file):
            raise CLIError("File {} not found.".format(source))
    # Validate the extension
    logger.debug('Validating the extension {}'.format(ext_file))
    try:
        _validate_whl_extension(ext_file)
    except AssertionError:
        logger.debug(traceback.format_exc())
        raise CLIError('The extension is invalid. Use --debug for more information.')
    except CLIError as e:
        raise e
    logger.debug('Validation successful on {}'.format(ext_file))
    # Install with pip
    extension_path = get_extension_path(extension_name)
    pip_args = ['install', '--target', extension_path, ext_file]
    logger.debug('Executing pip with args: %s', pip_args)
    with HomebrewPipPatch():
        pip_status_code = _run_pip(pip_args)
    if pip_status_code > 0:
        logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path)
        shutil.rmtree(extension_path, ignore_errors=True)
        raise CLIError('An error occurred. Pip failed with status code {}. '
                       'Use --debug for more information.'.format(pip_status_code))
    # Save the whl we used to install the extension in the extension dir.
    dst = os.path.join(extension_path, whl_filename)
    shutil.copyfile(ext_file, dst)
    logger.debug('Saved the whl to %s', dst)
Ejemplo n.º 9
0
def _add_whl_ext(source):  # pylint: disable=too-many-statements
    import pip
    url_parse_result = urlparse(source)
    is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https')
    logger.debug('Extension source is url? %s', is_url)
    whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source)
    parsed_filename = WHEEL_INFO_RE(whl_filename)
    extension_name = parsed_filename.groupdict().get('name') if parsed_filename else None
    if not extension_name:
        raise CLIError('Unable to determine extension name from {}. Is the file name correct?'.format(source))
    if extension_exists(extension_name):
        raise CLIError('The extension {} already exists.'.format(extension_name))
    ext_file = None
    if is_url:
        # Download from URL
        tmp_dir = tempfile.mkdtemp()
        ext_file = os.path.join(tmp_dir, whl_filename)
        logger.debug('Downloading %s to %s', source, ext_file)
        try:
            _whl_download_from_url(url_parse_result, ext_file)
        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
            raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err)))
        logger.debug('Downloaded to %s', ext_file)
    else:
        # Get file path
        ext_file = os.path.realpath(os.path.expanduser(source))
        if not os.path.isfile(ext_file):
            raise CLIError("File {} not found.".format(source))
    # Validate the extension
    logger.debug('Validating the extension {}'.format(ext_file))
    try:
        _validate_whl_extension(ext_file)
    except AssertionError:
        logger.debug(traceback.format_exc())
        raise CLIError('The extension is invalid. Use --debug for more information.')
    except CLIError as e:
        raise e
    logger.debug('Validation successful on {}'.format(ext_file))
    # Install with pip
    extension_path = get_extension_path(extension_name)
    pip_args = ['install', '--target', extension_path, ext_file]
    logger.debug('Executing pip with args: %s', pip_args)
    pip_status_code = _run_pip(pip, pip_args)
    if pip_status_code > 0:
        logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path)
        shutil.rmtree(extension_path, ignore_errors=True)
        raise CLIError('An error occurred. Pip failed with status code {}. '
                       'Use --debug for more information.'.format(pip_status_code))
    # Save the whl we used to install the extension in the extension dir.
    dst = os.path.join(extension_path, whl_filename)
    shutil.copyfile(ext_file, dst)
    logger.debug('Saved the whl to %s', dst)
Ejemplo n.º 10
0
def add_platforms(wheel_ctx, platforms):
    """Add platform tags `platforms` to a wheel

    Add any platform tags in `platforms` that are missing
    to wheel_ctx's filename and ``WHEEL`` file.

    Parameters
    ----------
    wheel_ctx : InWheelCtx
        An open wheel context
    platforms : iterable
        platform tags to add to wheel filename and WHEEL tags - e.g.
        ``('macosx_10_9_intel', 'macosx_10_9_x86_64')
    """
    info_fname = pjoin(_dist_info_dir(wheel_ctx.path), 'WHEEL')
    info = read_pkg_info(info_fname)
    if info['Root-Is-Purelib'] == 'true':
        raise WheelToolsError('Cannot add platforms to pure wheel')

    # Check what tags we have
    if wheel_ctx.out_wheel is None:
        in_wheel = wheel_ctx.in_wheel
    else:
        raise NotImplementedError()

    parsed_fname = WHEEL_INFO_RE(basename(in_wheel))
    in_fname_tags = parsed_fname.groupdict()['plat'].split('.')
    extra_fname_tags = [tag for tag in platforms if tag not in in_fname_tags]
    in_wheel_base, ext = splitext(basename(in_wheel))
    out_wheel_base = '.'.join([in_wheel_base] + list(extra_fname_tags))
    out_wheel = out_wheel_base + ext

    in_info_tags = [tag for name, tag in info.items() if name == 'Tag']
    # Python version, C-API version combinations
    pyc_apis = ['-'.join(tag.split('-')[:2]) for tag in in_info_tags]
    # unique Python version, C-API version combinations
    pyc_apis = unique_by_index(pyc_apis)
    # Add new platform tags for each Python version, C-API combination
    required_tags = ['-'.join(tup) for tup in product(pyc_apis, platforms)]
    needs_write = False
    for req_tag in required_tags:
        if req_tag in in_info_tags:
            continue
        needs_write = True
        info.add_header('Tag', req_tag)
    if needs_write:
        write_pkg_info(info_fname, info)
        # Tell context manager to write wheel on exit by setting filename
        wheel_ctx.out_wheel = out_wheel
    return wheel_ctx.out_wheel
Ejemplo n.º 11
0
def add_platforms(wheel_ctx, platforms):
    """Add platform tags `platforms` to a wheel

    Add any platform tags in `platforms` that are missing
    to wheel_ctx's filename and ``WHEEL`` file.

    Parameters
    ----------
    wheel_ctx : InWheelCtx
        An open wheel context
    platforms : iterable
        platform tags to add to wheel filename and WHEEL tags - e.g.
        ``('macosx_10_9_intel', 'macosx_10_9_x86_64')
    """
    info_fname = pjoin(_dist_info_dir(wheel_ctx.path), 'WHEEL')
    info = read_pkg_info(info_fname)
    if info['Root-Is-Purelib'] == 'true':
        raise WheelToolsError('Cannot add platforms to pure wheel')

    # Check what tags we have
    if wheel_ctx.out_wheel is None:
        in_wheel = wheel_ctx.in_wheel
    else:
        raise NotImplementedError()

    parsed_fname = WHEEL_INFO_RE(basename(in_wheel))
    in_fname_tags = parsed_fname.groupdict()['plat'].split('.')
    extra_fname_tags = [tag for tag in platforms if tag not in in_fname_tags]
    in_wheel_base, ext = splitext(basename(in_wheel))
    out_wheel_base = '.'.join([in_wheel_base] + list(extra_fname_tags))
    out_wheel = out_wheel_base + ext

    in_info_tags = [tag for name, tag in info.items() if name == 'Tag']
    # Python version, C-API version combinations
    pyc_apis = ['-'.join(tag.split('-')[:2]) for tag in in_info_tags]
    # unique Python version, C-API version combinations
    pyc_apis = unique_by_index(pyc_apis)
    # Add new platform tags for each Python version, C-API combination
    required_tags = ['-'.join(tup) for tup in product(pyc_apis, platforms)]
    needs_write = False
    for req_tag in required_tags:
        if req_tag in in_info_tags:
            continue
        needs_write = True
        info.add_header('Tag', req_tag)
    if needs_write:
        write_pkg_info(info_fname, info)
        # Tell context manager to write wheel on exit by setting filename
        wheel_ctx.out_wheel = out_wheel
    return wheel_ctx.out_wheel
Ejemplo n.º 12
0
Archivo: axle.py Proyecto: rob-b/belt
def copy_wheels_to_pypi(wheel_dir, local_pypi):
    for wheel in os.listdir(wheel_dir):
        match = WHEEL_INFO_RE(wheel)
        if match is None:
            continue
        tags = match.groupdict()
        path_to_wheel = os.path.abspath(os.path.join(wheel_dir, wheel))
        wheel_destination = WheelDestination(local_pypi, tags['name'], tags['ver'])
        local_wheel = os.path.join(wheel_destination.path, os.path.basename(path_to_wheel))

        if os.path.exists(local_wheel):
            continue

        mkdir_p(wheel_destination.path)
        shutil.copyfile(path_to_wheel, local_wheel)
Ejemplo n.º 13
0
 def test_source_wheels(self):
     # Test we can build all sources into wheels and that metadata from the wheel is valid
     built_whl_dir = tempfile.mkdtemp()
     source_extensions = [
         os.path.join(SRC_PATH, n) for n in os.listdir(SRC_PATH)
         if os.path.isdir(os.path.join(SRC_PATH, n))
     ]
     for s in source_extensions:
         if not os.path.isfile(os.path.join(s, 'setup.py')):
             continue
         try:
             check_call([
                 'python', 'setup.py', 'bdist_wheel', '-q', '-d',
                 built_whl_dir
             ],
                        cwd=s,
                        stdout=PIPE,
                        stderr=PIPE)
         except CalledProcessError as err:
             self.fail("Unable to build extension {} : {}".format(s, err))
     for filename in os.listdir(built_whl_dir):
         ext_file = os.path.join(built_whl_dir, filename)
         ext_dir = tempfile.mkdtemp(dir=built_whl_dir)
         ext_name = WHEEL_INFO_RE(filename).groupdict().get('name')
         metadata = get_ext_metadata(ext_dir, ext_file, ext_name)
         run_requires = metadata.get('run_requires')
         if run_requires and ext_name not in SKIP_DEP_CHECK:
             deps = run_requires[0]['requires']
             self.assertTrue(
                 all(not dep.startswith('azure-') for dep in deps),
                 "Dependencies of {} use disallowed extension dependencies. "
                 "Remove these dependencies: {}".format(filename, deps))
     shutil.rmtree(built_whl_dir)
Ejemplo n.º 14
0
 def get_metadata(self):
     from wheel.install import WHEEL_INFO_RE
     if not extension_exists(self.name):
         return None
     metadata = {}
     ext_dir = get_extension_path(self.name)
     dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')]
     azext_metadata = WheelExtension.get_azext_metadata(ext_dir)
     if azext_metadata:
         metadata.update(azext_metadata)
     for dist_info_dirname in dist_info_dirs:
         parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname)
         if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == self.name.replace('-', '_'):
             whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME)
             if os.path.isfile(whl_metadata_filepath):
                 with open(whl_metadata_filepath) as f:
                     metadata.update(json.load(f))
     return metadata
Ejemplo n.º 15
0
 def test_extension_filenames(self):
     for ext_name, exts in self.index['extensions'].items():
         self.assertEqual(ext_name.find('_'), -1, "Extension names should not contain underscores. "
                                                  "Found {}".format(ext_name))
         for item in exts:
             self.assertTrue(item['filename'].endswith('.whl'),
                             "Filename {} must end with .whl".format(item['filename']))
             self.assertEqual(ext_name, item['metadata']['name'],
                              "Extension name mismatch in extensions['{}']. "
                              "Found an extension in the list with name "
                              "{}".format(ext_name, item['metadata']['name']))
             parsed_filename = WHEEL_INFO_RE(item['filename'])
             p = parsed_filename.groupdict()
             self.assertTrue(p.get('name'), "Can't get name for {}".format(item['filename']))
             universal_wheel = p.get('pyver') == 'py2.py3' and p.get('abi') == 'none' and p.get('plat') == 'any'
             self.assertTrue(universal_wheel,
                             "{} of {} not universal (platform independent) wheel. "
                             "It should end in py2.py3-none-any.whl".format(item['filename'], ext_name))
Ejemplo n.º 16
0
def get_ext_metadata(ext_dir, ext_file, ext_name):
    # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L89
    WHL_METADATA_FILENAME = 'metadata.json'
    zip_ref = zipfile.ZipFile(ext_file, 'r')
    zip_ref.extractall(ext_dir)
    zip_ref.close()
    metadata = {}
    dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')]
    azext_metadata = _get_azext_metadata(ext_dir)
    if azext_metadata:
        metadata.update(azext_metadata)
    for dist_info_dirname in dist_info_dirs:
        parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname)
        if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'):
            whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME)
            if os.path.isfile(whl_metadata_filepath):
                with open(whl_metadata_filepath) as f:
                    metadata.update(json.load(f))
    return metadata
Ejemplo n.º 17
0
def execute(args, p):
    import os
    import sys
    from wheel.install import WHEEL_INFO_RE  # type: ignore
    from .wheeltools import InWheelCtx, add_platforms, WheelToolsError
    from .wheel_abi import analyze_wheel_abi

    wheel_abi = analyze_wheel_abi(args.WHEEL_FILE)

    parsed_fname = WHEEL_INFO_RE(basename(args.WHEEL_FILE))
    in_fname_tags = parsed_fname.groupdict()['plat'].split('.')

    print('%s recieves the following tag: "%s".' %
          (basename(args.WHEEL_FILE), wheel_abi.overall_tag))
    print('Use ``auditwheel show`` for more details')

    if wheel_abi.overall_tag in in_fname_tags:
        print('No tags to be added. Exiting.')
        return 1

    # todo: move more of this logic to separate file
    if not exists(args.WHEEL_DIR):
        os.makedirs(args.WHEEL_DIR)

    with InWheelCtx(args.WHEEL_FILE) as ctx:
        try:
            out_wheel = add_platforms(ctx, [wheel_abi.overall_tag])
        except WheelToolsError as e:
            print('\n%s.' % str(e), file=sys.stderr)
            return 1

        if out_wheel:
            # tell context manager to write wheel on exit with
            # the proper output directory
            ctx.out_wheel = join(args.WHEEL_DIR, basename(out_wheel))

    return 0
Ejemplo n.º 18
0
def execute(args, p):
    import os
    import sys
    from wheel.install import WHEEL_INFO_RE  # type: ignore
    from .wheeltools import InWheelCtx, add_platforms, WheelToolsError
    from .wheel_abi import analyze_wheel_abi

    wheel_abi = analyze_wheel_abi(args.WHEEL_FILE)

    parsed_fname = WHEEL_INFO_RE(basename(args.WHEEL_FILE))
    in_fname_tags = parsed_fname.groupdict()['plat'].split('.')

    print('%s recieves the following tag: "%s".' % (basename(args.WHEEL_FILE),
                                                    wheel_abi.overall_tag))
    print('Use ``auditwheel show`` for more details')

    if wheel_abi.overall_tag in in_fname_tags:
        print('No tags to be added. Exiting.')
        return 1

    # todo: move more of this logic to separate file
    if not exists(args.WHEEL_DIR):
        os.makedirs(args.WHEEL_DIR)

    with InWheelCtx(args.WHEEL_FILE) as ctx:
        try:
            out_wheel = add_platforms(ctx, [wheel_abi.overall_tag])
        except WheelToolsError as e:
            print('\n%s.' % str(e), file=sys.stderr)
            return 1

        if out_wheel:
            # tell context manager to write wheel on exit with
            # the proper output directory
            ctx.out_wheel = join(args.WHEEL_DIR, basename(out_wheel))

    return 0
Ejemplo n.º 19
0
    def get_metadata(self):
        from wheel.install import WHEEL_INFO_RE
        from glob import glob
        if not extension_exists(self.name):
            return None
        metadata = {}
        ext_dir = self.path or get_extension_path(self.name)
        info_dirs = glob(os.path.join(ext_dir, '*.*-info'))
        azext_metadata = WheelExtension.get_azext_metadata(ext_dir)
        if azext_metadata:
            metadata.update(azext_metadata)

        for dist_info_dirname in info_dirs:
            parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname)
            if parsed_dist_info_dir:
                parsed_dist_info_dir = parsed_dist_info_dir.groupdict().get('name')

            if os.path.split(parsed_dist_info_dir)[-1] == self.name.replace('-', '_'):
                whl_metadata_filepath = os.path.join(dist_info_dirname, WHL_METADATA_FILENAME)
                if os.path.isfile(whl_metadata_filepath):
                    with open(whl_metadata_filepath) as f:
                        metadata.update(json.loads(f.read()))

        return metadata
Ejemplo n.º 20
0
def _add_whl_ext(source, ext_sha256=None):  # pylint: disable=too-many-statements
    if not source.endswith('.whl'):
        raise ValueError(
            'Unknown extension type. Only Python wheels are supported.')
    url_parse_result = urlparse(source)
    is_url = (url_parse_result.scheme == 'http'
              or url_parse_result.scheme == 'https')
    logger.debug('Extension source is url? %s', is_url)
    whl_filename = os.path.basename(
        url_parse_result.path) if is_url else os.path.basename(source)
    parsed_filename = WHEEL_INFO_RE(whl_filename)
    # Extension names can have - but .whl format changes it to _ (PEP 0427). Undo this.
    extension_name = parsed_filename.groupdict().get('name').replace(
        '_', '-') if parsed_filename else None
    if not extension_name:
        raise CLIError(
            'Unable to determine extension name from {}. Is the file name correct?'
            .format(source))
    if extension_exists(extension_name):
        raise CLIError(
            'The extension {} already exists.'.format(extension_name))
    ext_file = None
    if is_url:
        # Download from URL
        tmp_dir = tempfile.mkdtemp()
        ext_file = os.path.join(tmp_dir, whl_filename)
        logger.debug('Downloading %s to %s', source, ext_file)
        try:
            _whl_download_from_url(url_parse_result, ext_file)
        except (requests.exceptions.ConnectionError,
                requests.exceptions.HTTPError) as err:
            raise CLIError(
                'Please ensure you have network connection. Error detail: {}'.
                format(str(err)))
        logger.debug('Downloaded to %s', ext_file)
    else:
        # Get file path
        ext_file = os.path.realpath(os.path.expanduser(source))
        if not os.path.isfile(ext_file):
            raise CLIError("File {} not found.".format(source))
    # Validate the extension
    logger.debug('Validating the extension %s', ext_file)
    if ext_sha256:
        valid_checksum, computed_checksum = is_valid_sha256sum(
            ext_file, ext_sha256)
        if valid_checksum:
            logger.debug("Checksum of %s is OK", ext_file)
        else:
            logger.debug(
                "Invalid checksum for %s. Expected '%s', computed '%s'.",
                ext_file, ext_sha256, computed_checksum)
            raise CLIError(
                "The checksum of the extension does not match the expected value. "
                "Use --debug for more information.")
    try:
        _validate_whl_extension(ext_file)
    except AssertionError:
        logger.debug(traceback.format_exc())
        raise CLIError(
            'The extension is invalid. Use --debug for more information.')
    except CLIError as e:
        raise e
    logger.debug('Validation successful on %s', ext_file)
    # Install with pip
    extension_path = get_extension_path(extension_name)
    pip_args = ['install', '--target', extension_path, ext_file]
    logger.debug('Executing pip with args: %s', pip_args)
    with HomebrewPipPatch():
        pip_status_code = _run_pip(pip_args)
    if pip_status_code > 0:
        logger.debug(
            'Pip failed so deleting anything we might have installed at %s',
            extension_path)
        shutil.rmtree(extension_path, ignore_errors=True)
        raise CLIError(
            'An error occurred. Pip failed with status code {}. '
            'Use --debug for more information.'.format(pip_status_code))
    # Save the whl we used to install the extension in the extension dir.
    dst = os.path.join(extension_path, whl_filename)
    shutil.copyfile(ext_file, dst)
    logger.debug('Saved the whl to %s', dst)
Ejemplo n.º 21
0
def add_platforms(wheel_ctx, platforms, remove_platforms=()):
    """Add platform tags `platforms` to a wheel

    Add any platform tags in `platforms` that are missing
    to wheel_ctx's filename and ``WHEEL`` file.

    Parameters
    ----------
    wheel_ctx : InWheelCtx
        An open wheel context
    platforms : iterable
        platform tags to add to wheel filename and WHEEL tags - e.g.
        ``('macosx_10_9_intel', 'macosx_10_9_x86_64')
    remove_platforms : iterable
        platform tags to remove to the wheel filename and WHEEL tags, e.g.
        ``('linux_x86_64',)`` when ``('manylinux_x86_64')`` is added
    """
    definitely_not_purelib = False

    info_fname = pjoin(_dist_info_dir(wheel_ctx.path), 'WHEEL')
    info = read_pkg_info(info_fname)
    # Check what tags we have
    if wheel_ctx.out_wheel is not None:
        out_dir = dirname(wheel_ctx.out_wheel)
        wheel_fname = basename(wheel_ctx.out_wheel)
    else:
        out_dir = '.'
        wheel_fname = basename(wheel_ctx.in_wheel)

    parsed_fname = WHEEL_INFO_RE(wheel_fname)
    fparts = parsed_fname.groupdict()
    original_fname_tags = fparts['plat'].split('.')
    logger.info('Previous filename tags: %s', ', '.join(original_fname_tags))
    fname_tags = {
        tag
        for tag in original_fname_tags if tag not in remove_platforms
    }
    fname_tags |= set(platforms)

    # Can't be 'any' and another platform
    if 'any' in fname_tags and len(fname_tags) > 1:
        fname_tags.remove('any')
        remove_platforms.append('any')
        definitely_not_purelib = True

    if fname_tags != original_fname_tags:
        logger.info('New filename tags: %s', ', '.join(fname_tags))
    else:
        logger.info('No filename tags change needed.')

    _, ext = splitext(wheel_fname)
    fparts['plat'] = '.'.join(fname_tags)
    fparts['ext'] = ext
    out_wheel_fname = "{namever}-{pyver}-{abi}-{plat}{ext}".format(**fparts)
    out_wheel = pjoin(out_dir, out_wheel_fname)

    in_info_tags = [tag for name, tag in info.items() if name == 'Tag']
    logger.info('Previous WHEEL info tags: %s', ', '.join(in_info_tags))
    # Python version, C-API version combinations
    pyc_apis = ['-'.join(tag.split('-')[:2]) for tag in in_info_tags]
    # unique Python version, C-API version combinations
    pyc_apis = unique_by_index(pyc_apis)
    # Add new platform tags for each Python version, C-API combination
    wanted_tags = ['-'.join(tup) for tup in product(pyc_apis, platforms)]
    new_tags = [tag for tag in wanted_tags if tag not in in_info_tags]
    unwanted_tags = [
        '-'.join(tup) for tup in product(pyc_apis, remove_platforms)
    ]
    updated_tags = [tag for tag in in_info_tags if tag not in unwanted_tags]
    updated_tags += new_tags
    if updated_tags != in_info_tags:
        del info['Tag']
        for tag in updated_tags:
            info.add_header('Tag', tag)

        if definitely_not_purelib:
            info['Root-Is-Purelib'] = 'False'
            logger.info('Changed wheel type to Platlib')

        logger.info('New WHEEL info tags: %s', ', '.join(info.get_all('Tag')))
        write_pkg_info(info_fname, info)
    else:
        logger.info('No WHEEL info change needed.')
    return out_wheel
Ejemplo n.º 22
0
def _add_whl_ext(source, ext_sha256=None, pip_extra_index_urls=None, pip_proxy=None):  # pylint: disable=too-many-statements
    if not source.endswith('.whl'):
        raise ValueError('Unknown extension type. Only Python wheels are supported.')
    url_parse_result = urlparse(source)
    is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https')
    logger.debug('Extension source is url? %s', is_url)
    whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source)
    parsed_filename = WHEEL_INFO_RE(whl_filename)
    # Extension names can have - but .whl format changes it to _ (PEP 0427). Undo this.
    extension_name = parsed_filename.groupdict().get('name').replace('_', '-') if parsed_filename else None
    if not extension_name:
        raise CLIError('Unable to determine extension name from {}. Is the file name correct?'.format(source))
    if extension_exists(extension_name):
        raise CLIError('The extension {} already exists.'.format(extension_name))
    ext_file = None
    if is_url:
        # Download from URL
        tmp_dir = tempfile.mkdtemp()
        ext_file = os.path.join(tmp_dir, whl_filename)
        logger.debug('Downloading %s to %s', source, ext_file)
        try:
            _whl_download_from_url(url_parse_result, ext_file)
        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
            raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err)))
        logger.debug('Downloaded to %s', ext_file)
    else:
        # Get file path
        ext_file = os.path.realpath(os.path.expanduser(source))
        if not os.path.isfile(ext_file):
            raise CLIError("File {} not found.".format(source))
    # Validate the extension
    logger.debug('Validating the extension %s', ext_file)
    if ext_sha256:
        valid_checksum, computed_checksum = is_valid_sha256sum(ext_file, ext_sha256)
        if valid_checksum:
            logger.debug("Checksum of %s is OK", ext_file)
        else:
            logger.debug("Invalid checksum for %s. Expected '%s', computed '%s'.",
                         ext_file, ext_sha256, computed_checksum)
            raise CLIError("The checksum of the extension does not match the expected value. "
                           "Use --debug for more information.")
    try:
        _validate_whl_extension(ext_file)
    except AssertionError:
        logger.debug(traceback.format_exc())
        raise CLIError('The extension is invalid. Use --debug for more information.')
    except CLIError as e:
        raise e
    logger.debug('Validation successful on %s', ext_file)
    # Check for distro consistency
    check_distro_consistency()
    # Install with pip
    extension_path = get_extension_path(extension_name)
    pip_args = ['install', '--target', extension_path, ext_file]

    if pip_proxy:
        pip_args = pip_args + ['--proxy', pip_proxy]
    if pip_extra_index_urls:
        for extra_index_url in pip_extra_index_urls:
            pip_args = pip_args + ['--extra-index-url', extra_index_url]

    logger.debug('Executing pip with args: %s', pip_args)
    with HomebrewPipPatch():
        pip_status_code = _run_pip(pip_args)
    if pip_status_code > 0:
        logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path)
        shutil.rmtree(extension_path, ignore_errors=True)
        raise CLIError('An error occurred. Pip failed with status code {}. '
                       'Use --debug for more information.'.format(pip_status_code))
    # Save the whl we used to install the extension in the extension dir.
    dst = os.path.join(extension_path, whl_filename)
    shutil.copyfile(ext_file, dst)
    logger.debug('Saved the whl to %s', dst)
Ejemplo n.º 23
0
def add_platforms(wheel_ctx, platforms, remove_platforms=()):
    """Add platform tags `platforms` to a wheel

    Add any platform tags in `platforms` that are missing
    to wheel_ctx's filename and ``WHEEL`` file.

    Parameters
    ----------
    wheel_ctx : InWheelCtx
        An open wheel context
    platforms : iterable
        platform tags to add to wheel filename and WHEEL tags - e.g.
        ``('macosx_10_9_intel', 'macosx_10_9_x86_64')
    remove_platforms : iterable
        platform tags to remove to the wheel filename and WHEEL tags, e.g.
        ``('linux_x86_64',)`` when ``('manylinux_x86_64')`` is added
    """
    info_fname = pjoin(_dist_info_dir(wheel_ctx.path), 'WHEEL')
    info = read_pkg_info(info_fname)
    if info['Root-Is-Purelib'] == 'true':
        print('No need to add platforms to pure wheel - Skipping {}'.format(wheel_ctx.in_wheel))
        return

    # Check what tags we have
    if wheel_ctx.out_wheel is not None:
        out_dir = dirname(wheel_ctx.out_wheel)
        wheel_fname = basename(wheel_ctx.out_wheel)
    else:
        out_dir = '.'
        wheel_fname = basename(wheel_ctx.in_wheel)

    parsed_fname = WHEEL_INFO_RE(wheel_fname)
    fparts = parsed_fname.groupdict()
    original_fname_tags = fparts['plat'].split('.')
    print('Previous filename tags:', ', '.join(original_fname_tags))
    fname_tags = [tag for tag in original_fname_tags
                  if tag not in remove_platforms]
    for platform in platforms:
        if platform not in fname_tags:
            fname_tags.append(platform)
    if fname_tags != original_fname_tags:
        print('New filename tags:', ', '.join(fname_tags))
    else:
        print('No filename tags change needed.')

    wheel_base, ext = splitext(wheel_fname)
    fparts['plat'] = '.'.join(fname_tags)
    fparts['ext'] = ext
    out_wheel_fname = "{namever}-{pyver}-{abi}-{plat}{ext}".format(**fparts)
    out_wheel = pjoin(out_dir, out_wheel_fname)

    in_info_tags = [tag for name, tag in info.items() if name == 'Tag']
    print('Previous WHEEL info tags:', ', '.join(in_info_tags))
    # Python version, C-API version combinations
    pyc_apis = ['-'.join(tag.split('-')[:2]) for tag in in_info_tags]
    # unique Python version, C-API version combinations
    pyc_apis = unique_by_index(pyc_apis)
    # Add new platform tags for each Python version, C-API combination
    wanted_tags = ['-'.join(tup) for tup in product(pyc_apis, platforms)]
    new_tags = [tag for tag in wanted_tags if tag not in in_info_tags]
    unwanted_tags = ['-'.join(tup)
                     for tup in product(pyc_apis, remove_platforms)]
    updated_tags = [tag for tag in in_info_tags if tag not in unwanted_tags]
    updated_tags += new_tags
    needs_write = updated_tags != in_info_tags
    if needs_write:
        del info['Tag']
        for tag in updated_tags:
            info.add_header('Tag', tag)
        print('New WHEEL info tags:', ', '.join(info.get_all('Tag')))
        write_pkg_info(info_fname, info)
    else:
        print('No WHEEL info change needed.')
    return out_wheel