Пример #1
0
def _is_url_like_archive(url):
    # type: (str) -> bool
    """Return whether the URL looks like an archive.
    """
    filename = Link(url).filename
    for bad_ext in ARCHIVE_EXTENSIONS:
        if filename.endswith(bad_ext):
            return True
    return False
Пример #2
0
    def collect_links(self, project_name):
        # type: (str) -> CollectedLinks
        """Find all available links for the given project name.

        :return: All the Link objects (unfiltered), as a CollectedLinks object.
        """
        search_scope = self.search_scope
        index_locations = search_scope.get_index_urls_locations(project_name)
        index_file_loc, index_url_loc = group_locations(index_locations)
        fl_file_loc, fl_url_loc = group_locations(
            self.find_links,
            expand_dir=True,
        )

        file_links = [
            Link(url) for url in itertools.chain(index_file_loc, fl_file_loc)
        ]

        # We trust every directly linked archive in find_links
        find_link_links = [Link(url, '-f') for url in self.find_links]

        # We trust every url that the user has given us whether it was given
        # via --index-url or --find-links.
        # We want to filter out anything that does not have a secure origin.
        url_locations = [
            link for link in itertools.chain(
                # Mark PyPI indices as "cache_link_parsing == False" -- this
                # will avoid caching the result of parsing the page for links.
                (Link(url, cache_link_parsing=False) for url in index_url_loc),
                (Link(url) for url in fl_url_loc),
            ) if self.session.is_secure_origin(link)
        ]

        url_locations = _remove_duplicate_links(url_locations)
        lines = [
            '{} location(s) to search for versions of {}:'.format(
                len(url_locations),
                project_name,
            ),
        ]
        for link in url_locations:
            lines.append('* {}'.format(link))
        logger.debug('\n'.join(lines))

        return CollectedLinks(
            files=file_links,
            find_links=find_link_links,
            project_urls=url_locations,
        )
Пример #3
0
def _create_link_from_element(
        anchor,  # type: HTMLElement
        page_url,  # type: str
        base_url,  # type: str
):
    # type: (...) -> Optional[Link]
    """
    Convert an anchor element in a simple repository page to a Link.
    """
    href = anchor.get("href")
    if not href:
        return None

    url = _clean_link(urllib_parse.urljoin(base_url, href))
    pyrequire = anchor.get('data-requires-python')
    pyrequire = unescape(pyrequire) if pyrequire else None

    yanked_reason = anchor.get('data-yanked')
    if yanked_reason:
        # This is a unicode string in Python 2 (and 3).
        yanked_reason = unescape(yanked_reason)

    link = Link(
        url,
        comes_from=page_url,
        requires_python=pyrequire,
        yanked_reason=yanked_reason,
    )

    return link
Пример #4
0
def build(
    requirements,  # type: Iterable[InstallRequirement]
    wheel_cache,  # type: WheelCache
    build_options,  # type: List[str]
    global_options,  # type: List[str]
):
    # type: (...) -> BuildResult
    """Build wheels.

    :return: The list of InstallRequirement that succeeded to build and
        the list of InstallRequirement that failed to build.
    """
    if not requirements:
        return [], []

    # Build the wheels.
    logger.info(
        'Building wheels for collected packages: %s',
        ', '.join(req.name for req in requirements),
    )

    with indent_log():
        build_successes, build_failures = [], []
        for req in requirements:
            cache_dir = _get_cache_dir(req, wheel_cache)
            wheel_file = _build_one(
                req, cache_dir, build_options, global_options
            )
            if wheel_file:
                # Update the link for this.
                req.link = Link(path_to_url(wheel_file))
                req.local_file_path = req.link.file_path
                assert req.link.is_wheel
                build_successes.append(req)
            else:
                build_failures.append(req)

    # notify success/failure
    if build_successes:
        logger.info(
            'Successfully built %s',
            ' '.join([req.name for req in build_successes]),
        )
    if build_failures:
        logger.info(
            'Failed to build %s',
            ' '.join([req.name for req in build_failures]),
        )
    # Return a list of requirements that failed to build
    return build_successes, build_failures
Пример #5
0
def parse_req_from_editable(editable_req):
    # type: (str) -> RequirementParts
    name, url, extras_override = parse_editable(editable_req)

    if name is not None:
        try:
            req = Requirement(name)
        except InvalidRequirement:
            raise InstallationError("Invalid requirement: '{}'".format(name))
    else:
        req = None

    link = Link(url)

    return RequirementParts(req, link, None, extras_override)
Пример #6
0
    def get(
            self,
            link,  # type: Link
            package_name,  # type: Optional[str]
            supported_tags,  # type: List[Tag]
    ):
        # type: (...) -> Link
        candidates = []

        if not package_name:
            return link

        canonical_package_name = canonicalize_name(package_name)
        for wheel_name, wheel_dir in self._get_candidates(
                link, canonical_package_name):
            try:
                wheel = Wheel(wheel_name)
            except InvalidWheelFilename:
                continue
            if canonicalize_name(wheel.name) != canonical_package_name:
                logger.debug(
                    "Ignoring cached wheel {} for {} as it "
                    "does not match the expected distribution name {}.".format(
                        wheel_name, link, package_name))
                continue
            if not wheel.supported(supported_tags):
                # Built for a different python/arch/etc
                continue
            candidates.append((
                wheel.support_index_min(supported_tags),
                wheel_name,
                wheel_dir,
            ))

        if not candidates:
            return link

        _, wheel_name, wheel_dir = min(candidates)
        return Link(path_to_url(os.path.join(wheel_dir, wheel_name)))
Пример #7
0
    def __init__(
        self,
        req,  # type: Optional[Requirement]
        comes_from,  # type: Optional[Union[str, InstallRequirement]]
        editable=False,  # type: bool
        link=None,  # type: Optional[Link]
        markers=None,  # type: Optional[Marker]
        use_pep517=None,  # type: Optional[bool]
        isolated=False,  # type: bool
        install_options=None,  # type: Optional[List[str]]
        global_options=None,  # type: Optional[List[str]]
        hash_options=None,  # type: Optional[Dict[str, List[str]]]
        constraint=False,  # type: bool
        extras=()  # type: Iterable[str]
    ):
        # type: (...) -> None
        assert req is None or isinstance(req, Requirement), req
        self.req = req
        self.comes_from = comes_from
        self.constraint = constraint
        self.editable = editable

        # source_dir is the local directory where the linked requirement is
        # located, or unpacked. In case unpacking is needed, creating and
        # populating source_dir is done by the RequirementPreparer. Note this
        # is not necessarily the directory where pyproject.toml or setup.py is
        # located - that one is obtained via unpacked_source_directory.
        self.source_dir = None  # type: Optional[str]
        if self.editable:
            assert link
            if link.is_file:
                self.source_dir = os.path.normpath(
                    os.path.abspath(link.file_path)
                )

        if link is None and req and req.url:
            # PEP 508 URL requirement
            link = Link(req.url)
        self.link = self.original_link = link
        self.original_link_is_in_wheel_cache = False

        # Path to any downloaded or already-existing package.
        self.local_file_path = None  # type: Optional[str]
        if self.link and self.link.is_file:
            self.local_file_path = self.link.file_path

        if extras:
            self.extras = extras
        elif req:
            self.extras = {
                pkg_resources.safe_extra(extra) for extra in req.extras
            }
        else:
            self.extras = set()
        if markers is None and req:
            markers = req.marker
        self.markers = markers

        # This holds the pkg_resources.Distribution object if this requirement
        # is already available:
        self.satisfied_by = None  # type: Optional[Distribution]
        # Whether the installation process should try to uninstall an existing
        # distribution before installing this requirement.
        self.should_reinstall = False
        # Temporary build location
        self._temp_build_dir = None  # type: Optional[TempDirectory]
        # Set to True after successful installation
        self.install_succeeded = None  # type: Optional[bool]
        # Supplied options
        self.install_options = install_options if install_options else []
        self.global_options = global_options if global_options else []
        self.hash_options = hash_options if hash_options else {}
        # Set to True after successful preparation of this requirement
        self.prepared = False
        self.is_direct = False

        # Set by the legacy resolver when the requirement has been downloaded
        # TODO: This introduces a strong coupling between the resolver and the
        #       requirement (the coupling was previously between the resolver
        #       and the requirement set). This should be refactored to allow
        #       the requirement to decide for itself when it has been
        #       successfully downloaded - but that is more tricky to get right,
        #       se we are making the change in stages.
        self.successfully_downloaded = False

        self.isolated = isolated
        self.build_env = NoOpBuildEnvironment()  # type: BuildEnvironment

        # For PEP 517, the directory where we request the project metadata
        # gets stored. We need this to pass to build_wheel, so the backend
        # can ensure that the wheel matches the metadata (see the PEP for
        # details).
        self.metadata_directory = None  # type: Optional[str]

        # The static build requirements (from pyproject.toml)
        self.pyproject_requires = None  # type: Optional[List[str]]

        # Build requirements that we will check are available
        self.requirements_to_check = []  # type: List[str]

        # The PEP 517 backend we should use to build the project
        self.pep517_backend = None  # type: Optional[Pep517HookCaller]

        # Are we using PEP 517 for this requirement?
        # After pyproject.toml has been loaded, the only valid values are True
        # and False. Before loading, None is valid (meaning "use the default").
        # Setting an explicit value before loading pyproject.toml is supported,
        # but after loading this flag should be treated as read only.
        self.use_pep517 = use_pep517
Пример #8
0
def parse_editable(editable_req):
    # type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]]
    """Parses an editable requirement into:
        - a requirement name
        - an URL
        - extras
        - editable options
    Accepted requirements:
        svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir
        .[some_extra]
    """

    url = editable_req

    # If a file path is specified with extras, strip off the extras.
    url_no_extras, extras = _strip_extras(url)

    if os.path.isdir(url_no_extras):
        if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):
            msg = (
                'File "setup.py" not found. Directory cannot be installed '
                'in editable mode: {}'.format(os.path.abspath(url_no_extras))
            )
            pyproject_path = make_pyproject_path(url_no_extras)
            if os.path.isfile(pyproject_path):
                msg += (
                    '\n(A "pyproject.toml" file was found, but editable '
                    'mode currently requires a setup.py based build.)'
                )
            raise InstallationError(msg)

        # Treating it as code that has already been checked out
        url_no_extras = path_to_url(url_no_extras)

    if url_no_extras.lower().startswith('file:'):
        package_name = Link(url_no_extras).egg_fragment
        if extras:
            return (
                package_name,
                url_no_extras,
                Requirement("placeholder" + extras.lower()).extras,
            )
        else:
            return package_name, url_no_extras, None

    for version_control in vcs:
        if url.lower().startswith('{}:'.format(version_control)):
            url = '{}+{}'.format(version_control, url)
            break

    if '+' not in url:
        raise InstallationError(
            '{} is not a valid editable requirement. '
            'It should either be a path to a local project or a VCS URL '
            '(beginning with svn+, git+, hg+, or bzr+).'.format(editable_req)
        )

    vc_type = url.split('+', 1)[0].lower()

    if not vcs.get_backend(vc_type):
        backends = ", ".join([bends.name + '+URL' for bends in vcs.backends])
        error_message = "For --editable={}, " \
                        "only {} are currently supported".format(
                            editable_req, backends)
        raise InstallationError(error_message)

    package_name = Link(url).egg_fragment
    if not package_name:
        raise InstallationError(
            "Could not detect requirement name for '{}', please specify one "
            "with #egg=your_package_name".format(editable_req)
        )
    return package_name, url, None
Пример #9
0
def parse_req_from_line(name, line_source):
    # type: (str, Optional[str]) -> RequirementParts
    if is_url(name):
        marker_sep = '; '
    else:
        marker_sep = ';'
    if marker_sep in name:
        name, markers_as_string = name.split(marker_sep, 1)
        markers_as_string = markers_as_string.strip()
        if not markers_as_string:
            markers = None
        else:
            markers = Marker(markers_as_string)
    else:
        markers = None
    name = name.strip()
    req_as_string = None
    path = os.path.normpath(os.path.abspath(name))
    link = None
    extras_as_string = None

    if is_url(name):
        link = Link(name)
    else:
        p, extras_as_string = _strip_extras(path)
        url = _get_url_from_path(p, name)
        if url is not None:
            link = Link(url)

    # it's a local file, dir, or url
    if link:
        # Handle relative file URLs
        if link.scheme == 'file' and re.search(r'\.\./', link.url):
            link = Link(
                path_to_url(os.path.normpath(os.path.abspath(link.path))))
        # wheel file
        if link.is_wheel:
            wheel = Wheel(link.filename)  # can raise InvalidWheelFilename
            req_as_string = "{wheel.name}=={wheel.version}".format(**locals())
        else:
            # set the req to the egg fragment.  when it's not there, this
            # will become an 'unnamed' requirement
            req_as_string = link.egg_fragment

    # a requirement specifier
    else:
        req_as_string = name

    extras = convert_extras(extras_as_string)

    def with_source(text):
        # type: (str) -> str
        if not line_source:
            return text
        return '{} (from {})'.format(text, line_source)

    if req_as_string is not None:
        try:
            req = Requirement(req_as_string)
        except InvalidRequirement:
            if os.path.sep in req_as_string:
                add_msg = "It looks like a path."
                add_msg += deduce_helpful_msg(req_as_string)
            elif ('=' in req_as_string and
                  not any(op in req_as_string for op in operators)):
                add_msg = "= is not a valid operator. Did you mean == ?"
            else:
                add_msg = ''
            msg = with_source(
                'Invalid requirement: {!r}'.format(req_as_string)
            )
            if add_msg:
                msg += '\nHint: {}'.format(add_msg)
            raise InstallationError(msg)
    else:
        req = None

    return RequirementParts(req, link, markers, extras)