def _get_url_from_path(path, name): # type: (str, str) -> str """ First, it checks whether a provided path is an installable directory (e.g. it has a setup.py). If it is, returns the path. If false, check if the path is an archive file (such as a .whl). The function checks if the path is a file. If false, if the path has an @, it will treat it as a PEP 440 URL requirement and return the path. """ if _looks_like_path(name) and os.path.isdir(path): if is_installable_dir(path): return path_to_url(path) raise InstallationError( "Directory %r is not installable. Neither 'setup.py' " "nor 'pyproject.toml' found." % name) if not is_archive_file(path): return None if os.path.isfile(path): return path_to_url(path) urlreq_parts = name.split('@', 1) if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]): # If the path contains '@' and the part before it does not look # like a path, try to treat it as a PEP 440 URL req instead. return None logger.warning( 'Requirement %r looks like a filename, but the ' 'file does not exist', name) return path_to_url(path)
def _get_url_from_path(path: str, name: str) -> Optional[str]: """ First, it checks whether a provided path is an installable directory. If it is, returns the path. If false, check if the path is an archive file (such as a .whl). The function checks if the path is a file. If false, if the path has an @, it will treat it as a PEP 440 URL requirement and return the path. """ if _looks_like_path(name) and os.path.isdir(path): if is_installable_dir(path): return path_to_url(path) # TODO: The is_installable_dir test here might not be necessary # now that it is done in load_pyproject_toml too. raise InstallationError( f"Directory {name!r} is not installable. Neither 'setup.py' " "nor 'pyproject.toml' found." ) if not is_archive_file(path): return None if os.path.isfile(path): return path_to_url(path) urlreq_parts = name.split("@", 1) if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]): # If the path contains '@' and the part before it does not look # like a path, try to treat it as a PEP 440 URL req instead. return None logger.warning( "Requirement %r looks like a filename, but the file does not exist", name, ) return path_to_url(path)
def get_remote_url(cls, location): url = cls.run_command(['showconfig', 'paths.default'], show_stdout=False, cwd=location).strip() if cls._is_local_repository(url): url = path_to_url(url) return url.strip()
def sort_path(path): # type: (str) -> None url = path_to_url(path) if mimetypes.guess_type(url, strict=False)[0] == 'text/html': urls.append(url) else: files.append(url)
def build_source( location: str, *, candidates_from_page: CandidatesFromPage, page_validator: PageValidator, expand_dir: bool, cache_link_parsing: bool, ) -> Tuple[Optional[str], Optional[LinkSource]]: path: Optional[str] = None url: Optional[str] = None if os.path.exists(location): # Is a local path. url = path_to_url(location) path = location elif location.startswith("file:"): # A file: URL. url = location path = url_to_path(location) elif is_url(location): url = location if url is None: msg = ("Location '%s' is ignored: " "it is either a non-existing path or lacks a specific scheme.") logger.warning(msg, location) return (None, None) if path is None: source: LinkSource = _RemoteFileSource( candidates_from_page=candidates_from_page, page_validator=page_validator, link=Link(url, cache_link_parsing=cache_link_parsing), ) return (url, source) if os.path.isdir(path): if expand_dir: source = _FlatDirectorySource( candidates_from_page=candidates_from_page, path=path, ) else: source = _IndexDirectorySource( candidates_from_page=candidates_from_page, link=Link(url, cache_link_parsing=cache_link_parsing), ) return (url, source) elif os.path.isfile(path): source = _LocalFileSource( candidates_from_page=candidates_from_page, link=Link(url, cache_link_parsing=cache_link_parsing), ) return (url, source) logger.warning( "Location '%s' is ignored: it is neither a file nor a directory.", location, ) return (url, None)
def build( requirements: Iterable[InstallRequirement], wheel_cache: WheelCache, verify: bool, build_options: List[str], global_options: List[str], ) -> BuildResult: """Build wheels. :return: The list of InstallRequirement that succeeded to build and the list of InstallRequirement that failed to build. """ if not requirements: return [], [] # Build the wheels. logger.info( "Building wheels for collected packages: %s", ", ".join(req.name for req in requirements), # type: ignore ) with indent_log(): build_successes, build_failures = [], [] for req in requirements: assert req.name cache_dir = _get_cache_dir(req, wheel_cache) wheel_file = _build_one( req, cache_dir, verify, build_options, global_options, req.editable and req.permit_editable_wheels, ) if wheel_file: # Update the link for this. req.link = Link(path_to_url(wheel_file)) req.local_file_path = req.link.file_path assert req.link.is_wheel build_successes.append(req) else: build_failures.append(req) # notify success/failure if build_successes: logger.info( "Successfully built %s", " ".join([req.name for req in build_successes]), # type: ignore ) if build_failures: logger.info( "Failed to build %s", " ".join([req.name for req in build_failures]), # type: ignore ) # Return a list of requirements that failed to build return build_successes, build_failures
def get_remote_url(cls, location: str) -> str: url = cls.run_command( ["showconfig", "paths.default"], show_stdout=False, stdout_only=True, cwd=location, ).strip() if cls._is_local_repository(url): url = path_to_url(url) return url.strip()
def get_remote_url(cls, location): urls = cls.run_command(['info'], show_stdout=False, cwd=location) for line in urls.splitlines(): line = line.strip() for x in ('checkout of branch: ', 'parent branch: '): if line.startswith(x): repo = line.split(x)[1] if cls._is_local_repository(repo): return path_to_url(repo) return repo return None
def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]: """Parses an editable requirement into: - a requirement name - an URL - extras - editable options Accepted requirements: svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir .[some_extra] """ url = editable_req # If a file path is specified with extras, strip off the extras. url_no_extras, extras = _strip_extras(url) if os.path.isdir(url_no_extras): # Treating it as code that has already been checked out url_no_extras = path_to_url(url_no_extras) if url_no_extras.lower().startswith("file:"): package_name = Link(url_no_extras).egg_fragment if extras: return ( package_name, url_no_extras, get_requirement("placeholder" + extras.lower()).extras, ) else: return package_name, url_no_extras, set() for version_control in vcs: if url.lower().startswith(f"{version_control}:"): url = f"{version_control}+{url}" break link = Link(url) if not link.is_vcs: backends = ", ".join(vcs.all_schemes) raise InstallationError( f"{editable_req} is not a valid editable requirement. " f"It should either be a path to a local project or a VCS URL " f"(beginning with {backends})." ) package_name = link.egg_fragment if not package_name: raise InstallationError( "Could not detect requirement name for '{}', please specify one " "with #egg=your_package_name".format(editable_req) ) return package_name, url, set()
def get_remote_url(cls, location: str) -> str: urls = cls.run_command(["info"], show_stdout=False, stdout_only=True, cwd=location) for line in urls.splitlines(): line = line.strip() for x in ("checkout of branch: ", "parent branch: "): if line.startswith(x): repo = line.split(x)[1] if cls._is_local_repository(repo): return path_to_url(repo) return repo raise RemoteNotFoundError
def build( requirements, # type: Iterable[InstallRequirement] wheel_cache, # type: WheelCache build_options, # type: List[str] global_options, # type: List[str] ): # type: (...) -> BuildResult """Build wheels. :return: The list of InstallRequirement that succeeded to build and the list of InstallRequirement that failed to build. """ if not requirements: return [], [] # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join(req.name for req in requirements), ) with indent_log(): build_successes, build_failures = [], [] for req in requirements: cache_dir = _get_cache_dir(req, wheel_cache) wheel_file = _build_one(req, cache_dir, build_options, global_options) if wheel_file: # Update the link for this. req.link = Link(path_to_url(wheel_file)) req.local_file_path = req.link.file_path assert req.link.is_wheel build_successes.append(req) else: build_failures.append(req) # notify success/failure if build_successes: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_successes]), ) if build_failures: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failures]), ) # Return a list of requirements that failed to build return build_successes, build_failures
def __init__( self, url: str, comes_from: Optional[Union[str, "HTMLPage"]] = None, requires_python: Optional[str] = None, yanked_reason: Optional[str] = None, cache_link_parsing: bool = True, ) -> None: """ :param url: url of the resource pointed to (href of the link) :param comes_from: instance of HTMLPage where the link was found, or string. :param requires_python: String containing the `Requires-Python` metadata field, specified in PEP 345. This may be specified by a data-requires-python attribute in the HTML link tag, as described in PEP 503. :param yanked_reason: the reason the file has been yanked, if the file has been yanked, or None if the file hasn't been yanked. This is the value of the "data-yanked" attribute, if present, in a simple repository HTML link. If the file has been yanked but no reason was provided, this should be the empty string. See PEP 592 for more information and the specification. :param cache_link_parsing: A flag that is used elsewhere to determine whether resources retrieved from this link should be cached. PyPI index urls should generally have this set to False, for example. """ # url can be a UNC windows share if url.startswith('\\\\'): url = path_to_url(url) self._parsed_url = urllib.parse.urlsplit(url) # Store the url as a private attribute to prevent accidentally # trying to set a new value. self._url = url self.comes_from = comes_from self.requires_python = requires_python if requires_python else None self.yanked_reason = yanked_reason super().__init__(key=url, defining_class=Link) self.cache_link_parsing = cache_link_parsing
def get( self, link, # type: Link package_name, # type: Optional[str] supported_tags, # type: List[Tag] ): # type: (...) -> Link candidates = [] if not package_name: return link canonical_package_name = canonicalize_name(package_name) for wheel_name, wheel_dir in self._get_candidates( link, canonical_package_name ): try: wheel = Wheel(wheel_name) except InvalidWheelFilename: continue if canonicalize_name(wheel.name) != canonical_package_name: logger.debug( "Ignoring cached wheel %s for %s as it " "does not match the expected distribution name %s.", wheel_name, link, package_name, ) continue if not wheel.supported(supported_tags): # Built for a different python/arch/etc continue candidates.append( ( wheel.support_index_min(supported_tags), wheel_name, wheel_dir, ) ) if not candidates: return link _, wheel_name, wheel_dir = min(candidates) return Link(path_to_url(os.path.join(wheel_dir, wheel_name)))
def __init__( self, url, # type: str comes_from=None, # type: Optional[Union[str, HTMLPage]] requires_python=None, # type: Optional[str] yanked_reason=None, # type: Optional[Text] ): # type: (...) -> None """ :param url: url of the resource pointed to (href of the link) :param comes_from: instance of HTMLPage where the link was found, or string. :param requires_python: String containing the `Requires-Python` metadata field, specified in PEP 345. This may be specified by a data-requires-python attribute in the HTML link tag, as described in PEP 503. :param yanked_reason: the reason the file has been yanked, if the file has been yanked, or None if the file hasn't been yanked. This is the value of the "data-yanked" attribute, if present, in a simple repository HTML link. If the file has been yanked but no reason was provided, this should be the empty string. See PEP 592 for more information and the specification. """ # url can be a UNC windows share if url.startswith('\\\\'): url = path_to_url(url) self._parsed_url = urllib_parse.urlsplit(url) # Store the url as a private attribute to prevent accidentally # trying to set a new value. self._url = url self.comes_from = comes_from self.requires_python = requires_python if requires_python else None self.yanked_reason = yanked_reason super(Link, self).__init__(key=url, defining_class=Link)
def direct_url_for_editable(source_dir: str) -> DirectUrl: return DirectUrl( url=path_to_url(source_dir), info=DirInfo(editable=True), )
def _link_for_candidate(self, link, candidate): # type: (Link, str) -> Link root = self.get_path_for_link(link) path = os.path.join(root, candidate) return Link(path_to_url(path))
def parse_editable(editable_req): # type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]] """Parses an editable requirement into: - a requirement name - an URL - extras - editable options Accepted requirements: svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir .[some_extra] """ url = editable_req # If a file path is specified with extras, strip off the extras. url_no_extras, extras = _strip_extras(url) if os.path.isdir(url_no_extras): if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): msg = ('File "setup.py" not found. Directory cannot be installed ' 'in editable mode: {}'.format( os.path.abspath(url_no_extras))) pyproject_path = make_pyproject_path(url_no_extras) if os.path.isfile(pyproject_path): msg += ('\n(A "pyproject.toml" file was found, but editable ' 'mode currently requires a setup.py based build.)') raise InstallationError(msg) # Treating it as code that has already been checked out url_no_extras = path_to_url(url_no_extras) if url_no_extras.lower().startswith('file:'): package_name = Link(url_no_extras).egg_fragment if extras: return ( package_name, url_no_extras, Requirement("placeholder" + extras.lower()).extras, ) else: return package_name, url_no_extras, None for version_control in vcs: if url.lower().startswith('%s:' % version_control): url = '%s+%s' % (version_control, url) break if '+' not in url: raise InstallationError( '{} is not a valid editable requirement. ' 'It should either be a path to a local project or a VCS URL ' '(beginning with svn+, git+, hg+, or bzr+).'.format(editable_req)) vc_type = url.split('+', 1)[0].lower() if not vcs.get_backend(vc_type): error_message = 'For --editable=%s only ' % editable_req + \ ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ ' is currently supported' raise InstallationError(error_message) package_name = Link(url).egg_fragment if not package_name: raise InstallationError( "Could not detect requirement name for '%s', please specify one " "with #egg=your_package_name" % editable_req) return package_name, url, None
def parse_req_from_line(name, line_source): # type: (str, Optional[str]) -> RequirementParts if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: name, markers_as_string = name.split(marker_sep, 1) markers_as_string = markers_as_string.strip() if not markers_as_string: markers = None else: markers = Marker(markers_as_string) else: markers = None name = name.strip() req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None extras_as_string = None if is_url(name): link = Link(name) else: p, extras_as_string = _strip_extras(path) url = _get_url_from_path(p, name) if url is not None: link = Link(url) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == 'file' and re.search(r'\.\./', link.url): link = Link( path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req_as_string = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req_as_string = link.egg_fragment # a requirement specifier else: req_as_string = name extras = convert_extras(extras_as_string) def with_source(text): if not line_source: return text return '{} (from {})'.format(text, line_source) if req_as_string is not None: try: req = Requirement(req_as_string) except InvalidRequirement: if os.path.sep in req_as_string: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req_as_string) elif ('=' in req_as_string and not any(op in req_as_string for op in operators)): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = '' msg = with_source( 'Invalid requirement: {!r}'.format(req_as_string)) if add_msg: msg += '\nHint: {}'.format(add_msg) raise InstallationError(msg) else: req = None return RequirementParts(req, link, markers, extras)
def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]: """Parses an editable requirement into: - a requirement name - an URL - extras - editable options Accepted requirements: svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir .[some_extra] """ url = editable_req # If a file path is specified with extras, strip off the extras. url_no_extras, extras = _strip_extras(url) if os.path.isdir(url_no_extras): setup_py = os.path.join(url_no_extras, 'setup.py') setup_cfg = os.path.join(url_no_extras, 'setup.cfg') if not os.path.exists(setup_py) and not os.path.exists(setup_cfg): msg = ( 'File "setup.py" or "setup.cfg" not found. Directory cannot be ' 'installed in editable mode: {}'.format( os.path.abspath(url_no_extras))) pyproject_path = make_pyproject_path(url_no_extras) if os.path.isfile(pyproject_path): msg += ('\n(A "pyproject.toml" file was found, but editable ' 'mode currently requires a setuptools-based build.)') raise InstallationError(msg) # Treating it as code that has already been checked out url_no_extras = path_to_url(url_no_extras) if url_no_extras.lower().startswith('file:'): package_name = Link(url_no_extras).egg_fragment if extras: return ( package_name, url_no_extras, Requirement("placeholder" + extras.lower()).extras, ) else: return package_name, url_no_extras, set() for version_control in vcs: if url.lower().startswith(f'{version_control}:'): url = f'{version_control}+{url}' break link = Link(url) if not link.is_vcs: backends = ", ".join(vcs.all_schemes) raise InstallationError( f'{editable_req} is not a valid editable requirement. ' f'It should either be a path to a local project or a VCS URL ' f'(beginning with {backends}).') package_name = link.egg_fragment if not package_name: raise InstallationError( "Could not detect requirement name for '{}', please specify one " "with #egg=your_package_name".format(editable_req)) return package_name, url, set()
def parse_req_from_line(name: str, line_source: Optional[str]) -> RequirementParts: if is_url(name): marker_sep = "; " else: marker_sep = ";" if marker_sep in name: name, markers_as_string = name.split(marker_sep, 1) markers_as_string = markers_as_string.strip() if not markers_as_string: markers = None else: markers = Marker(markers_as_string) else: markers = None name = name.strip() req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None extras_as_string = None if is_url(name): link = Link(name) else: p, extras_as_string = _strip_extras(path) url = _get_url_from_path(p, name) if url is not None: link = Link(url) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == "file" and re.search(r"\.\./", link.url): link = Link(path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req_as_string = f"{wheel.name}=={wheel.version}" else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req_as_string = link.egg_fragment # a requirement specifier else: req_as_string = name extras = convert_extras(extras_as_string) def with_source(text: str) -> str: if not line_source: return text return f"{text} (from {line_source})" def _parse_req_string(req_as_string: str) -> Requirement: try: req = get_requirement(req_as_string) except InvalidRequirement: if os.path.sep in req_as_string: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req_as_string) elif "=" in req_as_string and not any( op in req_as_string for op in operators ): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = "" msg = with_source(f"Invalid requirement: {req_as_string!r}") if add_msg: msg += f"\nHint: {add_msg}" raise InstallationError(msg) else: # Deprecate extras after specifiers: "name>=1.0[extras]" # This currently works by accident because _strip_extras() parses # any extras in the end of the string and those are saved in # RequirementParts for spec in req.specifier: spec_str = str(spec) if spec_str.endswith("]"): msg = f"Extras after version '{spec_str}'." raise InstallationError(msg) return req if req_as_string is not None: req: Optional[Requirement] = _parse_req_string(req_as_string) else: req = None return RequirementParts(req, link, markers, extras)
def build( self, requirements, # type: Iterable[InstallRequirement] should_unpack=False # type: bool ): # type: (...) -> List[InstallRequirement] """Build wheels. :param should_unpack: If True, after building the wheel, unpack it and replace the sdist with the unpacked version in preparation for installation. :return: True if all the wheels built correctly. """ # pip install uses should_unpack=True. # pip install never provides a _wheel_dir. # pip wheel uses should_unpack=False. # pip wheel always provides a _wheel_dir (via the preparer). assert ((should_unpack and not self._wheel_dir) or (not should_unpack and self._wheel_dir)) buildset = [] cache_available = bool(self.wheel_cache.cache_dir) for req in requirements: ephem_cache = should_use_ephemeral_cache( req, should_unpack=should_unpack, cache_available=cache_available, check_binary_allowed=self.check_binary_allowed, ) if ephem_cache is None: continue # Determine where the wheel should go. if should_unpack: if ephem_cache: output_dir = self.wheel_cache.get_ephem_path_for_link( req.link) else: output_dir = self.wheel_cache.get_path_for_link(req.link) else: output_dir = self._wheel_dir buildset.append((req, output_dir)) if not buildset: return [] # TODO by @pradyunsg # Should break up this method into 2 separate methods. # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for (req, _) in buildset]), ) python_tag = None if should_unpack: python_tag = pep425tags.implementation_tag with indent_log(): build_success, build_failure = [], [] for req, output_dir in buildset: try: ensure_dir(output_dir) except OSError as e: logger.warning( "Building wheel for %s failed: %s", req.name, e, ) build_failure.append(req) continue wheel_file = self._build_one( req, output_dir, python_tag=python_tag, ) if wheel_file: build_success.append(req) if should_unpack: # XXX: This is mildly duplicative with prepare_files, # but not close enough to pull out to a single common # method. # The code below assumes temporary source dirs - # prevent it doing bad things. if (req.source_dir and not has_delete_marker_file(req.source_dir)): raise AssertionError( "bad source dir - missing marker") # Delete the source we built the wheel from req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. req.source_dir = req.ensure_build_location( self.preparer.build_dir) # Update the link for this. req.link = Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_file(req.link.file_path, req.source_dir) else: build_failure.append(req) # notify success/failure if build_success: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_success]), ) if build_failure: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) # Return a list of requirements that failed to build return build_failure
def file_links(self) -> FoundLinks: for path in self._path.iterdir(): url = path_to_url(str(path)) if _is_html_file(url): continue yield Link(url)
def page_candidates(self) -> FoundCandidates: for path in self._path.iterdir(): url = path_to_url(str(path)) if not _is_html_file(url): continue yield from self._candidates_from_page(Link(url))