def parse_editable(editable_req): """Parses an editable requirement into: - a requirement name - an URL - extras - editable options Accepted requirements: svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir .[some_extra] """ url = editable_req # If a file path is specified with extras, strip off the extras. url_no_extras, extras = _strip_extras(url) if os.path.isdir(url_no_extras): if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): raise InstallationError( "Directory %r is not installable. File 'setup.py' not found." % url_no_extras) # Treating it as code that has already been checked out url_no_extras = path_to_url(url_no_extras) if url_no_extras.lower().startswith('file:'): package_name = Link(url_no_extras).egg_fragment if extras: return ( package_name, url_no_extras, Requirement("placeholder" + extras.lower()).extras, ) else: return package_name, url_no_extras, None for version_control in vcs: if url.lower().startswith('%s:' % version_control): url = '%s+%s' % (version_control, url) break if '+' not in url: raise InstallationError( '%s should either be a path to a local project or a VCS url ' 'beginning with svn+, git+, hg+, or bzr+' % editable_req) vc_type = url.split('+', 1)[0].lower() if not vcs.get_backend(vc_type): error_message = 'For --editable=%s only ' % editable_req + \ ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ ' is currently supported' raise InstallationError(error_message) package_name = Link(url).egg_fragment if not package_name: raise InstallationError( "Could not detect requirement name for '%s', please specify one " "with #egg=your_package_name" % editable_req) return package_name, url, None
def build_source( location: str, *, candidates_from_page: CandidatesFromPage, page_validator: PageValidator, expand_dir: bool, cache_link_parsing: bool, ) -> Tuple[Optional[str], Optional[LinkSource]]: path: Optional[str] = None url: Optional[str] = None if os.path.exists(location): # Is a local path. url = path_to_url(location) path = location elif location.startswith("file:"): # A file: URL. url = location path = url_to_path(location) elif is_url(location): url = location if url is None: msg = ("Location '%s' is ignored: " "it is either a non-existing path or lacks a specific scheme.") logger.warning(msg, location) return (None, None) if path is None: source: LinkSource = _RemoteFileSource( candidates_from_page=candidates_from_page, page_validator=page_validator, link=Link(url, cache_link_parsing=cache_link_parsing), ) return (url, source) if os.path.isdir(path): if expand_dir: source = _FlatDirectorySource( candidates_from_page=candidates_from_page, path=path, ) else: source = _IndexDirectorySource( candidates_from_page=candidates_from_page, link=Link(url, cache_link_parsing=cache_link_parsing), ) return (url, source) elif os.path.isfile(path): source = _LocalFileSource( candidates_from_page=candidates_from_page, link=Link(url, cache_link_parsing=cache_link_parsing), ) return (url, source) logger.warning( "Location '%s' is ignored: it is neither a file nor a directory.", location, ) return (url, None)
def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]: """Parses an editable requirement into: - a requirement name - an URL - extras - editable options Accepted requirements: svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir .[some_extra] """ url = editable_req # If a file path is specified with extras, strip off the extras. url_no_extras, extras = _strip_extras(url) if os.path.isdir(url_no_extras): # Treating it as code that has already been checked out url_no_extras = path_to_url(url_no_extras) if url_no_extras.lower().startswith("file:"): package_name = Link(url_no_extras).egg_fragment if extras: return ( package_name, url_no_extras, get_requirement("placeholder" + extras.lower()).extras, ) else: return package_name, url_no_extras, set() for version_control in vcs: if url.lower().startswith(f"{version_control}:"): url = f"{version_control}+{url}" break link = Link(url) if not link.is_vcs: backends = ", ".join(vcs.all_schemes) raise InstallationError( f"{editable_req} is not a valid editable requirement. " f"It should either be a path to a local project or a VCS URL " f"(beginning with {backends})." ) package_name = link.egg_fragment if not package_name: raise InstallationError( "Could not detect requirement name for '{}', please specify one " "with #egg=your_package_name".format(editable_req) ) return package_name, url, set()
def collect_links(self, project_name): # type: (str) -> CollectedLinks """Find all available links for the given project name. :return: All the Link objects (unfiltered), as a CollectedLinks object. """ search_scope = self.search_scope index_locations = search_scope.get_index_urls_locations(project_name) index_file_loc, index_url_loc = group_locations(index_locations) fl_file_loc, fl_url_loc = group_locations( self.find_links, expand_dir=True, ) file_links = [ Link(url) for url in itertools.chain(index_file_loc, fl_file_loc) ] # We trust every directly linked archive in find_links find_link_links = [Link(url, '-f') for url in self.find_links] # We trust every url that the user has given us whether it was given # via --index-url or --find-links. # We want to filter out anything that does not have a secure origin. url_locations = [ link for link in itertools.chain( (Link(url) for url in index_url_loc), (Link(url) for url in fl_url_loc), ) if self.session.is_secure_origin(link) ] url_locations = _remove_duplicate_links(url_locations) lines = [ '{} location(s) to search for versions of {}:'.format( len(url_locations), project_name, ), ] for link in url_locations: lines.append('* {}'.format(link)) logger.debug('\n'.join(lines)) pages_links = {} for page in self._get_pages(url_locations): pages_links[page.url] = list(parse_links(page)) return CollectedLinks( files=file_links, find_links=find_link_links, pages=pages_links, )
def _create_link_from_element( anchor, # type: HTMLElement page_url, # type: str base_url, # type: str ): # type: (...) -> Optional[Link] """ Convert an anchor element in a simple repository page to a Link. """ href = anchor.get("href") if not href: return None url = _clean_link(urllib_parse.urljoin(base_url, href)) pyrequire = anchor.get('data-requires-python') pyrequire = unescape(pyrequire) if pyrequire else None yanked_reason = anchor.get('data-yanked') if yanked_reason: # This is a unicode string in Python 2 (and 3). yanked_reason = unescape(yanked_reason) link = Link( url, comes_from=page_url, requires_python=pyrequire, yanked_reason=yanked_reason, ) return link
def _create_link_from_element( anchor: HTMLElement, page_url: str, base_url: str, ) -> Optional[Link]: """ Convert an anchor element in a simple repository page to a Link. """ href = anchor.get("href") if not href: return None url = _clean_link(urllib.parse.urljoin(base_url, href)) pyrequire = anchor.get('data-requires-python') pyrequire = html.unescape(pyrequire) if pyrequire else None yanked_reason = anchor.get('data-yanked') if yanked_reason: yanked_reason = html.unescape(yanked_reason) link = Link( url, comes_from=page_url, requires_python=pyrequire, yanked_reason=yanked_reason, ) return link
def install_req_from_editable(editable_req, comes_from=None, isolated=False, options=None, wheel_cache=None, constraint=False): name, url, extras_override = parse_editable(editable_req) if url.startswith('file:'): source_dir = url_to_path(url) else: source_dir = None if name is not None: try: req = Requirement(name) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % name) else: req = None return InstallRequirement( req, comes_from, source_dir=source_dir, editable=True, link=Link(url), constraint=constraint, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, extras=extras_override or (), )
def _is_url_like_archive(url): # type: (str) -> bool """Return whether the URL looks like an archive. """ filename = Link(url).filename for bad_ext in ARCHIVE_EXTENSIONS: if filename.endswith(bad_ext): return True return False
def build( requirements: Iterable[InstallRequirement], wheel_cache: WheelCache, verify: bool, build_options: List[str], global_options: List[str], ) -> BuildResult: """Build wheels. :return: The list of InstallRequirement that succeeded to build and the list of InstallRequirement that failed to build. """ if not requirements: return [], [] # Build the wheels. logger.info( "Building wheels for collected packages: %s", ", ".join(req.name for req in requirements), # type: ignore ) with indent_log(): build_successes, build_failures = [], [] for req in requirements: assert req.name cache_dir = _get_cache_dir(req, wheel_cache) wheel_file = _build_one( req, cache_dir, verify, build_options, global_options, req.editable and req.permit_editable_wheels, ) if wheel_file: # Update the link for this. req.link = Link(path_to_url(wheel_file)) req.local_file_path = req.link.file_path assert req.link.is_wheel build_successes.append(req) else: build_failures.append(req) # notify success/failure if build_successes: logger.info( "Successfully built %s", " ".join([req.name for req in build_successes]), # type: ignore ) if build_failures: logger.info( "Failed to build %s", " ".join([req.name for req in build_failures]), # type: ignore ) # Return a list of requirements that failed to build return build_successes, build_failures
def get_location(self, dist, dependency_links): for url in dependency_links: egg_fragment = Link(url).egg_fragment if not egg_fragment: continue if '-' in egg_fragment: # FIXME: will this work when a package has - in the name? key = '-'.join(egg_fragment.split('-')[:-1]).lower() else: key = egg_fragment if key == dist.key: return url.split('#', 1)[0] return None
def parse_req_from_editable(editable_req: str) -> RequirementParts: name, url, extras_override = parse_editable(editable_req) if name is not None: try: req: Optional[Requirement] = Requirement(name) except InvalidRequirement: raise InstallationError(f"Invalid requirement: '{name}'") else: req = None link = Link(url) return RequirementParts(req, link, None, extras_override)
def build( requirements, # type: Iterable[InstallRequirement] wheel_cache, # type: WheelCache build_options, # type: List[str] global_options, # type: List[str] ): # type: (...) -> BuildResult """Build wheels. :return: The list of InstallRequirement that succeeded to build and the list of InstallRequirement that failed to build. """ if not requirements: return [], [] # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join(req.name for req in requirements), ) with indent_log(): build_successes, build_failures = [], [] for req in requirements: cache_dir = _get_cache_dir(req, wheel_cache) wheel_file = _build_one(req, cache_dir, build_options, global_options) if wheel_file: # Update the link for this. req.link = Link(path_to_url(wheel_file)) req.local_file_path = req.link.file_path assert req.link.is_wheel build_successes.append(req) else: build_failures.append(req) # notify success/failure if build_successes: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_successes]), ) if build_failures: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failures]), ) # Return a list of requirements that failed to build return build_successes, build_failures
def iter_links(self): """Yields all links in the page""" document = html5lib.parse( self.content, transport_encoding=_get_encoding_from_headers(self.headers), namespaceHTMLElements=False, ) base_url = _determine_base_url(document, self.url) for anchor in document.findall(".//a"): if anchor.get("href"): href = anchor.get("href") url = _clean_link(urllib_parse.urljoin(base_url, href)) pyrequire = anchor.get('data-requires-python') pyrequire = unescape(pyrequire) if pyrequire else None yield Link(url, self.url, requires_python=pyrequire)
def parse_req_from_editable(editable_req): # type: (str) -> RequirementParts name, url, extras_override = parse_editable(editable_req) if name is not None: try: req = Requirement(name) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % name) else: req = None link = Link(url) return RequirementParts(req, link, None, extras_override)
def _get_html_response(url: str, session: PipSession) -> Response: """Access an HTML page with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotHTML` if it is not HTML. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got HTML, and raise `_NotHTML` otherwise. """ if is_archive_file(Link(url).filename): _ensure_html_response(url, session=session) logger.debug("Getting page %s", redact_auth_from_url(url)) resp = session.get( url, headers={ "Accept": "text/html", # We don't want to blindly returned cached data for # /simple/, because authors generally expecting that # twine upload && pip install will function, but if # they've done a pip install in the last ~10 minutes # it won't. Thus by setting this to zero we will not # blindly use any cached data, however the benefit of # using max-age=0 instead of no-cache, is that we will # still support conditional requests, so we will still # minimize traffic sent in cases where the page hasn't # changed at all, we will just always incur the round # trip for the conditional GET now instead of only # once per 10 minutes. # For more information, please see pypa/pip#5670. "Cache-Control": "max-age=0", }, ) raise_for_status(resp) # The check for archives above only works if the url ends with # something that looks like an archive. However that is not a # requirement of an url. Unless we issue a HEAD request on every # url we cannot know ahead of time for sure if something is HTML # or not. However we can check after we've downloaded it. _ensure_html_header(resp) return resp
def get( self, link, # type: Link package_name, # type: Optional[str] supported_tags, # type: List[Tag] ): # type: (...) -> Link candidates = [] if not package_name: return link canonical_package_name = canonicalize_name(package_name) for wheel_name, wheel_dir in self._get_candidates( link, canonical_package_name ): try: wheel = Wheel(wheel_name) except InvalidWheelFilename: continue if canonicalize_name(wheel.name) != canonical_package_name: logger.debug( "Ignoring cached wheel %s for %s as it " "does not match the expected distribution name %s.", wheel_name, link, package_name, ) continue if not wheel.supported(supported_tags): # Built for a different python/arch/etc continue candidates.append( ( wheel.support_index_min(supported_tags), wheel_name, wheel_dir, ) ) if not candidates: return link _, wheel_name, wheel_dir = min(candidates) return Link(path_to_url(os.path.join(wheel_dir, wheel_name)))
def _create_link_from_element( element_attribs: Dict[str, Optional[str]], page_url: str, base_url: str, ) -> Optional[Link]: """ Convert an anchor element's attributes in a simple repository page to a Link. """ href = element_attribs.get("href") if not href: return None url = _clean_link(urllib.parse.urljoin(base_url, href)) pyrequire = element_attribs.get("data-requires-python") yanked_reason = element_attribs.get("data-yanked") link = Link( url, comes_from=page_url, requires_python=pyrequire, yanked_reason=yanked_reason, ) return link
def install_req_from_editable( editable_req, # type: str comes_from=None, # type: Optional[str] use_pep517=None, # type: Optional[bool] isolated=False, # type: bool options=None, # type: Optional[Dict[str, Any]] wheel_cache=None, # type: Optional[WheelCache] constraint=False # type: bool ): # type: (...) -> InstallRequirement name, url, extras_override = parse_editable(editable_req) if url.startswith('file:'): source_dir = url_to_path(url) else: source_dir = None if name is not None: try: req = Requirement(name) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % name) else: req = None return InstallRequirement( req, comes_from, source_dir=source_dir, editable=True, link=Link(url), constraint=constraint, use_pep517=use_pep517, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, extras=extras_override or (), )
def page_candidates(self) -> FoundCandidates: for path in self._path.iterdir(): url = path_to_url(str(path)) if not _is_html_file(url): continue yield from self._candidates_from_page(Link(url))
def install_req_from_line(name, comes_from=None, isolated=False, options=None, wheel_cache=None, constraint=False): """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: name, markers = name.split(marker_sep, 1) markers = markers.strip() if not markers: markers = None else: markers = Marker(markers) else: markers = None name = name.strip() req = None path = os.path.normpath(os.path.abspath(name)) link = None extras = None if is_url(name): link = Link(name) else: p, extras = _strip_extras(path) looks_like_dir = os.path.isdir(p) and (os.path.sep in name or (os.path.altsep is not None and os.path.altsep in name) or name.startswith('.')) if looks_like_dir: if not is_installable_dir(p): raise InstallationError( "Directory %r is not installable. Neither 'setup.py' " "nor 'pyproject.toml' found." % name) link = Link(path_to_url(p)) elif is_archive_file(p): if not os.path.isfile(p): logger.warning( 'Requirement %r looks like a filename, but the ' 'file does not exist', name) link = Link(path_to_url(p)) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == 'file' and re.search(r'\.\./', link.url): link = Link( path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req = link.egg_fragment # a requirement specifier else: req = name if extras: extras = Requirement("placeholder" + extras.lower()).extras else: extras = () if req is not None: try: req = Requirement(req) except InvalidRequirement: if os.path.sep in req: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req) elif '=' in req and not any(op in req for op in operators): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = traceback.format_exc() raise InstallationError("Invalid requirement: '%s'\n%s" % (req, add_msg)) return InstallRequirement( req, comes_from, link=link, markers=markers, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, constraint=constraint, extras=extras, )
def __init__( self, req: Optional[Requirement], comes_from: Optional[Union[str, "InstallRequirement"]], editable: bool = False, link: Optional[Link] = None, markers: Optional[Marker] = None, use_pep517: Optional[bool] = None, isolated: bool = False, install_options: Optional[List[str]] = None, global_options: Optional[List[str]] = None, hash_options: Optional[Dict[str, List[str]]] = None, constraint: bool = False, extras: Collection[str] = (), user_supplied: bool = False, permit_editable_wheels: bool = False, ) -> None: assert req is None or isinstance(req, Requirement), req self.req = req self.comes_from = comes_from self.constraint = constraint self.editable = editable self.permit_editable_wheels = permit_editable_wheels self.legacy_install_reason: Optional[int] = None # source_dir is the local directory where the linked requirement is # located, or unpacked. In case unpacking is needed, creating and # populating source_dir is done by the RequirementPreparer. Note this # is not necessarily the directory where pyproject.toml or setup.py is # located - that one is obtained via unpacked_source_directory. self.source_dir: Optional[str] = None if self.editable: assert link if link.is_file: self.source_dir = os.path.normpath( os.path.abspath(link.file_path)) if link is None and req and req.url: # PEP 508 URL requirement link = Link(req.url) self.link = self.original_link = link self.original_link_is_in_wheel_cache = False # Path to any downloaded or already-existing package. self.local_file_path: Optional[str] = None if self.link and self.link.is_file: self.local_file_path = self.link.file_path if extras: self.extras = extras elif req: self.extras = {safe_extra(extra) for extra in req.extras} else: self.extras = set() if markers is None and req: markers = req.marker self.markers = markers # This holds the Distribution object if this requirement is already installed. self.satisfied_by: Optional[BaseDistribution] = None # Whether the installation process should try to uninstall an existing # distribution before installing this requirement. self.should_reinstall = False # Temporary build location self._temp_build_dir: Optional[TempDirectory] = None # Set to True after successful installation self.install_succeeded: Optional[bool] = None # Supplied options self.install_options = install_options if install_options else [] self.global_options = global_options if global_options else [] self.hash_options = hash_options if hash_options else {} # Set to True after successful preparation of this requirement self.prepared = False # User supplied requirement are explicitly requested for installation # by the user via CLI arguments or requirements files, as opposed to, # e.g. dependencies, extras or constraints. self.user_supplied = user_supplied self.isolated = isolated self.build_env: BuildEnvironment = NoOpBuildEnvironment() # For PEP 517, the directory where we request the project metadata # gets stored. We need this to pass to build_wheel, so the backend # can ensure that the wheel matches the metadata (see the PEP for # details). self.metadata_directory: Optional[str] = None # The static build requirements (from pyproject.toml) self.pyproject_requires: Optional[List[str]] = None # Build requirements that we will check are available self.requirements_to_check: List[str] = [] # The PEP 517 backend we should use to build the project self.pep517_backend: Optional[Pep517HookCaller] = None # Are we using PEP 517 for this requirement? # After pyproject.toml has been loaded, the only valid values are True # and False. Before loading, None is valid (meaning "use the default"). # Setting an explicit value before loading pyproject.toml is supported, # but after loading this flag should be treated as read only. self.use_pep517 = use_pep517 # This requirement needs more preparation before it can be built self.needs_more_preparation = False
def build( self, requirements, # type: Iterable[InstallRequirement] session, # type: PipSession autobuilding=False # type: bool ): # type: (...) -> List[InstallRequirement] """Build wheels. :param unpack: If True, replace the sdist we built from with the newly built wheel, in preparation for installation. :return: True if all the wheels built correctly. """ buildset = [] format_control = self.finder.format_control # Whether a cache directory is available for autobuilding=True. cache_available = bool(self._wheel_dir or self.wheel_cache.cache_dir) for req in requirements: ephem_cache = should_use_ephemeral_cache( req, format_control=format_control, autobuilding=autobuilding, cache_available=cache_available, ) if ephem_cache is None: continue buildset.append((req, ephem_cache)) if not buildset: return [] # Is any wheel build not using the ephemeral cache? if any(not ephem_cache for _, ephem_cache in buildset): have_directory_for_build = self._wheel_dir or ( autobuilding and self.wheel_cache.cache_dir) assert have_directory_for_build # TODO by @pradyunsg # Should break up this method into 2 separate methods. # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for (req, _) in buildset]), ) _cache = self.wheel_cache # shorter name with indent_log(): build_success, build_failure = [], [] for req, ephem in buildset: python_tag = None if autobuilding: python_tag = pep425tags.implementation_tag if ephem: output_dir = _cache.get_ephem_path_for_link(req.link) else: output_dir = _cache.get_path_for_link(req.link) try: ensure_dir(output_dir) except OSError as e: logger.warning("Building wheel for %s failed: %s", req.name, e) build_failure.append(req) continue else: output_dir = self._wheel_dir wheel_file = self._build_one( req, output_dir, python_tag=python_tag, ) if wheel_file: build_success.append(req) if autobuilding: # XXX: This is mildly duplicative with prepare_files, # but not close enough to pull out to a single common # method. # The code below assumes temporary source dirs - # prevent it doing bad things. if req.source_dir and not os.path.exists( os.path.join(req.source_dir, PIP_DELETE_MARKER_FILENAME)): raise AssertionError( "bad source dir - missing marker") # Delete the source we built the wheel from req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. req.source_dir = req.build_location( self.preparer.build_dir) # Update the link for this. req.link = Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_url( req.link, req.source_dir, None, False, session=session, ) else: build_failure.append(req) # notify success/failure if build_success: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_success]), ) if build_failure: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) # Return a list of requirements that failed to build return build_failure
def parse_editable(editable_req): # type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]] """Parses an editable requirement into: - a requirement name - an URL - extras - editable options Accepted requirements: svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir .[some_extra] """ url = editable_req # If a file path is specified with extras, strip off the extras. url_no_extras, extras = _strip_extras(url) if os.path.isdir(url_no_extras): if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): msg = ('File "setup.py" not found. Directory cannot be installed ' 'in editable mode: {}'.format( os.path.abspath(url_no_extras))) pyproject_path = make_pyproject_path(url_no_extras) if os.path.isfile(pyproject_path): msg += ('\n(A "pyproject.toml" file was found, but editable ' 'mode currently requires a setup.py based build.)') raise InstallationError(msg) # Treating it as code that has already been checked out url_no_extras = path_to_url(url_no_extras) if url_no_extras.lower().startswith('file:'): package_name = Link(url_no_extras).egg_fragment if extras: return ( package_name, url_no_extras, Requirement("placeholder" + extras.lower()).extras, ) else: return package_name, url_no_extras, None for version_control in vcs: if url.lower().startswith('%s:' % version_control): url = '%s+%s' % (version_control, url) break if '+' not in url: raise InstallationError( '{} is not a valid editable requirement. ' 'It should either be a path to a local project or a VCS URL ' '(beginning with svn+, git+, hg+, or bzr+).'.format(editable_req)) vc_type = url.split('+', 1)[0].lower() if not vcs.get_backend(vc_type): error_message = 'For --editable=%s only ' % editable_req + \ ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ ' is currently supported' raise InstallationError(error_message) package_name = Link(url).egg_fragment if not package_name: raise InstallationError( "Could not detect requirement name for '%s', please specify one " "with #egg=your_package_name" % editable_req) return package_name, url, None
def parse_req_from_line(name, line_source): # type: (str, Optional[str]) -> RequirementParts if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: name, markers_as_string = name.split(marker_sep, 1) markers_as_string = markers_as_string.strip() if not markers_as_string: markers = None else: markers = Marker(markers_as_string) else: markers = None name = name.strip() req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None extras_as_string = None if is_url(name): link = Link(name) else: p, extras_as_string = _strip_extras(path) url = _get_url_from_path(p, name) if url is not None: link = Link(url) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == 'file' and re.search(r'\.\./', link.url): link = Link( path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req_as_string = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req_as_string = link.egg_fragment # a requirement specifier else: req_as_string = name extras = convert_extras(extras_as_string) def with_source(text): if not line_source: return text return '{} (from {})'.format(text, line_source) if req_as_string is not None: try: req = Requirement(req_as_string) except InvalidRequirement: if os.path.sep in req_as_string: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req_as_string) elif ('=' in req_as_string and not any(op in req_as_string for op in operators)): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = '' msg = with_source( 'Invalid requirement: {!r}'.format(req_as_string)) if add_msg: msg += '\nHint: {}'.format(add_msg) raise InstallationError(msg) else: req = None return RequirementParts(req, link, markers, extras)
def find_all_candidates(self, project_name): # type: (str) -> List[Optional[InstallationCandidate]] """Find all available InstallationCandidate for project_name This checks index_urls and find_links. All versions found are returned as an InstallationCandidate list. See _link_package_versions for details on which files are accepted """ index_locations = self._get_index_urls_locations(project_name) index_file_loc, index_url_loc = self._sort_locations(index_locations) fl_file_loc, fl_url_loc = self._sort_locations( self.find_links, expand_dir=True, ) file_locations = (Link(url) for url in itertools.chain( index_file_loc, fl_file_loc, )) # We trust every url that the user has given us whether it was given # via --index-url or --find-links. # We want to filter out any thing which does not have a secure origin. url_locations = [ link for link in itertools.chain( (Link(url) for url in index_url_loc), (Link(url) for url in fl_url_loc), ) if self._validate_secure_origin(logger, link) ] logger.debug('%d location(s) to search for versions of %s:', len(url_locations), project_name) for location in url_locations: logger.debug('* %s', location) canonical_name = canonicalize_name(project_name) formats = self.format_control.get_allowed_formats(canonical_name) search = Search(project_name, canonical_name, formats) find_links_versions = self._package_versions( # We trust every directly linked archive in find_links (Link(url, '-f') for url in self.find_links), search) page_versions = [] for page in self._get_pages(url_locations, project_name): try: logger.debug('Analyzing links from page %s', page.url) except AttributeError: continue with indent_log(): page_versions.extend( self._package_versions(page.iter_links(), search)) file_versions = self._package_versions(file_locations, search) if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.location.url) for candidate in file_versions ])) # This is an intentional priority ordering return file_versions + find_links_versions + page_versions
def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]: """Parses an editable requirement into: - a requirement name - an URL - extras - editable options Accepted requirements: svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir .[some_extra] """ url = editable_req # If a file path is specified with extras, strip off the extras. url_no_extras, extras = _strip_extras(url) if os.path.isdir(url_no_extras): setup_py = os.path.join(url_no_extras, 'setup.py') setup_cfg = os.path.join(url_no_extras, 'setup.cfg') if not os.path.exists(setup_py) and not os.path.exists(setup_cfg): msg = ( 'File "setup.py" or "setup.cfg" not found. Directory cannot be ' 'installed in editable mode: {}'.format( os.path.abspath(url_no_extras))) pyproject_path = make_pyproject_path(url_no_extras) if os.path.isfile(pyproject_path): msg += ('\n(A "pyproject.toml" file was found, but editable ' 'mode currently requires a setuptools-based build.)') raise InstallationError(msg) # Treating it as code that has already been checked out url_no_extras = path_to_url(url_no_extras) if url_no_extras.lower().startswith('file:'): package_name = Link(url_no_extras).egg_fragment if extras: return ( package_name, url_no_extras, Requirement("placeholder" + extras.lower()).extras, ) else: return package_name, url_no_extras, set() for version_control in vcs: if url.lower().startswith(f'{version_control}:'): url = f'{version_control}+{url}' break link = Link(url) if not link.is_vcs: backends = ", ".join(vcs.all_schemes) raise InstallationError( f'{editable_req} is not a valid editable requirement. ' f'It should either be a path to a local project or a VCS URL ' f'(beginning with {backends}).') package_name = link.egg_fragment if not package_name: raise InstallationError( "Could not detect requirement name for '{}', please specify one " "with #egg=your_package_name".format(editable_req)) return package_name, url, set()
def parse_req_from_line(name: str, line_source: Optional[str]) -> RequirementParts: if is_url(name): marker_sep = "; " else: marker_sep = ";" if marker_sep in name: name, markers_as_string = name.split(marker_sep, 1) markers_as_string = markers_as_string.strip() if not markers_as_string: markers = None else: markers = Marker(markers_as_string) else: markers = None name = name.strip() req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None extras_as_string = None if is_url(name): link = Link(name) else: p, extras_as_string = _strip_extras(path) url = _get_url_from_path(p, name) if url is not None: link = Link(url) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == "file" and re.search(r"\.\./", link.url): link = Link(path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req_as_string = f"{wheel.name}=={wheel.version}" else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req_as_string = link.egg_fragment # a requirement specifier else: req_as_string = name extras = convert_extras(extras_as_string) def with_source(text: str) -> str: if not line_source: return text return f"{text} (from {line_source})" def _parse_req_string(req_as_string: str) -> Requirement: try: req = get_requirement(req_as_string) except InvalidRequirement: if os.path.sep in req_as_string: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req_as_string) elif "=" in req_as_string and not any( op in req_as_string for op in operators ): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = "" msg = with_source(f"Invalid requirement: {req_as_string!r}") if add_msg: msg += f"\nHint: {add_msg}" raise InstallationError(msg) else: # Deprecate extras after specifiers: "name>=1.0[extras]" # This currently works by accident because _strip_extras() parses # any extras in the end of the string and those are saved in # RequirementParts for spec in req.specifier: spec_str = str(spec) if spec_str.endswith("]"): msg = f"Extras after version '{spec_str}'." raise InstallationError(msg) return req if req_as_string is not None: req: Optional[Requirement] = _parse_req_string(req_as_string) else: req = None return RequirementParts(req, link, markers, extras)
def _link_for_candidate(self, link, candidate): # type: (Link, str) -> Link root = self.get_path_for_link(link) path = os.path.join(root, candidate) return Link(path_to_url(path))
def __init__( self, req, # type: Optional[Requirement] comes_from, # type: Optional[Union[str, InstallRequirement]] source_dir=None, # type: Optional[str] editable=False, # type: bool link=None, # type: Optional[Link] update=True, # type: bool markers=None, # type: Optional[Marker] use_pep517=None, # type: Optional[bool] isolated=False, # type: bool options=None, # type: Optional[Dict[str, Any]] wheel_cache=None, # type: Optional[WheelCache] constraint=False, # type: bool extras=() # type: Iterable[str] ): # type: (...) -> None assert req is None or isinstance(req, Requirement), req self.req = req self.comes_from = comes_from self.constraint = constraint if source_dir is not None: self.source_dir = os.path.normpath(os.path.abspath(source_dir)) else: self.source_dir = None self.editable = editable self._wheel_cache = wheel_cache if link is None and req and req.url: # PEP 508 URL requirement link = Link(req.url) self.link = self.original_link = link if extras: self.extras = extras elif req: self.extras = { pkg_resources.safe_extra(extra) for extra in req.extras } else: self.extras = set() if markers is None and req: markers = req.marker self.markers = markers self._egg_info_path = None # type: Optional[str] # This holds the pkg_resources.Distribution object if this requirement # is already available: self.satisfied_by = None # This hold the pkg_resources.Distribution object if this requirement # conflicts with another installed distribution: self.conflicts_with = None # Temporary build location self._temp_build_dir = TempDirectory(kind="req-build") # Used to store the global directory where the _temp_build_dir should # have been created. Cf _correct_build_location method. self._ideal_build_dir = None # type: Optional[str] # True if the editable should be updated: self.update = update # Set to True after successful installation self.install_succeeded = None # type: Optional[bool] # UninstallPathSet of uninstalled distribution (for possible rollback) self.uninstalled_pathset = None self.options = options if options else {} # Set to True after successful preparation of this requirement self.prepared = False self.is_direct = False self.isolated = isolated self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment # For PEP 517, the directory where we request the project metadata # gets stored. We need this to pass to build_wheel, so the backend # can ensure that the wheel matches the metadata (see the PEP for # details). self.metadata_directory = None # type: Optional[str] # The static build requirements (from pyproject.toml) self.pyproject_requires = None # type: Optional[List[str]] # Build requirements that we will check are available self.requirements_to_check = [] # type: List[str] # The PEP 517 backend we should use to build the project self.pep517_backend = None # type: Optional[Pep517HookCaller] # Are we using PEP 517 for this requirement? # After pyproject.toml has been loaded, the only valid values are True # and False. Before loading, None is valid (meaning "use the default"). # Setting an explicit value before loading pyproject.toml is supported, # but after loading this flag should be treated as read only. self.use_pep517 = use_pep517
def file_links(self) -> FoundLinks: for path in self._path.iterdir(): url = path_to_url(str(path)) if _is_html_file(url): continue yield Link(url)