def test_url_to_path(url, win_expected, non_win_expected): if sys.platform == 'win32': expected_path = win_expected else: expected_path = non_win_expected if expected_path is None: with pytest.raises(ValueError): url_to_path(url) else: assert url_to_path(url) == expected_path
def test_url_to_path(url: str, win_expected: str, non_win_expected: str) -> None: if sys.platform == "win32": expected_path = win_expected else: expected_path = non_win_expected if expected_path is None: with pytest.raises(ValueError): url_to_path(url) else: assert url_to_path(url) == expected_path
def get_file_content(url, session, comes_from=None): # type: (str, PipSession, Optional[str]) -> Tuple[str, Text] """Gets the content of a file; it may be a filename, file: URL, or http: URL. Returns (location, content). Content is unicode. Respects # -*- coding: declarations on the retrieved files. :param url: File path or url. :param session: PipSession instance. :param comes_from: Origin description of requirements. """ scheme = get_url_scheme(url) if scheme in ['http', 'https']: # FIXME: catch some errors resp = session.get(url) raise_for_status(resp) return resp.url, resp.text elif scheme == 'file': if comes_from and comes_from.startswith('http'): raise InstallationError( 'Requirements file {} references URL {}, ' 'which is local'.format(comes_from, url) ) url = url_to_path(url) try: with open(url, 'rb') as f: content = auto_decode(f.read()) except IOError as exc: raise InstallationError( 'Could not open requirements file: {}'.format(exc) ) return url, content
def send(self, request, stream=None, timeout=None, verify=None, cert=None, proxies=None): pathname = url_to_path(request.url) resp = Response() resp.status_code = 200 resp.url = request.url try: stats = os.stat(pathname) except OSError as exc: resp.status_code = 404 resp.raw = exc else: modified = email.utils.formatdate(stats.st_mtime, usegmt=True) content_type = mimetypes.guess_type(pathname)[0] or "text/plain" resp.headers = CaseInsensitiveDict({ "Content-Type": content_type, "Content-Length": stats.st_size, "Last-Modified": modified, }) resp.raw = open(pathname, "rb") resp.close = resp.raw.close return resp
def install_req_from_editable( editable_req, # type: str comes_from=None, # type: Optional[str] use_pep517=None, # type: Optional[bool] isolated=False, # type: bool options=None, # type: Optional[Dict[str, Any]] wheel_cache=None, # type: Optional[WheelCache] constraint=False # type: bool ): # type: (...) -> InstallRequirement name, url, extras_override = parse_editable(editable_req) if url.startswith('file:'): source_dir = url_to_path(url) else: source_dir = None if name is not None: try: req = Requirement(name) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % name) else: req = None return InstallRequirement( req, comes_from, source_dir=source_dir, editable=True, link=Link(url), constraint=constraint, use_pep517=use_pep517, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, extras=extras_override or (), )
def open_local_or_remote_file(link, session): """ Open local or remote file for reading. :type link: pip.index.Link :type session: requests.Session :raises ValueError: If link points to a local directory. :return: a context manager to a FileStream with the opened file-like object """ url = link.url_without_fragment if link.is_file: # Local URL local_path = url_to_path(url) if os.path.isdir(local_path): raise ValueError(f"Cannot open directory for read: {url}") else: st = os.stat(local_path) with open(local_path, "rb") as local_file: yield FileStream(stream=local_file, size=st.st_size) else: # Remote URL headers = {"Accept-Encoding": "identity"} response = session.get(url, headers=headers, stream=True) # Content length must be int or None try: content_length = int(response.headers["content-length"]) except (ValueError, KeyError, TypeError): content_length = None try: yield FileStream(stream=response.raw, size=content_length) finally: response.close()
def send( self, request: PreparedRequest, stream: bool = False, timeout: Optional[Union[float, Tuple[float, float]]] = None, verify: Union[bool, str] = True, cert: Optional[Union[str, Tuple[str, str]]] = None, proxies: Optional[Mapping[str, str]] = None, ) -> Response: pathname = url_to_path(request.url) resp = Response() resp.status_code = 200 resp.url = request.url try: stats = os.stat(pathname) except OSError as exc: resp.status_code = 404 resp.raw = exc else: modified = email.utils.formatdate(stats.st_mtime, usegmt=True) content_type = mimetypes.guess_type(pathname)[0] or "text/plain" resp.headers = CaseInsensitiveDict( { "Content-Type": content_type, "Content-Length": stats.st_size, "Last-Modified": modified, } ) resp.raw = open(pathname, "rb") resp.close = resp.raw.close return resp
def group_locations(locations, expand_dir=False): # type: (Sequence[str], bool) -> Tuple[List[str], List[str]] """ Divide a list of locations into two groups: "files" (archives) and "urls." :return: A pair of lists (files, urls). """ files = [] urls = [] # puts the url for the given file path into the appropriate list def sort_path(path): # type: (str) -> None url = path_to_url(path) if mimetypes.guess_type(url, strict=False)[0] == 'text/html': urls.append(url) else: files.append(url) for url in locations: is_local_path = os.path.exists(url) is_file_url = url.startswith('file:') if is_local_path or is_file_url: if is_local_path: path = url else: path = url_to_path(url) if os.path.isdir(path): if expand_dir: path = os.path.realpath(path) for item in os.listdir(path): sort_path(os.path.join(path, item)) elif is_file_url: urls.append(url) else: logger.warning( "Path '%s' is ignored: it is a directory.", path, ) elif os.path.isfile(path): sort_path(path) else: logger.warning( "Url '%s' is ignored: it is neither a file " "nor a directory.", url, ) elif is_url(url): # Only add url with clear scheme urls.append(url) else: logger.warning( "Url '%s' is ignored. It is either a non-existing " "path or lacks a specific scheme.", url, ) return files, urls
def send(self, request, stream=None, timeout=None, verify=None, cert=None, proxies=None): pathname = url_to_path(request.url) resp = Response() resp.status_code = 200 resp."""PipSession and supporting code, containing all pip-specific
def get_file_content(url, session): # type: (str, PipSession) -> Tuple[str, str] """Gets the content of a file; it may be a filename, file: URL, or http: URL. Returns (location, content). Content is unicode. Respects # -*- coding: declarations on the retrieved files. :param url: File path or url. :param session: PipSession instance. """ scheme = get_url_scheme(url) if scheme in ['http', 'https']: # FIXME: catch some errors resp = session.get(url) raise_for_status(resp) return resp.url, resp.text elif scheme == 'file': url = url_to_path(url) try: with open(url, 'rb') as f: content = auto_decode(f.read()) except OSError as exc: raise InstallationError(f'Could not open requirements file: {exc}') return url, content
def build_source( location: str, *, candidates_from_page: CandidatesFromPage, page_validator: PageValidator, expand_dir: bool, cache_link_parsing: bool, ) -> Tuple[Optional[str], Optional[LinkSource]]: path: Optional[str] = None url: Optional[str] = None if os.path.exists(location): # Is a local path. url = path_to_url(location) path = location elif location.startswith("file:"): # A file: URL. url = location path = url_to_path(location) elif is_url(location): url = location if url is None: msg = ( "Location '%s' is ignored: " "it is either a non-existing path or lacks a specific scheme." ) logger.warning(msg, location) return (None, None) if path is None: source: LinkSource = _RemoteFileSource( candidates_from_page=candidates_from_page, page_validator=page_validator, link=Link(url, cache_link_parsing=cache_link_parsing), ) return (url, source) if os.path.isdir(path): if expand_dir: source = _FlatDirectorySource( candidates_from_page=candidates_from_page, path=path, ) else: source = _IndexDirectorySource( candidates_from_page=candidates_from_page, link=Link(url, cache_link_parsing=cache_link_parsing), ) return (url, source) elif os.path.isfile(path): source = _LocalFileSource( candidates_from_page=candidates_from_page, link=Link(url, cache_link_parsing=cache_link_parsing), ) return (url, source) logger.warning( "Location '%s' is ignored: it is neither a file nor a directory.", location, ) return (url, None)
def is_dir_url(link): # type: (Link) -> bool """Return whether a file:// Link points to a directory. ``link`` must not have any other scheme but file://. Call is_file_url() first. """ link_path = url_to_path(link.url_without_fragment) return os.path.isdir(link_path)
def unpack_file_url( link, # type: Link location, # type: str download_dir=None, # type: Optional[str] hashes=None # type: Optional[Hashes] ): # type: (...) -> None """Unpack link into location. If download_dir is provided and link points to a file, make a copy of the link file inside download_dir. """ link_path = url_to_path(link.url_without_fragment) # If it's a url to a local directory if is_dir_url(link): if os.path.isdir(location): rmtree(location) _copy_source_tree(link_path, location) if download_dir: logger.info('Link is a directory, ignoring download_dir') return # If --require-hashes is off, `hashes` is either empty, the # link's embedded hash, or MissingHashes; it is required to # match. If --require-hashes is on, we are satisfied by any # hash in `hashes` matching: a URL-based or an option-based # one; no internet-sourced hash will be in `hashes`. if hashes: hashes.check_against_path(link_path) # If a download dir is specified, is the file already there and valid? already_downloaded_path = None if download_dir: already_downloaded_path = _check_download_dir(link, download_dir, hashes) if already_downloaded_path: from_path = already_downloaded_path else: from_path = link_path content_type = mimetypes.guess_type(from_path)[0] # unpack the archive to the build dir location. even when only downloading # archives, they have to be unpacked to parse dependencies unpack_file(from_path, location, content_type, link) # a download dir is specified and not already downloaded if download_dir and not already_downloaded_path: _copy_file(from_path, download_dir, link)
def find_all_candidates(self, project_name): # type: (str) -> List[InstallationCandidate] """Find all available InstallationCandidate for project_name This checks index_urls and find_links. All versions found are returned as an InstallationCandidate list. See LinkEvaluator.evaluate_link() for details on which files are accepted. """ collected_links = self._link_collector.collect_links(project_name) link_evaluator = self.make_link_evaluator(project_name) find_links_versions = self.evaluate_links( link_evaluator, links=collected_links.find_links, ) page_versions = [] for page_url, page_links in collected_links.pages.items(): logger.debug('Analyzing links from page %s', page_url) with indent_log(): new_versions = self.evaluate_links( link_evaluator, links=page_links, ) page_versions.extend(new_versions) file_versions = self.evaluate_links( link_evaluator, links=collected_links.files, ) if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.link.url) for candidate in file_versions ]) ) # This is an intentional priority ordering return file_versions + find_links_versions + page_versions
def find_all_candidates(self, project_name: str) -> List[InstallationCandidate]: """Find all available InstallationCandidate for project_name This checks index_urls and find_links. All versions found are returned as an InstallationCandidate list. See LinkEvaluator.evaluate_link() for details on which files are accepted. """ link_evaluator = self.make_link_evaluator(project_name) collected_sources = self._link_collector.collect_sources( project_name=project_name, candidates_from_page=functools.partial( self.process_project_url, link_evaluator=link_evaluator, ), ) page_candidates_it = itertools.chain.from_iterable( source.page_candidates() for sources in collected_sources for source in sources if source is not None ) page_candidates = list(page_candidates_it) file_links_it = itertools.chain.from_iterable( source.file_links() for sources in collected_sources for source in sources if source is not None ) file_candidates = self.evaluate_links( link_evaluator, sorted(file_links_it, reverse=True), ) if logger.isEnabledFor(logging.DEBUG) and file_candidates: paths = [url_to_path(c.link.url) for c in file_candidates] logger.debug("Local files found: %s", ", ".join(paths)) # This is an intentional priority ordering return file_candidates + page_candidates
def find_all_candidates(self, project_name): # type: (str) -> List[InstallationCandidate] """Find all available InstallationCandidate for project_name This checks index_urls and find_links. All versions found are returned as an InstallationCandidate list. See LinkEvaluator.evaluate_link() for details on which files are accepted. """ collected_links = self._link_collector.collect_links(project_name) link_evaluator = self.make_link_evaluator(project_name) find_links_versions = self.evaluate_links( link_evaluator, links=collected_links.find_links, ) page_versions = [] for project_url in collected_links.project_urls: package_links = self.process_project_url( project_url, link_evaluator=link_evaluator, ) page_versions.extend(package_links) file_versions = self.evaluate_links( link_evaluator, links=collected_links.files, ) if file_versions: file_versions.sort(reverse=True) logger.debug( "Local files found: %s", ", ".join([ url_to_path(candidate.link.url) for candidate in file_versions ]), ) # This is an intentional priority ordering return file_versions + find_links_versions + page_versions
def editable_project_location(self) -> Optional[str]: """The project location for editable distributions. This is the directory where pyproject.toml or setup.py is located. None if the distribution is not installed in editable mode. """ # TODO: this property is relatively costly to compute, memoize it ? direct_url = self.direct_url if direct_url: if direct_url.is_local_editable(): return url_to_path(direct_url.url) else: # Search for an .egg-link file by walking sys.path, as it was # done before by dist_is_editable(). egg_link_path = egg_link_path_from_sys_path(self.raw_name) if egg_link_path: # TODO: get project location from second line of egg_link file # (https://github.com/pypa/pip/issues/10243) return self.location return None
<<<<<<< HEAD ======= :param comes_from: Origin description of requirements. >>>>>>> 74c061954d5e927be4caafbd793e96a50563c265 """ scheme = get_url_scheme(url) if scheme in ['http', 'https']: # FIXME: catch some errors resp = session.get(url) raise_for_status(resp) return resp.url, resp.text elif scheme == 'file': <<<<<<< HEAD url = url_to_path(url) ======= if comes_from and comes_from.startswith('http'): raise InstallationError( 'Requirements file {} references URL {}, ' 'which is local'.format(comes_from, url) ) path = url.split(':', 1)[1] path = path.replace('\\', '/') match = _url_slash_drive_re.match(path) if match: path = match.group(1) + ':' + path.split('|', 1)[1] path = urllib_parse.unquote(path) if path.startswith('/'): path = '/' + path.lstrip('/')
def test_url_to_path_path_to_url_symmetry_win() -> None: path = r"C:\tmp\file" assert url_to_path(path_to_url(path)) == path unc_path = r"\\unc\share\path" assert url_to_path(path_to_url(unc_path)) == unc_path
def test_url_to_path_path_to_url_symmetry_win(): path = r'C:\tmp\file' assert url_to_path(path_to_url(path)) == path unc_path = r'\\unc\share\path' assert url_to_path(path_to_url(unc_path)) == unc_path
def file_path(self): # type: () -> str return url_to_path(self.url)
def file_path(self) -> str: return url_to_path(self.url)
def prepare_linked_requirement( self, req, # type: InstallRequirement session, # type: PipSession finder, # type: PackageFinder require_hashes, # type: bool ): # type: (...) -> AbstractDistribution """Prepare a requirement that would be obtained from req.link """ assert req.link link = req.link # TODO: Breakup into smaller functions if link.scheme == 'file': path = url_to_path(link.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req.req or req) with indent_log(): # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory req.ensure_has_source_dir(self.build_dir) # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req.source_dir` if os.path.exists(os.path.join(req.source_dir, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing build directory (%s). This is " "likely due to a previous installation that failed" ". pip is being responsible and not assuming it " "can delete this. Please delete it and try again." % (req, req.source_dir) ) # Now that we have the real link, we can tell what kind of # requirements we have and raise some more informative errors # than otherwise. (For example, we can raise VcsHashUnsupported # for a VCS URL rather than HashMissing.) if require_hashes: # We could check these first 2 conditions inside # unpack_url and save repetition of conditions, but then # we would report less-useful error messages for # unhashable requirements, complaining that there's no # hash provided. if link.is_vcs: raise VcsHashUnsupported() elif is_file_url(link) and is_dir_url(link): raise DirectoryUrlHashUnsupported() if not req.original_link and not req.is_pinned: # Unpinned packages are asking for trouble when a new # version is uploaded. This isn't a security check, but # it saves users a surprising hash mismatch in the # future. # # file:/// URLs aren't pinnable, so don't complain # about them not being pinned. raise HashUnpinned() hashes = req.hashes(trust_internet=not require_hashes) if require_hashes and not hashes: # Known-good hashes are missing for this requirement, so # shim it with a facade object that will provoke hash # computation and then raise a HashMissing exception # showing the user what the hash should be. hashes = MissingHashes() download_dir = self.download_dir if link.is_wheel and self.wheel_download_dir: # when doing 'pip wheel` we download wheels to a # dedicated dir. download_dir = self.wheel_download_dir try: unpack_url( link, req.source_dir, download_dir, session=session, hashes=hashes, progress_bar=self.progress_bar ) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because of error %s', req, exc, ) raise InstallationError( 'Could not install requirement %s because of HTTP ' 'error %s for URL %s' % (req, exc, link) ) if link.is_wheel: if download_dir: # When downloading, we only unpack wheels to get # metadata. autodelete_unpacked = True else: # When installing a wheel, we use the unpacked # wheel. autodelete_unpacked = False else: # We always delete unpacked sdists after pip runs. autodelete_unpacked = True if autodelete_unpacked: write_delete_marker_file(req.source_dir) abstract_dist = _get_prepared_distribution( req, self.req_tracker, finder, self.build_isolation, ) if self._download_should_save: # Make a .zip of the source_dir we already created. if not link.is_artifact: req.archive(self.download_dir) return abstract_dist