def request(self, host, handler, request_body, verbose=False): # type: (str, str, Dict[str, str], bool) -> None parts = (self._scheme, host, handler, None, None, None) url = urllib_parse.urlunparse(parts) try: headers = {'Content-Type': 'text/xml'} response = self._session.post(url, data=request_body, headers=headers, stream=True) raise_for_status(response) self.verbose = verbose return self.parse_response(response.raw) except NetworkConnectionError as exc: assert exc.response logger.critical( "HTTP error %s while getting %s", exc.response.status_code, url, ) raise
def get_file_content(url, session, comes_from=None): # type: (str, PipSession, Optional[str]) -> Tuple[str, Text] """Gets the content of a file; it may be a filename, file: URL, or http: URL. Returns (location, content). Content is unicode. Respects # -*- coding: declarations on the retrieved files. :param url: File path or url. :param session: PipSession instance. :param comes_from: Origin description of requirements. """ scheme = get_url_scheme(url) if scheme in ['http', 'https']: # FIXME: catch some errors resp = session.get(url) raise_for_status(resp) return resp.url, resp.text elif scheme == 'file': if comes_from and comes_from.startswith('http'): raise InstallationError('Requirements file {} references URL {}, ' 'which is local'.format(comes_from, url)) path = url.split(':', 1)[1] path = path.replace('\\', '/') match = _url_slash_drive_re.match(path) if match: path = match.group(1) + ':' + path.split('|', 1)[1] path = urllib_parse.unquote(path) if path.startswith('/'): path = '/' + path.lstrip('/') url = path try: with open(url, 'rb') as f: content = auto_decode(f.read()) except IOError as exc: raise InstallationError( 'Could not open requirements file: {}'.format(exc)) return url, content
def get_file_content(url: str, session: PipSession) -> Tuple[str, str]: """Gets the content of a file; it may be a filename, file: URL, or http: URL. Returns (location, content). Content is unicode. Respects # -*- coding: declarations on the retrieved files. :param url: File path or url. :param session: PipSession instance. """ scheme = get_url_scheme(url) # Pip has special support for file:// URLs (LocalFSAdapter). if scheme in ["http", "https", "file"]: resp = session.get(url) raise_for_status(resp) return resp.url, resp.text # Assume this is a bare path. try: with open(url, "rb") as f: content = auto_decode(f.read()) except OSError as exc: raise InstallationError(f"Could not open requirements file: {exc}") return url, content
def _http_get_download(session, link): # type: (PipSession, Link) -> Response target_url = link.url.split("#", 1)[0] resp = session.get(target_url, headers=HEADERS, stream=True) raise_for_status(resp) return resp
def _get_simple_response(url: str, session: PipSession) -> Response: """Access an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. """ if is_archive_file(Link(url).filename): _ensure_api_response(url, session=session) logger.debug("Getting page %s", redact_auth_from_url(url)) resp = session.get( url, headers={ "Accept": ", ".join([ "application/vnd.pypi.simple.v1+json", "application/vnd.pypi.simple.v1+html; q=0.1", "text/html; q=0.01", ]), # We don't want to blindly returned cached data for # /simple/, because authors generally expecting that # twine upload && pip install will function, but if # they've done a pip install in the last ~10 minutes # it won't. Thus by setting this to zero we will not # blindly use any cached data, however the benefit of # using max-age=0 instead of no-cache, is that we will # still support conditional requests, so we will still # minimize traffic sent in cases where the page hasn't # changed at all, we will just always incur the round # trip for the conditional GET now instead of only # once per 10 minutes. # For more information, please see pypa/pip#5670. "Cache-Control": "max-age=0", }, ) raise_for_status(resp) # The check for archives above only works if the url ends with # something that looks like an archive. However that is not a # requirement of an url. Unless we issue a HEAD request on every # url we cannot know ahead of time for sure if something is a # Simple API response or not. However we can check after we've # downloaded it. _ensure_api_header(resp) logger.debug( "Fetched page %s as %s", redact_auth_from_url(url), resp.headers.get("Content-Type", "Unknown"), ) return resp
# type: (str, PipSession, Optional[str]) -> Tuple[str, Text] """Gets the content of a file; it may be a filename, file: URL, or http: URL. Returns (location, content). Content is unicode. Respects # -*- coding: declarations on the retrieved files. :param url: File path or url. :param session: PipSession instance. :param comes_from: Origin description of requirements. """ scheme = get_url_scheme(url) if scheme in ['http', 'https']: # FIXME: catch some errors resp = session.get(url) <<<<<<< HEAD raise_for_status(resp) ======= resp.raise_for_status() >>>>>>> b66a76afa15ab74019740676a52a071b85ed8f71 return resp.url, resp.text elif scheme == 'file': if comes_from and comes_from.startswith('http'): raise InstallationError( 'Requirements file {} references URL {}, ' 'which is local'.format(comes_from, url) ) path = url.split(':', 1)[1] path = path.replace('\\', '/') match = _url_slash_drive_re.match(path)
self._scheme = index_parts.scheme self._session = session def request(self, host, handler, request_body, verbose=False): <<<<<<< HEAD # type: (str, str, Dict[str, str], bool) -> None ======= >>>>>>> b66a76afa15ab74019740676a52a071b85ed8f71 parts = (self._scheme, host, handler, None, None, None) url = urllib_parse.urlunparse(parts) try: headers = {'Content-Type': 'text/xml'} response = self._session.post(url, data=request_body, headers=headers, stream=True) <<<<<<< HEAD raise_for_status(response) self.verbose = verbose return self.parse_response(response.raw) except NetworkConnectionError as exc: assert exc.response ======= response.raise_for_status() self.verbose = verbose return self.parse_response(response.raw) except requests.HTTPError as exc: >>>>>>> b66a76afa15ab74019740676a52a071b85ed8f71 logger.critical( "HTTP error %s while getting %s", exc.response.status_code, url, ) raise