def get_rev_options_args(url): """ Return the extra arguments to pass to RevOptions. """ r = urllib_parse.urlsplit(url) if hasattr(r, 'username'): # >= Python-2.5 username, password = r.username, r.password else: netloc = r[1] if '@' in netloc: auth = netloc.split('@')[0] if ':' in auth: username, password = auth.split(':', 1) else: username, password = auth, None else: username, password = None, None extra_args = [] if username: extra_args += ['--username', username] if password: extra_args += ['--password', password] return extra_args
def _transform_url(url, transform_netloc): purl = urllib_parse.urlsplit(url) netloc = transform_netloc(purl.netloc) # stripped url url_pieces = (purl.scheme, netloc, purl.path, purl.query, purl.fragment) surl = urllib_parse.urlunsplit(url_pieces) return surl
def filename(self): # type: () -> str _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) name = posixpath.basename(path.rstrip('/')) or netloc name = urllib_parse.unquote(name) assert name, ('URL %r produced no filename' % self.url) return name
def get_rev_options(vcs, url, rev): """ Return a RevOptions object. """ r = urllib_parse.urlsplit(url) if hasattr(r, 'username'): # >= Python-2.5 username, password = r.username, r.password else: netloc = r[1] if '@' in netloc: auth = netloc.split('@')[0] if ':' in auth: username, password = auth.split(':', 1) else: username, password = auth, None else: username, password = None, None extra_args = [] if username: extra_args += ['--username', username] if password: extra_args += ['--password', password] return vcs.make_rev_options(rev, extra_args=extra_args)
def get_rev_options(vcs, url, rev): """ Return a RevOptions object. """ r = urllib_parse.urlsplit(url) if hasattr(r, 'username'): # >= Python-2.5 username, password = r.username, r.password else: netloc = r[1] if '@' in netloc: auth = netloc.split('@')[0] if ':' in auth: username, password = auth.split(':', 1) else: username, password = auth, None else: username, password = None, None extra_args = [] if username: extra_args += ['--username', username] if password: extra_args += ['--password', password] return vcs.make_rev_options(rev, extra_args=extra_args)
def __init__(self, url, file_storage_domain): super(PackageIndex, self).__init__() self.url = url self.netloc = urllib_parse.urlsplit(url).netloc self.simple_url = self._url_for_path('simple') self.pypi_url = self._url_for_path('pypi') # This is part of a temporary hack used to block installs of PyPI # packages which depend on external urls only necessary until PyPI can # block such packages themselves self.file_storage_domain = file_storage_domain
def _get_content_type(url, session): """Get the Content-Type of the given url, using a HEAD request""" scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) if scheme not in {'http', 'https'}: # FIXME: some warning or something? # assertion error? return '' resp = session.head(url, allow_redirects=True) resp.raise_for_status() return resp.headers.get("Content-Type", "")
def _get_content_type(url, session): """Get the Content-Type of the given url, using a HEAD request""" scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) if scheme not in {'http', 'https'}: # FIXME: some warning or something? # assertion error? return '' resp = session.head(url, allow_redirects=True) resp.raise_for_status() return resp.headers.get("Content-Type", "")
def remove_auth_from_url(url): # Return a copy of url with 'username:password@' removed. # username/pass params are passed to subversion through flags # and are not recognized in the url. # parsed url purl = urllib_parse.urlsplit(url) netloc, user_pass = split_auth_from_netloc(purl.netloc) # stripped url url_pieces = (purl.scheme, netloc, purl.path, purl.query, purl.fragment) surl = urllib_parse.urlunsplit(url_pieces) return surl
def url_to_path(url): """ Convert a file: URL to a path. """ assert url.startswith('file:'), ( "You can only turn file: urls into filenames (not %r)" % url) _, netloc, path, _, _ = urllib_parse.urlsplit(url) # if we have a UNC path, prepend UNC share notation if netloc: netloc = '\\\\' + netloc path = urllib_request.url2pathname(netloc + path) return path
def _ensure_html_response(url, session): # type: (str, PipSession) -> None """Send a HEAD request to the URL, and ensure the response contains HTML. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotHTML` if the content type is not text/html. """ scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) if scheme not in {'http', 'https'}: raise _NotHTTP() resp = session.head(url, allow_redirects=True) resp.raise_for_status() _ensure_html_header(resp)
def url_to_path(url): """ Convert a file: URL to a path. """ assert url.startswith('file:'), ( "You can only turn file: urls into filenames (not %r)" % url) _, netloc, path, _, _ = urllib_parse.urlsplit(url) # if we have a UNC path, prepend UNC share notation if netloc: netloc = '\\\\' + netloc path = urllib_request.url2pathname(netloc + path) return path
def get_url_rev(self): """ Returns the correct repository URL and revision by parsing the given repository URL """ error_message = ("Sorry, '%s' is a malformed VCS url. " "The format is <vcs>+<protocol>://<url>, " "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp") assert '+' in self.url, error_message % self.url url = self.url.split('+', 1)[1] scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) rev = None if '@' in path: path, rev = path.rsplit('@', 1) url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) return url, rev
def remove_auth_from_url(url): # Return a copy of url with 'username:password@' removed. # username/pass params are passed to subversion through flags # and are not recognized in the url. # parsed url purl = urllib_parse.urlsplit(url) stripped_netloc = \ purl.netloc.split('@')[-1] # stripped url url_pieces = ( purl.scheme, stripped_netloc, purl.path, purl.query, purl.fragment ) surl = urllib_parse.urlunsplit(url_pieces) return surl
def _transform_url(url, transform_netloc): """Transform and replace netloc in a url. transform_netloc is a function taking the netloc and returning a tuple. The first element of this tuple is the new netloc. The entire tuple is returned. Returns a tuple containing the transformed url as item 0 and the original tuple returned by transform_netloc as item 1. """ purl = urllib_parse.urlsplit(url) netloc_tuple = transform_netloc(purl.netloc) # stripped url url_pieces = (purl.scheme, netloc_tuple[0], purl.path, purl.query, purl.fragment) surl = urllib_parse.urlunsplit(url_pieces) return surl, netloc_tuple
def get_url_rev(self): """ Returns the correct repository URL and revision by parsing the given repository URL """ error_message = ( "Sorry, '%s' is a malformed VCS url. " "The format is <vcs>+<protocol>://<url>, " "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp" ) assert '+' in self.url, error_message % self.url url = self.url.split('+', 1)[1] scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) rev = None if '@' in path: path, rev = path.rsplit('@', 1) url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) return url, rev
def get_url_rev_and_auth(self, url): """ Parse the repository URL to use, and return the URL, revision, and auth info to use. Returns: (url, rev, (username, password)). """ scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) if '+' not in scheme: raise ValueError( "Sorry, {!r} is a malformed VCS url. " "The format is <vcs>+<protocol>://<url>, " "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url)) # Remove the vcs prefix. scheme = scheme.split('+', 1)[1] netloc, user_pass = self.get_netloc_and_auth(netloc, scheme) rev = None if '@' in path: path, rev = path.rsplit('@', 1) url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) return url, rev, user_pass
def url_to_path(url): # type: (str) -> str """ Convert a file: URL to a path. """ assert url.startswith('file:'), ( "You can only turn file: urls into filenames (not %r)" % url) _, netloc, path, _, _ = urllib_parse.urlsplit(url) if not netloc or netloc == 'localhost': # According to RFC 8089, same as empty authority. netloc = '' elif sys.platform == 'win32': # If we have a UNC path, prepend UNC share notation. netloc = '\\\\' + netloc else: raise ValueError( 'non-local file URIs are not supported on this platform: %r' % url) path = urllib_request.url2pathname(netloc + path) return path
def __init__( self, url, # type: str comes_from=None, # type: Optional[Union[str, HTMLPage]] requires_python=None, # type: Optional[str] yanked_reason=None, # type: Optional[Text] ): # type: (...) -> None """ :param url: url of the resource pointed to (href of the link) :param comes_from: instance of HTMLPage where the link was found, or string. :param requires_python: String containing the `Requires-Python` metadata field, specified in PEP 345. This may be specified by a data-requires-python attribute in the HTML link tag, as described in PEP 503. :param yanked_reason: the reason the file has been yanked, if the file has been yanked, or None if the file hasn't been yanked. This is the value of the "data-yanked" attribute, if present, in a simple repository HTML link. If the file has been yanked but no reason was provided, this should be the empty string. See PEP 592 for more information and the specification. """ # url can be a UNC windows share if url.startswith('\\\\'): url = path_to_url(url) self._parsed_url = urllib_parse.urlsplit(url) # Store the url as a private attribute to prevent accidentally # trying to set a new value. self._url = url self.comes_from = comes_from self.requires_python = requires_python if requires_python else None self.yanked_reason = yanked_reason super(Link, self).__init__(key=url, defining_class=Link)
def netloc(self): return urllib_parse.urlsplit(self.url)[1]
def __init__(self, url): self.url = url self.netloc = urllib_parse.urlsplit(url).netloc self.simple_url = self.url_to_path('simple') self.pypi_url = self.url_to_path('pypi')
def path(self): return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2])
def url_without_fragment(self): scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) return urllib_parse.urlunsplit((scheme, netloc, path, query, None))
def scheme(self): return urllib_parse.urlsplit(self.url)[0]
def netloc(self): return urllib_parse.urlsplit(self.url)[1]
def url_without_fragment(self): # type: () -> str scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) return urllib_parse.urlunsplit((scheme, netloc, path, query, None))
def filename(self): _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) name = posixpath.basename(path.rstrip('/')) or netloc name = urllib_parse.unquote(name) assert name, ('URL %r produced no filename' % self.url) return name
def path(self): # type: () -> str return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2])
def netloc(self): # type: () -> str return urllib_parse.urlsplit(self.url)[1]
def scheme(self): # type: () -> str return urllib_parse.urlsplit(self.url)[0]
def scheme(self): return urllib_parse.urlsplit(self.url)[0]
def path(self): return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2])