def _set_query(self, value):
query = utils.urlencode(value)
scheme, netloc, path, params, _, fragment = urllib.parse.urlparse(
self.url)
_, _, _, self.path = utils.parse_url(
urllib.parse.urlunparse(
[scheme, netloc, path, params, query, fragment]))
def query(self, odict):
query = utils.urlencode(odict.lst)
scheme, netloc, path, params, _, fragment = urllib.parse.urlparse(
self.url)
_, _, _, self.path = utils.parse_url(
urllib.parse.urlunparse(
[scheme, netloc, path, params, query, fragment]))
def path_components(self, components):
components = map(lambda x: urllib.parse.quote(x, safe=""), components)
path = "/" + "/".join(components)
scheme, netloc, _, params, query, fragment = urllib.parse.urlparse(
self.url)
_, _, _, self.path = utils.parse_url(
urllib.parse.urlunparse(
[scheme, netloc, path, params, query, fragment]))
def _parse_init_proxy(self, line):
v = self._parse_init(line)
if not v:
return None
method, url, httpversion = v
parts = utils.parse_url(url)
if not parts:
return None
scheme, host, port, path = parts
return method, scheme, host, port, path, httpversion
def url(self, url):
"""
Parses a URL specification, and updates the Request's information
accordingly.
Returns False if the URL was invalid, True if the request succeeded.
"""
parts = utils.parse_url(url)
if not parts:
raise ValueError("Invalid URL: %s" % url)
self.scheme, self.host, self.port, self.path = parts
def _parse_init_proxy(self, line):
v = self._parse_init(line)
if not v:
return None
method, url, httpversion = v
parts = utils.parse_url(url)
if not parts:
return None
scheme, host, port, path = parts
return method, scheme, host, port, path, httpversion
def url(self, url):
"""
Parses a URL specification, and updates the Request's information
accordingly.
Returns False if the URL was invalid, True if the request succeeded.
"""
parts = utils.parse_url(url)
if not parts:
raise ValueError("Invalid URL: %s" % url)
self.scheme, self.host, self.port, self.path = parts
def read_request(self):
self.request_data_finished.wait()
authority = self.request_headers.get(':authority', '')
method = self.request_headers.get(':method', 'GET')
scheme = self.request_headers.get(':scheme', 'https')
path = self.request_headers.get(':path', '/')
host = None
port = None
if path == '*' or path.startswith("/"):
first_line_format = "relative"
elif method == 'CONNECT': # pragma: no cover
raise NotImplementedError("CONNECT over HTTP/2 is not implemented.")
else: # pragma: no cover
first_line_format = "absolute"
# FIXME: verify if path or :host contains what we need
scheme, host, port, _ = parse_url(path)
if authority:
host, _, port = authority.partition(':')
if not host:
host = 'localhost'
if not port:
port = 443 if scheme == 'https' else 80
port = int(port)
data = []
while self.request_data_queue.qsize() > 0:
data.append(self.request_data_queue.get())
data = b"".join(data)
return HTTPRequest(
first_line_format,
method,
scheme,
host,
port,
path,
b"HTTP/2.0",
self.request_headers,
data,
timestamp_start=self.timestamp_start,
timestamp_end=self.timestamp_end,
)
def url(self, url):
self.scheme, self.host, self.port, self.path = utils.parse_url(url)
def read_request(
self,
include_body=True,
body_size_limit=None,
allow_empty=False,
):
if body_size_limit is not None:
raise NotImplementedError()
self.perform_connection_preface()
timestamp_start = time.time()
if hasattr(self.tcp_handler.rfile, "reset_timestamps"):
self.tcp_handler.rfile.reset_timestamps()
stream_id, headers, body = self._receive_transmission(
include_body=include_body,
)
if hasattr(self.tcp_handler.rfile, "first_byte_timestamp"):
# more accurate timestamp_start
timestamp_start = self.tcp_handler.rfile.first_byte_timestamp
timestamp_end = time.time()
authority = headers.get(':authority', '')
method = headers.get(':method', 'GET')
scheme = headers.get(':scheme', 'https')
path = headers.get(':path', '/')
host = None
port = None
if path == '*' or path.startswith("/"):
form_in = "relative"
elif method == 'CONNECT':
form_in = "authority"
if ":" in authority:
host, port = authority.split(":", 1)
else:
host = authority
else:
form_in = "absolute"
# FIXME: verify if path or :host contains what we need
scheme, host, port, _ = utils.parse_url(path)
if host is None:
host = 'localhost'
if port is None:
port = 80 if scheme == 'http' else 443
port = int(port)
request = http.Request(
form_in,
method,
scheme,
host,
port,
path,
(2, 0),
headers,
body,
timestamp_start,
timestamp_end,
)
# FIXME: We should not do this.
request.stream_id = stream_id
return request
def test_parse_url():
assert not utils.parse_url("")
u = "http://foo.com:8888/test"
s, h, po, pa = utils.parse_url(u)
assert s == "http"
assert h == "foo.com"
assert po == 8888
assert pa == "/test"
s, h, po, pa = utils.parse_url("http://foo/bar")
assert s == "http"
assert h == "foo"
assert po == 80
assert pa == "/bar"
s, h, po, pa = utils.parse_url("http://*****:*****@foo/bar")
assert s == "http"
assert h == "foo"
assert po == 80
assert pa == "/bar"
s, h, po, pa = utils.parse_url("http://foo")
assert pa == "/"
s, h, po, pa = utils.parse_url("https://foo")
assert po == 443
assert not utils.parse_url("https://foo:bar")
assert not utils.parse_url("https://foo:")
# Invalid IDNA
assert not utils.parse_url("http://\xfafoo")
# Invalid PATH
assert not utils.parse_url("http:/\xc6/localhost:56121")
# Null byte in host
assert not utils.parse_url("http://foo\0")
# Port out of range
assert not utils.parse_url("http://foo:999999")
# Invalid IPv6 URL - see http://www.ietf.org/rfc/rfc2732.txt
assert not utils.parse_url('http://lo[calhost')
def path_components(self, components):
components = map(lambda x: urllib.parse.quote(x, safe=""), components)
path = "/" + "/".join(components)
scheme, netloc, _, params, query, fragment = urllib.parse.urlparse(self.url)
_, _, _, self.path = utils.parse_url(
urllib.parse.urlunparse([scheme, netloc, path, params, query, fragment]))
def url(self, url):
self.scheme, self.host, self.port, self.path = utils.parse_url(url)
def test_parse_url():
with tutils.raises(ValueError):
utils.parse_url("")
s, h, po, pa = utils.parse_url(b"http://foo.com:8888/test")
assert s == b"http"
assert h == b"foo.com"
assert po == 8888
assert pa == b"/test"
s, h, po, pa = utils.parse_url("http://foo/bar")
assert s == b"http"
assert h == b"foo"
assert po == 80
assert pa == b"/bar"
s, h, po, pa = utils.parse_url(b"http://*****:*****@foo/bar")
assert s == b"http"
assert h == b"foo"
assert po == 80
assert pa == b"/bar"
s, h, po, pa = utils.parse_url(b"http://foo")
assert pa == b"/"
s, h, po, pa = utils.parse_url(b"https://foo")
assert po == 443
with tutils.raises(ValueError):
utils.parse_url(b"https://foo:bar")
# Invalid IDNA
with tutils.raises(ValueError):
utils.parse_url("http://\xfafoo")
# Invalid PATH
with tutils.raises(ValueError):
utils.parse_url("http:/\xc6/localhost:56121")
# Null byte in host
with tutils.raises(ValueError):
utils.parse_url("http://foo\0")
# Port out of range
_, _, port, _ = utils.parse_url("http://foo:999999")
assert port == 80
# Invalid IPv6 URL - see http://www.ietf.org/rfc/rfc2732.txt
with tutils.raises(ValueError):
utils.parse_url('http://lo[calhost')
def test_parse_url():
with tutils.raises(ValueError):
utils.parse_url("")
s, h, po, pa = utils.parse_url(b"http://foo.com:8888/test")
assert s == b"http"
assert h == b"foo.com"
assert po == 8888
assert pa == b"/test"
s, h, po, pa = utils.parse_url("http://foo/bar")
assert s == b"http"
assert h == b"foo"
assert po == 80
assert pa == b"/bar"
s, h, po, pa = utils.parse_url(b"http://*****:*****@foo/bar")
assert s == b"http"
assert h == b"foo"
assert po == 80
assert pa == b"/bar"
s, h, po, pa = utils.parse_url(b"http://foo")
assert pa == b"/"
s, h, po, pa = utils.parse_url(b"https://foo")
assert po == 443
with tutils.raises(ValueError):
utils.parse_url(b"https://foo:bar")
# Invalid IDNA
with tutils.raises(ValueError):
utils.parse_url("http://\xfafoo")
# Invalid PATH
with tutils.raises(ValueError):
utils.parse_url("http:/\xc6/localhost:56121")
# Null byte in host
with tutils.raises(ValueError):
utils.parse_url("http://foo\0")
# Port out of range
_, _, port, _ = utils.parse_url("http://foo:999999")
assert port == 80
# Invalid IPv6 URL - see http://www.ietf.org/rfc/rfc2732.txt
with tutils.raises(ValueError):
utils.parse_url('http://lo[calhost')
def test_parse_url():
assert not utils.parse_url("")
u = "http://foo.com:8888/test"
s, h, po, pa = utils.parse_url(u)
assert s == "http"
assert h == "foo.com"
assert po == 8888
assert pa == "/test"
s, h, po, pa = utils.parse_url("http://foo/bar")
assert s == "http"
assert h == "foo"
assert po == 80
assert pa == "/bar"
s, h, po, pa = utils.parse_url("http://*****:*****@foo/bar")
assert s == "http"
assert h == "foo"
assert po == 80
assert pa == "/bar"
s, h, po, pa = utils.parse_url("http://foo")
assert pa == "/"
s, h, po, pa = utils.parse_url("https://foo")
assert po == 443
assert not utils.parse_url("https://foo:bar")
assert not utils.parse_url("https://foo:")
# Invalid IDNA
assert not utils.parse_url("http://\xfafoo")
# Invalid PATH
assert not utils.parse_url("http:/\xc6/localhost:56121")
# Null byte in host
assert not utils.parse_url("http://foo\0")
# Port out of range
assert not utils.parse_url("http://foo:999999")
# Invalid IPv6 URL - see http://www.ietf.org/rfc/rfc2732.txt
assert not utils.parse_url('http://lo[calhost')
def query(self, odict):
query = utils.urlencode(odict.lst)
scheme, netloc, path, params, _, fragment = urllib.parse.urlparse(self.url)
_, _, _, self.path = utils.parse_url(
urllib.parse.urlunparse([scheme, netloc, path, params, query, fragment]))
def read_request(
self,
include_body=True,
body_size_limit=None,
allow_empty=False,
):
if body_size_limit is not None:
raise NotImplementedError()
self.perform_connection_preface()
timestamp_start = time.time()
if hasattr(self.tcp_handler.rfile, "reset_timestamps"):
self.tcp_handler.rfile.reset_timestamps()
stream_id, headers, body = self._receive_transmission(
include_body=include_body, )
if hasattr(self.tcp_handler.rfile, "first_byte_timestamp"):
# more accurate timestamp_start
timestamp_start = self.tcp_handler.rfile.first_byte_timestamp
timestamp_end = time.time()
authority = headers.get(':authority', '')
method = headers.get(':method', 'GET')
scheme = headers.get(':scheme', 'https')
path = headers.get(':path', '/')
host = None
port = None
if path == '*' or path.startswith("/"):
form_in = "relative"
elif method == 'CONNECT':
form_in = "authority"
if ":" in authority:
host, port = authority.split(":", 1)
else:
host = authority
else:
form_in = "absolute"
# FIXME: verify if path or :host contains what we need
scheme, host, port, _ = utils.parse_url(path)
if host is None:
host = 'localhost'
if port is None:
port = 80 if scheme == 'http' else 443
port = int(port)
request = http.Request(
form_in,
method,
scheme,
host,
port,
path,
(2, 0),
headers,
body,
timestamp_start,
timestamp_end,
)
# FIXME: We should not do this.
request.stream_id = stream_id
return request