def test_metrics(self):
prometheus_metrics = create_metrics(self.metrics_definitions)
resource = http.PrometheusMetricsResource(prometheus_metrics)
request = Request(DummyChannel(), False)
content = resource.render_GET(request).decode('utf-8')
self.assertIn('TYPE sample_histogram histogram', content)
self.assertIn('TYPE sample_counter counter', content)
def test_renderRealRequest(self):
"""
The request managed by L{WebSocketsResource.render} doesn't contain
unnecessary HTTP headers like I{Content-Type} or I{Transfer-Encoding}.
"""
channel = DummyChannel()
channel.transport = StringTransportWithDisconnection()
channel.transport.protocol = channel
request = Request(channel, False)
headers = {
"upgrade": "Websocket",
"connection": "Upgrade",
"sec-websocket-key": "secure",
"sec-websocket-version": "13"
}
for key, value in headers.items():
request.requestHeaders.setRawHeaders(key, [value])
request.method = "GET"
request.clientproto = "HTTP/1.1"
result = self.resource.render(request)
self.assertEqual(NOT_DONE_YET, result)
self.assertEqual(
[("Connection", ["Upgrade"]), ("Upgrade", ["WebSocket"]),
("Sec-Websocket-Accept", ["oYBv54i42V5dw6KnZqOFroecUTc="])],
list(request.responseHeaders.getAllRawHeaders()))
self.assertEqual(
"HTTP/1.1 101 Switching Protocols\r\n"
"Connection: Upgrade\r\n"
"Upgrade: WebSocket\r\n"
"Sec-Websocket-Accept: oYBv54i42V5dw6KnZqOFroecUTc=\r\n\r\n",
channel.transport.value())
self.assertEqual(101, request.code)
self.assertIdentical(None, request.transport)
def test_render_GET_logs_node_event_with_original_path_ip(self):
path = factory.make_name("path")
ip = factory.make_ip_address()
request = Request(DummyChannel(), False)
request.requestHeaders = Headers({
"X-Original-URI": [path],
"X-Original-Remote-IP": [ip]
})
log_info = self.patch(http.log, "info")
mock_deferLater = self.patch(http, "deferLater")
mock_deferLater.side_effect = always_succeed_with(None)
resource = http.HTTPLogResource()
resource.render_GET(request)
self.assertThat(
log_info,
MockCalledOnceWith("{path} requested by {remote_host}",
path=path,
remote_host=ip),
)
self.assertThat(
mock_deferLater,
MockCalledOnceWith(
ANY,
0,
http.send_node_event_ip_address,
event_type=EVENT_TYPES.NODE_HTTP_REQUEST,
ip_address=ip,
description=path,
),
)
def test_updateSession(self):
session = self.site.makeSession()
class FakeChannel(object):
transport = None
def isSecure(self):
return False
def getPeer(self):
return None
def getHost(self):
return None
request = Request(FakeChannel(), False)
request.sitepath = [b"bb"]
session.updateSession(request)
self.assertEqual(len(request.cookies), 1)
name, value = request.cookies[0].split(b";")[0].split(b"=")
decoded = jwt.decode(value,
self.SECRET,
algorithms=[service.SESSION_SECRET_ALGORITHM])
self.assertEqual(decoded['user_info'], {'anonymous': True})
self.assertIn('exp', decoded)
def test_render_GET_logs_node_event_status_message(self):
path = factory.make_name("squashfs")
ip = factory.make_ip_address()
request = Request(DummyChannel(), False)
request.requestHeaders = Headers({
"X-Original-URI": [path],
"X-Original-Remote-IP": [ip]
})
mock_deferLater = self.patch(http, "deferLater")
mock_deferLater.side_effect = always_succeed_with(None)
mock_send_node_event_ip_address = self.patch(
http, "send_node_event_ip_address")
resource = http.HTTPLogResource()
resource.render_GET(request)
self.assertThat(
mock_deferLater,
MockCalledOnceWith(
ANY,
0,
http.send_node_event_ip_address,
event_type=EVENT_TYPES.NODE_HTTP_REQUEST,
ip_address=ip,
description=path,
),
)
self.assertThat(
mock_send_node_event_ip_address,
MockCalledOnceWith(event_type=EVENT_TYPES.LOADING_EPHEMERAL,
ip_address=ip),
)
def test_renderRealRequest(self):
"""
The request managed by L{WebSocketsResource.render} doesn't contain
unnecessary HTTP headers like I{Content-Type}.
"""
channel = DummyChannel()
channel.transport = StringTransportWithDisconnection()
channel.transport.protocol = channel
request = Request(channel, False)
headers = {
b"upgrade": b"Websocket",
b"connection": b"Upgrade",
b"sec-websocket-key": b"secure",
b"sec-websocket-version": b"13"
}
for key, value in headers.items():
request.requestHeaders.setRawHeaders(key, [value])
request.method = b"GET"
request.clientproto = b"HTTP/1.1"
result = self.resource.render(request)
self.assertEqual(NOT_DONE_YET, result)
self.assertEqual(
[(b"Connection", [b"Upgrade"]),
(b"Sec-Websocket-Accept", [b"oYBv54i42V5dw6KnZqOFroecUTc="]),
(b"Upgrade", [b"WebSocket"])],
sorted(request.responseHeaders.getAllRawHeaders()))
self.assertThat(
channel.transport.value(),
StartsWith(b"HTTP/1.1 101 Switching Protocols\r\n"
b"Transfer-Encoding: chunked\r\n"))
self.assertEqual(101, request.code)
self.assertIdentical(None, request.transport)
def test_returns_normal_resources(self):
r = APIResource()
a = Resource()
r.putChild(b'a', a)
req = Request(FakeChannel(), None)
a_ = r.getChild(b'a', req)
self.assertEqual(a, a_)
def setUp(self):
self.channel = DummyChannel()
request = Request(self.channel, False)
transport = WebSocketTransport(request)
handler = TestHandler(transport)
transport._attachHandler(handler)
self.decoder = WebSocketFrameDecoder(request, handler)
self.decoder.MAX_LENGTH = 100
def setUp(self):
self.channel = DummyChannel()
self.request = request = Request(self.channel, False)
# Simulate request handling
request.startedWriting = True
transport = WebSocketTransport(request)
self.handler = TestHandler(transport)
transport._attachHandler(self.handler)
def test_metrics_disabled(self):
prometheus_metrics = create_metrics(
None, registry=prometheus_client.CollectorRegistry())
resource = http.PrometheusMetricsResource(prometheus_metrics)
request = Request(DummyChannel(), False)
content = resource.render_GET(request).decode('utf-8')
self.assertEqual(request.code, 404)
self.assertEqual(content, '')
def test_redirectToUnicodeURL(self):
"""
L{redirectTo} will raise TypeError if unicode object is passed in URL
"""
request = Request(DummyChannel(), True)
request.method = b"GET"
targetURL = "http://target.example.com/4321"
self.assertRaises(TypeError, redirectTo, targetURL, request)
def test_requestFeature(self):
self.log.addFeature(request)
req = Request(DummyChannel(), False)
req.method = 'GET'
req.uri = '/foo'
self.log.request(req).info('handling request')
self.assertIn('method=GET', self.out.getvalue())
self.assertIn('uri=/foo', self.out.getvalue())
def getRequest(method, url):
import warnings
msg = 'These tests will be removed in a future release of the txrestserver'
warnings.warn(msg, DeprecationWarning)
req = Request(FakeChannel(), None)
req.method = method
req.path = url
return req
def fake_twisted_request(*args, **kwargs):
"""
Create a fake Twisted `Request` instance for the purposes of testing.
"""
kwargs.setdefault('Request',
lambda channel: Request(channel=channel, queued=False))
request = fake_nevow_request(*args, **kwargs)
request.finish = lambda: next(request.finish.counter)
request.finish.counter = itertools.count()
return request
def test_metrics(self):
prometheus_metrics = create_metrics(
self.metrics_definitions,
registry=prometheus_client.CollectorRegistry(),
)
resource = http.PrometheusMetricsResource(prometheus_metrics)
request = Request(DummyChannel(), False)
content = resource.render_GET(request).decode("utf-8")
self.assertEqual(request.code, 200)
self.assertIn("TYPE sample_histogram histogram", content)
self.assertIn("TYPE sample_counter counter", content)
def test_updateSession(self):
session = self.site.makeSession()
request = Request(FakeChannel(), False)
request.sitepath = [b"bb"]
session.updateSession(request)
self.assertEqual(len(request.cookies), 1)
_, value = request.cookies[0].split(b";")[0].split(b"=")
decoded = jwt.decode(value,
self.SECRET,
algorithms=[service.SESSION_SECRET_ALGORITHM])
self.assertEqual(decoded['user_info'], {'anonymous': True})
self.assertIn('exp', decoded)
def getTextOfPage(root, page, args=None, return_request=False):
"""This perpetrates several awful hacks."""
if args is not None:
page += '?' + urllib.urlencode(args)
channel = DummyChannel()
channel.site = Site(root)
r = Request(channel, 0)
r.content = StringIO()
r.requestReceived("GET", "/" + page, "1.1")
if return_request:
return channel.transport.written.getvalue(), r
else:
return channel.transport.written.getvalue()
def test_headersAndCode(self):
"""
L{redirectTo} will set the C{Location} and C{Content-Type} headers on
its request, and set the response code to C{FOUND}, so the browser will
be redirected.
"""
request = Request(DummyChannel(), True)
request.method = 'GET'
targetURL = "http://target.example.com/4321"
redirectTo(targetURL, request)
self.assertEqual(request.code, FOUND)
self.assertEqual(request.responseHeaders.getRawHeaders('location'),
[targetURL])
self.assertEqual(request.responseHeaders.getRawHeaders('content-type'),
['text/html; charset=utf-8'])
def doLocationTest(self, requestPath: bytes):
"""
Render a response to a request with path *requestPath*
@param requestPath: A slash-separated path like C{b'/foo/bar'}.
@returns: The value of the I{Location} header.
"""
request = Request(DummyChannel(), True)
request.method = b"GET"
request.prepath = requestPath.lstrip(b"/").split(b"/")
resource = ParentRedirect()
resource.render(request)
[location] = request.responseHeaders.getRawHeaders(b"Location")
return location
def test_absentServerHeader(self):
request = Request(FakeChannel(), False)
self.assertEqual(request.responseHeaders.hasHeader('Server'), False)
def fix_twisted_web_http_Request():
"""Fix broken IPv6 handling in twisted.web.http.request.Request."""
from netaddr import IPAddress
from netaddr.core import AddrFormatError
from twisted.internet import address
from twisted.python.compat import intToBytes, networkString
import twisted.web.http
from twisted.web.server import Request
from twisted.web.test.requesthelper import DummyChannel
def new_getRequestHostname(self):
# Unlike upstream, support/require IPv6 addresses to be
# [ip:v6:add:ress]:port, with :port being optional.
# IPv6 IP addresses are wrapped in [], to disambigate port numbers.
host = self.getHeader(b"host")
if host:
if host.startswith(b"[") and b"]" in host:
if host.find(b"]") < host.rfind(b":"):
# The format is: [ip:add:ress]:port.
return host[:host.rfind(b":")]
else:
# no :port after [...]
return host
# No brackets, so it must be host:port or IPv4:port.
return host.split(b":", 1)[0]
host = self.getHost().host
try:
if isinstance(host, str):
ip = IPAddress(host)
else:
ip = IPAddress(host.decode("idna"))
except AddrFormatError:
# If we could not convert the hostname to an IPAddress, assume that
# it is a hostname.
return networkString(host)
if ip.version == 4:
return networkString(host)
else:
return networkString("[" + host + "]")
def new_setHost(self, host, port, ssl=0):
try:
ip = IPAddress(host.decode("idna"))
except AddrFormatError:
ip = None # `host` is a host or domain name.
self._forceSSL = ssl # set first so isSecure will work
if self.isSecure():
default = 443
else:
default = 80
if ip is None:
hostHeader = host
elif ip.version == 4:
hostHeader = host
else:
hostHeader = b"[" + host + b"]"
if port != default:
hostHeader += b":" + intToBytes(port)
self.requestHeaders.setRawHeaders(b"host", [hostHeader])
if ip is None:
# Pretend that a host or domain name is an IPv4 address.
self.host = address.IPv4Address("TCP", host, port)
elif ip.version == 4:
self.host = address.IPv4Address("TCP", host, port)
else:
self.host = address.IPv6Address("TCP", host, port)
request = Request(DummyChannel(), False)
request.client = address.IPv6Address("TCP", "fe80::1", "80")
request.setHost(b"fe80::1", 1234)
if isinstance(request.host, address.IPv4Address):
# Buggy code calls fe80::1 an IPv4Address.
twisted.web.http.Request.setHost = new_setHost
if request.getRequestHostname() == b"fe80":
# The fe80::1 test address above was incorrectly interpreted as
# address='fe80', port = ':1', because it does host.split(':', 1)[0].
twisted.web.http.Request.getRequestHostname = new_getRequestHostname
def getRequest(method, url):
req = Request(FakeChannel(), None)
req.method = method
req.path = url
return req
def fix_twisted_web_http_Request():
"""Add ipv6 support to Request.getClientIP()
Specifically, IPv6 IP addresses need to be wrapped in [], and return
address.IPv6Address when needed.
See https://bugs.launchpad.net/ubuntu/+source/twisted/+bug/1604608
"""
from netaddr import IPAddress
from netaddr.core import AddrFormatError
from twisted.internet import address
from twisted.python.compat import (
intToBytes,
networkString,
)
import twisted.web.http
from twisted.web.server import Request
from twisted.web.test.requesthelper import DummyChannel
def new_getClientIP(self):
from twisted.internet import address
# upstream doesn't check for address.IPv6Address
if isinstance(self.client, address.IPv4Address):
return self.client.host
elif isinstance(self.client, address.IPv6Address):
return self.client.host
else:
return None
def new_getRequestHostname(self):
# Unlike upstream, support/require IPv6 addresses to be
# [ip:v6:add:ress]:port, with :port being optional.
# IPv6 IP addresses are wrapped in [], to disambigate port numbers.
host = self.getHeader(b'host')
if host:
if host.startswith(b'[') and b']' in host:
if host.find(b']') < host.rfind(b':'):
# The format is: [ip:add:ress]:port.
return host[:host.rfind(b':')]
else:
# no :port after [...]
return host
# No brackets, so it must be host:port or IPv4:port.
return host.split(b':', 1)[0]
host = self.getHost().host
try:
if isinstance(host, str):
ip = IPAddress(host)
else:
ip = IPAddress(host.decode("idna"))
except AddrFormatError:
# If we could not convert the hostname to an IPAddress, assume that
# it is a hostname.
return networkString(host)
if ip.version == 4:
return networkString(host)
else:
return networkString('[' + host + ']')
def new_setHost(self, host, port, ssl=0):
try:
ip = IPAddress(host.decode("idna"))
except AddrFormatError:
ip = None # `host` is a host or domain name.
self._forceSSL = ssl # set first so isSecure will work
if self.isSecure():
default = 443
else:
default = 80
if ip is None:
hostHeader = host
elif ip.version == 4:
hostHeader = host
else:
hostHeader = b"[" + host + b"]"
if port != default:
hostHeader += b":" + intToBytes(port)
self.requestHeaders.setRawHeaders(b"host", [hostHeader])
if ip is None:
# Pretend that a host or domain name is an IPv4 address.
self.host = address.IPv4Address("TCP", host, port)
elif ip.version == 4:
self.host = address.IPv4Address("TCP", host, port)
else:
self.host = address.IPv6Address("TCP", host, port)
request = Request(DummyChannel(), False)
request.client = address.IPv6Address('TCP', 'fe80::1', '80')
request.setHost(b"fe80::1", 1234)
if request.getClientIP() is None:
# Buggy code returns None for IPv6 addresses.
twisted.web.http.Request.getClientIP = new_getClientIP
if isinstance(request.host, address.IPv4Address):
# Buggy code calls fe80::1 an IPv4Address.
twisted.web.http.Request.setHost = new_setHost
if request.getRequestHostname() == b'fe80':
# The fe80::1 test address above was incorrectly interpreted as
# address='fe80', port = ':1', because it does host.split(':', 1)[0].
twisted.web.http.Request.getRequestHostname = new_getRequestHostname
