class HostnameMappingTestCase(AsyncHTTPTestCase):
def setUp(self):
super(HostnameMappingTestCase, self).setUp()
self.http_client = SimpleAsyncHTTPClient(
self.io_loop,
hostname_mapping={
'www.example.com': '127.0.0.1',
('foo.example.com', 8000): ('127.0.0.1', self.get_http_port()),
})
def get_app(self):
return Application([url("/hello", HelloWorldHandler), ])
def test_hostname_mapping(self):
self.http_client.fetch(
'http://www.example.com:%d/hello' % self.get_http_port(), self.stop)
response = self.wait()
response.rethrow()
self.assertEqual(response.body, b'Hello world!')
def test_port_mapping(self):
self.http_client.fetch('http://foo.example.com:8000/hello', self.stop)
response = self.wait()
response.rethrow()
self.assertEqual(response.body, b'Hello world!')
class BaseSSLTest(AsyncHTTPTestCase, LogTrapTestCase):
def get_ssl_version(self):
raise NotImplementedError()
def setUp(self):
super(BaseSSLTest, self).setUp()
# Replace the client defined in the parent class.
# Some versions of libcurl have deadlock bugs with ssl,
# so always run these tests with SimpleAsyncHTTPClient.
self.http_client = SimpleAsyncHTTPClient(io_loop=self.io_loop,
force_instance=True)
def get_app(self):
return Application([('/', HelloWorldRequestHandler,
dict(protocol="https"))])
def get_httpserver_options(self):
# Testing keys were generated with:
# openssl req -new -keyout tornado/test/test.key -out tornado/test/test.crt -nodes -days 3650 -x509
test_dir = os.path.dirname(__file__)
return dict(ssl_options=dict(
certfile=os.path.join(test_dir, 'test.crt'),
keyfile=os.path.join(test_dir, 'test.key'),
ssl_version=self.get_ssl_version()))
def fetch(self, path, **kwargs):
self.http_client.fetch(self.get_url(path).replace('http', 'https'),
self.stop,
validate_cert=False,
**kwargs)
return self.wait()
def test_connection_limit(self):
client = SimpleAsyncHTTPClient(self.io_loop, max_clients=2,
force_instance=True)
self.assertEqual(client.max_clients, 2)
seen = []
# Send 4 requests. Two can be sent immediately, while the others
# will be queued
for i in range(4):
client.fetch(self.get_url("/trigger"),
lambda response, i=i: (seen.append(i), self.stop()))
self.wait(condition=lambda: len(self.triggers) == 2)
self.assertEqual(len(client.queue), 2)
# Finish the first two requests and let the next two through
self.triggers.popleft()()
self.triggers.popleft()()
self.wait(condition=lambda: (len(self.triggers) == 2 and
len(seen) == 2))
self.assertEqual(set(seen), set([0, 1]))
self.assertEqual(len(client.queue), 0)
# Finish all the pending requests
self.triggers.popleft()()
self.triggers.popleft()()
self.wait(condition=lambda: len(seen) == 4)
self.assertEqual(set(seen), set([0, 1, 2, 3]))
self.assertEqual(len(self.triggers), 0)
def fetch(self, request, callback, **kwargs):
self.request_queue.append((request, callback, kwargs))
while self.available_requests > 0 and len(self.request_queue) > 0:
request, callback, kwargs = self.request_queue.popleft()
SimpleAsyncHTTPClient.fetch(self, request, callback, **kwargs)
self.available_requests -= 1
yield gen.Task(self.io_loop.add_timeout, self.period)
self.available_requests += 1
def test_redirect_connection_limit(self):
# following redirects should not consume additional connections
client = SimpleAsyncHTTPClient(self.io_loop, max_clients=1,
force_instance=True)
client.fetch(self.get_url('/countdown/3'), self.stop,
max_redirects=3)
response = self.wait()
response.rethrow()
def raw_fetch(self, headers, body):
client = SimpleAsyncHTTPClient(self.io_loop)
conn = RawRequestHTTPConnection(
self.io_loop, client, httpclient.HTTPRequest(self.get_url("/")), None, self.stop, 1024 * 1024
)
conn.set_request(b("\r\n").join(headers + [utf8("Content-Length: %d\r\n" % len(body))]) + b("\r\n") + body)
response = self.wait()
client.close()
response.rethrow()
return response
def setUp(self):
super(SSLTest, self).setUp()
# Replace the client defined in the parent class.
# Some versions of libcurl have deadlock bugs with ssl,
# so always run these tests with SimpleAsyncHTTPClient.
self.http_client = SimpleAsyncHTTPClient(io_loop=self.io_loop,
force_instance=True)
def setUp(self):
super(HostnameMappingTestCase, self).setUp()
self.http_client = SimpleAsyncHTTPClient(
hostname_mapping={
'www.example.com': '127.0.0.1',
('foo.example.com', 8000): ('127.0.0.1', self.get_http_port()),
})
class SSLTest(AsyncHTTPTestCase, LogTrapTestCase):
def setUp(self):
super(SSLTest, self).setUp()
# Replace the client defined in the parent class.
# Some versions of libcurl have deadlock bugs with ssl,
# so always run these tests with SimpleAsyncHTTPClient.
self.http_client = SimpleAsyncHTTPClient(io_loop=self.io_loop,
force_instance=True)
def get_app(self):
return Application([('/', HelloWorldRequestHandler,
dict(protocol="https"))])
def get_httpserver_options(self):
# Testing keys were generated with:
# openssl req -new -keyout tornado/test/test.key -out tornado/test/test.crt -nodes -days 3650 -x509
test_dir = os.path.dirname(__file__)
return dict(ssl_options=dict(
certfile=os.path.join(test_dir, 'test.crt'),
keyfile=os.path.join(test_dir, 'test.key')))
def fetch(self, path, **kwargs):
self.http_client.fetch(self.get_url(path).replace('http', 'https'),
self.stop,
validate_cert=False,
**kwargs)
return self.wait()
def test_ssl(self):
response = self.fetch('/')
self.assertEqual(response.body, b("Hello world"))
def test_large_post(self):
response = self.fetch('/',
method='POST',
body='A'*5000)
self.assertEqual(response.body, b("Got 5000 bytes in POST"))
def test_non_ssl_request(self):
# Make sure the server closes the connection when it gets a non-ssl
# connection, rather than waiting for a timeout or otherwise
# misbehaving.
self.http_client.fetch(self.get_url("/"), self.stop,
request_timeout=3600,
connect_timeout=3600)
response = self.wait()
self.assertEqual(response.code, 599)
def raw_fetch(self, headers, body):
client = SimpleAsyncHTTPClient(self.io_loop)
conn = RawRequestHTTPConnection(
self.io_loop, client,
httpclient._RequestProxy(
httpclient.HTTPRequest(self.get_url("/")),
dict(httpclient.HTTPRequest._DEFAULTS)),
None, self.stop,
1024 * 1024, Resolver(io_loop=self.io_loop))
conn.set_request(
b"\r\n".join(headers +
[utf8("Content-Length: %d\r\n" % len(body))]) +
b"\r\n" + body)
response = self.wait()
client.close()
response.rethrow()
return response
def setUp(self):
super(HostnameMappingTestCase, self).setUp()
self.http_client = SimpleAsyncHTTPClient(
self.io_loop,
hostname_mapping={
"www.example.com": "127.0.0.1",
("foo.example.com", 8000): ("127.0.0.1", self.get_http_port()),
},
)
def __init__(self, link: str, loop: IOLoop = None, timeout: int = 50):
self.link = link
self.client = SimpleAsyncHTTPClient(loop)
self.timeout = timeout
if loop is None:
self.loop = IOLoop.current(False)
else:
self.loop = loop
self.header_only = False
class AsyncHandler:
def __init__(self, link: str, loop: IOLoop = None, timeout: int = 50):
self.link = link
self.client = SimpleAsyncHTTPClient(loop)
self.timeout = timeout
if loop is None:
self.loop = IOLoop.current(False)
else:
self.loop = loop
self.header_only = False
def get_response(self, header_only=False):
self.header_only = header_only
# result = self.inner_get_response()
result = self.loop.run_sync(self.inner_get_response)
self.client.close()
return result
@gen.coroutine
def inner_get_response(self) -> (int, dict, str):
method = "GET"
if self.header_only:
method = "HEAD"
try:
req = HTTPRequest(
self.link,
headers=WebRequestCommonHeader.get_html_header(),
request_timeout=self.timeout,
follow_redirects=True,
method=method,
)
response = yield self.client.fetch(req)
# self.client.close()
return response.code, response.headers, response.body
except HTTPError as ex:
result = ex.response
# self.loop.stop()
return result.code, None, ""
except:
# self.loop.stop()
return 999, None, ""
finally:
pass
def estimate(self, source, destination, force_refresh=False):
self._count += 1
self.debug('Estimating (Total {0} requested so far)'.format(
self._count))
estimate = self._get_cache(source, destination)
if estimate:
raise gen.Return(estimate)
data = {
'startX': source[0],
'startY': source[1],
'endX': destination[0],
'endY': destination[1],
'reqCoordType': 'WGS84GEO',
'resCoordType': 'WGS84GEO',
'speed': int(self.speed),
}
headers = {
'accept': 'application/json',
'appkey': settings.TMAP_API_KEY,
}
body = urllib.urlencode(data)
client = SimpleAsyncHTTPClient()
request = HTTPRequest(self.url, method='POST', headers=headers,
body=body)
r = yield client.fetch(request)
data = json.loads(r.body)
error = data.get('error')
if error:
raise TmapException(error)
prop = data['features'][0]['properties']
estimate = Estimate(int(prop['totalDistance']), int(prop['totalTime']))
self._set_cache(source, destination, estimate)
raise gen.Return(estimate)
def main():
parse_command_line()
app = Application([('/', ChunkHandler)])
app.listen(options.port, address='127.0.0.1')
def callback(response):
response.rethrow()
assert len(response.body) == (options.num_chunks * options.chunk_size)
logging.warning("fetch completed in %s seconds", response.request_time)
IOLoop.instance().stop()
logging.warning("Starting fetch with curl client")
curl_client = CurlAsyncHTTPClient()
curl_client.fetch('http://localhost:%d/' % options.port,
callback=callback)
IOLoop.instance().start()
logging.warning("Starting fetch with simple client")
simple_client = SimpleAsyncHTTPClient()
simple_client.fetch('http://localhost:%d/' % options.port,
callback=callback)
IOLoop.instance().start()
def test_ip(ip):
hostname = APP_ID + '.appspot.com'
url = 'https://' + hostname + '/2'
request = HTTPRequest(url, request_timeout=5)
try:
client = SimpleAsyncHTTPClient(force_instance=True,
hostname_mapping={hostname: ip})
res = yield client.fetch(request, raise_error=False)
if isinstance(res.error, OSError):
if res.error.errno == errno.EMFILE:
warning_msg = ("Too many open files. You should increase the"
" maximum allowed number of network connections in you"
" system or decrease the CONCURRENCY setting.")
warnings.warn(warning_msg)
if res.code == 200:
return True
else:
return False
finally:
client.close()
def get_http_client(self):
return SimpleAsyncHTTPClient(io_loop=self.io_loop,
max_header_size=1024)
class SimpleHTTPClientTestCase(AsyncHTTPTestCase, LogTrapTestCase):
def setUp(self):
super(SimpleHTTPClientTestCase, self).setUp()
self.http_client = SimpleAsyncHTTPClient(self.io_loop)
def get_app(self):
# callable objects to finish pending /trigger requests
self.triggers = collections.deque()
return Application([
url("/trigger", TriggerHandler, dict(queue=self.triggers,
wake_callback=self.stop)),
url("/chunk", ChunkHandler),
url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
url("/hang", HangHandler),
url("/hello", HelloWorldHandler),
url("/content_length", ContentLengthHandler),
url("/head", HeadHandler),
url("/no_content", NoContentHandler),
url("/303_post", SeeOther303PostHandler),
url("/303_get", SeeOther303GetHandler),
], gzip=True)
def test_singleton(self):
# Class "constructor" reuses objects on the same IOLoop
self.assertTrue(SimpleAsyncHTTPClient(self.io_loop) is
SimpleAsyncHTTPClient(self.io_loop))
# unless force_instance is used
self.assertTrue(SimpleAsyncHTTPClient(self.io_loop) is not
SimpleAsyncHTTPClient(self.io_loop,
force_instance=True))
# different IOLoops use different objects
io_loop2 = IOLoop()
self.assertTrue(SimpleAsyncHTTPClient(self.io_loop) is not
SimpleAsyncHTTPClient(io_loop2))
def test_connection_limit(self):
client = SimpleAsyncHTTPClient(self.io_loop, max_clients=2,
force_instance=True)
self.assertEqual(client.max_clients, 2)
seen = []
# Send 4 requests. Two can be sent immediately, while the others
# will be queued
for i in range(4):
client.fetch(self.get_url("/trigger"),
lambda response, i=i: (seen.append(i), self.stop()))
self.wait(condition=lambda: len(self.triggers) == 2)
self.assertEqual(len(client.queue), 2)
# Finish the first two requests and let the next two through
self.triggers.popleft()()
self.triggers.popleft()()
self.wait(condition=lambda: (len(self.triggers) == 2 and
len(seen) == 2))
self.assertEqual(set(seen), set([0, 1]))
self.assertEqual(len(client.queue), 0)
# Finish all the pending requests
self.triggers.popleft()()
self.triggers.popleft()()
self.wait(condition=lambda: len(seen) == 4)
self.assertEqual(set(seen), set([0, 1, 2, 3]))
self.assertEqual(len(self.triggers), 0)
def test_redirect_connection_limit(self):
# following redirects should not consume additional connections
client = SimpleAsyncHTTPClient(self.io_loop, max_clients=1,
force_instance=True)
client.fetch(self.get_url('/countdown/3'), self.stop,
max_redirects=3)
response = self.wait()
response.rethrow()
def test_default_certificates_exist(self):
open(_DEFAULT_CA_CERTS).close()
def test_gzip(self):
# All the tests in this file should be using gzip, but this test
# ensures that it is in fact getting compressed.
# Setting Accept-Encoding manually bypasses the client's
# decompression so we can see the raw data.
response = self.fetch("/chunk", use_gzip=False,
headers={"Accept-Encoding": "gzip"})
self.assertEqual(response.headers["Content-Encoding"], "gzip")
self.assertNotEqual(response.body, b("asdfqwer"))
# Our test data gets bigger when gzipped. Oops. :)
self.assertEqual(len(response.body), 34)
f = gzip.GzipFile(mode="r", fileobj=response.buffer)
self.assertEqual(f.read(), b("asdfqwer"))
def test_max_redirects(self):
response = self.fetch("/countdown/5", max_redirects=3)
self.assertEqual(302, response.code)
# We requested 5, followed three redirects for 4, 3, 2, then the last
# unfollowed redirect is to 1.
self.assertTrue(response.request.url.endswith("/countdown/5"))
self.assertTrue(response.effective_url.endswith("/countdown/2"))
self.assertTrue(response.headers["Location"].endswith("/countdown/1"))
def test_303_redirect(self):
response = self.fetch("/303_post", method="POST", body="blah")
self.assertEqual(200, response.code)
self.assertTrue(response.request.url.endswith("/303_post"))
self.assertTrue(response.effective_url.endswith("/303_get"))
#request is the original request, is a POST still
self.assertEqual("POST", response.request.method)
def test_request_timeout(self):
response = self.fetch('/hang', request_timeout=0.1)
self.assertEqual(response.code, 599)
self.assertTrue(0.099 < response.request_time < 0.11, response.request_time)
self.assertEqual(str(response.error), "HTTP 599: Timeout")
def test_ipv6(self):
if not socket.has_ipv6:
# python compiled without ipv6 support, so skip this test
return
try:
self.http_server.listen(self.get_http_port(), address='::1')
except socket.gaierror as e:
if e.args[0] == socket.EAI_ADDRFAMILY:
# python supports ipv6, but it's not configured on the network
# interface, so skip this test.
return
raise
url = self.get_url("/hello").replace("localhost", "[::1]")
# ipv6 is currently disabled by default and must be explicitly requested
self.http_client.fetch(url, self.stop)
response = self.wait()
self.assertEqual(response.code, 599)
self.http_client.fetch(url, self.stop, allow_ipv6=True)
response = self.wait()
self.assertEqual(response.body, b("Hello world!"))
def test_multiple_content_length_accepted(self):
response = self.fetch("/content_length?value=2,2")
self.assertEqual(response.body, b("ok"))
response = self.fetch("/content_length?value=2,%202,2")
self.assertEqual(response.body, b("ok"))
response = self.fetch("/content_length?value=2,4")
self.assertEqual(response.code, 599)
response = self.fetch("/content_length?value=2,%202,3")
self.assertEqual(response.code, 599)
def test_head_request(self):
response = self.fetch("/head", method="HEAD")
self.assertEqual(response.code, 200)
self.assertEqual(response.headers["content-length"], "7")
self.assertFalse(response.body)
def test_no_content(self):
response = self.fetch("/no_content")
self.assertEqual(response.code, 204)
# 204 status doesn't need a content-length, but tornado will
# add a zero content-length anyway.
self.assertEqual(response.headers["Content-length"], "0")
# 204 status with non-zero content length is malformed
response = self.fetch("/no_content?error=1")
self.assertEqual(response.code, 599)
def setUp(self):
super(SimpleHTTPClientTestCase, self).setUp()
# replace the client defined in the parent class
self.http_client = SimpleAsyncHTTPClient(io_loop=self.io_loop,
force_instance=True)
def __init__(self, *args, **kwargs):
self._warcout = WarcOutputSingleton()
SimpleAsyncHTTPClient.__init__(self, *args, **kwargs)
def __init__(self, *args, **kwargs):
#self._warcout = WarcOutputSingleton()
SimpleAsyncHTTPClient.__init__(self, *args, **kwargs)
from tornado.ioloop import IOLoop
from tornado.netutil import Resolver, OverrideResolver
from tornado.simple_httpclient import SimpleAsyncHTTPClient
from tornado.httpclient import HTTPRequest
from tornado import gen
request = HTTPRequest('http://www.baidu.com', validate_cert=False)
reslover = OverrideResolver(Resolver(),
mapping={"www.baidu.com": "111.13.101.208"})
client = SimpleAsyncHTTPClient(resolver=reslover)
@gen.coroutine
def get_response():
xxx = yield reslover.resolve('www.baidu.com', 80)
print xxx
rs = yield client.fetch(request)
print rs.body
IOLoop.current().run_sync(get_response)
def get_http_client(self):
# These tests require HTTP/1; force the use of SimpleAsyncHTTPClient.
return SimpleAsyncHTTPClient()
def get_http_client(self):
return SimpleAsyncHTTPClient(max_header_size=1024)
def get_http_client(self):
# simple_httpclient only: curl doesn't expose the reason string
return SimpleAsyncHTTPClient(io_loop=self.io_loop)
def get_http_client(self):
return SimpleAsyncHTTPClient(io_loop=self.io_loop,
max_body_size=1024 * 64)
def get_http_client(self):
return SimpleAsyncHTTPClient()
def get_http_client(self):
# 100KB body with 64KB buffer
return SimpleAsyncHTTPClient(max_body_size=1024 * 100,
max_buffer_size=1024 * 64)
def get_http_client(self):
return SimpleAsyncHTTPClient(max_body_size=1024 * 64)
def setUp(self):
super(SimpleHTTPClientTestCase, self).setUp()
self.http_client = SimpleAsyncHTTPClient(self.io_loop)
def initialize(self, io_loop=None, max_clients=10, period=timedelta(seconds=1), **kwargs):
SimpleAsyncHTTPClient.initialize(self, io_loop=io_loop, max_clients=max_clients, **kwargs)
self.period = period
self.available_requests = max_clients
self.request_queue = deque()
class KubeSpawner(Spawner):
"""
Implement a JupyterHub spawner to spawn pods in a Kubernetes Cluster.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# By now, all the traitlets have been set, so we can use them to compute
# other attributes
# Use curl HTTPClient if available, else fall back to Simple one
try:
from tornado.curl_httpclient import CurlAsyncHTTPClient
self.httpclient = CurlAsyncHTTPClient(max_clients=64)
except ImportError:
from tornado.simple_httpclient import SimpleAsyncHTTPClient
self.httpclient = SimpleAsyncHTTPClient(max_clients=64)
# FIXME: Support more than just kubeconfig
self.request = request_maker()
self.pod_name = self._expand_user_properties(self.pod_name_template)
self.pvc_name = self._expand_user_properties(self.pvc_name_template)
if self.hub_connect_ip:
scheme, netloc, path, params, query, fragment = urlparse(
self.hub.api_url)
netloc = '{ip}:{port}'.format(
ip=self.hub_connect_ip,
port=self.hub_connect_port,
)
self.accessible_hub_api_url = urlunparse(
(scheme, netloc, path, params, query, fragment))
else:
self.accessible_hub_api_url = self.hub.api_url
if self.port == 0:
# Our default port is 8888
self.port = 8888
namespace = Unicode(config=True,
help="""
Kubernetes namespace to spawn user pods in.
If running inside a kubernetes cluster with service accounts enabled,
defaults to the current namespace. If not, defaults to 'default'
""")
def _namespace_default(self):
"""
Set namespace default to current namespace if running in a k8s cluster
If not in a k8s cluster with service accounts enabled, default to
'default'
"""
ns_path = '/var/run/secrets/kubernetes.io/serviceaccount/namespace'
if os.path.exists(ns_path):
with open(ns_path) as f:
return f.read().strip()
return 'default'
ip = Unicode('0.0.0.0',
help="""
The IP address (or hostname) the single-user server should listen on.
We override this from the parent so we can set a more sane default for
the Kubernetes setup.
""").tag(config=True)
cmd = Command(None,
allow_none=True,
minlen=0,
help="""
The command used for starting the single-user server.
Provide either a string or a list containing the path to the startup script command. Extra arguments,
other than this path, should be provided via `args`.
This is usually set if you want to start the single-user server in a different python
environment (with virtualenv/conda) than JupyterHub itself.
Some spawners allow shell-style expansion here, allowing you to use environment variables.
Most, including the default, do not. Consult the documentation for your spawner to verify!
If set to None, Kubernetes will start the CMD that is specified in the Docker image being started.
""").tag(config=True)
pod_name_template = Unicode('jupyter-{username}-{userid}',
config=True,
help="""
Template to use to form the name of user's pods.
{username} and {userid} are expanded to the escaped, dns-label safe
username & integer user id respectively.
This must be unique within the namespace the pods are being spawned
in, so if you are running multiple jupyterhubs spawning in the
same namespace, consider setting this to be something more unique.
""")
pvc_name_template = Unicode('claim-{username}-{userid}',
config=True,
help="""
Template to use to form the name of user's pvc.
{username} and {userid} are expanded to the escaped, dns-label safe
username & integer user id respectively.
This must be unique within the namespace the pvc are being spawned
in, so if you are running multiple jupyterhubs spawning in the
same namespace, consider setting this to be something more unique.
""")
hub_connect_ip = Unicode(None,
config=True,
allow_none=True,
help="""
IP/DNS hostname to be used by pods to reach out to the hub API.
Defaults to `None`, in which case the `hub_ip` config is used.
In kubernetes contexts, this is often not the same as `hub_ip`,
since the hub runs in a pod which is fronted by a service. This IP
should be something that pods can access to reach the hub process.
This can also be through the proxy - API access is authenticated
with a token that is passed only to the hub, so security is fine.
Usually set to the service IP / DNS name of the service that fronts
the hub pod (deployment/replicationcontroller/replicaset)
Used together with `hub_connect_port` configuration.
""")
hub_connect_port = Integer(config=True,
help="""
Port to use by pods to reach out to the hub API.
Defaults to be the same as `hub_port`.
In kubernetes contexts, this is often not the same as `hub_port`,
since the hub runs in a pod which is fronted by a service. This
allows easy port mapping, and some systems take advantage of it.
This should be set to the `port` attribute of a service that is
fronting the hub pod.
""")
def _hub_connect_port_default(self):
"""
Set default port on which pods connect to hub to be the hub port
The hub needs to be accessible to the pods at this port. We default
to the port the hub is listening on. This would be overriden in case
some amount of port mapping is happening.
"""
return self.hub.server.port
singleuser_extra_labels = Dict({},
config=True,
help="""
Extra kubernetes labels to set on the spawned single-user pods.
The keys and values specified here would be set as labels on the spawned single-user
kubernetes pods. The keys and values must both be strings that match the kubernetes
label key / value constraints.
See https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ for more
info on what labels are and why you might want to use them!
""")
singleuser_image_spec = Unicode('jupyter/singleuser:latest',
config=True,
help="""
Docker image spec to use for spawning user's containers.
Defaults to `jupyter/singleuser:latest`
Name of the container + a tag, same as would be used with
a `docker pull` command. If tag is set to `latest`, kubernetes will
check the registry each time a new user is spawned to see if there
is a newer image available. If available, new image will be pulled.
Note that this could cause long delays when spawning, especially
if the image is large. If you do not specify a tag, whatever version
of the image is first pulled on the node will be used, thus possibly
leading to inconsistent images on different nodes. For all these
reasons, it is recommended to specify a specific immutable tag
for the imagespec.
If your image is very large, you might need to increase the timeout
for starting the single user container from the default. You can
set this with:
```
c.KubeSpawner.start_timeout = 60 * 5 # Upto 5 minutes
```
""")
singleuser_image_pull_policy = Unicode('IfNotPresent',
config=True,
help="""
The image pull policy of the docker container specified in
singleuser_image_spec.
Defaults to `IfNotPresent` which causes the Kubelet to NOT pull the image
specified in singleuser_image_spec if it already exists, except if the tag
is :latest. For more information on image pull policy, refer to
http://kubernetes.io/docs/user-guide/images/
This configuration is primarily used in development if you are
actively changing the singleuser_image_spec and would like to pull the image
whenever a user container is spawned.
""")
singleuser_image_pull_secrets = Unicode(None,
allow_none=True,
config=True,
help="""
The kubernetes secret to use for pulling images from private repository.
Set this to the name of a Kubernetes secret containing the docker configuration
required to pull the image specified in singleuser_image_spec.
https://kubernetes.io/docs/user-guide/images/#specifying-imagepullsecrets-on-a-pod
has more information on when and why this might need to be set, and what it
should be set to.
""")
singleuser_uid = Union([Integer(), Callable()],
allow_none=True,
config=True,
help="""
The UID to run the single-user server containers as.
This UID should ideally map to a user that already exists in the container
image being used. Running as root is discouraged.
Instead of an integer, this could also be a callable that takes as one
parameter the current spawner instance and returns an integer. The callable
will be called asynchronously if it returns a future. Note that
the interface of the spawner class is not deemed stable across versions,
so using this functionality might cause your JupyterHub or kubespawner
upgrades to break.
If set to `None`, the user specified with the `USER` directive in the
container metadata is used.
""")
singleuser_fs_gid = Union([Integer(), Callable()],
allow_none=True,
config=True,
help="""
The GID of the group that should own any volumes that are created & mounted.
A special supplemental group that applies primarily to the volumes mounted
in the single-user server. In volumes from supported providers, the following
things happen:
1. The owning GID will be the this GID
2. The setgid bit is set (new files created in the volume will be owned by
this GID)
3. The permission bits are OR’d with rw-rw
The single-user server will also be run with this gid as part of its supplemental
groups.
Instead of an integer, this could also be a callable that takes as one
parameter the current spawner instance and returns an integer. The callable will
be called asynchronously if it returns a future, rather than an int. Note that
the interface of the spawner class is not deemed stable across versions,
so using this functionality might cause your JupyterHub or kubespawner
upgrades to break.
You'll *have* to set this if you are using auto-provisioned volumes with most
cloud providers. See [fsGroup](http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_podsecuritycontext)
for more details.
""")
volumes = List([],
config=True,
help="""
List of Kubernetes Volume specifications that will be mounted in the user pod.
This list will be directly added under `volumes` in the kubernetes pod spec,
so you should use the same structure. Each item in the list must have the
following two keys:
- name
Name that'll be later used in the `volume_mounts` config to mount this
volume at a specific path.
- <name-of-a-supported-volume-type> (such as `hostPath`, `persistentVolumeClaim`,
etc)
The key name determines the type of volume to mount, and the value should
be an object specifying the various options available for that kind of
volume.
See http://kubernetes.io/docs/user-guide/volumes/ for more information on the
various kinds of volumes available and their options. Your kubernetes cluster
must already be configured to support the volume types you want to use.
{username} and {userid} are expanded to the escaped, dns-label safe
username & integer user id respectively, wherever they are used.
""")
volume_mounts = List([],
config=True,
help="""
List of paths on which to mount volumes in the user notebook's pod.
This list will be added to the values of the `volumeMounts` key under the user's
container in the kubernetes pod spec, so you should use the same structure as that.
Each item in the list should be a dictionary with at least these two keys:
- mountPath
The path on the container in which we want to mount the volume.
- name
The name of the volume we want to mount, as specified in the `volumes`
config.
See http://kubernetes.io/docs/user-guide/volumes/ for more information on how
the volumeMount item works.
{username} and {userid} are expanded to the escaped, dns-label safe
username & integer user id respectively, wherever they are used.
""")
user_storage_capacity = Unicode(None,
config=True,
allow_none=True,
help="""
The ammount of storage space to request from the volume that the pvc will
mount to. This ammount will be the ammount of storage space the user has
to work with on their notebook. If left blank, the kubespawner will not
create a pvc for the pod.
This will be added to the `resources: requests: storage:` in the k8s pod spec.
See http://kubernetes.io/docs/user-guide/persistent-volumes/#persistentvolumeclaims
for more information on how storage works.
Quantities can be represented externally as unadorned integers, or as fixed-point
integers with one of these SI suffices (E, P, T, G, M, K, m) or their power-of-two
equivalents (Ei, Pi, Ti, Gi, Mi, Ki). For example, the following represent roughly
'the same value: 128974848, "129e6", "129M" , "123Mi".
(https://github.com/kubernetes/kubernetes/blob/master/docs/design/resources.md)
""")
user_storage_class = Unicode(None,
config=True,
allow_none=True,
help="""
The storage class that the pvc will use. If left blank, the kubespawner will not
create a pvc for the pod.
This will be added to the `annotations: volume.beta.kubernetes.io/storage-class:`
in the pvc metadata.
This will determine what type of volume the pvc will request to use. If one exists
that matches the criteria of the StorageClass, the pvc will mount to that. Otherwise,
b/c it has a storage class, k8s will dynamicallly spawn a pv for the pvc to bind to
and a machine in the cluster for the pv to bind to.
See http://kubernetes.io/docs/user-guide/persistent-volumes/#storageclasses for
more information on how StorageClasses work.
""")
user_storage_access_modes = List(["ReadWriteOnce"],
config=True,
help="""
List of access modes the user has for the pvc.
The access modes are:
The access modes are:
ReadWriteOnce – the volume can be mounted as read-write by a single node
ReadOnlyMany – the volume can be mounted read-only by many nodes
ReadWriteMany – the volume can be mounted as read-write by many nodes
See http://kubernetes.io/docs/user-guide/persistent-volumes/#access-modes for
more information on how access modes work.
""")
def _expand_user_properties(self, template):
# Make sure username matches the restrictions for DNS labels
safe_chars = set(string.ascii_lowercase + string.digits)
safe_username = ''.join(
[s if s in safe_chars else '-' for s in self.user.name.lower()])
return template.format(userid=self.user.id, username=safe_username)
def _expand_all(self, src):
if isinstance(src, list):
return [self._expand_all(i) for i in src]
elif isinstance(src, dict):
return {k: self._expand_all(v) for k, v in src.items()}
elif isinstance(src, str):
return self._expand_user_properties(src)
else:
return src
@gen.coroutine
def get_pod_manifest(self):
"""
Make a pod manifest that will spawn current user's notebook pod.
"""
# Add a hack to ensure that no service accounts are mounted in spawned pods
# This makes sure that we don't accidentally give access to the whole
# kubernetes API to the users in the spawned pods.
# See https://github.com/kubernetes/kubernetes/issues/16779#issuecomment-157460294
hack_volumes = [{'name': 'no-api-access-please', 'emptyDir': {}}]
hack_volume_mounts = [{
'name': 'no-api-access-please',
'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount',
'readOnly': True
}]
if callable(self.singleuser_uid):
singleuser_uid = yield gen.maybe_future(self.singleuser_uid(self))
else:
singleuser_uid = self.singleuser_uid
if callable(self.singleuser_fs_gid):
singleuser_fs_gid = yield gen.maybe_future(
self.singleuser_fs_gid(self))
else:
singleuser_fs_gid = self.singleuser_fs_gid
if self.cmd:
real_cmd = self.cmd + self.get_args()
else:
real_cmd = None
return make_pod_spec(
self.pod_name,
self.singleuser_image_spec,
self.singleuser_image_pull_policy,
self.singleuser_image_pull_secrets,
self.port,
real_cmd,
singleuser_uid,
singleuser_fs_gid,
self.get_env(),
self._expand_all(self.volumes) + hack_volumes,
self._expand_all(self.volume_mounts) + hack_volume_mounts,
self.singleuser_extra_labels,
self.cpu_limit,
self.cpu_guarantee,
self.mem_limit,
self.mem_guarantee,
)
def get_pvc_manifest(self):
"""
Make a pvc manifest that will spawn current user's pvc.
"""
return make_pvc_spec(self.pvc_name, self.user_storage_class,
self.user_storage_access_modes,
self.user_storage_capacity)
@gen.coroutine
def get_pod_info(self, pod_name):
"""
Fetch info about a specific pod with the given pod name in current namespace
Return `None` if pod with given name does not exist in current namespace
"""
try:
response = yield self.httpclient.fetch(
self.request(k8s_url(
self.namespace,
'pods',
pod_name,
)))
except HTTPError as e:
if e.code == 404:
return None
raise
data = response.body.decode('utf-8')
return json.loads(data)
@gen.coroutine
def get_pvc_info(self, pvc_name):
"""
Fetch info about a specific pvc with the given pvc name in current namespace
Return `None` if pvc with given name does not exist in current namespace
"""
try:
response = yield self.httpclient.fetch(
self.request(
k8s_url(
self.namespace,
'persistentvolumeclaims',
pvc_name,
)))
except HTTPError as e:
if e.code == 404:
return None
raise
data = response.body.decode('utf-8')
return json.loads(data)
def is_pod_running(self, pod):
"""
Check if the given pod is running
pod must be a dictionary representing a Pod kubernetes API object.
"""
return pod['status']['phase'] == 'Running'
def get_state(self):
"""
Save state required to reinstate this user's pod from scratch
We save the pod_name, even though we could easily compute it,
because JupyterHub requires you save *some* state! Otherwise
it assumes your server is dead. This works around that.
It's also useful for cases when the pod_template changes between
restarts - this keeps the old pods around.
"""
state = super().get_state()
state['pod_name'] = self.pod_name
return state
def load_state(self, state):
"""
Load state from storage required to reinstate this user's pod
Since this runs after __init__, this will override the generated pod_name
if there's one we have saved in state. These are the same in most cases,
but if the pod_template has changed in between restarts, it will no longer
be the case. This allows us to continue serving from the old pods with
the old names.
"""
if 'pod_name' in state:
self.pod_name = state['pod_name']
@gen.coroutine
def poll(self):
"""
Check if the pod is still running.
Returns None if it is, and 1 if it isn't. These are the return values
JupyterHub expects.
"""
data = yield self.get_pod_info(self.pod_name)
if data is not None and self.is_pod_running(data):
return None
return 1
@gen.coroutine
def start(self):
if self.user_storage_class is not None and self.user_storage_capacity is not None:
pvc_manifest = self.get_pvc_manifest()
try:
yield self.httpclient.fetch(
self.request(url=k8s_url(self.namespace,
'persistentvolumeclaims'),
body=json.dumps(pvc_manifest),
method='POST',
headers={'Content-Type': 'application/json'}))
except:
self.log.info("Pvc " + self.pvc_name +
" already exists, so did not create new pvc.")
# If we run into a 409 Conflict error, it means a pod with the
# same name already exists. We stop it, wait for it to stop, and
# try again. We try 4 times, and if it still fails we give up.
# FIXME: Have better / cleaner retry logic!
retry_times = 4
pod_manifest = yield self.get_pod_manifest()
for i in range(retry_times):
try:
yield self.httpclient.fetch(
self.request(url=k8s_url(self.namespace, 'pods'),
body=json.dumps(pod_manifest),
method='POST',
headers={'Content-Type': 'application/json'}))
break
except HTTPError as e:
if e.code != 409:
# We only want to handle 409 conflict errors
self.log.exception("Failed for %s",
json.dumps(pod_manifest))
raise
self.log.info('Found existing pod %s, attempting to kill',
self.pod_name)
yield self.stop(True)
self.log.info(
'Killed pod %s, will try starting singleuser pod again',
self.pod_name)
else:
raise Exception(
'Can not create user pod %s already exists & could not be deleted'
% self.pod_name)
while True:
data = yield self.get_pod_info(self.pod_name)
if data is not None and self.is_pod_running(data):
break
yield gen.sleep(1)
return (data['status']['podIP'], self.port)
@gen.coroutine
def stop(self, now=False):
body = {
'kind': "DeleteOptions",
'apiVersion': 'v1',
'gracePeriodSeconds': 0
}
yield self.httpclient.fetch(
self.request(
url=k8s_url(self.namespace, 'pods', self.pod_name),
method='DELETE',
body=json.dumps(body),
headers={'Content-Type': 'application/json'},
# Tornado's client thinks DELETE requests shouldn't have a body
# which is a bogus restriction
allow_nonstandard_methods=True,
))
if not now:
# If now is true, just return immediately, do not wait for
# shut down to complete
while True:
data = yield self.get_pod_info(self.pod_name)
if data is None:
break
yield gen.sleep(1)
def _env_keep_default(self):
return []
def get_args(self):
args = super(KubeSpawner, self).get_args()
# HACK: we wanna replace --hub-api-url=self.hub.api_url with
# self.accessible_hub_api_url. This is required in situations where
# the IP the hub is listening on (such as 0.0.0.0) is not the IP where
# it can be reached by the pods (such as the service IP used for the hub!)
# FIXME: Make this better?
to_replace = '--hub-api-url="%s"' % (self.hub.api_url)
for i in range(len(args)):
if args[i] == to_replace:
args[i] = '--hub-api-url="%s"' % (self.accessible_hub_api_url)
break
return args
def get_env(self):
# HACK: This is deprecated, and should be removed soon.
# We set these to be compatible with DockerSpawner and earlie KubeSpawner
env = super(KubeSpawner, self).get_env()
env.update({
'JPY_USER': self.user.name,
'JPY_COOKIE_NAME': self.user.server.cookie_name,
'JPY_BASE_URL': self.user.server.base_url,
'JPY_HUB_PREFIX': self.hub.server.base_url,
'JPY_HUB_API_URL': self.accessible_hub_api_url
})
return env
async def make_client():
await gen.sleep(0)
return SimpleAsyncHTTPClient()
def setUp(self):
super(SimpleHTTPClientTestCase, self).setUp()
# replace the client defined in the parent class
self.http_client = SimpleAsyncHTTPClient(io_loop=self.io_loop,
force_instance=True)
class SimpleHTTPClientTestCase(AsyncHTTPTestCase):
def setUp(self):
super(SimpleHTTPClientTestCase, self).setUp()
self.http_client = SimpleAsyncHTTPClient(self.io_loop)
def get_app(self):
# callable objects to finish pending /trigger requests
self.triggers = collections.deque()
return Application([
url("/trigger", TriggerHandler, dict(queue=self.triggers,
wake_callback=self.stop)),
url("/chunk", ChunkHandler),
url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
url("/hang", HangHandler),
url("/hello", HelloWorldHandler),
url("/content_length", ContentLengthHandler),
url("/head", HeadHandler),
url("/options", OptionsHandler),
url("/no_content", NoContentHandler),
url("/see_other_post", SeeOtherPostHandler),
url("/see_other_get", SeeOtherGetHandler),
url("/host_echo", HostEchoHandler),
], gzip=True)
def test_singleton(self):
# Class "constructor" reuses objects on the same IOLoop
self.assertTrue(SimpleAsyncHTTPClient(self.io_loop) is
SimpleAsyncHTTPClient(self.io_loop))
# unless force_instance is used
self.assertTrue(SimpleAsyncHTTPClient(self.io_loop) is not
SimpleAsyncHTTPClient(self.io_loop,
force_instance=True))
# different IOLoops use different objects
io_loop2 = IOLoop()
self.assertTrue(SimpleAsyncHTTPClient(self.io_loop) is not
SimpleAsyncHTTPClient(io_loop2))
def test_connection_limit(self):
client = SimpleAsyncHTTPClient(self.io_loop, max_clients=2,
force_instance=True)
self.assertEqual(client.max_clients, 2)
seen = []
# Send 4 requests. Two can be sent immediately, while the others
# will be queued
for i in range(4):
client.fetch(self.get_url("/trigger"),
lambda response, i=i: (seen.append(i), self.stop()))
self.wait(condition=lambda: len(self.triggers) == 2)
self.assertEqual(len(client.queue), 2)
# Finish the first two requests and let the next two through
self.triggers.popleft()()
self.triggers.popleft()()
self.wait(condition=lambda: (len(self.triggers) == 2 and
len(seen) == 2))
self.assertEqual(set(seen), set([0, 1]))
self.assertEqual(len(client.queue), 0)
# Finish all the pending requests
self.triggers.popleft()()
self.triggers.popleft()()
self.wait(condition=lambda: len(seen) == 4)
self.assertEqual(set(seen), set([0, 1, 2, 3]))
self.assertEqual(len(self.triggers), 0)
def test_redirect_connection_limit(self):
# following redirects should not consume additional connections
client = SimpleAsyncHTTPClient(self.io_loop, max_clients=1,
force_instance=True)
client.fetch(self.get_url('/countdown/3'), self.stop,
max_redirects=3)
response = self.wait()
response.rethrow()
def test_default_certificates_exist(self):
open(_DEFAULT_CA_CERTS).close()
def test_gzip(self):
# All the tests in this file should be using gzip, but this test
# ensures that it is in fact getting compressed.
# Setting Accept-Encoding manually bypasses the client's
# decompression so we can see the raw data.
response = self.fetch("/chunk", use_gzip=False,
headers={"Accept-Encoding": "gzip"})
self.assertEqual(response.headers["Content-Encoding"], "gzip")
self.assertNotEqual(response.body, b("asdfqwer"))
# Our test data gets bigger when gzipped. Oops. :)
self.assertEqual(len(response.body), 34)
f = gzip.GzipFile(mode="r", fileobj=response.buffer)
self.assertEqual(f.read(), b("asdfqwer"))
def test_max_redirects(self):
response = self.fetch("/countdown/5", max_redirects=3)
self.assertEqual(302, response.code)
# We requested 5, followed three redirects for 4, 3, 2, then the last
# unfollowed redirect is to 1.
self.assertTrue(response.request.url.endswith("/countdown/5"))
self.assertTrue(response.effective_url.endswith("/countdown/2"))
self.assertTrue(response.headers["Location"].endswith("/countdown/1"))
def test_header_reuse(self):
# Apps may reuse a headers object if they are only passing in constant
# headers like user-agent. The header object should not be modified.
headers = HTTPHeaders({'User-Agent': 'Foo'})
self.fetch("/hello", headers=headers)
self.assertEqual(list(headers.get_all()), [('User-Agent', 'Foo')])
def test_see_other_redirect(self):
for code in (302, 303):
response = self.fetch("/see_other_post", method="POST", body="%d" % code)
self.assertEqual(200, response.code)
self.assertTrue(response.request.url.endswith("/see_other_post"))
self.assertTrue(response.effective_url.endswith("/see_other_get"))
#request is the original request, is a POST still
self.assertEqual("POST", response.request.method)
def test_request_timeout(self):
with ExpectLog(gen_log, "uncaught exception"):
response = self.fetch('/trigger?wake=false', request_timeout=0.1)
self.assertEqual(response.code, 599)
self.assertTrue(0.099 < response.request_time < 0.12, response.request_time)
self.assertEqual(str(response.error), "HTTP 599: Timeout")
# trigger the hanging request to let it clean up after itself
self.triggers.popleft()()
@unittest.skipIf(not socket.has_ipv6, 'ipv6 support not present')
def test_ipv6(self):
try:
self.http_server.listen(self.get_http_port(), address='::1')
except socket.gaierror as e:
if e.args[0] == socket.EAI_ADDRFAMILY:
# python supports ipv6, but it's not configured on the network
# interface, so skip this test.
return
raise
url = self.get_url("/hello").replace("localhost", "[::1]")
# ipv6 is currently disabled by default and must be explicitly requested
with ExpectLog(gen_log, "uncaught exception"):
self.http_client.fetch(url, self.stop)
response = self.wait()
self.assertEqual(response.code, 599)
self.http_client.fetch(url, self.stop, allow_ipv6=True)
response = self.wait()
self.assertEqual(response.body, b("Hello world!"))
def test_multiple_content_length_accepted(self):
response = self.fetch("/content_length?value=2,2")
self.assertEqual(response.body, b("ok"))
response = self.fetch("/content_length?value=2,%202,2")
self.assertEqual(response.body, b("ok"))
with ExpectLog(gen_log, "uncaught exception"):
response = self.fetch("/content_length?value=2,4")
self.assertEqual(response.code, 599)
response = self.fetch("/content_length?value=2,%202,3")
self.assertEqual(response.code, 599)
def test_head_request(self):
response = self.fetch("/head", method="HEAD")
self.assertEqual(response.code, 200)
self.assertEqual(response.headers["content-length"], "7")
self.assertFalse(response.body)
def test_options_request(self):
response = self.fetch("/options", method="OPTIONS")
self.assertEqual(response.code, 200)
self.assertEqual(response.headers["content-length"], "2")
self.assertEqual(response.headers["access-control-allow-origin"], "*")
self.assertEqual(response.body, b("ok"))
def test_no_content(self):
response = self.fetch("/no_content")
self.assertEqual(response.code, 204)
# 204 status doesn't need a content-length, but tornado will
# add a zero content-length anyway.
self.assertEqual(response.headers["Content-length"], "0")
# 204 status with non-zero content length is malformed
with ExpectLog(gen_log, "uncaught exception"):
response = self.fetch("/no_content?error=1")
self.assertEqual(response.code, 599)
def test_host_header(self):
host_re = re.compile(b("^localhost:[0-9]+$"))
response = self.fetch("/host_echo")
self.assertTrue(host_re.match(response.body))
url = self.get_url("/host_echo").replace("http://", "http://*****:*****@")
self.http_client.fetch(url, self.stop)
response = self.wait()
self.assertTrue(host_re.match(response.body), response.body)
def test_connection_refused(self):
server_socket, port = bind_unused_port()
server_socket.close()
with ExpectLog(gen_log, ".*"):
self.http_client.fetch("http://localhost:%d/" % port, self.stop)
response = self.wait()
self.assertEqual(599, response.code)
if sys.platform != 'cygwin':
# cygwin returns EPERM instead of ECONNREFUSED here
self.assertTrue(str(errno.ECONNREFUSED) in str(response.error),
response.error)
# This is usually "Connection refused".
# On windows, strerror is broken and returns "Unknown error".
expected_message = os.strerror(errno.ECONNREFUSED)
self.assertTrue(expected_message in str(response.error),
response.error)
class SimpleHTTPClientTestCase(AsyncHTTPTestCase, LogTrapTestCase):
def get_app(self):
# callable objects to finish pending /trigger requests
self.triggers = collections.deque()
return Application([
url("/hello", HelloWorldHandler),
url("/post", PostHandler),
url("/chunk", ChunkHandler),
url("/auth", AuthHandler),
url("/hang", HangHandler),
url("/trigger", TriggerHandler,
dict(queue=self.triggers, wake_callback=self.stop)),
url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
],
gzip=True)
def setUp(self):
super(SimpleHTTPClientTestCase, self).setUp()
# replace the client defined in the parent class
self.http_client = SimpleAsyncHTTPClient(io_loop=self.io_loop,
force_instance=True)
def test_hello_world(self):
response = self.fetch("/hello")
self.assertEqual(response.code, 200)
self.assertEqual(response.headers["Content-Type"], "text/plain")
self.assertEqual(response.body, "Hello world!")
response = self.fetch("/hello?name=Ben")
self.assertEqual(response.body, "Hello Ben!")
def test_streaming_callback(self):
# streaming_callback is also tested in test_chunked
chunks = []
response = self.fetch("/hello", streaming_callback=chunks.append)
# with streaming_callback, data goes to the callback and not response.body
self.assertEqual(chunks, ["Hello world!"])
self.assertFalse(response.body)
def test_post(self):
response = self.fetch("/post", method="POST", body="arg1=foo&arg2=bar")
self.assertEqual(response.code, 200)
self.assertEqual(response.body, "Post arg1: foo, arg2: bar")
def test_chunked(self):
response = self.fetch("/chunk")
self.assertEqual(response.body, "asdfqwer")
chunks = []
response = self.fetch("/chunk", streaming_callback=chunks.append)
self.assertEqual(chunks, ["asdf", "qwer"])
self.assertFalse(response.body)
def test_basic_auth(self):
self.assertEqual(
self.fetch("/auth",
auth_username="******",
auth_password="******").body,
"Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==")
def test_gzip(self):
# All the tests in this file should be using gzip, but this test
# ensures that it is in fact getting compressed.
# Setting Accept-Encoding manually bypasses the client's
# decompression so we can see the raw data.
response = self.fetch("/chunk",
use_gzip=False,
headers={"Accept-Encoding": "gzip"})
self.assertEqual(response.headers["Content-Encoding"], "gzip")
self.assertNotEqual(response.body, "asdfqwer")
# Our test data gets bigger when gzipped. Oops. :)
self.assertEqual(len(response.body), 34)
f = gzip.GzipFile(mode="r", fileobj=response.buffer)
self.assertEqual(f.read(), "asdfqwer")
def test_connect_timeout(self):
# create a socket and bind it to a port, but don't
# call accept so the connection will timeout.
#get_unused_port()
port = get_unused_port()
with closing(socket.socket()) as sock:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(('', port))
sock.listen(1)
self.http_client.fetch("http://*****:*****@")
self.http_client.fetch(url, self.stop)
response = self.wait()
self.assertEqual("Basic " + base64.b64encode("me:secret"),
response.body)
def get_http_client(self):
# Some versions of libcurl have deadlock bugs with ssl,
# so always run these tests with SimpleAsyncHTTPClient.
return SimpleAsyncHTTPClient(io_loop=self.io_loop,
force_instance=True,
defaults=dict(validate_cert=False))
def get_http_client(self):
return SimpleAsyncHTTPClient(io_loop=self.io_loop,
force_instance=True)
def create_client(self, **kwargs):
return SimpleAsyncHTTPClient(self.io_loop, force_instance=True,
**kwargs)
class SimpleHTTPClientTestCase(AsyncHTTPTestCase, LogTrapTestCase):
def get_app(self):
# callable objects to finish pending /trigger requests
self.triggers = collections.deque()
return Application([
url("/hello", HelloWorldHandler),
url("/post", PostHandler),
url("/chunk", ChunkHandler),
url("/auth", AuthHandler),
url("/hang", HangHandler),
url("/trigger", TriggerHandler, dict(queue=self.triggers,
wake_callback=self.stop)),
url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
], gzip=True)
def setUp(self):
super(SimpleHTTPClientTestCase, self).setUp()
# replace the client defined in the parent class
self.http_client = SimpleAsyncHTTPClient(io_loop=self.io_loop,
force_instance=True)
def test_hello_world(self):
response = self.fetch("/hello")
self.assertEqual(response.code, 200)
self.assertEqual(response.headers["Content-Type"], "text/plain")
self.assertEqual(response.body, "Hello world!")
response = self.fetch("/hello?name=Ben")
self.assertEqual(response.body, "Hello Ben!")
def test_streaming_callback(self):
# streaming_callback is also tested in test_chunked
chunks = []
response = self.fetch("/hello",
streaming_callback=chunks.append)
# with streaming_callback, data goes to the callback and not response.body
self.assertEqual(chunks, ["Hello world!"])
self.assertFalse(response.body)
def test_post(self):
response = self.fetch("/post", method="POST",
body="arg1=foo&arg2=bar")
self.assertEqual(response.code, 200)
self.assertEqual(response.body, "Post arg1: foo, arg2: bar")
def test_chunked(self):
response = self.fetch("/chunk")
self.assertEqual(response.body, "asdfqwer")
chunks = []
response = self.fetch("/chunk",
streaming_callback=chunks.append)
self.assertEqual(chunks, ["asdf", "qwer"])
self.assertFalse(response.body)
def test_basic_auth(self):
self.assertEqual(self.fetch("/auth", auth_username="******",
auth_password="******").body,
"Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==")
def test_gzip(self):
# All the tests in this file should be using gzip, but this test
# ensures that it is in fact getting compressed.
# Setting Accept-Encoding manually bypasses the client's
# decompression so we can see the raw data.
response = self.fetch("/chunk", use_gzip=False,
headers={"Accept-Encoding": "gzip"})
self.assertEqual(response.headers["Content-Encoding"], "gzip")
self.assertNotEqual(response.body, "asdfqwer")
# Our test data gets bigger when gzipped. Oops. :)
self.assertEqual(len(response.body), 34)
f = gzip.GzipFile(mode="r", fileobj=response.buffer)
self.assertEqual(f.read(), "asdfqwer")
def test_connect_timeout(self):
# create a socket and bind it to a port, but don't
# call accept so the connection will timeout.
#get_unused_port()
port = get_unused_port()
with closing(socket.socket()) as sock:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(('', port))
sock.listen(1)
self.http_client.fetch("http://localhost:%d/" % port,
self.stop,
connect_timeout=0.1)
response = self.wait()
self.assertEqual(response.code, 599)
self.assertEqual(str(response.error), "HTTP 599: Timeout")
def test_request_timeout(self):
response = self.fetch('/hang', request_timeout=0.1)
self.assertEqual(response.code, 599)
self.assertEqual(str(response.error), "HTTP 599: Timeout")
def test_singleton(self):
# Class "constructor" reuses objects on the same IOLoop
self.assertTrue(SimpleAsyncHTTPClient(self.io_loop) is
SimpleAsyncHTTPClient(self.io_loop))
# unless force_instance is used
self.assertTrue(SimpleAsyncHTTPClient(self.io_loop) is not
SimpleAsyncHTTPClient(self.io_loop,
force_instance=True))
# different IOLoops use different objects
io_loop2 = IOLoop()
self.assertTrue(SimpleAsyncHTTPClient(self.io_loop) is not
SimpleAsyncHTTPClient(io_loop2))
def test_connection_limit(self):
client = SimpleAsyncHTTPClient(self.io_loop, max_clients=2,
force_instance=True)
self.assertEqual(client.max_clients, 2)
seen = []
# Send 4 requests. Two can be sent immediately, while the others
# will be queued
for i in range(4):
client.fetch(self.get_url("/trigger"),
lambda response, i=i: (seen.append(i), self.stop()))
self.wait(condition=lambda: len(self.triggers) == 2)
self.assertEqual(len(client.queue), 2)
# Finish the first two requests and let the next two through
self.triggers.popleft()()
self.triggers.popleft()()
self.wait(condition=lambda: (len(self.triggers) == 2 and
len(seen) == 2))
self.assertEqual(set(seen), set([0, 1]))
self.assertEqual(len(client.queue), 0)
# Finish all the pending requests
self.triggers.popleft()()
self.triggers.popleft()()
self.wait(condition=lambda: len(seen) == 4)
self.assertEqual(set(seen), set([0, 1, 2, 3]))
self.assertEqual(len(self.triggers), 0)
def test_follow_redirect(self):
response = self.fetch("/countdown/2", follow_redirects=False)
self.assertEqual(302, response.code)
self.assertTrue(response.headers["Location"].endswith("/countdown/1"))
response = self.fetch("/countdown/2")
self.assertEqual(200, response.code)
self.assertTrue(response.effective_url.endswith("/countdown/0"))
self.assertEqual("Zero", response.body)
def test_max_redirects(self):
response = self.fetch("/countdown/5", max_redirects=3)
self.assertEqual(302, response.code)
# We requested 5, followed three redirects for 4, 3, 2, then the last
# unfollowed redirect is to 1.
self.assertTrue(response.request.url.endswith("/countdown/5"))
self.assertTrue(response.effective_url.endswith("/countdown/2"))
self.assertTrue(response.headers["Location"].endswith("/countdown/1"))
def test_default_certificates_exist(self):
open(_DEFAULT_CA_CERTS)
def create_client(self, **kwargs):
return SimpleAsyncHTTPClient(self.io_loop, force_instance=True,
defaults=dict(validate_cert=False),
**kwargs)
def get_http_client(self):
client = SimpleAsyncHTTPClient(force_instance=True)
self.assertTrue(isinstance(client, SimpleAsyncHTTPClient))
return client
def setUp(self):
super(SimpleHTTPClientTestCase, self).setUp()
self.http_client = SimpleAsyncHTTPClient(self.io_loop)
def post(self):
url = self.get_argument('url')
http = SimpleAsyncHTTPClient()
http.fetch(url, callback=self.on_fetch_complete)
def get_http_client(self):
# body_producer doesn't work on curl_httpclient, so override the
# configured AsyncHTTPClient implementation.
return SimpleAsyncHTTPClient(io_loop=self.io_loop)
