def test_require_encryption():
"""
Functional test for "require_encryption" setting.
"""
@gen.coroutine
def handle_comm(comm):
comm.abort()
c = {
'tls': {
'ca-file': ca_file,
'scheduler': {
'key': key1,
'cert': cert1,
},
'worker': {
'cert': keycert1,
},
},
}
with new_config(c):
sec = Security()
c['require-encryption'] = True
with new_config(c):
sec2 = Security()
for listen_addr in ['inproc://', 'tls://']:
with listen(listen_addr, handle_comm,
connection_args=sec.get_listen_args('scheduler')) as listener:
comm = yield connect(listener.contact_address,
connection_args=sec2.get_connection_args('worker'))
comm.abort()
with listen(listen_addr, handle_comm,
connection_args=sec2.get_listen_args('scheduler')) as listener:
comm = yield connect(listener.contact_address,
connection_args=sec2.get_connection_args('worker'))
comm.abort()
@contextmanager
def check_encryption_error():
with pytest.raises(RuntimeError) as excinfo:
yield
assert "encryption required" in str(excinfo.value)
for listen_addr in ['tcp://']:
with listen(listen_addr, handle_comm,
connection_args=sec.get_listen_args('scheduler')) as listener:
comm = yield connect(listener.contact_address,
connection_args=sec.get_connection_args('worker'))
comm.abort()
with pytest.raises(RuntimeError):
yield connect(listener.contact_address,
connection_args=sec2.get_connection_args('worker'))
with pytest.raises(RuntimeError):
listen(listen_addr, handle_comm,
connection_args=sec2.get_listen_args('scheduler'))
def test_require_encryption():
"""
Functional test for "require_encryption" setting.
"""
@gen.coroutine
def handle_comm(comm):
comm.abort()
c = {
'tls': {
'ca-file': ca_file,
'scheduler': {
'key': key1,
'cert': cert1,
},
'worker': {
'cert': keycert1,
},
},
}
with new_config(c):
sec = Security()
c['require-encryption'] = True
with new_config(c):
sec2 = Security()
for listen_addr in ['inproc://', 'tls://']:
with listen(listen_addr, handle_comm,
connection_args=sec.get_listen_args('scheduler')) as listener:
comm = yield connect(listener.contact_address,
connection_args=sec2.get_connection_args('worker'))
comm.abort()
with listen(listen_addr, handle_comm,
connection_args=sec2.get_listen_args('scheduler')) as listener:
comm = yield connect(listener.contact_address,
connection_args=sec2.get_connection_args('worker'))
comm.abort()
@contextmanager
def check_encryption_error():
with pytest.raises(RuntimeError) as excinfo:
yield
assert "encryption required" in str(excinfo.value)
for listen_addr in ['tcp://']:
with listen(listen_addr, handle_comm,
connection_args=sec.get_listen_args('scheduler')) as listener:
comm = yield connect(listener.contact_address,
connection_args=sec.get_connection_args('worker'))
comm.abort()
with pytest.raises(RuntimeError):
yield connect(listener.contact_address,
connection_args=sec2.get_connection_args('worker'))
with pytest.raises(RuntimeError):
listen(listen_addr, handle_comm,
connection_args=sec2.get_listen_args('scheduler'))
def test_tls_reject_certificate():
cli_ctx = get_client_ssl_context()
serv_ctx = get_server_ssl_context()
# These certs are not signed by our test CA
bad_cert_key = ('tls-self-signed-cert.pem', 'tls-self-signed-key.pem')
bad_cli_ctx = get_client_ssl_context(*bad_cert_key)
bad_serv_ctx = get_server_ssl_context(*bad_cert_key)
@gen.coroutine
def handle_comm(comm):
scheme, loc = parse_address(comm.peer_address)
assert scheme == 'tls'
yield comm.close()
# Listener refuses a connector not signed by the CA
listener = listen('tls://',
handle_comm,
connection_args={'ssl_context': serv_ctx})
listener.start()
with pytest.raises(EnvironmentError) as excinfo:
yield connect(listener.contact_address,
timeout=0.5,
connection_args={'ssl_context': bad_cli_ctx})
# The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028
if sys.version_info >= (3, ) and os.name != 'nt':
try:
# See https://serverfault.com/questions/793260/what-does-tlsv1-alert-unknown-ca-mean
assert "unknown ca" in str(excinfo.value)
except AssertionError:
if os.name == 'nt':
assert "An existing connection was forcibly closed" in str(
excinfo.value)
else:
raise
# Sanity check
comm = yield connect(listener.contact_address,
timeout=0.5,
connection_args={'ssl_context': cli_ctx})
yield comm.close()
# Connector refuses a listener not signed by the CA
listener = listen('tls://',
handle_comm,
connection_args={'ssl_context': bad_serv_ctx})
listener.start()
with pytest.raises(EnvironmentError) as excinfo:
yield connect(listener.contact_address,
timeout=0.5,
connection_args={'ssl_context': cli_ctx})
# The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028
if sys.version_info >= (3, ):
assert "certificate verify failed" in str(excinfo.value)
def test_tls_listen_connect():
"""
Functional test for TLS connection args.
"""
@gen.coroutine
def handle_comm(comm):
peer_addr = comm.peer_address
assert peer_addr.startswith("tls://")
yield comm.write("hello")
yield comm.close()
c = {
"tls": {
"ca-file": ca_file,
"scheduler": {
"key": key1,
"cert": cert1
},
"worker": {
"cert": keycert1
},
}
}
with new_config(c):
sec = Security()
c["tls"]["ciphers"] = FORCED_CIPHER
with new_config(c):
forced_cipher_sec = Security()
with listen("tls://",
handle_comm,
connection_args=sec.get_listen_args("scheduler")) as listener:
comm = yield connect(listener.contact_address,
connection_args=sec.get_connection_args("worker"))
msg = yield comm.read()
assert msg == "hello"
comm.abort()
# No SSL context for client
with pytest.raises(TypeError):
yield connect(
listener.contact_address,
connection_args=sec.get_connection_args("client"),
)
# Check forced cipher
comm = yield connect(
listener.contact_address,
connection_args=forced_cipher_sec.get_connection_args("worker"),
)
cipher, _, _, = comm.extra_info["cipher"]
assert cipher in [FORCED_CIPHER] + TLS_13_CIPHERS
comm.abort()
def test_tls_listen_connect():
"""
Functional test for TLS connection args.
"""
@gen.coroutine
def handle_comm(comm):
peer_addr = comm.peer_address
assert peer_addr.startswith('tls://')
yield comm.write('hello')
yield comm.close()
c = {
'tls': {
'ca-file': ca_file,
'scheduler': {
'key': key1,
'cert': cert1,
},
'worker': {
'cert': keycert1,
},
},
}
with new_config(c):
sec = Security()
c['tls']['ciphers'] = FORCED_CIPHER
with new_config(c):
forced_cipher_sec = Security()
with listen('tls://',
handle_comm,
connection_args=sec.get_listen_args('scheduler')) as listener:
comm = yield connect(listener.contact_address,
connection_args=sec.get_connection_args('worker'))
msg = yield comm.read()
assert msg == 'hello'
comm.abort()
# No SSL context for client
with pytest.raises(TypeError):
yield connect(listener.contact_address,
connection_args=sec.get_connection_args('client'))
# Check forced cipher
comm = yield connect(
listener.contact_address,
connection_args=forced_cipher_sec.get_connection_args('worker'))
cipher, _, _, = comm.extra_info['cipher']
assert cipher in [FORCED_CIPHER] + TLS_13_CIPHERS
comm.abort()
def test_tls_reject_certificate():
cli_ctx = get_client_ssl_context()
serv_ctx = get_server_ssl_context()
# These certs are not signed by our test CA
bad_cert_key = ('tls-self-signed-cert.pem', 'tls-self-signed-key.pem')
bad_cli_ctx = get_client_ssl_context(*bad_cert_key)
bad_serv_ctx = get_server_ssl_context(*bad_cert_key)
@gen.coroutine
def handle_comm(comm):
scheme, loc = parse_address(comm.peer_address)
assert scheme == 'tls'
yield comm.close()
# Listener refuses a connector not signed by the CA
listener = listen('tls://', handle_comm,
connection_args={'ssl_context': serv_ctx})
listener.start()
with pytest.raises(EnvironmentError) as excinfo:
yield connect(listener.contact_address, timeout=0.5,
connection_args={'ssl_context': bad_cli_ctx})
# The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028
if sys.version_info >= (3,) and os.name != 'nt':
try:
# See https://serverfault.com/questions/793260/what-does-tlsv1-alert-unknown-ca-mean
assert "unknown ca" in str(excinfo.value)
except AssertionError:
if os.name == 'nt':
assert "An existing connection was forcibly closed" in str(excinfo.value)
else:
raise
# Sanity check
comm = yield connect(listener.contact_address, timeout=0.5,
connection_args={'ssl_context': cli_ctx})
yield comm.close()
# Connector refuses a listener not signed by the CA
listener = listen('tls://', handle_comm,
connection_args={'ssl_context': bad_serv_ctx})
listener.start()
with pytest.raises(EnvironmentError) as excinfo:
yield connect(listener.contact_address, timeout=0.5,
connection_args={'ssl_context': cli_ctx})
# The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028
if sys.version_info >= (3,):
assert "certificate verify failed" in str(excinfo.value)
def test_inproc_comm_closed_explicit_2():
listener_errors = []
@gen.coroutine
def handle_comm(comm):
# Wait
try:
yield comm.read()
except CommClosedError:
assert comm.closed()
listener_errors.append(True)
else:
yield comm.close()
listener = listen("inproc://", handle_comm)
listener.start()
contact_addr = listener.contact_address
comm = yield connect(contact_addr)
yield comm.close()
assert comm.closed()
start = time()
while len(listener_errors) < 1:
assert time() < start + 1
yield gen.sleep(0.01)
assert len(listener_errors) == 1
with pytest.raises(CommClosedError):
yield comm.read()
with pytest.raises(CommClosedError):
yield comm.write("foo")
comm = yield connect(contact_addr)
yield comm.write("foo")
with pytest.raises(CommClosedError):
yield comm.read()
with pytest.raises(CommClosedError):
yield comm.write("foo")
assert comm.closed()
comm = yield connect(contact_addr)
yield comm.write("foo")
start = time()
while not comm.closed():
yield gen.sleep(0.01)
assert time() < start + 2
yield comm.close()
yield comm.close()
def test_inproc_comm_closed_explicit_2():
listener_errors = []
@gen.coroutine
def handle_comm(comm):
# Wait
try:
yield comm.read()
except CommClosedError:
assert comm.closed()
listener_errors.append(True)
else:
comm.close()
listener = listen('inproc://', handle_comm)
listener.start()
contact_addr = listener.contact_address
comm = yield connect(contact_addr)
comm.close()
assert comm.closed()
start = time()
while len(listener_errors) < 1:
assert time() < start + 1
yield gen.sleep(0.01)
assert len(listener_errors) == 1
with pytest.raises(CommClosedError):
yield comm.read()
with pytest.raises(CommClosedError):
yield comm.write("foo")
comm = yield connect(contact_addr)
comm.write("foo")
with pytest.raises(CommClosedError):
yield comm.read()
with pytest.raises(CommClosedError):
yield comm.write("foo")
assert comm.closed()
comm = yield connect(contact_addr)
comm.write("foo")
start = time()
while not comm.closed():
yield gen.sleep(0.01)
assert time() < start + 2
comm.close()
comm.close()
def test_tls_listen_connect():
"""
Functional test for TLS connection args.
"""
@gen.coroutine
def handle_comm(comm):
peer_addr = comm.peer_address
assert peer_addr.startswith('tls://')
yield comm.write('hello')
yield comm.close()
c = {
'tls': {
'ca-file': ca_file,
'scheduler': {
'key': key1,
'cert': cert1,
},
'worker': {
'cert': keycert1,
},
},
}
with new_config(c):
sec = Security()
c['tls']['ciphers'] = FORCED_CIPHER
with new_config(c):
forced_cipher_sec = Security()
with listen('tls://', handle_comm,
connection_args=sec.get_listen_args('scheduler')) as listener:
comm = yield connect(listener.contact_address,
connection_args=sec.get_connection_args('worker'))
msg = yield comm.read()
assert msg == 'hello'
comm.abort()
# No SSL context for client
with pytest.raises(TypeError):
yield connect(listener.contact_address,
connection_args=sec.get_connection_args('client'))
# Check forced cipher
comm = yield connect(listener.contact_address,
connection_args=forced_cipher_sec.get_connection_args('worker'))
cipher, _, _, = comm.extra_info['cipher']
assert cipher in [FORCED_CIPHER] + TLS_13_CIPHERS
comm.abort()
def check_comm_closed_implicit(addr, delay=None):
@gen.coroutine
def handle_comm(comm):
yield comm.close()
listener = listen(addr, handle_comm)
listener.start()
contact_addr = listener.contact_address
comm = yield connect(contact_addr)
with pytest.raises(CommClosedError):
yield comm.write({})
comm = yield connect(contact_addr)
with pytest.raises(CommClosedError):
yield comm.read()
def check_comm_closed_implicit(addr, delay=None, listen_args=None,
connect_args=None):
@gen.coroutine
def handle_comm(comm):
yield comm.close()
listener = listen(addr, handle_comm, connection_args=listen_args)
listener.start()
contact_addr = listener.contact_address
comm = yield connect(contact_addr, connection_args=connect_args)
with pytest.raises(CommClosedError):
yield comm.write({})
comm = yield connect(contact_addr, connection_args=connect_args)
with pytest.raises(CommClosedError):
yield comm.read()
async def connect(self, addr, timeout=None):
"""
Get a Comm to the given address. For internal use.
"""
available = self.available[addr]
occupied = self.occupied[addr]
while available:
comm = available.pop()
if comm.closed():
self.semaphore.release()
else:
occupied.add(comm)
return comm
if self.semaphore.locked():
self.collect()
self._n_connecting += 1
await self.semaphore.acquire()
fut = None
try:
if self.status != Status.running:
raise CommClosedError(
f"ConnectionPool not running. Status: {self.status}"
)
fut = asyncio.create_task(
connect(
addr,
timeout=timeout or self.timeout,
deserialize=self.deserialize,
**self.connection_args,
)
)
self._connecting.add(fut)
comm = await fut
comm.name = "ConnectionPool"
comm._pool = weakref.ref(self)
comm.allow_offload = self.allow_offload
self._created.add(comm)
occupied.add(comm)
return comm
except asyncio.CancelledError as exc:
self.semaphore.release()
raise CommClosedError(
f"ConnectionPool not running. Status: {self.status}"
) from exc
except Exception:
self.semaphore.release()
raise
finally:
self._connecting.discard(fut)
self._n_connecting -= 1
def _main(self, address, obj, n_transfers, **kwargs):
listener = listen(address, partial(self._handle_comm, n_transfers), **kwargs)
yield listener.start()
comm = yield connect(listener.contact_address, **kwargs)
for i in range(n_transfers):
yield comm.write(obj)
# Read back to ensure that the round-trip is complete
for i in range(n_transfers):
yield comm.read()
yield comm.close()
listener.stop()
def test_comm_failure_threading():
"""
When we fail to connect, make sure we don't make a lot
of threads.
We only assert for PY3, because the thread limit only is
set for python 3. See github PR #2403 discussion for info.
"""
@gen.coroutine
def sleep_for_60ms():
max_thread_count = 0
for x in range(60):
yield gen.sleep(0.001)
thread_count = threading.active_count()
if thread_count > max_thread_count:
max_thread_count = thread_count
raise gen.Return(max_thread_count)
original_thread_count = threading.active_count()
# tcp.TCPConnector()
sleep_future = sleep_for_60ms()
with pytest.raises(IOError):
yield connect("tcp://localhost:28400", 0.052)
max_thread_count = yield sleep_future
# 2 is the number set by BaseTCPConnector.executor (ThreadPoolExecutor)
if PY3:
assert max_thread_count <= 2 + original_thread_count
# tcp.TLSConnector()
sleep_future = sleep_for_60ms()
with pytest.raises(IOError):
yield connect(
"tls://localhost:28400",
0.052,
connection_args={"ssl_context": get_client_ssl_context()},
)
max_thread_count = yield sleep_future
if PY3:
assert max_thread_count <= 2 + original_thread_count
def client_communicate(key, delay=0):
comm = yield connect(listener.contact_address, connection_args=connect_args)
assert comm.peer_address == listener.contact_address
yield comm.write({"op": "ping", "data": key})
yield comm.write({"op": "foobar"})
if delay:
yield gen.sleep(delay)
msg = yield comm.read()
assert msg == {"op": "pong", "data": key}
l.append(key)
yield comm.close()
def client_communicate(key, delay=0):
comm = yield connect(listener.contact_address)
assert comm.peer_address == listener.contact_address
yield comm.write({'op': 'ping', 'data': key})
yield comm.write({'op': 'foobar'})
if delay:
yield gen.sleep(delay)
msg = yield comm.read()
assert msg == {'op': 'pong', 'data': key}
l.append(key)
yield comm.close()
def client_communicate(key, delay=0):
comm = yield connect(listener.contact_address,
connection_args=connect_args)
assert comm.peer_address == listener.contact_address
yield comm.write({'op': 'ping', 'data': key})
yield comm.write({'op': 'foobar'})
if delay:
yield gen.sleep(delay)
msg = yield comm.read()
assert msg == {'op': 'pong', 'data': key}
l.append(key)
yield comm.close()
def get_comm_pair(listen_addr, listen_args=None, connect_args=None):
q = queues.Queue()
def handle_comm(comm):
q.put(comm)
listener = listen(listen_addr, handle_comm, connection_args=listen_args)
listener.start()
comm = yield connect(listener.contact_address,
connection_args=connect_args)
serv_comm = yield q.get()
raise gen.Return((comm, serv_comm))
def check_comm_closed_explicit(addr):
@gen.coroutine
def handle_comm(comm):
# Wait
try:
yield comm.read()
except CommClosedError:
pass
listener = listen(addr, handle_comm)
listener.start()
contact_addr = listener.contact_address
comm = yield connect(contact_addr)
comm.close()
with pytest.raises(CommClosedError):
yield comm.write({})
comm = yield connect(contact_addr)
comm.close()
with pytest.raises(CommClosedError):
yield comm.read()
def check_deserialize_eoferror(addr):
"""
EOFError when deserializing should close the comm.
"""
@gen.coroutine
def handle_comm(comm):
yield comm.write({'data': to_serialize(_EOFRaising())})
with pytest.raises(CommClosedError):
yield comm.read()
with listen(addr, handle_comm) as listener:
comm = yield connect(listener.contact_address, deserialize=deserialize)
with pytest.raises(CommClosedError):
yield comm.read()
def check_deserialize_eoferror(addr):
"""
EOFError when deserializing should close the comm.
"""
@gen.coroutine
def handle_comm(comm):
yield comm.write({"data": to_serialize(_EOFRaising())})
with pytest.raises(CommClosedError):
yield comm.read()
with listen(addr, handle_comm) as listener:
comm = yield connect(listener.contact_address, deserialize=deserialize)
with pytest.raises(CommClosedError):
yield comm.read()
def get_comm_pair(listen_addr, listen_args=None, connect_args=None,
**kwargs):
q = queues.Queue()
def handle_comm(comm):
q.put(comm)
listener = listen(listen_addr, handle_comm,
connection_args=listen_args, **kwargs)
listener.start()
comm = yield connect(listener.contact_address,
connection_args=connect_args, **kwargs)
serv_comm = yield q.get()
raise gen.Return((comm, serv_comm))
def check_connector_deserialize(addr, deserialize, in_value, check_out):
done = locks.Event()
@gen.coroutine
def handle_comm(comm):
yield comm.write(in_value)
yield done.wait()
yield comm.close()
with listen(addr, handle_comm) as listener:
comm = yield connect(listener.contact_address, deserialize=deserialize)
out_value = yield comm.read()
done.set()
yield comm.close()
check_out(out_value)
def check_connector_deserialize(addr, deserialize, in_value, check_out):
q = queues.Queue()
@gen.coroutine
def handle_comm(comm):
msg = yield q.get()
yield comm.write(msg)
yield comm.close()
with listen(addr, handle_comm) as listener:
comm = yield connect(listener.contact_address, deserialize=deserialize)
q.put_nowait(in_value)
out_value = yield comm.read()
yield comm.close()
check_out(out_value)
def check_connector_deserialize(addr, deserialize, in_value, check_out):
done = locks.Event()
@gen.coroutine
def handle_comm(comm):
yield comm.write(in_value)
yield done.wait()
yield comm.close()
with listen(addr, handle_comm) as listener:
comm = yield connect(listener.contact_address, deserialize=deserialize)
out_value = yield comm.read()
done.set()
yield comm.close()
check_out(out_value)
def check_listener_deserialize(addr, deserialize, in_value, check_out):
q = queues.Queue()
@gen.coroutine
def handle_comm(comm):
msg = yield comm.read()
q.put_nowait(msg)
yield comm.close()
with listen(addr, handle_comm, deserialize=deserialize) as listener:
comm = yield connect(listener.contact_address)
yield comm.write(in_value)
out_value = yield q.get()
check_out(out_value)
yield comm.close()
def test_tls_temporary_credentials_functional():
pytest.importorskip("cryptography")
@gen.coroutine
def handle_comm(comm):
peer_addr = comm.peer_address
assert peer_addr.startswith("tls://")
yield comm.write("hello")
yield comm.close()
sec = Security.temporary()
with listen("tls://",
handle_comm,
connection_args=sec.get_listen_args("scheduler")) as listener:
comm = yield connect(listener.contact_address,
connection_args=sec.get_connection_args("worker"))
msg = yield comm.read()
assert msg == "hello"
comm.abort()
def check_connect_timeout(addr):
t1 = time()
with pytest.raises(IOError):
yield connect(addr, timeout=0.15)
dt = time() - t1
assert 1 >= dt >= 0.1
def check_connect_timeout(addr):
t1 = time()
with pytest.raises(IOError):
yield connect(addr, timeout=0.15)
dt = time() - t1
assert 1 >= dt >= 0.1
def test_require_encryption():
"""
Functional test for "require_encryption" setting.
"""
@gen.coroutine
def handle_comm(comm):
comm.abort()
c = {
"distributed.comm.tls.ca-file": ca_file,
"distributed.comm.tls.scheduler.key": key1,
"distributed.comm.tls.scheduler.cert": cert1,
"distributed.comm.tls.worker.cert": keycert1,
}
with dask.config.set(c):
sec = Security()
c["distributed.comm.require-encryption"] = True
with dask.config.set(c):
sec2 = Security()
for listen_addr in ["inproc://", "tls://"]:
with listen(
listen_addr, handle_comm, connection_args=sec.get_listen_args("scheduler")
) as listener:
comm = yield connect(
listener.contact_address,
connection_args=sec2.get_connection_args("worker"),
)
comm.abort()
with listen(
listen_addr, handle_comm, connection_args=sec2.get_listen_args("scheduler")
) as listener:
comm = yield connect(
listener.contact_address,
connection_args=sec2.get_connection_args("worker"),
)
comm.abort()
@contextmanager
def check_encryption_error():
with pytest.raises(RuntimeError) as excinfo:
yield
assert "encryption required" in str(excinfo.value)
for listen_addr in ["tcp://"]:
with listen(
listen_addr, handle_comm, connection_args=sec.get_listen_args("scheduler")
) as listener:
comm = yield connect(
listener.contact_address,
connection_args=sec.get_connection_args("worker"),
)
comm.abort()
with pytest.raises(RuntimeError):
yield connect(
listener.contact_address,
connection_args=sec2.get_connection_args("worker"),
)
with pytest.raises(RuntimeError):
listen(
listen_addr,
handle_comm,
connection_args=sec2.get_listen_args("scheduler"),
)
def _connect_close(self, addr):
comm = yield connect(addr)
yield comm.close()
