def test_require_encryption(): """ Functional test for "require_encryption" setting. """ @gen.coroutine def handle_comm(comm): comm.abort() c = { 'tls': { 'ca-file': ca_file, 'scheduler': { 'key': key1, 'cert': cert1, }, 'worker': { 'cert': keycert1, }, }, } with new_config(c): sec = Security() c['require-encryption'] = True with new_config(c): sec2 = Security() for listen_addr in ['inproc://', 'tls://']: with listen(listen_addr, handle_comm, connection_args=sec.get_listen_args('scheduler')) as listener: comm = yield connect(listener.contact_address, connection_args=sec2.get_connection_args('worker')) comm.abort() with listen(listen_addr, handle_comm, connection_args=sec2.get_listen_args('scheduler')) as listener: comm = yield connect(listener.contact_address, connection_args=sec2.get_connection_args('worker')) comm.abort() @contextmanager def check_encryption_error(): with pytest.raises(RuntimeError) as excinfo: yield assert "encryption required" in str(excinfo.value) for listen_addr in ['tcp://']: with listen(listen_addr, handle_comm, connection_args=sec.get_listen_args('scheduler')) as listener: comm = yield connect(listener.contact_address, connection_args=sec.get_connection_args('worker')) comm.abort() with pytest.raises(RuntimeError): yield connect(listener.contact_address, connection_args=sec2.get_connection_args('worker')) with pytest.raises(RuntimeError): listen(listen_addr, handle_comm, connection_args=sec2.get_listen_args('scheduler'))
def test_require_encryption(): """ Functional test for "require_encryption" setting. """ @gen.coroutine def handle_comm(comm): comm.abort() c = { 'tls': { 'ca-file': ca_file, 'scheduler': { 'key': key1, 'cert': cert1, }, 'worker': { 'cert': keycert1, }, }, } with new_config(c): sec = Security() c['require-encryption'] = True with new_config(c): sec2 = Security() for listen_addr in ['inproc://', 'tls://']: with listen(listen_addr, handle_comm, connection_args=sec.get_listen_args('scheduler')) as listener: comm = yield connect(listener.contact_address, connection_args=sec2.get_connection_args('worker')) comm.abort() with listen(listen_addr, handle_comm, connection_args=sec2.get_listen_args('scheduler')) as listener: comm = yield connect(listener.contact_address, connection_args=sec2.get_connection_args('worker')) comm.abort() @contextmanager def check_encryption_error(): with pytest.raises(RuntimeError) as excinfo: yield assert "encryption required" in str(excinfo.value) for listen_addr in ['tcp://']: with listen(listen_addr, handle_comm, connection_args=sec.get_listen_args('scheduler')) as listener: comm = yield connect(listener.contact_address, connection_args=sec.get_connection_args('worker')) comm.abort() with pytest.raises(RuntimeError): yield connect(listener.contact_address, connection_args=sec2.get_connection_args('worker')) with pytest.raises(RuntimeError): listen(listen_addr, handle_comm, connection_args=sec2.get_listen_args('scheduler'))
def test_tls_reject_certificate(): cli_ctx = get_client_ssl_context() serv_ctx = get_server_ssl_context() # These certs are not signed by our test CA bad_cert_key = ('tls-self-signed-cert.pem', 'tls-self-signed-key.pem') bad_cli_ctx = get_client_ssl_context(*bad_cert_key) bad_serv_ctx = get_server_ssl_context(*bad_cert_key) @gen.coroutine def handle_comm(comm): scheme, loc = parse_address(comm.peer_address) assert scheme == 'tls' yield comm.close() # Listener refuses a connector not signed by the CA listener = listen('tls://', handle_comm, connection_args={'ssl_context': serv_ctx}) listener.start() with pytest.raises(EnvironmentError) as excinfo: yield connect(listener.contact_address, timeout=0.5, connection_args={'ssl_context': bad_cli_ctx}) # The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028 if sys.version_info >= (3, ) and os.name != 'nt': try: # See https://serverfault.com/questions/793260/what-does-tlsv1-alert-unknown-ca-mean assert "unknown ca" in str(excinfo.value) except AssertionError: if os.name == 'nt': assert "An existing connection was forcibly closed" in str( excinfo.value) else: raise # Sanity check comm = yield connect(listener.contact_address, timeout=0.5, connection_args={'ssl_context': cli_ctx}) yield comm.close() # Connector refuses a listener not signed by the CA listener = listen('tls://', handle_comm, connection_args={'ssl_context': bad_serv_ctx}) listener.start() with pytest.raises(EnvironmentError) as excinfo: yield connect(listener.contact_address, timeout=0.5, connection_args={'ssl_context': cli_ctx}) # The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028 if sys.version_info >= (3, ): assert "certificate verify failed" in str(excinfo.value)
def test_tls_listen_connect(): """ Functional test for TLS connection args. """ @gen.coroutine def handle_comm(comm): peer_addr = comm.peer_address assert peer_addr.startswith("tls://") yield comm.write("hello") yield comm.close() c = { "tls": { "ca-file": ca_file, "scheduler": { "key": key1, "cert": cert1 }, "worker": { "cert": keycert1 }, } } with new_config(c): sec = Security() c["tls"]["ciphers"] = FORCED_CIPHER with new_config(c): forced_cipher_sec = Security() with listen("tls://", handle_comm, connection_args=sec.get_listen_args("scheduler")) as listener: comm = yield connect(listener.contact_address, connection_args=sec.get_connection_args("worker")) msg = yield comm.read() assert msg == "hello" comm.abort() # No SSL context for client with pytest.raises(TypeError): yield connect( listener.contact_address, connection_args=sec.get_connection_args("client"), ) # Check forced cipher comm = yield connect( listener.contact_address, connection_args=forced_cipher_sec.get_connection_args("worker"), ) cipher, _, _, = comm.extra_info["cipher"] assert cipher in [FORCED_CIPHER] + TLS_13_CIPHERS comm.abort()
def test_tls_listen_connect(): """ Functional test for TLS connection args. """ @gen.coroutine def handle_comm(comm): peer_addr = comm.peer_address assert peer_addr.startswith('tls://') yield comm.write('hello') yield comm.close() c = { 'tls': { 'ca-file': ca_file, 'scheduler': { 'key': key1, 'cert': cert1, }, 'worker': { 'cert': keycert1, }, }, } with new_config(c): sec = Security() c['tls']['ciphers'] = FORCED_CIPHER with new_config(c): forced_cipher_sec = Security() with listen('tls://', handle_comm, connection_args=sec.get_listen_args('scheduler')) as listener: comm = yield connect(listener.contact_address, connection_args=sec.get_connection_args('worker')) msg = yield comm.read() assert msg == 'hello' comm.abort() # No SSL context for client with pytest.raises(TypeError): yield connect(listener.contact_address, connection_args=sec.get_connection_args('client')) # Check forced cipher comm = yield connect( listener.contact_address, connection_args=forced_cipher_sec.get_connection_args('worker')) cipher, _, _, = comm.extra_info['cipher'] assert cipher in [FORCED_CIPHER] + TLS_13_CIPHERS comm.abort()
def test_tls_reject_certificate(): cli_ctx = get_client_ssl_context() serv_ctx = get_server_ssl_context() # These certs are not signed by our test CA bad_cert_key = ('tls-self-signed-cert.pem', 'tls-self-signed-key.pem') bad_cli_ctx = get_client_ssl_context(*bad_cert_key) bad_serv_ctx = get_server_ssl_context(*bad_cert_key) @gen.coroutine def handle_comm(comm): scheme, loc = parse_address(comm.peer_address) assert scheme == 'tls' yield comm.close() # Listener refuses a connector not signed by the CA listener = listen('tls://', handle_comm, connection_args={'ssl_context': serv_ctx}) listener.start() with pytest.raises(EnvironmentError) as excinfo: yield connect(listener.contact_address, timeout=0.5, connection_args={'ssl_context': bad_cli_ctx}) # The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028 if sys.version_info >= (3,) and os.name != 'nt': try: # See https://serverfault.com/questions/793260/what-does-tlsv1-alert-unknown-ca-mean assert "unknown ca" in str(excinfo.value) except AssertionError: if os.name == 'nt': assert "An existing connection was forcibly closed" in str(excinfo.value) else: raise # Sanity check comm = yield connect(listener.contact_address, timeout=0.5, connection_args={'ssl_context': cli_ctx}) yield comm.close() # Connector refuses a listener not signed by the CA listener = listen('tls://', handle_comm, connection_args={'ssl_context': bad_serv_ctx}) listener.start() with pytest.raises(EnvironmentError) as excinfo: yield connect(listener.contact_address, timeout=0.5, connection_args={'ssl_context': cli_ctx}) # The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028 if sys.version_info >= (3,): assert "certificate verify failed" in str(excinfo.value)
def test_inproc_comm_closed_explicit_2(): listener_errors = [] @gen.coroutine def handle_comm(comm): # Wait try: yield comm.read() except CommClosedError: assert comm.closed() listener_errors.append(True) else: yield comm.close() listener = listen("inproc://", handle_comm) listener.start() contact_addr = listener.contact_address comm = yield connect(contact_addr) yield comm.close() assert comm.closed() start = time() while len(listener_errors) < 1: assert time() < start + 1 yield gen.sleep(0.01) assert len(listener_errors) == 1 with pytest.raises(CommClosedError): yield comm.read() with pytest.raises(CommClosedError): yield comm.write("foo") comm = yield connect(contact_addr) yield comm.write("foo") with pytest.raises(CommClosedError): yield comm.read() with pytest.raises(CommClosedError): yield comm.write("foo") assert comm.closed() comm = yield connect(contact_addr) yield comm.write("foo") start = time() while not comm.closed(): yield gen.sleep(0.01) assert time() < start + 2 yield comm.close() yield comm.close()
def test_inproc_comm_closed_explicit_2(): listener_errors = [] @gen.coroutine def handle_comm(comm): # Wait try: yield comm.read() except CommClosedError: assert comm.closed() listener_errors.append(True) else: comm.close() listener = listen('inproc://', handle_comm) listener.start() contact_addr = listener.contact_address comm = yield connect(contact_addr) comm.close() assert comm.closed() start = time() while len(listener_errors) < 1: assert time() < start + 1 yield gen.sleep(0.01) assert len(listener_errors) == 1 with pytest.raises(CommClosedError): yield comm.read() with pytest.raises(CommClosedError): yield comm.write("foo") comm = yield connect(contact_addr) comm.write("foo") with pytest.raises(CommClosedError): yield comm.read() with pytest.raises(CommClosedError): yield comm.write("foo") assert comm.closed() comm = yield connect(contact_addr) comm.write("foo") start = time() while not comm.closed(): yield gen.sleep(0.01) assert time() < start + 2 comm.close() comm.close()
def test_tls_listen_connect(): """ Functional test for TLS connection args. """ @gen.coroutine def handle_comm(comm): peer_addr = comm.peer_address assert peer_addr.startswith('tls://') yield comm.write('hello') yield comm.close() c = { 'tls': { 'ca-file': ca_file, 'scheduler': { 'key': key1, 'cert': cert1, }, 'worker': { 'cert': keycert1, }, }, } with new_config(c): sec = Security() c['tls']['ciphers'] = FORCED_CIPHER with new_config(c): forced_cipher_sec = Security() with listen('tls://', handle_comm, connection_args=sec.get_listen_args('scheduler')) as listener: comm = yield connect(listener.contact_address, connection_args=sec.get_connection_args('worker')) msg = yield comm.read() assert msg == 'hello' comm.abort() # No SSL context for client with pytest.raises(TypeError): yield connect(listener.contact_address, connection_args=sec.get_connection_args('client')) # Check forced cipher comm = yield connect(listener.contact_address, connection_args=forced_cipher_sec.get_connection_args('worker')) cipher, _, _, = comm.extra_info['cipher'] assert cipher in [FORCED_CIPHER] + TLS_13_CIPHERS comm.abort()
def check_comm_closed_implicit(addr, delay=None): @gen.coroutine def handle_comm(comm): yield comm.close() listener = listen(addr, handle_comm) listener.start() contact_addr = listener.contact_address comm = yield connect(contact_addr) with pytest.raises(CommClosedError): yield comm.write({}) comm = yield connect(contact_addr) with pytest.raises(CommClosedError): yield comm.read()
def check_comm_closed_implicit(addr, delay=None, listen_args=None, connect_args=None): @gen.coroutine def handle_comm(comm): yield comm.close() listener = listen(addr, handle_comm, connection_args=listen_args) listener.start() contact_addr = listener.contact_address comm = yield connect(contact_addr, connection_args=connect_args) with pytest.raises(CommClosedError): yield comm.write({}) comm = yield connect(contact_addr, connection_args=connect_args) with pytest.raises(CommClosedError): yield comm.read()
async def connect(self, addr, timeout=None): """ Get a Comm to the given address. For internal use. """ available = self.available[addr] occupied = self.occupied[addr] while available: comm = available.pop() if comm.closed(): self.semaphore.release() else: occupied.add(comm) return comm if self.semaphore.locked(): self.collect() self._n_connecting += 1 await self.semaphore.acquire() fut = None try: if self.status != Status.running: raise CommClosedError( f"ConnectionPool not running. Status: {self.status}" ) fut = asyncio.create_task( connect( addr, timeout=timeout or self.timeout, deserialize=self.deserialize, **self.connection_args, ) ) self._connecting.add(fut) comm = await fut comm.name = "ConnectionPool" comm._pool = weakref.ref(self) comm.allow_offload = self.allow_offload self._created.add(comm) occupied.add(comm) return comm except asyncio.CancelledError as exc: self.semaphore.release() raise CommClosedError( f"ConnectionPool not running. Status: {self.status}" ) from exc except Exception: self.semaphore.release() raise finally: self._connecting.discard(fut) self._n_connecting -= 1
def _main(self, address, obj, n_transfers, **kwargs): listener = listen(address, partial(self._handle_comm, n_transfers), **kwargs) yield listener.start() comm = yield connect(listener.contact_address, **kwargs) for i in range(n_transfers): yield comm.write(obj) # Read back to ensure that the round-trip is complete for i in range(n_transfers): yield comm.read() yield comm.close() listener.stop()
def test_comm_failure_threading(): """ When we fail to connect, make sure we don't make a lot of threads. We only assert for PY3, because the thread limit only is set for python 3. See github PR #2403 discussion for info. """ @gen.coroutine def sleep_for_60ms(): max_thread_count = 0 for x in range(60): yield gen.sleep(0.001) thread_count = threading.active_count() if thread_count > max_thread_count: max_thread_count = thread_count raise gen.Return(max_thread_count) original_thread_count = threading.active_count() # tcp.TCPConnector() sleep_future = sleep_for_60ms() with pytest.raises(IOError): yield connect("tcp://localhost:28400", 0.052) max_thread_count = yield sleep_future # 2 is the number set by BaseTCPConnector.executor (ThreadPoolExecutor) if PY3: assert max_thread_count <= 2 + original_thread_count # tcp.TLSConnector() sleep_future = sleep_for_60ms() with pytest.raises(IOError): yield connect( "tls://localhost:28400", 0.052, connection_args={"ssl_context": get_client_ssl_context()}, ) max_thread_count = yield sleep_future if PY3: assert max_thread_count <= 2 + original_thread_count
def client_communicate(key, delay=0): comm = yield connect(listener.contact_address, connection_args=connect_args) assert comm.peer_address == listener.contact_address yield comm.write({"op": "ping", "data": key}) yield comm.write({"op": "foobar"}) if delay: yield gen.sleep(delay) msg = yield comm.read() assert msg == {"op": "pong", "data": key} l.append(key) yield comm.close()
def client_communicate(key, delay=0): comm = yield connect(listener.contact_address) assert comm.peer_address == listener.contact_address yield comm.write({'op': 'ping', 'data': key}) yield comm.write({'op': 'foobar'}) if delay: yield gen.sleep(delay) msg = yield comm.read() assert msg == {'op': 'pong', 'data': key} l.append(key) yield comm.close()
def client_communicate(key, delay=0): comm = yield connect(listener.contact_address, connection_args=connect_args) assert comm.peer_address == listener.contact_address yield comm.write({'op': 'ping', 'data': key}) yield comm.write({'op': 'foobar'}) if delay: yield gen.sleep(delay) msg = yield comm.read() assert msg == {'op': 'pong', 'data': key} l.append(key) yield comm.close()
def get_comm_pair(listen_addr, listen_args=None, connect_args=None): q = queues.Queue() def handle_comm(comm): q.put(comm) listener = listen(listen_addr, handle_comm, connection_args=listen_args) listener.start() comm = yield connect(listener.contact_address, connection_args=connect_args) serv_comm = yield q.get() raise gen.Return((comm, serv_comm))
def check_comm_closed_explicit(addr): @gen.coroutine def handle_comm(comm): # Wait try: yield comm.read() except CommClosedError: pass listener = listen(addr, handle_comm) listener.start() contact_addr = listener.contact_address comm = yield connect(contact_addr) comm.close() with pytest.raises(CommClosedError): yield comm.write({}) comm = yield connect(contact_addr) comm.close() with pytest.raises(CommClosedError): yield comm.read()
def check_deserialize_eoferror(addr): """ EOFError when deserializing should close the comm. """ @gen.coroutine def handle_comm(comm): yield comm.write({'data': to_serialize(_EOFRaising())}) with pytest.raises(CommClosedError): yield comm.read() with listen(addr, handle_comm) as listener: comm = yield connect(listener.contact_address, deserialize=deserialize) with pytest.raises(CommClosedError): yield comm.read()
def check_deserialize_eoferror(addr): """ EOFError when deserializing should close the comm. """ @gen.coroutine def handle_comm(comm): yield comm.write({"data": to_serialize(_EOFRaising())}) with pytest.raises(CommClosedError): yield comm.read() with listen(addr, handle_comm) as listener: comm = yield connect(listener.contact_address, deserialize=deserialize) with pytest.raises(CommClosedError): yield comm.read()
def get_comm_pair(listen_addr, listen_args=None, connect_args=None, **kwargs): q = queues.Queue() def handle_comm(comm): q.put(comm) listener = listen(listen_addr, handle_comm, connection_args=listen_args, **kwargs) listener.start() comm = yield connect(listener.contact_address, connection_args=connect_args, **kwargs) serv_comm = yield q.get() raise gen.Return((comm, serv_comm))
def check_connector_deserialize(addr, deserialize, in_value, check_out): done = locks.Event() @gen.coroutine def handle_comm(comm): yield comm.write(in_value) yield done.wait() yield comm.close() with listen(addr, handle_comm) as listener: comm = yield connect(listener.contact_address, deserialize=deserialize) out_value = yield comm.read() done.set() yield comm.close() check_out(out_value)
def check_connector_deserialize(addr, deserialize, in_value, check_out): q = queues.Queue() @gen.coroutine def handle_comm(comm): msg = yield q.get() yield comm.write(msg) yield comm.close() with listen(addr, handle_comm) as listener: comm = yield connect(listener.contact_address, deserialize=deserialize) q.put_nowait(in_value) out_value = yield comm.read() yield comm.close() check_out(out_value)
def check_connector_deserialize(addr, deserialize, in_value, check_out): done = locks.Event() @gen.coroutine def handle_comm(comm): yield comm.write(in_value) yield done.wait() yield comm.close() with listen(addr, handle_comm) as listener: comm = yield connect(listener.contact_address, deserialize=deserialize) out_value = yield comm.read() done.set() yield comm.close() check_out(out_value)
def check_listener_deserialize(addr, deserialize, in_value, check_out): q = queues.Queue() @gen.coroutine def handle_comm(comm): msg = yield comm.read() q.put_nowait(msg) yield comm.close() with listen(addr, handle_comm, deserialize=deserialize) as listener: comm = yield connect(listener.contact_address) yield comm.write(in_value) out_value = yield q.get() check_out(out_value) yield comm.close()
def test_tls_temporary_credentials_functional(): pytest.importorskip("cryptography") @gen.coroutine def handle_comm(comm): peer_addr = comm.peer_address assert peer_addr.startswith("tls://") yield comm.write("hello") yield comm.close() sec = Security.temporary() with listen("tls://", handle_comm, connection_args=sec.get_listen_args("scheduler")) as listener: comm = yield connect(listener.contact_address, connection_args=sec.get_connection_args("worker")) msg = yield comm.read() assert msg == "hello" comm.abort()
def check_connect_timeout(addr): t1 = time() with pytest.raises(IOError): yield connect(addr, timeout=0.15) dt = time() - t1 assert 1 >= dt >= 0.1
def check_connect_timeout(addr): t1 = time() with pytest.raises(IOError): yield connect(addr, timeout=0.15) dt = time() - t1 assert 1 >= dt >= 0.1
def test_require_encryption(): """ Functional test for "require_encryption" setting. """ @gen.coroutine def handle_comm(comm): comm.abort() c = { "distributed.comm.tls.ca-file": ca_file, "distributed.comm.tls.scheduler.key": key1, "distributed.comm.tls.scheduler.cert": cert1, "distributed.comm.tls.worker.cert": keycert1, } with dask.config.set(c): sec = Security() c["distributed.comm.require-encryption"] = True with dask.config.set(c): sec2 = Security() for listen_addr in ["inproc://", "tls://"]: with listen( listen_addr, handle_comm, connection_args=sec.get_listen_args("scheduler") ) as listener: comm = yield connect( listener.contact_address, connection_args=sec2.get_connection_args("worker"), ) comm.abort() with listen( listen_addr, handle_comm, connection_args=sec2.get_listen_args("scheduler") ) as listener: comm = yield connect( listener.contact_address, connection_args=sec2.get_connection_args("worker"), ) comm.abort() @contextmanager def check_encryption_error(): with pytest.raises(RuntimeError) as excinfo: yield assert "encryption required" in str(excinfo.value) for listen_addr in ["tcp://"]: with listen( listen_addr, handle_comm, connection_args=sec.get_listen_args("scheduler") ) as listener: comm = yield connect( listener.contact_address, connection_args=sec.get_connection_args("worker"), ) comm.abort() with pytest.raises(RuntimeError): yield connect( listener.contact_address, connection_args=sec2.get_connection_args("worker"), ) with pytest.raises(RuntimeError): listen( listen_addr, handle_comm, connection_args=sec2.get_listen_args("scheduler"), )
def _connect_close(self, addr): comm = yield connect(addr) yield comm.close()