async def test_comm_objs():
comm, serv_comm = await get_comm_pair()
scheme, loc = parse_address(comm.peer_address)
assert scheme == "ucx"
scheme, loc = parse_address(serv_comm.peer_address)
assert scheme == "ucx"
assert comm.peer_address == serv_comm.local_address
def uri_from_host_port(host_arg, port_arg, default_port):
"""
Process the *host* and *port* CLI options.
Return a URI.
"""
# Much of distributed depends on a well-known IP being assigned to
# each entity (Worker, Scheduler, etc.), so avoid "universal" addresses
# like '' which would listen on all registered IPs and interfaces.
scheme, loc = parse_address(host_arg or "")
host, port = parse_host_port(
loc, port_arg if port_arg is not None else default_port)
if port is None and port_arg is None:
port_arg = default_port
if port and port_arg and port != port_arg:
raise ValueError("port number given twice in options: "
"host %r and port %r" % (host_arg, port_arg))
if port is None and port_arg is not None:
port = port_arg
# Note `port = 0` means "choose a random port"
if port is None:
port = default_port
loc = unparse_host_port(host, port)
addr = unparse_address(scheme, loc)
return addr
def handle_comm(comm):
scheme, loc = parse_address(comm.peer_address)
assert scheme == bound_scheme
msg = yield comm.read()
assert msg['op'] == 'ping'
msg['op'] = 'pong'
yield comm.write(msg)
msg = yield comm.read()
assert msg['op'] == 'foobar'
yield comm.close()
def handle_comm(comm):
scheme, loc = parse_address(comm.peer_address)
assert scheme == bound_scheme
msg = yield comm.read()
assert msg["op"] == "ping"
msg["op"] = "pong"
yield comm.write(msg)
msg = yield comm.read()
assert msg["op"] == "foobar"
yield comm.close()
def handle_comm(comm):
scheme, loc = parse_address(comm.peer_address)
assert scheme == bound_scheme
msg = yield comm.read()
assert msg['op'] == 'ping'
msg['op'] = 'pong'
yield comm.write(msg)
msg = yield comm.read()
assert msg['op'] == 'foobar'
yield comm.close()
async def test_inproc_handshakes_concurrently():
async def handle_comm():
pass
async with listen("inproc://", handle_comm) as listener:
addr = listener.listen_address
scheme, loc = parse_address(addr)
connector = get_backend(scheme).get_connector()
comm1 = await connector.connect(loc)
comm2 = await connector.connect(loc)
await comm1.close()
await comm2.close()
async def test_inproc_continues_listening_after_handshake_error():
async def handle_comm():
pass
async with listen("inproc://", handle_comm) as listener:
addr = listener.listen_address
scheme, loc = parse_address(addr)
connector = get_backend(scheme).get_connector()
comm = await connector.connect(loc)
await comm.close()
comm = await connector.connect(loc)
await comm.close()
def handle_comm(comm):
scheme, loc = parse_address(comm.peer_address)
assert scheme == 'tls'
yield comm.close()
def check_client_server(addr, check_listen_addr=None, check_contact_addr=None,
listen_args=None, connect_args=None):
"""
Abstract client / server test.
"""
@gen.coroutine
def handle_comm(comm):
scheme, loc = parse_address(comm.peer_address)
assert scheme == bound_scheme
msg = yield comm.read()
assert msg['op'] == 'ping'
msg['op'] = 'pong'
yield comm.write(msg)
msg = yield comm.read()
assert msg['op'] == 'foobar'
yield comm.close()
# Arbitrary connection args should be ignored
listen_args = listen_args or {'xxx': 'bar'}
connect_args = connect_args or {'xxx': 'foo'}
listener = listen(addr, handle_comm, connection_args=listen_args)
listener.start()
# Check listener properties
bound_addr = listener.listen_address
bound_scheme, bound_loc = parse_address(bound_addr)
assert bound_scheme in ('inproc', 'tcp', 'tls')
assert bound_scheme == parse_address(addr)[0]
if check_listen_addr is not None:
check_listen_addr(bound_loc)
contact_addr = listener.contact_address
contact_scheme, contact_loc = parse_address(contact_addr)
assert contact_scheme == bound_scheme
if check_contact_addr is not None:
check_contact_addr(contact_loc)
else:
assert contact_addr == bound_addr
# Check client <-> server comms
l = []
@gen.coroutine
def client_communicate(key, delay=0):
comm = yield connect(listener.contact_address,
connection_args=connect_args)
assert comm.peer_address == listener.contact_address
yield comm.write({'op': 'ping', 'data': key})
yield comm.write({'op': 'foobar'})
if delay:
yield gen.sleep(delay)
msg = yield comm.read()
assert msg == {'op': 'pong', 'data': key}
l.append(key)
yield comm.close()
yield client_communicate(key=1234)
# Many clients at once
futures = [client_communicate(key=i, delay=0.05) for i in range(20)]
yield futures
assert set(l) == {1234} | set(range(20))
listener.stop()
def handle_comm(comm):
scheme, loc = parse_address(comm.peer_address)
assert scheme == "tls"
yield comm.close()
def check_client_server(
addr,
check_listen_addr=None,
check_contact_addr=None,
listen_args=None,
connect_args=None,
):
"""
Abstract client / server test.
"""
@gen.coroutine
def handle_comm(comm):
scheme, loc = parse_address(comm.peer_address)
assert scheme == bound_scheme
msg = yield comm.read()
assert msg["op"] == "ping"
msg["op"] = "pong"
yield comm.write(msg)
msg = yield comm.read()
assert msg["op"] == "foobar"
yield comm.close()
# Arbitrary connection args should be ignored
listen_args = listen_args or {"xxx": "bar"}
connect_args = connect_args or {"xxx": "foo"}
listener = listen(addr, handle_comm, connection_args=listen_args)
listener.start()
# Check listener properties
bound_addr = listener.listen_address
bound_scheme, bound_loc = parse_address(bound_addr)
assert bound_scheme in backends
assert bound_scheme == parse_address(addr)[0]
if check_listen_addr is not None:
check_listen_addr(bound_loc)
contact_addr = listener.contact_address
contact_scheme, contact_loc = parse_address(contact_addr)
assert contact_scheme == bound_scheme
if check_contact_addr is not None:
check_contact_addr(contact_loc)
else:
assert contact_addr == bound_addr
# Check client <-> server comms
l = []
@gen.coroutine
def client_communicate(key, delay=0):
comm = yield connect(listener.contact_address,
connection_args=connect_args)
assert comm.peer_address == listener.contact_address
yield comm.write({"op": "ping", "data": key})
yield comm.write({"op": "foobar"})
if delay:
yield gen.sleep(delay)
msg = yield comm.read()
assert msg == {"op": "pong", "data": key}
l.append(key)
yield comm.close()
yield client_communicate(key=1234)
# Many clients at once
futures = [client_communicate(key=i, delay=0.05) for i in range(20)]
yield futures
assert set(l) == {1234} | set(range(20))
listener.stop()
async def handle_comm(comm):
scheme, loc = parse_address(comm.peer_address)
assert scheme == "tls"
await comm.close()
def check_client_server(addr, check_listen_addr=None, check_contact_addr=None):
"""
Abstract client / server test.
"""
@gen.coroutine
def handle_comm(comm):
scheme, loc = parse_address(comm.peer_address)
assert scheme == bound_scheme
msg = yield comm.read()
assert msg['op'] == 'ping'
msg['op'] = 'pong'
yield comm.write(msg)
msg = yield comm.read()
assert msg['op'] == 'foobar'
yield comm.close()
listener = listen(addr, handle_comm)
listener.start()
# Check listener properties
bound_addr = listener.listen_address
bound_scheme, bound_loc = parse_address(bound_addr)
assert bound_scheme in ('inproc', 'tcp', 'zmq')
assert bound_scheme == parse_address(addr)[0]
if check_listen_addr is not None:
check_listen_addr(bound_loc)
contact_addr = listener.contact_address
contact_scheme, contact_loc = parse_address(contact_addr)
assert contact_scheme == bound_scheme
if check_contact_addr is not None:
check_contact_addr(contact_loc)
else:
assert contact_addr == bound_addr
# Check client <-> server comms
l = []
@gen.coroutine
def client_communicate(key, delay=0):
comm = yield connect(listener.contact_address)
assert comm.peer_address == listener.contact_address
yield comm.write({'op': 'ping', 'data': key})
yield comm.write({'op': 'foobar'})
if delay:
yield gen.sleep(delay)
msg = yield comm.read()
assert msg == {'op': 'pong', 'data': key}
l.append(key)
yield comm.close()
yield client_communicate(key=1234)
# Many clients at once
futures = [client_communicate(key=i, delay=0.05) for i in range(20)]
yield futures
assert set(l) == {1234} | set(range(20))
listener.stop()
