Exemplo n.º 1
0
async def test_connection_pool_detects_remote_close():
    server = Server({"ping": pingpong})
    await server.listen("tcp://")

    # open a connection, use it and give it back to the pool
    p = ConnectionPool(limit=10)
    conn = await p.connect(server.address)
    await send_recv(conn, op="ping")
    p.reuse(server.address, conn)

    # now close this connection on the *server*
    assert len(server._comms) == 1
    server_conn = list(server._comms.keys())[0]
    await server_conn.close()

    # give the ConnectionPool some time to realize that the connection is closed
    await asyncio.sleep(0.1)

    # the connection pool should not hand out `conn` again
    conn2 = await p.connect(server.address)
    assert conn2 is not conn
    p.reuse(server.address, conn2)
    # check that `conn` has ben removed from the internal data structures
    assert p.open == 1 and p.active == 0

    # check connection pool invariants hold even after it detects a closed connection
    # while creating conn2:
    p._validate()
    p.close()
Exemplo n.º 2
0
def test_connection_pool_tls():
    """
    Make sure connection args are supported.
    """
    sec = tls_security()
    connection_args = sec.get_connection_args('client')
    listen_args = sec.get_listen_args('scheduler')

    @gen.coroutine
    def ping(comm, delay=0.01):
        yield gen.sleep(delay)
        raise gen.Return('pong')

    servers = [Server({'ping': ping}) for i in range(10)]
    for server in servers:
        server.listen('tls://', listen_args=listen_args)

    rpc = ConnectionPool(limit=5, connection_args=connection_args)

    yield [rpc(s.address).ping() for s in servers[:5]]
    yield [rpc(s.address).ping() for s in servers[::2]]
    yield [rpc(s.address).ping() for s in servers]
    assert rpc.active == 0

    rpc.close()
Exemplo n.º 3
0
def test_connection_pool_remove():
    @gen.coroutine
    def ping(comm, delay=0.01):
        yield gen.sleep(delay)
        raise gen.Return('pong')

    servers = [Server({'ping': ping}) for i in range(5)]
    for server in servers:
        server.listen(0)

    rpc = ConnectionPool(limit=10)
    serv = servers.pop()
    yield [rpc(s.address).ping() for s in servers]
    yield [rpc(serv.address).ping() for i in range(3)]
    yield rpc.connect(serv.address)
    assert sum(map(len, rpc.available.values())) == 6
    assert sum(map(len, rpc.occupied.values())) == 1
    assert rpc.active == 1
    assert rpc.open == 7

    rpc.remove(serv.address)
    assert serv.address not in rpc.available
    assert serv.address not in rpc.occupied
    assert sum(map(len, rpc.available.values())) == 4
    assert sum(map(len, rpc.occupied.values())) == 0
    assert rpc.active == 0
    assert rpc.open == 4

    rpc.collect()
    comm = yield rpc.connect(serv.address)
    rpc.remove(serv.address)
    rpc.reuse(serv.address, comm)

    rpc.close()
Exemplo n.º 4
0
def test_gather_from_workers_permissive(c, s, a, b):
    rpc = ConnectionPool()
    x = yield c.scatter({"x": 1}, workers=a.address)

    data, missing, bad_workers = yield gather_from_workers(
        {"x": [a.address], "y": [b.address]}, rpc=rpc
    )

    assert data == {"x": 1}
    assert list(missing) == ["y"]
Exemplo n.º 5
0
async def test_connection_pool():
    async def ping(comm, delay=0.1):
        await asyncio.sleep(delay)
        return "pong"

    servers = [Server({"ping": ping}) for i in range(10)]
    for server in servers:
        await server.listen(0)

    rpc = ConnectionPool(limit=5)

    # Reuse connections
    await asyncio.gather(
        *[rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[:5]])
    await asyncio.gather(*[rpc(s.address).ping() for s in servers[:5]])
    await asyncio.gather(
        *[rpc("127.0.0.1:%d" % s.port).ping() for s in servers[:5]])
    await asyncio.gather(
        *[rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[:5]])
    assert sum(map(len, rpc.available.values())) == 5
    assert sum(map(len, rpc.occupied.values())) == 0
    assert rpc.active == 0
    assert rpc.open == 5

    # Clear out connections to make room for more
    await asyncio.gather(
        *[rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[5:]])
    assert rpc.active == 0
    assert rpc.open == 5

    s = servers[0]
    await asyncio.gather(
        *[rpc(ip="127.0.0.1", port=s.port).ping(delay=0.1) for i in range(3)])
    assert len(rpc.available["tcp://127.0.0.1:%d" % s.port]) == 3

    # Explicitly clear out connections
    rpc.collect()
    start = time()
    while any(rpc.available.values()):
        await asyncio.sleep(0.01)
        assert time() < start + 2

    rpc.close()
Exemplo n.º 6
0
async def test_connection_pool_respects_limit():

    limit = 5

    async def ping(comm, delay=0.01):
        await asyncio.sleep(delay)
        return "pong"

    async def do_ping(pool, port):
        assert pool.open <= limit
        await pool(ip="127.0.0.1", port=port).ping()
        assert pool.open <= limit

    servers = [Server({"ping": ping}) for i in range(10)]
    for server in servers:
        await server.listen(0)

    pool = ConnectionPool(limit=limit)

    await asyncio.gather(*[do_ping(pool, s.port) for s in servers])
Exemplo n.º 7
0
def test_connection_pool():

    @gen.coroutine
    def ping(comm, delay=0.1):
        yield gen.sleep(delay)
        raise gen.Return('pong')

    servers = [Server({'ping': ping}) for i in range(10)]
    for server in servers:
        server.listen(0)

    rpc = ConnectionPool(limit=5)

    # Reuse connections
    yield [rpc(ip='127.0.0.1', port=s.port).ping() for s in servers[:5]]
    yield [rpc(s.address).ping() for s in servers[:5]]
    yield [rpc('127.0.0.1:%d' % s.port).ping() for s in servers[:5]]
    yield [rpc(ip='127.0.0.1', port=s.port).ping() for s in servers[:5]]
    assert sum(map(len, rpc.available.values())) == 5
    assert sum(map(len, rpc.occupied.values())) == 0
    assert rpc.active == 0
    assert rpc.open == 5

    # Clear out connections to make room for more
    yield [rpc(ip='127.0.0.1', port=s.port).ping() for s in servers[5:]]
    assert rpc.active == 0
    assert rpc.open == 5

    s = servers[0]
    yield [rpc(ip='127.0.0.1', port=s.port).ping(delay=0.1) for i in range(3)]
    assert len(rpc.available['tcp://127.0.0.1:%d' % s.port]) == 3

    # Explicitly clear out connections
    rpc.collect()
    start = time()
    while any(rpc.available.values()):
        yield gen.sleep(0.01)
        assert time() < start + 2

    rpc.close()
Exemplo n.º 8
0
async def test_connection_pool_remove():
    async def ping(comm, delay=0.01):
        await asyncio.sleep(delay)
        return "pong"

    servers = [Server({"ping": ping}) for i in range(5)]
    for server in servers:
        await server.listen(0)

    rpc = ConnectionPool(limit=10)
    serv = servers.pop()
    await asyncio.gather(*[rpc(s.address).ping() for s in servers])
    await asyncio.gather(*[rpc(serv.address).ping() for i in range(3)])
    await rpc.connect(serv.address)
    assert sum(map(len, rpc.available.values())) == 6
    assert sum(map(len, rpc.occupied.values())) == 1
    assert rpc.active == 1
    assert rpc.open == 7

    rpc.remove(serv.address)
    assert serv.address not in rpc.available
    assert serv.address not in rpc.occupied
    assert sum(map(len, rpc.available.values())) == 4
    assert sum(map(len, rpc.occupied.values())) == 0
    assert rpc.active == 0
    assert rpc.open == 4

    rpc.collect()

    # this pattern of calls (esp. `reuse` after `remove`)
    # can happen in case of worker failures:
    comm = await rpc.connect(serv.address)
    rpc.remove(serv.address)
    rpc.reuse(serv.address, comm)

    rpc.close()
Exemplo n.º 9
0
async def test_connection_pool_tls():
    """
    Make sure connection args are supported.
    """
    sec = tls_security()
    connection_args = sec.get_connection_args("client")
    listen_args = sec.get_listen_args("scheduler")

    async def ping(comm, delay=0.01):
        await asyncio.sleep(delay)
        return "pong"

    servers = [Server({"ping": ping}) for i in range(10)]
    for server in servers:
        await server.listen("tls://", listen_args=listen_args)

    rpc = ConnectionPool(limit=5, connection_args=connection_args)

    await asyncio.gather(*[rpc(s.address).ping() for s in servers[:5]])
    await asyncio.gather(*[rpc(s.address).ping() for s in servers[::2]])
    await asyncio.gather(*[rpc(s.address).ping() for s in servers])
    assert rpc.active == 0

    rpc.close()