def update_graph():
weights = [
lb.get_dip_weight(POOL_HANDLE, ip, p)
for (ip, p) in sorted(server_IPs.keys())
]
loads = yield all_results(
[server.callRemote('get_load', loadavg) for server in servers])
conns = yield all_results(
[server.callRemote('get_conn_count') for server in servers])
now = (datetime.now() - demo_start).total_seconds()
data = [now] + weights + loads + conns
with open('./data.tsv', 'a') as f:
f.write('{}\n'.format('\t'.join([str(x) for x in data])))
f.flush()
def test_dont_drop(remote_module, p4run):
lb, client, server1 = yield all_results([
LoadBalancer.get_initialised('s1', topology_db_file=p4run.topo_path),
remote_module('myutils.client', host='h3'),
remote_module('myutils.server', 8001, host='h1'),
])
server1_ip, server2_ip = [p4run.topo.get_host_ip(h) for h in ('h1', 'h2')]
pool_handle = yield lb.add_pool('10.0.0.1', 8000)
print(' --------- v1: {server1} ---------')
yield lb.add_dip(pool_handle, server1_ip, 8001)
yield lb.commit()
yield client.callRemote('start_echo_clients', '10.0.0.1', 8000, count=5)
time.sleep(0.5) # make sure the clients have connected
print(' --------- v2: {server2} ---------')
yield lb.rm_dip(pool_handle, server1_ip, 8001)
yield lb.add_dip(pool_handle, server2_ip, 8001)
yield lb.commit()
time.sleep(0.5) # give it time to notice the breakage, if any
# test #1: this does not explode :D -- this would throw if something bad happened to the connections
yield client.callRemote('close_all_connections')
nconns = yield server1.callRemote('get_conn_count')
assert nconns == 5
def test_connections_break(remote_module, p4run):
"""
Tests that connections break when the pool changes. Exists to avoid false positives.
Same as `l4_loadbalancer/keep_connections_test.py:test_old_versions`,
but uses the unversioned thing and expects it to fail."""
lb, client, server1, server2 = yield all_results([
LoadBalancerUnversioned.get_initialised('s1', topology_db_file=p4run.topo_path),
remote_module('myutils.client', host='h3'),
remote_module('myutils.server', 8001, host='h1'),
remote_module('myutils.server', 8001, host='h2'),
])
yield sleep(0.5)
server1_ip, server2_ip = [p4run.topo.get_host_ip(h) for h in ('h1', 'h2')]
pool_handle = yield lb.add_pool('10.0.0.1', 8000)
print(' --------- create a pool with server1 ---------')
yield lb.add_dip(pool_handle, server1_ip, 8001)
yield client.callRemote('start_echo_clients', '10.0.0.1', 8000, count=5)
yield sleep(0.5) # make sure the clients have connected
print(' --------- break it: change the pool ---------')
yield lb.rm_dip(pool_handle, server1_ip, 8001)
yield lb.add_dip(pool_handle, server2_ip, 8001)
yield sleep(0.5) # give time to notice the breakage
with pytest.raises(pb.RemoteError) as excinfo:
# this should throw a ConnectionLost, because we broke the connections
yield client.callRemote('close_all_connections')
assert 'ConnectionLost' in str(excinfo)
def adjust_weights(self):
for pool, dips_ in self.pool_hashes.items():
dips = sorted(dips_.keys())
loads = yield all_results(
[defer.maybeDeferred(self.get_metrics, *dip) for dip in dips])
wanted = self.normalise(self.metrics_to_weights(loads))
for (dip, dport), w in zip(dips, wanted):
yield self.set_dip_weight(pool, dip, dport, w)
yield self.commit()
def prefill_mac_table(self):
# * MAC associations to hosts
ds = []
for h in self.topo.get_hosts_connected_to(self.sw_name):
mac = self.topo.get_host_mac(h)
port = self.topo.node_to_node_port_num(self.sw_name, h)
ds.append(
self.controller.table_add("mac", "forward", [mac],
[str(port)]))
yield all_results(ds)
def test_echo_client(remote_module):
client, server = yield all_results([
remote_module('myutils.client', host=None),
remote_module('myutils.server', 8000, host=None),
])
yield client.callRemote('start_echo_clients', 'localhost', 8000, count=5)
yield sleep(0.5) # make sure the clients have connected
yield client.callRemote('close_all_connections')
nconns1 = yield server.callRemote('get_conn_count')
assert nconns1 == 5
def test_add_dip(remote_module, p4run):
print(' --------- prepare server, client, and loadbalancer ---------')
client, server, lb = yield all_results([
remote_module('myutils.client', host='h1'),
remote_module('myutils.server', 8001, host='h2'),
LoadBalancer.get_initialised('s1', topology_db_file=p4run.topo_path),
])
print(' --------- set up the pool ---------')
pool_h = yield lb.add_pool('10.0.0.1', 8000)
yield lb.add_dip(pool_h, p4run.topo.get_host_ip('h2'), 8001)
print(' --------- check that it worked ---------')
yield client.callRemote('make_connections', '10.0.0.1', 8000, count=47)
num_conns = yield server.callRemote('get_conn_count')
print('{}/47 connections successful'.format(num_conns))
assert num_conns == 47
def test_equal_balancing(remote_module, p4run):
NUM_CONNS = 1000
TOLERANCE = 0.8
pools = {
('10.0.0.1', 8000): [('h1', 8001), ('h2', 8002), ('h3', 8003)],
('10.0.0.1', 7000): [('h1', 7000)]
}
# create pools
lb = yield LoadBalancer.get_initialised('s1',
topology_db_file=p4run.topo_path)
for vip, dips in pools.items():
pool_h = yield lb.add_pool(*vip)
for dip in dips:
yield lb.add_dip(pool_h, p4run.topo.get_host_ip(dip[0]), dip[1])
print(' ----- vips + inverse: -----')
pprint(lb.vips.data)
pprint(lb.vips_inverse.data)
print(' ----- dips + inverse: -----')
pprint(lb.dips.data)
pprint(lb.dips_inverse.data)
# run the servers
servers = {} # vip => [server remote]
for vip, dips in pools.items():
servers[vip] = []
server_ds = []
for dip in dips:
dhost, dport = dip
server_ds.append(remote_module('myutils.server', dport,
host=dhost))
servers[vip] = yield all_results(server_ds)
# run the client
client = yield remote_module('myutils.client', host='h4')
for vip in pools:
yield client.callRemote('make_connections', *vip, count=NUM_CONNS)
# check the servers' connection counts
for vip, dips in pools.items():
expected_conns = TOLERANCE * NUM_CONNS / len(dips)
for server in servers[vip]:
num_conns = yield server.callRemote('get_conn_count')
assert num_conns >= expected_conns, "[FLAKY] distribution should be approximately uniform"
def test_rm_dip(remote_module, p4run):
print(' --------- prepare server, client, and loadbalancer ---------')
client, server, lb = yield all_results([
remote_module('myutils.client', host='h1'),
remote_module('myutils.server', 8001, host='h2'),
LoadBalancer.get_initialised('s1', topology_db_file=p4run.topo_path),
])
print(' --------- set up the pool ---------')
pool_h = yield lb.add_pool('10.0.0.1', 8000)
yield lb.add_dip(pool_h, p4run.topo.get_host_ip('h3'),
8001) # will remove this later
yield lb.add_dip(pool_h, p4run.topo.get_host_ip('h2'), 8001)
yield lb.rm_dip(pool_h, p4run.topo.get_host_ip('h3'), 8001) # tadaaa :D
print(' ----- dips: -----')
pprint(lb.dips.data)
print(' --------- check that it worked ---------')
yield client.callRemote('make_connections', '10.0.0.1', 8000, count=47)
num_conns = yield server.callRemote('get_conn_count')
assert num_conns == 47, "everything should go to h2 because h3 was removed"
def test_rollover_active_conns(remote_module, p4run):
lb, client1, client2, server1, server2 = yield all_results([
LoadBalancer.get_initialised('s1', topology_db_file=p4run.topo_path),
remote_module('myutils.client', host='h3'),
remote_module('myutils.client', host='h4'),
remote_module('myutils.server', 8001, host='h1'),
remote_module('myutils.server', 8001, host='h2'),
])
server1_ip, server2_ip = [p4run.topo.get_host_ip(h) for h in ('h1', 'h2')]
pool_handle = yield lb.add_pool('10.0.0.1', 8000)
for i in range(5):
print(' --------- v1: server1 ---------')
yield lb.add_dip(pool_handle, server1_ip, 8001)
yield lb.commit()
yield client1.callRemote('start_echo_clients',
'10.0.0.1',
8000,
count=5)
yield sleep(0.5) # make sure the clients have connected
print(' --------- v2: server2 ---------')
yield lb.rm_dip(pool_handle, server1_ip, 8001)
yield lb.add_dip(pool_handle, server2_ip, 8001)
# roll over the version => eat old pending tables
for i in range(5):
yield lb.commit()
yield client2.callRemote('start_echo_clients',
'10.0.0.1',
8000,
count=3)
yield sleep(0.5) # give it time to notice the breakage, if any
# this would throw if something bad happened to the connections
yield client1.callRemote('close_all_connections')
yield client2.callRemote('close_all_connections')
yield check_conn_count(server1, 5)
yield check_conn_count(server2, 3)
yield lb.rm_dip(pool_handle, server2_ip,
8001) # clean up before next iteration
def test_direct_conn(remote_module, p4run):
print(' --------- prepare server, client, and loadbalancer ---------')
client, server, lb = yield all_results([
remote_module('myutils.client', host='h1'),
remote_module('myutils.server', 8000, host='h2'),
LoadBalancer.get_initialised('s1', topology_db_file=p4run.topo_path),
])
print(
" --------- add a random pool: unused, just to make sure it doesn't mess things up ---------"
)
pool_h = yield lb.add_pool('10.0.0.1', 4700)
yield lb.add_dip(pool_h, p4run.topo.get_host_ip('h1'), 4700)
yield lb.add_dip(pool_h, p4run.topo.get_host_ip('h2'), 4700)
yield lb.commit()
print(' --------- check that it worked ---------')
yield client.callRemote('make_connections',
p4run.topo.get_host_ip('h2'),
8000,
count=47)
num_conns = yield server.callRemote('get_conn_count')
assert num_conns == 47
def demo(reactor):
lines = WaitForLines()
stdio.StandardIO(lines)
##### Preparation #######################################################
# These are our servers.
server_hosts = [
# host, port, num CPUs
('h1', 9000, 1),
('h1', 9001, 2),
('h2', 9002, 4),
('h2', 9003, 8),
]
# Run the server and client programs and get a "remote control" to them.
servers = yield all_results([
remote_module('myutils.server', port, ncpus, host=host)
for host, port, ncpus in server_hosts
])
clients = yield all_results([
remote_module('myutils.client', host='h3'),
remote_module('myutils.client', host='h4'),
])
# Build an (ip, port) => server remote dict to use later.
topo = Topology('./topology.db')
server_IPs = {(topo.get_host_ip(h), p): remote
for (h, p, _), remote in zip(server_hosts, servers)}
# Teach my load balancer controller how to get load from the servers.
# In real life I could be e.g. SSHing into the servers, or using my
# monitoring infrastructure.
def get_load(ip, port):
return server_IPs[(ip, port)].callRemote('get_load', 20)
def set_weights(loads):
return [1.0 / load for load in loads]
# And start the controller.
lb = yield MetricsLoadBalancer.get_initialised(
's1', get_metrics=get_load, metrics_to_weights=set_weights)
# Create a server pool on the loadbalancer.
pool_handle = yield lb.add_pool('10.0.0.1', 8000)
for ip, port in server_IPs.keys():
yield lb.add_dip(pool_handle, ip, port)
yield lb.commit()
##### Now the fun begins ################################################
setup_graph(server_IPs, lb, 10)
print('---------------- press Enter to start clients -----------------')
yield lines.line_received
@defer.inlineCallbacks
def client0():
"""Client 0 will send long-running requests: closes after 10 seconds."""
print('client0 running')
yield clients[0].callRemote('start_echo_clients',
'10.0.0.1',
8000,
count=4)
yield sleep(10)
yield clients[0].callRemote('close_all_connections')
@defer.inlineCallbacks
def client1():
"""Client 1 will send bursts of short connections (2s)."""
print('client1 running')
yield clients[1].callRemote('start_echo_clients',
'10.0.0.1',
8000,
count=20)
yield sleep(2)
yield clients[1].callRemote('close_all_connections')
# Run client0 every 13 seconds.
task.LoopingCall(client0).start(13)
# Run client1 every 4 seconds.
task.LoopingCall(client1).start(3)
print(
'---------------- press Enter to start adjusting weights ----------------'
)
yield lines.line_received
lb.start_loop()