def test_remove_while_request_not_in_progress():
sim = SimulatorKernel(outputDirectory = None)
server1 = MockServer(sim, latency = 0.1)
server2 = MockServer(sim, latency = 0.1)
lb = LoadBalancer(sim)
lb.addBackend(server1)
lb.addBackend(server2)
onShutdownCompleted = Mock()
def remove_active_server():
if server1.numSeenRequests:
lb.removeBackend(server1, onShutdownCompleted)
else:
lb.removeBackend(server2, onShutdownCompleted)
r1 = Request()
r1.onCompleted = Mock()
sim.add(0, lambda: lb.request(r1))
sim.add(1, lambda: remove_active_server())
sim.add(1, lambda: lb.request(Request()))
sim.add(2, lambda: lb.request(Request()))
sim.add(2, lambda: lb.request(Request()))
sim.run()
r1.onCompleted.assert_called_once_with()
onShutdownCompleted.assert_called_once_with()
assert server1.numSeenRequests == 1 or server2.numSeenRequests == 1
assert server1.numSeenRequests == 3 or server2.numSeenRequests == 3
def test_request_hooks():
sim = SimulatorKernel(outputDirectory = None)
loadBalancer = MockLoadBalancer(sim, latency = 1)
autoScalerController = mock.Mock()
autoScalerController.controlInterval = 1
autoScalerController.onRequest = mock.Mock(return_value=0)
autoScalerController.onCompleted = mock.Mock(return_value=0)
autoScalerController.onControlPeriod = mock.Mock(return_value=0)
autoScalerController.onStatus = mock.Mock(return_value=0)
autoScaler = AutoScaler(sim, loadBalancer, controller = autoScalerController)
assert str(autoScaler)
server1 = mock.Mock(name = 'server1')
server2 = mock.Mock(name = 'server2')
autoScaler.addBackend(server1)
autoScaler.addBackend(server2)
r = Request()
autoScaler.request(r)
sim.add(100, lambda: autoScaler.scaleUp())
sim.run(until = 1000)
# TODO: Check exact call parameters
assert autoScalerController.onRequest.call_count == 1, autoScalerController.onRequest.call_count
assert autoScalerController.onCompleted.call_count == 1, autoScalerController.onCompleted.call_count
assert autoScalerController.onStatus.call_count == 3, autoScalerController.onStatus.call_count
assert autoScalerController.onControlPeriod.call_count == 1000, autoScalerController.onControlPeriod.call_count
def test_with_controller_always_yes():
completedRequests = []
controller = Mock()
controller.withOptional.return_value = True, 1
sim = SimulatorKernel(outputDirectory = None)
server = Server(sim,
serviceTimeY = 10, serviceTimeYVariance = 0,
serviceTimeN = 1, serviceTimeNVariance = 0)
server.controller = controller
r = Request()
r.onCompleted = lambda: completedRequests.append(r)
sim.add(0, lambda: server.request(r))
r2 = Request()
r2.onCompleted = lambda: completedRequests.append(r2)
sim.add(0, lambda: server.request(r2))
sim.run()
controller.withOptional.assert_called_with()
assert set(completedRequests) == set([ r, r2 ])
assert abs(server.getActiveTime() - 20.0) < eps, server.getActiveTime()
def test_remove_two_servers_while_request_in_progress():
sim = SimulatorKernel(outputDirectory = None)
server1 = MockServer(sim, latency = 10)
server2 = MockServer(sim, latency = 10)
server3 = MockServer(sim, latency = 10)
lb = LoadBalancer(sim)
lb.algorithm = 'SQF'
lb.addBackend(server1)
lb.addBackend(server2)
lb.addBackend(server3)
onShutdownCompleted = Mock()
r1 = Request()
r1.onCompleted = Mock()
r2 = Request()
r2.onCompleted = Mock()
r3 = Request()
r3.onCompleted = Mock()
sim.add(0, lambda: lb.request(r1))
sim.add(0, lambda: lb.request(r2))
sim.add(0, lambda: lb.request(r3))
sim.add(1, lambda: lb.removeBackend(server1, onShutdownCompleted))
sim.add(2, lambda: lb.removeBackend(server2, onShutdownCompleted))
sim.run()
r1.onCompleted.assert_called_once_with()
r2.onCompleted.assert_called_once_with()
r3.onCompleted.assert_called_once_with()
assert onShutdownCompleted.call_count == 2
assert server1.numSeenRequests == 1
assert server2.numSeenRequests == 1
assert server3.numSeenRequests == 1
def test_closed_client_off():
sim = SimulatorKernel()
server = MockServer(sim)
client = ClosedLoopClient(sim, server)
sim.add(10, lambda: client.deactivate())
sim.run(until=100)
assert server.numSeenRequests < 20, server.numSeenRequests
def test_open_client_off():
sim = SimulatorKernel()
server = MockServer(sim)
clients = OpenLoopClient(sim, server)
clients.setRate(10)
sim.add(10, lambda: clients.setRate(0))
sim.run(until=100)
assert server.numSeenRequests < 110, server.numSeenRequests
def test_scale_up_error():
sim = SimulatorKernel(outputDirectory = None)
loadBalancer = MockLoadBalancer(sim, latency = 1)
autoScaler = AutoScaler(sim, loadBalancer, startupDelay = 60)
assert str(autoScaler)
server1 = mock.Mock(name = 'server1')
autoScaler.addBackend(server1)
autoScaler.scaleUp()
autoScaler.scaleUp()
def test():
sim = SimulatorKernel(outputDirectory = None)
server1 = Mock()
server2 = Mock()
lb = LoadBalancer(sim)
lb.addBackend(server1)
lb.addBackend(server2)
lb.removeBackend(server2)
assert str(lb)
def test_closed_client():
sim = SimulatorKernel()
server = MockServer(sim)
client = ClosedLoopClient(sim, server)
sim.run(until=100)
# TODO: Better statistical test
assert server.numSeenRequests > 70, server.numSeenRequests
assert server.numSeenRequests < 130, server.numSeenRequests
assert client.numCompletedRequests == server.numSeenRequests
assert client.numCompletedRequestsWithOptional == 1
assert len(client.responseTimes) == client.numCompletedRequests
assert max(client.responseTimes) == 0
def check_autoscaler_controller_factory(autoscalerControllerFactory):
parser = argparse.ArgumentParser()
autoscalerControllerFactory.addCommandLine(parser)
args = parser.parse_args(args=[])
autoscalerControllerFactory.parseCommandLine(args)
sim = SimulatorKernel(outputDirectory=None)
controller = autoscalerControllerFactory.newInstance(sim, "blah")
# TODO: smoke test
sim.run()
assert str(controller) == "blah"
def test_closed_client_with_latency():
sim = SimulatorKernel()
server = MockServer(sim, latency=1)
client = ClosedLoopClient(sim, server)
sim.run(until=100)
# TODO: Better statistical test
assert server.numSeenRequests > 40, server.numSeenRequests
assert server.numSeenRequests < 60, server.numSeenRequests
assert server.numSeenRequests - client.numCompletedRequests <= 1
assert client.numCompletedRequestsWithOptional == 1
assert len(client.responseTimes) == client.numCompletedRequests
assert abs(max(client.responseTimes) - 1.0) < eps, max(
client.responseTimes)
def test_open_client_with_latency():
sim = SimulatorKernel()
server = MockServer(sim, latency=1)
clients = OpenLoopClient(sim, server)
clients.setRate(10)
sim.run(until=100)
# TODO: Better statistical test
assert server.numSeenRequests > 900, server.numSeenRequests
assert server.numSeenRequests < 1100, server.numSeenRequests
assert server.numSeenRequests - clients.numCompletedRequests <= 20, server.numSeenRequests - clients.numCompletedRequests
assert clients.numCompletedRequestsWithOptional == 1
assert len(clients.responseTimes) == clients.numCompletedRequests
assert abs(max(clients.responseTimes) - 1) < eps
def test_scale_by():
sim = SimulatorKernel(outputDirectory = None)
loadBalancer = MockLoadBalancer(sim, latency = 1)
autoScaler = AutoScaler(sim, loadBalancer, startupDelay = 60)
assert str(autoScaler)
server1 = mock.Mock(name = 'server1')
server2 = mock.Mock(name = 'server2')
server3 = mock.Mock(name = 'server3')
server4 = mock.Mock(name = 'server4')
autoScaler.addBackend(server1)
autoScaler.addBackend(server2)
autoScaler.addBackend(server3)
autoScaler.addBackend(server4)
sim.add(0, lambda: assert_autoscaler_status_is(autoScaler, 4, 0, 0, 0))
sim.add(10, lambda: autoScaler.scaleBy(2))
sim.add(10, lambda: assert_autoscaler_status_is(autoScaler, 2, 2, 0, 0))
sim.add(10+59, lambda: loadBalancer.addBackend.assert_not_called()) # startup delay
sim.add(10+59, lambda: assert_autoscaler_status_is(autoScaler, 2, 2, 0, 0))
sim.add(10+60+eps, lambda: assert_autoscaler_status_is(autoScaler, 2, 0, 2, 0))
calls1 = [mock.call(server1), mock.call(server2)]
sim.add(10+60+eps, lambda: loadBalancer.addBackend.assert_has_calls(calls1))
sim.add(100, lambda: autoScaler.scaleBy(2))
sim.add(100, lambda: assert_autoscaler_status_is(autoScaler, 0, 2, 2, 0))
sim.add(100+59, lambda: assert_autoscaler_status_is(autoScaler, 0, 2, 2, 0))
sim.add(100+60+eps, lambda: assert_autoscaler_status_is(autoScaler, 0, 0, 4, 0))
calls2 = [mock.call(server3), mock.call(server4)]
sim.add(100+60+eps, lambda: loadBalancer.addBackend.assert_has_calls(calls2))
sim.add(200, lambda: autoScaler.scaleBy(-3))
sim.add(200, lambda: assert_autoscaler_status_is(autoScaler, 0, 0, 1, 3))
sim.add(200+1-eps, lambda: assert_autoscaler_status_is(autoScaler, 0, 0, 1, 3))
sim.add(200+1+eps, lambda: assert_autoscaler_status_is(autoScaler, 3, 0, 1, 0))
sim.run()
assert loadBalancer.lastRemovedBackend == server2
def test_without_controller():
completedRequests = []
sim = SimulatorKernel(outputDirectory = None)
server = Server(sim, serviceTimeY = 1, serviceTimeYVariance = 0)
r = Request()
r.onCompleted = lambda: completedRequests.append(r)
sim.add(0, lambda: server.request(r))
r2 = Request()
r2.onCompleted = lambda: completedRequests.append(r2)
sim.add(0, lambda: server.request(r2))
sim.run()
assert set(completedRequests) == set([ r, r2 ])
assert server.getActiveTime() == 2.0, server.getActiveTime()
def check_replica_controller_factory(replicaControllerFactory):
parser = argparse.ArgumentParser()
# TODO: avoid code duplication
parser.add_argument('--rcSetpoint', default=1.0)
parser.add_argument('--rcPercentile', default=95)
replicaControllerFactory.addCommandLine(parser)
args = parser.parse_args(args=[])
replicaControllerFactory.parseCommandLine(args)
sim = SimulatorKernel(outputDirectory=None)
controller = replicaControllerFactory.newInstance(sim, "blah")
# smoke test
controller.reportData(1, 10, 2, 2)
sim.run()
assert str(controller) == "blah"
assert controller.withOptional()[0] in [False, True]
assert 0 <= controller.withOptional()[1] <= 1
def test_scale_up_and_down():
sim = SimulatorKernel(outputDirectory = None)
loadBalancer = MockLoadBalancer(sim, latency = 1)
autoScaler = AutoScaler(sim, loadBalancer, startupDelay = 60)
assert str(autoScaler)
server1 = mock.Mock(name = 'server1')
server2 = mock.Mock(name = 'server2')
autoScaler.addBackend(server1)
autoScaler.addBackend(server2)
sim.add(0, lambda: assert_autoscaler_status_is(autoScaler, 2, 0, 0, 0))
sim.add(10, lambda: autoScaler.scaleUp())
sim.add(10, lambda: assert_autoscaler_status_is(autoScaler, 1, 1, 0, 0))
sim.add(10+59, lambda: loadBalancer.addBackend.assert_not_called()) # startup delay
sim.add(10+59, lambda: assert_autoscaler_status_is(autoScaler, 1, 1, 0, 0))
sim.add(10+60+eps, lambda: assert_autoscaler_status_is(autoScaler, 1, 0, 1, 0))
sim.add(10+60+eps, lambda: loadBalancer.addBackend.assert_called_with(server1))
sim.add(100, lambda: autoScaler.scaleUp())
sim.add(100, lambda: assert_autoscaler_status_is(autoScaler, 0, 1, 1, 0))
sim.add(100+59, lambda: assert_autoscaler_status_is(autoScaler, 0, 1, 1, 0))
sim.add(100+60+eps, lambda: assert_autoscaler_status_is(autoScaler, 0, 0, 2, 0))
sim.add(100+60+eps, lambda: loadBalancer.addBackend.assert_called_with(server2))
sim.add(200, lambda: autoScaler.scaleDown())
sim.add(200, lambda: assert_autoscaler_status_is(autoScaler, 0, 0, 1, 1))
sim.add(200+1-eps, lambda: assert_autoscaler_status_is(autoScaler, 0, 0, 1, 1))
sim.add(200+1+eps, lambda: assert_autoscaler_status_is(autoScaler, 1, 0, 1, 0))
sim.run()
assert loadBalancer.lastRemovedBackend == server2
def test_invalid_action():
sim = SimulatorKernel(outputDirectory = None)
loadBalancer = MockLoadBalancer(sim, latency = 1)
autoScalerController = mock.Mock()
autoScalerController.controlInterval = 1
autoScalerController.onRequest = mock.Mock(return_value=-2)
autoScalerController.onCompleted = mock.Mock(return_value=0)
autoScalerController.onControlPeriod = mock.Mock(return_value=0)
autoScalerController.onStatus = mock.Mock(return_value=0)
autoScaler = AutoScaler(sim, loadBalancer, controller = autoScalerController)
assert str(autoScaler)
server1 = mock.Mock(name = 'server1')
server2 = mock.Mock(name = 'server2')
autoScaler.addBackend(server1)
autoScaler.addBackend(server2)
r = Request()
autoScaler.request(r)
sim.add(100, lambda: autoScaler.scaleUp())
sim.run()
def test_random_startup_delay(average=60, variance=10, values_to_test=10):
rng = random.Random()
startupDelayFunc = lambda: rng.normalvariate(average, variance)
# "predict" future random delays by resetting the PRNG's seed
random_delays = []
rng.seed(1)
for _ in range(values_to_test):
random_delays.append(startupDelayFunc())
rng.seed(1)
sim = SimulatorKernel(outputDirectory = None)
loadBalancer = MockLoadBalancer(sim, latency = 1)
autoScaler = AutoScaler(sim, loadBalancer, startupDelay = startupDelayFunc)
assert str(autoScaler)
for i in range(values_to_test):
server = mock.Mock(name = 'server' + str(i))
autoScaler.addBackend(server)
# NOTE: `lambda i=i` is required to capture the current value of `i` inside
# the lambda.
currentTime = 0
for i in range(values_to_test):
currentTime += 1
sim.add(currentTime, lambda i=i: autoScaler.scaleTo(i+1))
currentTime += random_delays[i]
sim.add(currentTime-eps, lambda i=i:
assert_autoscaler_status_is(autoScaler, values_to_test-i-1, 1, i, 0))
sim.add(currentTime+eps, lambda i=i:
assert_autoscaler_status_is(autoScaler, values_to_test-i-1, 0, i+1, 0))
sim.run()
assert_autoscaler_status_is(autoScaler, 0, 0, values_to_test, 0)
def test_open_client_str():
sim = SimulatorKernel()
server = MockServer(sim)
clients = OpenLoopClient(sim, server)
assert str(clients)
def runSingleSimulation(outdir, autoScalerControllerFactory,
replicaControllerFactory, scenario, timeSlice,
loadBalancingAlgorithm, equal_theta_gain,
equal_thetas_fast_gain, startupDelay):
startupDelayRng = random.Random()
startupDelayFunc = lambda: \
getattr(startupDelayRng, startupDelay[0])(*startupDelay[1:])
assert startupDelayFunc() # ensure the PRNG works
startupDelayRng.seed(1)
sim = SimulatorKernel(outputDirectory=outdir)
servers = []
clients = []
loadBalancer = LoadBalancer(sim, controlPeriod=1.0)
autoScaler = AutoScaler(sim,
loadBalancer,
controller=autoScalerControllerFactory.newInstance(
sim, 'as-ctr'),
startupDelay=startupDelayFunc)
openLoopClient = OpenLoopClient(sim, autoScaler)
loadBalancer.algorithm = loadBalancingAlgorithm
loadBalancer.equal_theta_gain = equal_theta_gain
loadBalancer.equal_thetas_fast_gain = equal_thetas_fast_gain
# Define verbs for scenarios
def addClients(at, n):
def addClientsHandler():
for _ in range(0, n):
clients.append(ClosedLoopClient(sim, loadBalancer))
sim.add(at, addClientsHandler)
def delClients(at, n):
def delClientsHandler():
for _ in range(0, n):
client = clients.pop()
client.deactivate()
sim.add(at, delClientsHandler)
def changeServiceTime(at, serverId, y, n):
def changeServiceTimeHandler():
server = servers[serverId]
server.serviceTimeY = y
server.serviceTimeN = n
sim.add(at, changeServiceTimeHandler)
def addServer(y, n, autoScale=False):
server = Server(sim, \
serviceTimeY = y, serviceTimeN = n, \
timeSlice = timeSlice)
newReplicaController = replicaControllerFactory.newInstance(
sim,
str(server) + "-ctl")
server.controller = newReplicaController
servers.append(server)
if autoScale:
autoScaler.addBackend(server)
else:
loadBalancer.addBackend(server)
def setRate(at, rate):
sim.add(at, lambda: openLoopClient.setRate(rate))
def endOfSimulation(at):
otherParams['simulateUntil'] = at
# Load scenario
otherParams = {}
execfile(scenario)
# For weighted-RR algorithm set the weights
if loadBalancingAlgorithm == 'weighted-RR':
serviceRates = np.array([1.0 / x.serviceTimeY for x in servers])
sumServiceRates = sum(serviceRates)
loadBalancer.weights = list(np.array(serviceRates / sumServiceRates))
if 'simulateUntil' not in otherParams:
raise Exception("Scenario does not define end-of-simulation")
sim.run(until=otherParams['simulateUntil'])
# Report end results
responseTimes = reduce(lambda x, y: x + y,
[client.responseTimes for client in clients],
[]) + openLoopClient.responseTimes
numRequestsWithOptional = sum([
client.numCompletedRequestsWithOptional for client in clients
]) + openLoopClient.numCompletedRequestsWithOptional
toReport = []
toReport.append(("autoScalerAlgorithm",
autoScalerControllerFactory.getName().ljust(20)))
toReport.append(
("loadBalancingAlgorithm", loadBalancingAlgorithm.ljust(20)))
toReport.append(
("replicaAlgorithm", replicaControllerFactory.getName().ljust(20)))
toReport.append(("numRequests", str(len(responseTimes)).rjust(7)))
toReport.append(
("numRequestsWithOptional", str(numRequestsWithOptional).rjust(7)))
toReport.append(
("optionalRatio",
"{:.3f}".format(numRequestsWithOptional / len(responseTimes))))
toReport.append(("avgResponseTime", "{:.3f}".format(avg(responseTimes))))
toReport.append(
("p95ResponseTime", "{:.3f}".format(np.percentile(responseTimes, 95))))
toReport.append(
("p99ResponseTime", "{:.3f}".format(np.percentile(responseTimes, 99))))
toReport.append(("maxResponseTime", "{:.3f}".format(max(responseTimes))))
toReport.append(
("stddevResponseTime", "{:.3f}".format(np.std(responseTimes))))
print(*[k for k, v in toReport], sep=', ')
print(*[v for k, v in toReport], sep=', ')
sim.output('final-results', ', '.join([k for k, v in toReport]))
sim.output('final-results', ', '.join([v for k, v in toReport]))
def test_closed_client_str():
sim = SimulatorKernel()
server = MockServer(sim)
client = ClosedLoopClient(sim, server)
assert str(client)