def test_evaluation_of_queue_settings(self):
settings = Settings(queue=LIFO())
simulation = Simulation(InMemoryDataStorage(None))
simulation.evaluate(settings)
task_pool = simulation.environment.look_up(Symbols.QUEUE).delegate.delegate.delegate
self.assertIsInstance(task_pool, LIFOTaskPool)
class ServiceTests(TestCase):
"""
Factor out the creation of the simulation, as well as a few helpers needed to create services that do
reject requests, or services that always fail, etc.
"""
def setUp(self):
self.simulation = Simulation(InMemoryDataStorage(None))
def define(self, symbol, value):
self.simulation.environment.define(symbol, value)
return value
def look_up(self, symbol):
return self.simulation.environment.look_up(symbol)
def evaluate(self, expression, continuation=lambda x: x):
return self.simulation.evaluate(expression, continuation)
def simulate_until(self, end):
self.simulation.run_until(end)
def query(self,
service_name,
operation,
on_success=lambda: None,
on_error=lambda: None):
request = Query(self.a_running_task(), operation, 1, lambda s: None)
request.on_success = on_success
request.on_error = on_error
self.send(request, service_name)
return request
def a_running_task(self):
task = Task(self.fake_client())
task.status = TaskStatus.BLOCKED # A regular evaluation would block it after sending the request
return task
def trigger(self,
service_name,
operation,
on_success=lambda: None,
on_error=lambda: None):
request = Trigger(self.a_running_task(), operation, 1, lambda s: None)
request.on_success = on_success
request.on_error = on_error
self.send(request, service_name)
return request
def send(self, request, service_name):
service_name = self.look_up(service_name)
request.send_to(service_name)
def fake_client(self):
fake_client = MagicMock()
fake_client.schedule = self.simulation.schedule
fake_client.next_request_id = MagicMock(
side_effect=list(range(1, 1000)))
return fake_client
def test_evaluation_of_queue_settings(self):
settings = Settings(queue=LIFO())
simulation = Simulation(InMemoryDataStorage(None))
simulation.evaluate(settings)
task_pool = simulation.environment.look_up(
Symbols.QUEUE).delegate.delegate.delegate
self.assertIsInstance(task_pool, LIFOTaskPool)
def test_evaluation_of_autoscaling_settings(self):
PERIOD = 23
settings = Settings(autoscaling=Autoscaling(PERIOD, limits=(3, 5)))
simulation = Simulation(InMemoryDataStorage(None))
simulation.evaluate(settings)
autoscaler = simulation.environment.look_up(Symbols.AUTOSCALING)
self.assertIsInstance(autoscaler, AutoScaler)
self.assertEqual(PERIOD, autoscaler.period)
self.assertEqual((3, 5), autoscaler.limits)
def test_evaluation_of_autoscaling_settings(self):
PERIOD = 23
settings = Settings(autoscaling=Autoscaling(PERIOD, limits=(3, 5)))
simulation = Simulation(InMemoryDataStorage(None))
simulation.evaluate(settings)
autoscaler = simulation.environment.look_up(Symbols.AUTOSCALING)
self.assertIsInstance(autoscaler, AutoScaler)
self.assertEqual(PERIOD, autoscaler.period)
self.assertEqual((3, 5), autoscaler.limits)
class ServiceTests(TestCase):
"""
Factor out the creation of the simulation, as well as a few helpers needed to create services that do
reject requests, or services that always fail, etc.
"""
def setUp(self):
self.simulation = Simulation(InMemoryDataStorage(None))
def define(self, symbol, value):
self.simulation.environment.define(symbol, value)
return value
def look_up(self, symbol):
return self.simulation.environment.look_up(symbol)
def evaluate(self, expression, continuation=lambda x:x):
return self.simulation.evaluate(expression, continuation)
def simulate_until(self, end):
self.simulation.run_until(end)
def query(self, service_name, operation, on_success=lambda:None, on_error=lambda:None):
request = Query(self.a_running_task(), operation, 1, lambda s:None)
request.on_success = on_success
request.on_error = on_error
self.send(request, service_name)
return request
def a_running_task(self):
task = Task(self.fake_client())
task.status = TaskStatus.BLOCKED # A regular evaluation would block it after sending the request
return task
def trigger(self, service_name, operation, on_success=lambda:None, on_error=lambda:None):
request = Trigger(self.a_running_task(), operation, 1, lambda s:None)
request.on_success = on_success
request.on_error = on_error
self.send(request, service_name)
return request
def send(self, request, service_name):
service_name = self.look_up(service_name)
request.send_to(service_name)
def fake_client(self):
fake_client = MagicMock()
fake_client.schedule = self.simulation.schedule
fake_client.next_request_id = MagicMock(side_effect=range(1, 1000))
return fake_client
def setUp(self):
self.simulation = Simulation(InMemoryDataStorage(None))
class TestInterpreter(TestCase):
def setUp(self):
self.simulation = Simulation(InMemoryDataStorage(None))
def define(self, symbol, value):
self.simulation.environment.define(symbol, value)
return value
def look_up(self, symbol):
return self.simulation.environment.look_up(symbol)
def evaluate(self, expression, continuation=lambda x: x):
return self.simulation.evaluate(expression, continuation)
def verify_definition(self, symbol, kind):
value = self.look_up(symbol)
self.assertTrue(isinstance(value, kind))
def send_request(self,
service_name,
operation_name,
on_success=lambda: None,
on_error=lambda: None):
service = self.look_up(service_name)
request = self.fake_request(operation_name,
on_success=on_success,
on_error=on_error)
request.send_to(service)
return request
def simulate_until(self, end):
self.simulation.run_until(end)
def test_evaluate_blocking_service_invocation(self):
db = self.define("DB", self.fake_service())
self.evaluate(
DefineService("Front-end",
DefineOperation("checkout", Query("DB", "op"))))
self.send_request("Front-end", "checkout")
self.simulate_until(10)
self.assertEqual(db.process.call_count, 1)
def test_trigger_rejection(self):
db = self.define("DB", self._a_service_that_rejects_requests())
front_end = self.evaluate(
DefineService("Front-end",
DefineOperation("checkout", Trigger("DB",
"op")))).value
request = self.send_request("Front-end", "checkout")
from mad.simulation.monitoring import Statistics
with patch.object(Statistics, "reset"):
self.simulate_until(10)
self.assertTrue(request.status == RequestStatus.ERROR)
monitor = front_end.look_up(Symbols.MONITOR)
self.assertEqual(0., monitor._reliability())
def test_query_make_services_busy(self):
db = self.evaluate(
DefineService("DB", DefineOperation("Select", Think(2)))).value
front_end = self.evaluate(
DefineService("Front-end",
DefineOperation("show", Query("DB",
"Select")))).value
request = self.send_request("Front-end", "show")
self.simulate_until(1)
self.assertEqual(
100.0,
front_end.workers.utilisation) # The front-end should be busy
self.simulate_until(2)
self.assertEqual(
0.0,
front_end.workers.utilisation) # The front-end should be released
self.assertEqual(
0.0, db.workers.utilisation) # DB has not yet received the request
self.simulate_until(3)
self.assertEqual(
0.0,
front_end.workers.utilisation) # The front-end should be released
self.assertEqual(100.0, db.workers.utilisation) # DB busy thinking
self.simulate_until(5)
self.assertEqual(
0.0, front_end.workers.utilisation) # Front end has nothing to do
self.assertEqual(100.0,
db.workers.utilisation) # DB busy sending answer
self.simulate_until(6)
self.assertEqual(
0.0, front_end.workers.utilisation) # Front end has nothing to do
self.assertEqual(
0.0, db.workers.utilisation) # DB has nothing to do anymore
self.simulate_until(7)
self.assertEqual(
100.0, front_end.workers.utilisation) # Front end busy replying
self.assertEqual(
0.0, db.workers.utilisation) # DB has nothing to do anymore
self.simulate_until(8)
self.assertEqual(
0.0, front_end.workers.utilisation) # Front end busy replying
self.assertEqual(
0.0, db.workers.utilisation) # DB has nothing to do anymore
self.simulate_until(9)
self.assertTrue(request.status == RequestStatus.OK)
def test_trigger_make_services_busy(self):
db = self.evaluate(
DefineService("DB", DefineOperation("Select", Think(2)))).value
front_end = self.evaluate(
DefineService("Front-end",
DefineOperation("show", Trigger("DB",
"Select")))).value
request = self.send_request("Front-end", "show")
self.simulate_until(1)
self.assertEqual(
100.0,
front_end.workers.utilisation) # The front-end should be busy
self.assertEqual(0.0, db.workers.utilisation) # DB has nothing to do
self.simulate_until(2)
self.assertEqual(
0.0,
front_end.workers.utilisation) # The front-end should be released
self.assertEqual(
0.0, db.workers.utilisation) # DB has not yet received the request
self.simulate_until(3)
self.assertEqual(
0.0,
front_end.workers.utilisation) # The front-end should be released
self.assertEqual(100.0, db.workers.utilisation) # DB busy thinking
self.simulate_until(4)
self.assertEqual(
100.0, front_end.workers.utilisation
) # Front end has received the acceptance ack and is replying
self.assertEqual(100.0,
db.workers.utilisation) # DB busy sending answer
self.simulate_until(5)
self.assertEqual(
0.0, front_end.workers.utilisation) # Front end has nothing to do
self.assertEqual(
0.0, db.workers.utilisation) # DB has nothing to do anymore
self.simulate_until(6)
self.assertTrue(request.status == RequestStatus.OK)
def test_rejected_query_make_services_busy(self):
db = self._a_service_that_rejects_requests()
self.define("DB", db)
front_end = self.evaluate(
DefineService("Front-end",
DefineOperation("show", Query("DB",
"Select")))).value
request = self.send_request("Front-end", "show")
self.simulate_until(1)
self.assertEqual(
100.0,
front_end.workers.utilisation) # The front-end should be busy
self.simulate_until(2)
self.assertEqual(
0.0,
front_end.workers.utilisation) # The front-end should be released
self.simulate_until(3)
self.assertEqual(
0.0,
front_end.workers.utilisation) # The front-end should be released
self.simulate_until(4)
self.assertEqual(
100.0, front_end.workers.utilisation
) # Front end has received the acceptance ack and is replying
self.simulate_until(5)
self.assertEqual(
0.0, front_end.workers.utilisation) # Front end has nothing to do
self.simulate_until(6)
self.assertTrue(request.status == RequestStatus.ERROR)
def test_failed_query_make_services_busy(self):
db = self._a_service_that_accepts_but_fails(after=3)
self.define("DB", db)
front_end = self.evaluate(
DefineService("Front-end",
DefineOperation("show", Query("DB",
"Select")))).value
request = self.send_request("Front-end", "show")
self.simulate_until(1)
self.assertEqual(
100.0,
front_end.workers.utilisation) # The front-end should be busy
self.simulate_until(2)
self.assertEqual(
0.0,
front_end.workers.utilisation) # The front-end should be released
self.simulate_until(3)
self.assertEqual(
0.0,
front_end.workers.utilisation) # The front-end should be released
self.simulate_until(7)
self.assertEqual(
100.0,
front_end.workers.utilisation) # Front-end forward the error
self.simulate_until(8)
self.assertEqual(
0.0,
front_end.workers.utilisation) # The front-end should be released
self.simulate_until(9)
self.assertTrue(request.status == RequestStatus.ERROR)
def test_rejected_trigger_make_services_busy(self):
db = self._a_service_that_rejects_requests()
self.define("DB", db)
front_end = self.evaluate(
DefineService("Front-end",
DefineOperation("show", Trigger("DB",
"Select")))).value
request = self.send_request("Front-end", "show")
self.simulate_until(1)
self.assertEqual(
100.0,
front_end.workers.utilisation) # The front-end should be busy
self.simulate_until(2)
self.assertEqual(
0.0,
front_end.workers.utilisation) # The front-end should be released
self.simulate_until(3)
self.assertEqual(
0.0,
front_end.workers.utilisation) # The front-end should be released
self.simulate_until(4)
self.assertEqual(
100.0, front_end.workers.utilisation
) # Front end has received the acceptance ack and is replying
self.simulate_until(5)
self.assertEqual(
0.0, front_end.workers.utilisation) # Front end has nothing to do
self.simulate_until(6)
self.assertTrue(request.status == RequestStatus.ERROR)
def test_evaluate_timeout_queries(self):
db = self.evaluate(DefineService("DB",
DefineOperation("op",
Think(8)))).value
front_end = self.evaluate(
DefineService(
"Front-end",
DefineOperation("checkout", Query("DB", "op",
timeout=5)))).value
request = self.send_request("Front-end", "checkout")
self.simulate_until(8)
self.assertTrue(request.status == RequestStatus.ERROR)
self.simulate_until(12)
self.assertEqual(RequestStatus.ERROR, request.status)
self.assertEqual(
0, len(front_end.tasks.delegate.delegate.delegate.paused))
self.assertEqual(0, front_end.tasks.blocked_count)
self.assertTrue(front_end.workers.are_available)
self.assertEqual(0, db.tasks.blocked_count)
self.assertEqual(0, len(db.tasks.delegate.delegate.delegate.paused))
self.assertTrue(db.workers.are_available)
def test_evaluate_timeout_on_query(self):
"""
Test that workers are released when a task is resumed, but its associated request has been discarded
"""
db = self.evaluate(
DefineService("DB", DefineOperation("insert", Think(8)))).value
storage = self.evaluate(
DefineService("Storage",
DefineOperation("store", Query("DB",
"insert")))).value
notifier = self.evaluate(
DefineService(
"Notifier",
DefineOperation("notify", Query("Storage", "store",
timeout=5)))).value
request = self.send_request("Notifier", "notify")
self.simulate_until(8)
self.assertTrue(request.status == RequestStatus.ERROR)
self.simulate_until(50) # Should have been release by then
self.assertTrue(storage.workers.are_available)
self.assertEqual(0,
len(storage.tasks.delegate.delegate.delegate.paused))
def test_evaluate_timeout_no_expiration(self):
db = self.evaluate(DefineService("DB",
DefineOperation("op",
Think(1)))).value
front_end = self.evaluate(
DefineService(
"Front-end",
DefineOperation(
"checkout",
Retry(expression=Query("DB", "op", timeout=1),
delay=Delay(2, "constant"),
limit=5)))).value
request = self.send_request("Front-end", "checkout")
self.simulate_until(25)
self.assertEqual(RequestStatus.ERROR, request.status)
def test_sequence_evaluation(self):
db = self.define("DB", self._a_service_that_accepts_but_fails(after=2))
self.evaluate(
DefineService(
"Front-end",
DefineOperation(
"checkout",
Sequence(Trigger("DB", "op"), Trigger("DB", "op")))))
self.send_request("Front-end", "checkout")
self.simulate_until(20)
self.assertEqual(db.process.call_count, 2)
def test_operation_invocation(self):
# TODO: Check whether this test is still useful
db = self.define("DB", self.service_that_always_fails())
self.evaluate(
DefineService("Front-end",
DefineOperation("checkout", Trigger("DB", "op"))))
self.send_request("Front-end", "checkout")
self.simulate_until(20)
self.assertEqual(db.process.call_count, 1)
def test_thinking(self):
db = self.define("DB", self.service_that_always_fails())
self.evaluate(
DefineService(
"Front-end",
DefineOperation("checkout",
Sequence(Think(5), Trigger("DB", "op")))))
self.send_request("Front-end", "checkout")
self.assertEqual(db.process.call_count, 0)
self.simulate_until(10)
self.assertEqual(db.process.call_count, 1)
def test_retry_and_succeed(self):
db = self.define("DB", self.service_that_succeeds_at_attempt(2))
self.evaluate(
DefineService(
"Front-end",
DefineOperation(
"checkout",
Retry(Query("DB", "insert"),
limit=5,
delay=Delay(10, "constant")))))
def test_fail():
self.fail("Expected failed request")
self.send_request("Front-end", "checkout", on_error=test_fail)
self.simulate_until(200)
self.assertEqual(db.process.call_count, 2)
def test_retry_and_fail(self):
db = self.define("DB", self.service_that_succeeds_at_attempt(15))
self.evaluate(
DefineService(
"Front-end",
DefineOperation(
"checkout",
Retry(Query("DB", "insert"),
limit=5,
delay=Delay(10, "constant")))))
def test_fail():
self.fail("Expected successful request")
self.send_request("Front-end", "checkout", on_success=test_fail)
self.simulate_until(200)
self.assertEqual(db.process.call_count, 5)
def test_fail(self):
self.evaluate(DefineService("DB", DefineOperation("Select", Fail())))
def test_failed():
self.fail("Expected successful request")
self.send_request("DB", "Select", on_success=test_failed)
self.simulate_until(20)
def test_ignore_error(self):
db1 = self.define("DB1", self.service_that_always_fails())
db2 = self.define("DB2", self.fake_service())
self.evaluate(
DefineService(
"Front-end",
DefineOperation(
"checkout",
Sequence(IgnoreError(Query("DB1", "op")),
Trigger("DB2", "op")))))
self.send_request("Front-end", "checkout")
self.simulate_until(20)
self.assertEqual(db1.process.call_count, 1)
self.assertEqual(db2.process.call_count, 1)
def test_operation_definition(self):
self.evaluate(DefineOperation("op", Trigger("serviceX", "op")))
self.verify_definition("op", Operation)
def test_service_request(self):
fake_service = self.define("serviceX", self.fake_service())
service = self.evaluate(
DefineService("my-service",
DefineOperation("op", Trigger("serviceX",
"op")))).value
request = self.fake_request("op")
request.send_to(service)
self.simulate_until(10)
self.assertEqual(fake_service.process.call_count, 1)
def test_service_definition(self):
fake_service = self.define("service Y", self.fake_service())
self.evaluate(
DefineService("Service X",
DefineOperation("op", Trigger("service Y", "foo"))))
self.verify_definition("Service X", Service)
service = self.look_up("Service X")
request = self.fake_request("op")
request.send_to(service)
self.simulate_until(10)
self.assertEqual(fake_service.process.call_count, 1)
def test_client_stub_definition(self):
client = self.evaluate(
Sequence(
DefineService("Service X", DefineOperation("op", Think(2))),
DefineClientStub("Client", 10, Query("Service X",
"op")))).value
client.on_success = MagicMock()
self.simulate_until(26)
monitor = client.look_up(Symbols.MONITOR)
self.assertEqual(monitor.tasks.successful, 2)
def fake_request(self,
operation,
on_success=lambda: None,
on_error=lambda: None):
request = SQuery(Task(self.fake_client()), operation, 1,
lambda s: None)
request.on_error = on_error
request.on_success = on_success
return request
def fake_service(self, name="DB"):
fake_service = MagicMock()
fake_service.name = name
fake_service.process = MagicMock()
fake_service.schedule = self.simulation.schedule
return fake_service
def fake_client(self):
fake_client = MagicMock()
fake_client.schedule = self.simulation.schedule
fake_client.next_request_id = MagicMock(side_effect=range(1, 100))
return fake_client
def _a_service_that_rejects_requests(self):
def always_reject(request):
request.reject()
service = self.fake_service()
service.process.side_effect = always_reject
return service
def _a_service_that_accepts_but_fails(self, after=4):
def always_accept(request):
def do_fail():
request.reply_error()
request.accept()
self.simulation._scheduler.after(after, do_fail)
service = self.fake_service()
service.process.side_effect = always_accept
return service
def _a_service_that_replies_after(self, after=4):
def always_accept(request):
def do_succeed():
request.reply_success()
request.accept()
self.simulation._scheduler.after(after, do_succeed)
service = self.fake_service()
service.process.side_effect = always_accept
return service
def service_that_always_fails(self):
def always_fail(request):
request.reply_error()
service = self.fake_service()
service.process.side_effect = always_fail
return service
def service_that_succeeds_at_attempt(self, successful_attempt):
attempt = 0
def respond(request):
nonlocal attempt
attempt += 1
if attempt == successful_attempt:
request.reply_success()
else:
request.reply_error()
service = self.fake_service()
service.process.side_effect = respond
service.schedule = self.simulation.schedule
return service
def evaluate(self, expression):
simulation = Simulation(InMemoryDataStorage(None))
simulation.evaluate(expression)
return simulation
def _simulate(self, expression, arguments):
simulation = Simulation(self.storage)
simulation.evaluate(expression)
simulation.run_until(arguments._time_limit, self.display)
self.display.simulation_complete(arguments)
return simulation
def setUp(self):
self.simulation = Simulation(InMemoryDataStorage(None))
class TestInterpreter(TestCase):
def setUp(self):
self.simulation = Simulation(InMemoryDataStorage(None))
def define(self, symbol, value):
self.simulation.environment.define(symbol, value)
return value
def look_up(self, symbol):
return self.simulation.environment.look_up(symbol)
def evaluate(self, expression, continuation=lambda x: x):
return self.simulation.evaluate(expression, continuation)
def verify_definition(self, symbol, kind):
value = self.look_up(symbol)
self.assertTrue(isinstance(value, kind))
def send_request(self, service_name, operation_name, on_success=lambda: None, on_error=lambda: None):
service = self.look_up(service_name)
request = self.fake_request(operation_name, on_success=on_success, on_error=on_error)
request.send_to(service)
return request
def simulate_until(self, end):
self.simulation.run_until(end)
def test_evaluate_blocking_service_invocation(self):
db = self.define("DB", self.fake_service())
self.evaluate(DefineService("Front-end", DefineOperation("checkout", Query("DB", "op"))))
self.send_request("Front-end", "checkout")
self.simulate_until(10)
self.assertEqual(db.process.call_count, 1)
def test_trigger_rejection(self):
db = self.define("DB", self._a_service_that_rejects_requests())
front_end = self.evaluate(DefineService("Front-end", DefineOperation("checkout", Trigger("DB", "op")))).value
request = self.send_request("Front-end", "checkout")
from mad.simulation.monitoring import Statistics
with patch.object(Statistics, "reset"):
self.simulate_until(10)
self.assertTrue(request.status == RequestStatus.ERROR)
monitor = front_end.look_up(Symbols.MONITOR)
self.assertEqual(0.0, monitor._reliability())
def test_query_make_services_busy(self):
db = self.evaluate(DefineService("DB", DefineOperation("Select", Think(2)))).value
front_end = self.evaluate(DefineService("Front-end", DefineOperation("show", Query("DB", "Select")))).value
request = self.send_request("Front-end", "show")
self.simulate_until(1)
self.assertEqual(100.0, front_end.workers.utilisation) # The front-end should be busy
self.simulate_until(2)
self.assertEqual(0.0, front_end.workers.utilisation) # The front-end should be released
self.assertEqual(0.0, db.workers.utilisation) # DB has not yet received the request
self.simulate_until(3)
self.assertEqual(0.0, front_end.workers.utilisation) # The front-end should be released
self.assertEqual(100.0, db.workers.utilisation) # DB busy thinking
self.simulate_until(5)
self.assertEqual(0.0, front_end.workers.utilisation) # Front end has nothing to do
self.assertEqual(100.0, db.workers.utilisation) # DB busy sending answer
self.simulate_until(6)
self.assertEqual(0.0, front_end.workers.utilisation) # Front end has nothing to do
self.assertEqual(0.0, db.workers.utilisation) # DB has nothing to do anymore
self.simulate_until(7)
self.assertEqual(100.0, front_end.workers.utilisation) # Front end busy replying
self.assertEqual(0.0, db.workers.utilisation) # DB has nothing to do anymore
self.simulate_until(8)
self.assertEqual(0.0, front_end.workers.utilisation) # Front end busy replying
self.assertEqual(0.0, db.workers.utilisation) # DB has nothing to do anymore
self.simulate_until(9)
self.assertTrue(request.status == RequestStatus.OK)
def test_trigger_make_services_busy(self):
db = self.evaluate(DefineService("DB", DefineOperation("Select", Think(2)))).value
front_end = self.evaluate(DefineService("Front-end", DefineOperation("show", Trigger("DB", "Select")))).value
request = self.send_request("Front-end", "show")
self.simulate_until(1)
self.assertEqual(100.0, front_end.workers.utilisation) # The front-end should be busy
self.assertEqual(0.0, db.workers.utilisation) # DB has nothing to do
self.simulate_until(2)
self.assertEqual(0.0, front_end.workers.utilisation) # The front-end should be released
self.assertEqual(0.0, db.workers.utilisation) # DB has not yet received the request
self.simulate_until(3)
self.assertEqual(0.0, front_end.workers.utilisation) # The front-end should be released
self.assertEqual(100.0, db.workers.utilisation) # DB busy thinking
self.simulate_until(4)
self.assertEqual(
100.0, front_end.workers.utilisation
) # Front end has received the acceptance ack and is replying
self.assertEqual(100.0, db.workers.utilisation) # DB busy sending answer
self.simulate_until(5)
self.assertEqual(0.0, front_end.workers.utilisation) # Front end has nothing to do
self.assertEqual(0.0, db.workers.utilisation) # DB has nothing to do anymore
self.simulate_until(6)
self.assertTrue(request.status == RequestStatus.OK)
def test_rejected_query_make_services_busy(self):
db = self._a_service_that_rejects_requests()
self.define("DB", db)
front_end = self.evaluate(DefineService("Front-end", DefineOperation("show", Query("DB", "Select")))).value
request = self.send_request("Front-end", "show")
self.simulate_until(1)
self.assertEqual(100.0, front_end.workers.utilisation) # The front-end should be busy
self.simulate_until(2)
self.assertEqual(0.0, front_end.workers.utilisation) # The front-end should be released
self.simulate_until(3)
self.assertEqual(0.0, front_end.workers.utilisation) # The front-end should be released
self.simulate_until(4)
self.assertEqual(
100.0, front_end.workers.utilisation
) # Front end has received the acceptance ack and is replying
self.simulate_until(5)
self.assertEqual(0.0, front_end.workers.utilisation) # Front end has nothing to do
self.simulate_until(6)
self.assertTrue(request.status == RequestStatus.ERROR)
def test_failed_query_make_services_busy(self):
db = self._a_service_that_accepts_but_fails(after=3)
self.define("DB", db)
front_end = self.evaluate(DefineService("Front-end", DefineOperation("show", Query("DB", "Select")))).value
request = self.send_request("Front-end", "show")
self.simulate_until(1)
self.assertEqual(100.0, front_end.workers.utilisation) # The front-end should be busy
self.simulate_until(2)
self.assertEqual(0.0, front_end.workers.utilisation) # The front-end should be released
self.simulate_until(3)
self.assertEqual(0.0, front_end.workers.utilisation) # The front-end should be released
self.simulate_until(7)
self.assertEqual(100.0, front_end.workers.utilisation) # Front-end forward the error
self.simulate_until(8)
self.assertEqual(0.0, front_end.workers.utilisation) # The front-end should be released
self.simulate_until(9)
self.assertTrue(request.status == RequestStatus.ERROR)
def test_rejected_trigger_make_services_busy(self):
db = self._a_service_that_rejects_requests()
self.define("DB", db)
front_end = self.evaluate(DefineService("Front-end", DefineOperation("show", Trigger("DB", "Select")))).value
request = self.send_request("Front-end", "show")
self.simulate_until(1)
self.assertEqual(100.0, front_end.workers.utilisation) # The front-end should be busy
self.simulate_until(2)
self.assertEqual(0.0, front_end.workers.utilisation) # The front-end should be released
self.simulate_until(3)
self.assertEqual(0.0, front_end.workers.utilisation) # The front-end should be released
self.simulate_until(4)
self.assertEqual(
100.0, front_end.workers.utilisation
) # Front end has received the acceptance ack and is replying
self.simulate_until(5)
self.assertEqual(0.0, front_end.workers.utilisation) # Front end has nothing to do
self.simulate_until(6)
self.assertTrue(request.status == RequestStatus.ERROR)
def test_evaluate_timeout_queries(self):
db = self.evaluate(DefineService("DB", DefineOperation("op", Think(8)))).value
front_end = self.evaluate(
DefineService("Front-end", DefineOperation("checkout", Query("DB", "op", timeout=5)))
).value
request = self.send_request("Front-end", "checkout")
self.simulate_until(8)
self.assertTrue(request.status == RequestStatus.ERROR)
self.simulate_until(12)
self.assertEqual(RequestStatus.ERROR, request.status)
self.assertEqual(0, len(front_end.tasks.delegate.delegate.delegate.paused))
self.assertEqual(0, front_end.tasks.blocked_count)
self.assertTrue(front_end.workers.are_available)
self.assertEqual(0, db.tasks.blocked_count)
self.assertEqual(0, len(db.tasks.delegate.delegate.delegate.paused))
self.assertTrue(db.workers.are_available)
def test_evaluate_timeout_on_query(self):
"""
Test that workers are released when a task is resumed, but its associated request has been discarded
"""
db = self.evaluate(DefineService("DB", DefineOperation("insert", Think(8)))).value
storage = self.evaluate(DefineService("Storage", DefineOperation("store", Query("DB", "insert")))).value
notifier = self.evaluate(
DefineService("Notifier", DefineOperation("notify", Query("Storage", "store", timeout=5)))
).value
request = self.send_request("Notifier", "notify")
self.simulate_until(8)
self.assertTrue(request.status == RequestStatus.ERROR)
self.simulate_until(50) # Should have been release by then
self.assertTrue(storage.workers.are_available)
self.assertEqual(0, len(storage.tasks.delegate.delegate.delegate.paused))
def test_evaluate_timeout_no_expiration(self):
db = self.evaluate(DefineService("DB", DefineOperation("op", Think(1)))).value
front_end = self.evaluate(
DefineService(
"Front-end",
DefineOperation(
"checkout", Retry(expression=Query("DB", "op", timeout=1), delay=Delay(2, "constant"), limit=5)
),
)
).value
request = self.send_request("Front-end", "checkout")
self.simulate_until(25)
self.assertEqual(RequestStatus.ERROR, request.status)
def test_sequence_evaluation(self):
db = self.define("DB", self._a_service_that_accepts_but_fails(after=2))
self.evaluate(
DefineService("Front-end", DefineOperation("checkout", Sequence(Trigger("DB", "op"), Trigger("DB", "op"))))
)
self.send_request("Front-end", "checkout")
self.simulate_until(20)
self.assertEqual(db.process.call_count, 2)
def test_operation_invocation(self):
# TODO: Check whether this test is still useful
db = self.define("DB", self.service_that_always_fails())
self.evaluate(DefineService("Front-end", DefineOperation("checkout", Trigger("DB", "op"))))
self.send_request("Front-end", "checkout")
self.simulate_until(20)
self.assertEqual(db.process.call_count, 1)
def test_thinking(self):
db = self.define("DB", self.service_that_always_fails())
self.evaluate(DefineService("Front-end", DefineOperation("checkout", Sequence(Think(5), Trigger("DB", "op")))))
self.send_request("Front-end", "checkout")
self.assertEqual(db.process.call_count, 0)
self.simulate_until(10)
self.assertEqual(db.process.call_count, 1)
def test_retry_and_succeed(self):
db = self.define("DB", self.service_that_succeeds_at_attempt(2))
self.evaluate(
DefineService(
"Front-end",
DefineOperation("checkout", Retry(Query("DB", "insert"), limit=5, delay=Delay(10, "constant"))),
)
)
def test_fail():
self.fail("Expected failed request")
self.send_request("Front-end", "checkout", on_error=test_fail)
self.simulate_until(200)
self.assertEqual(db.process.call_count, 2)
def test_retry_and_fail(self):
db = self.define("DB", self.service_that_succeeds_at_attempt(15))
self.evaluate(
DefineService(
"Front-end",
DefineOperation("checkout", Retry(Query("DB", "insert"), limit=5, delay=Delay(10, "constant"))),
)
)
def test_fail():
self.fail("Expected successful request")
self.send_request("Front-end", "checkout", on_success=test_fail)
self.simulate_until(200)
self.assertEqual(db.process.call_count, 5)
def test_fail(self):
self.evaluate(DefineService("DB", DefineOperation("Select", Fail())))
def test_failed():
self.fail("Expected successful request")
self.send_request("DB", "Select", on_success=test_failed)
self.simulate_until(20)
def test_ignore_error(self):
db1 = self.define("DB1", self.service_that_always_fails())
db2 = self.define("DB2", self.fake_service())
self.evaluate(
DefineService(
"Front-end",
DefineOperation("checkout", Sequence(IgnoreError(Query("DB1", "op")), Trigger("DB2", "op"))),
)
)
self.send_request("Front-end", "checkout")
self.simulate_until(20)
self.assertEqual(db1.process.call_count, 1)
self.assertEqual(db2.process.call_count, 1)
def test_operation_definition(self):
self.evaluate(DefineOperation("op", Trigger("serviceX", "op")))
self.verify_definition("op", Operation)
def test_service_request(self):
fake_service = self.define("serviceX", self.fake_service())
service = self.evaluate(DefineService("my-service", DefineOperation("op", Trigger("serviceX", "op")))).value
request = self.fake_request("op")
request.send_to(service)
self.simulate_until(10)
self.assertEqual(fake_service.process.call_count, 1)
def test_service_definition(self):
fake_service = self.define("service Y", self.fake_service())
self.evaluate(DefineService("Service X", DefineOperation("op", Trigger("service Y", "foo"))))
self.verify_definition("Service X", Service)
service = self.look_up("Service X")
request = self.fake_request("op")
request.send_to(service)
self.simulate_until(10)
self.assertEqual(fake_service.process.call_count, 1)
def test_client_stub_definition(self):
client = self.evaluate(
Sequence(
DefineService("Service X", DefineOperation("op", Think(2))),
DefineClientStub("Client", 10, Query("Service X", "op")),
)
).value
client.on_success = MagicMock()
self.simulate_until(26)
monitor = client.look_up(Symbols.MONITOR)
self.assertEqual(monitor.tasks.successful, 2)
def fake_request(self, operation, on_success=lambda: None, on_error=lambda: None):
request = SQuery(Task(self.fake_client()), operation, 1, lambda s: None)
request.on_error = on_error
request.on_success = on_success
return request
def fake_service(self, name="DB"):
fake_service = MagicMock()
fake_service.name = name
fake_service.process = MagicMock()
fake_service.schedule = self.simulation.schedule
return fake_service
def fake_client(self):
fake_client = MagicMock()
fake_client.schedule = self.simulation.schedule
fake_client.next_request_id = MagicMock(side_effect=range(1, 100))
return fake_client
def _a_service_that_rejects_requests(self):
def always_reject(request):
request.reject()
service = self.fake_service()
service.process.side_effect = always_reject
return service
def _a_service_that_accepts_but_fails(self, after=4):
def always_accept(request):
def do_fail():
request.reply_error()
request.accept()
self.simulation._scheduler.after(after, do_fail)
service = self.fake_service()
service.process.side_effect = always_accept
return service
def _a_service_that_replies_after(self, after=4):
def always_accept(request):
def do_succeed():
request.reply_success()
request.accept()
self.simulation._scheduler.after(after, do_succeed)
service = self.fake_service()
service.process.side_effect = always_accept
return service
def service_that_always_fails(self):
def always_fail(request):
request.reply_error()
service = self.fake_service()
service.process.side_effect = always_fail
return service
def service_that_succeeds_at_attempt(self, successful_attempt):
attempt = 0
def respond(request):
nonlocal attempt
attempt += 1
if attempt == successful_attempt:
request.reply_success()
else:
request.reply_error()
service = self.fake_service()
service.process.side_effect = respond
service.schedule = self.simulation.schedule
return service
def __init__(self):
self.simulation = Simulation(InMemoryDataStorage(None))
def evaluate(self, expression):
simulation = Simulation(InMemoryDataStorage(None))
simulation.evaluate(expression)
return simulation
def _simulate(self, expression, arguments):
simulation = Simulation(self.storage)
simulation.evaluate(expression)
simulation.run_until(arguments._time_limit, self.display)
self.display.simulation_complete(arguments)
return simulation
class ServiceMonitoring(TestCase):
def setUp(self):
self.simulation = Simulation(InMemoryDataStorage(None))
def define(self, symbol, value):
self.simulation.environment.define(symbol, value)
return value
def look_up(self, symbol):
return self.simulation.environment.look_up(symbol)
def evaluate(self, expression, continuation=lambda x: x):
return self.simulation.evaluate(expression, continuation)
def simulate_until(self, end):
self.simulation.run_until(end)
def test_notifies_rejection(self):
db = self.evaluate(
DefineService("DB", DefineOperation("Select", Think(5)))).value
fake_task_pool = MagicMock(ThrottlingPolicyDecorator)
fake_task_pool._accepts = MagicMock(return_value=False)
db.tasks = ThrottlingWrapper(db.environment, task_pool=fake_task_pool)
listener = MagicMock(Listener)
db.environment.look_up(Symbols.LISTENER).register(listener)
request1 = self.send_request("DB", "Select")
request2 = self.send_request("DB", "Select")
self.simulate_until(10)
expected_calls = [
call.arrival_of(request1),
call.arrival_of(request2),
call.rejection_of(request2),
call.success_replied_to(request1)
]
self.assertEqual(expected_calls, listener.method_calls,
listener.method_calls)
def test_notifies_arrival_and_success(self):
db = self.evaluate(
DefineService("DB", DefineOperation("Select", Think(5)))).value
listener = MagicMock(Listener)
db.environment.look_up(Symbols.LISTENER).register(listener)
request = self.send_request("DB", "Select")
self.simulate_until(10)
expected_calls = [
call.arrival_of(request),
call.success_replied_to(request)
]
self.assertEqual(expected_calls, listener.method_calls,
listener.method_calls)
def test_notifies_arrival_and_failure(self):
db = self.evaluate(
DefineService("DB", DefineOperation("Select", Fail()))).value
listener = MagicMock(Listener)
db.environment.look_up(Symbols.LISTENER).register(listener)
request = self.send_request("DB", "Select")
self.simulate_until(10)
expected_calls = [
call.arrival_of(request),
call.error_replied_to(request)
]
self.assertEqual(expected_calls, listener.method_calls,
listener.method_calls)
def test_notifies_when_request_timeout(self):
db = self.evaluate(
DefineService("DB", DefineOperation("Select", Fail()))).value
listener = MagicMock(Listener)
db.environment.look_up(Symbols.LISTENER).register(listener)
request = self.send_request("DB", "Select")
request.reply_error()
self.simulate_until(40)
expected_calls = [
call.arrival_of(request),
call.error_replied_to(request)
]
self.assertEqual(expected_calls, listener.method_calls,
listener.method_calls)
def send_request(self,
service_name,
operation_name,
on_success=lambda: None,
on_error=lambda: None):
service = self.look_up(service_name)
request = self.fake_request(operation_name,
on_success=on_success,
on_error=on_error)
request.send_to(service)
return request
def fake_request(self,
operation,
on_success=lambda: None,
on_error=lambda: None):
return Request(self.fake_client(),
0,
operation,
1,
on_success=on_success,
on_error=on_error)
def fake_client(self):
fake_client = MagicMock()
fake_client.schedule = self.simulation.schedule
return fake_client
