async def scale_all_apps(app: App):
cloud_interface = AsgardInterface()
state_checker = PeriodicStateChecker(cloud_interface)
decision_maker = DecisionComponent()
logger.debug({"AUTOSCALER": "iniciando autoscaler"})
apps_stats = await state_checker.get_scalable_apps_stats()
logger.debug({"AUTOSCALER_FETCH_APPS": [app.id for app in apps_stats]})
scaling_decisions = decision_maker.decide_scaling_actions(apps_stats)
await cloud_interface.apply_decisions(scaling_decisions)
async def test_does_not_make_decision_when_there_are_no_stats(self):
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
cpu_threshold=0.2,
app_stats=None,
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(0, len(decisions), "decision was made")
async def test_does_not_make_any_decision_when_everything_is_ignored(self):
apps = [
ScalableApp(
"test",
cpu_allocated=1.0,
mem_allocated=1.0,
cpu_threshold=None,
mem_threshold=None,
app_stats=AppStats(cpu_usage=0.8, mem_usage=0.3),
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(0, len(decisions), "does not return any decisions")
async def test_scales_memory_to_correct_value(self):
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
mem_threshold=0.2,
app_stats=AppStats(cpu_usage=0.4129, mem_usage=0.6262),
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(1, len(decisions), "did not return any decisions")
self.assertEqual(400.768, decisions[0].mem)
async def test_does_not_make_decision_when_app_is_using_max_mem_and_decision_would_upscale(
self):
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
mem_threshold=0.75,
app_stats=AppStats(cpu_usage=0.1, mem_usage=1.0),
max_mem_scale_limit=128,
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(0, len(decisions), "a decision was made")
async def test_does_not_scale_app_when_difference_less_than_5_percent(
self):
apps = [
ScalableApp(
"test",
cpu_allocated=1.0,
mem_allocated=1.0,
cpu_threshold=0.5,
mem_threshold=0.7,
app_stats=AppStats(cpu_usage=0.4501, mem_usage=0.7499),
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(0, len(decisions))
async def test_scales_app_when_difference_greater_than_5_percent(self):
apps = [
ScalableApp(
"test",
cpu_allocated=1.0,
mem_allocated=1.0,
cpu_threshold=0.5,
mem_threshold=0.7,
app_stats=AppStats(cpu_usage=0.4499, mem_usage=0.7501),
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(1, len(decisions))
self.assertEqual("test", decisions[0].id)
def test_logs_memory_downscaling_decisions(self):
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
mem_threshold=1,
app_stats=AppStats(cpu_usage=0.1, mem_usage=0.1),
)
]
mock_logger = NonCallableMock()
decider = DecisionComponent(logger=mock_logger)
decisions = decider.decide_scaling_actions(apps)
mock_logger.info.assert_called()
logged_dict = mock_logger.info.call_args[0][0]
self.assertIn("appname", logged_dict, "did not log correct app id")
self.assertEqual(apps[0].id, logged_dict["appname"],
"did not log correct app id")
self.assertIn("event", logged_dict, "did not log an event")
self.assertEqual(
DecisionEvents.MEM_SCALE_DOWN,
logged_dict["event"],
"did not log correct event",
)
self.assertIn("previous_value", logged_dict,
"did not log previous memory value")
self.assertEqual(
apps[0].mem_allocated,
logged_dict["previous_value"],
"did not log correct previous memory value",
)
self.assertIn("new_value", logged_dict, "did not log new memory value")
self.assertEqual(
decisions[0].mem,
logged_dict["new_value"],
"did not log correct new memory value",
)
async def test_logs_cpu_upscaling_decisions(self):
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
cpu_threshold=0.2,
app_stats=AppStats(cpu_usage=100, mem_usage=100),
)
]
mock_logger = NonCallableMock()
decider = DecisionComponent(logger=mock_logger)
decisions = decider.decide_scaling_actions(apps)
mock_logger.info.assert_called()
logged_dict = mock_logger.info.call_args[0][0]
self.assertIn("appname", logged_dict, "did not log correct app id")
self.assertEqual(apps[0].id, logged_dict["appname"],
"did not log correct app id")
self.assertIn("event", logged_dict, "did not log an event")
self.assertEqual(
DecisionEvents.CPU_SCALE_UP,
logged_dict["event"],
"did not log correct event",
)
self.assertIn("previous_value", logged_dict,
"did not log previous CPU value")
self.assertEqual(
logged_dict["previous_value"],
apps[0].cpu_allocated,
"did not log correct previous CPU value",
)
self.assertIn("new_value", logged_dict, "did not log new CPU value")
self.assertEqual(
logged_dict["new_value"],
decisions[0].cpu,
"did not log correct new CPU value",
)
async def test_does_not_make_memory_decision_when_memory_is_ignored(self):
apps = [
ScalableApp(
"test",
cpu_allocated=1.0,
mem_allocated=1.0,
cpu_threshold=0.7,
mem_threshold=None,
app_stats=AppStats(cpu_usage=0.8, mem_usage=0.3),
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(1, len(decisions), "makes decision for the app only")
self.assertEqual("test", decisions[0].id, "returns the correct app")
self.assertNotEqual(None, decisions[0].cpu, "returns cpu decision")
self.assertEqual(None, decisions[0].mem,
"does not return memory decision")
async def test_scales_cpu_and_mem_to_correct_value(self):
apps = [
ScalableApp(
"test1",
cpu_allocated=3.5,
mem_allocated=1.0,
cpu_threshold=0.1,
mem_threshold=0.1,
app_stats=AppStats(cpu_usage=1.0, mem_usage=1.0),
),
ScalableApp(
"test2",
cpu_allocated=3.5,
mem_allocated=1.0,
cpu_threshold=0.5,
mem_threshold=0.7,
app_stats=AppStats(cpu_usage=1.0, mem_usage=1.0),
),
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(2, len(decisions), "makes decisions for both apps")
self.assertEqual("test1", decisions[0].id,
"returns correct apps in correct order")
self.assertEqual("test2", decisions[1].id,
"returns correct apps in correct order")
self.assertEqual(35, decisions[0].cpu,
"decides correct values for cpu")
self.assertEqual(10, decisions[0].mem,
"decides correct values for memory")
self.assertEqual(7, decisions[1].cpu, "decides correct values for cpu")
self.assertEqual(
1.4286,
round(decisions[1].mem, 4),
"decides correct values for memory",
)
async def test_does_not_scale_cpu_above_max_scale_limit(self):
max_cpu_limit = float("-inf")
min_cpu_limit = float("-inf")
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
cpu_threshold=0.2,
app_stats=AppStats(cpu_usage=0.4129, mem_usage=0.8),
max_cpu_scale_limit=max_cpu_limit,
min_cpu_scale_limit=min_cpu_limit,
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(1, len(decisions), "did not return any decisions")
self.assertLessEqual(
max_cpu_limit,
decisions[0].cpu,
"cpu value is greater than the max limit",
)
async def test_does_not_scale_mem_below_min_scale_limit(self):
min_mem_limit = float("inf")
max_mem_limit = float("inf")
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
mem_threshold=0.5,
app_stats=AppStats(cpu_usage=0.4129, mem_usage=0.35),
min_mem_scale_limit=min_mem_limit,
max_mem_scale_limit=max_mem_limit,
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(1, len(decisions), "did not return any decisions")
self.assertGreaterEqual(
min_mem_limit,
decisions[0].mem,
"mem value is less than the min limit",
)
def test_logs_cpu_and_memory_scaling_decisions(self):
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
mem_threshold=1,
cpu_threshold=0.2,
app_stats=AppStats(cpu_usage=1.0, mem_usage=0.1),
)
]
mock_logger = NonCallableMock()
decider = DecisionComponent(logger=mock_logger)
decisions = decider.decide_scaling_actions(apps)
mock_logger.info.assert_called()
logger_calls = [call[0][0] for call in mock_logger.info.call_args_list]
self.assertEqual(len(logger_calls), 2, "did not call log.info 2 times")
logger_calls.sort(key=lambda call: call["event"])
cpu_log_dict = logger_calls[0]
mem_log_dict = logger_calls[1]
self.assertIn("appname", cpu_log_dict, "did not log correct app id")
self.assertEqual(mem_log_dict["appname"], apps[0].id,
"did not log correct app id")
self.assertIn("event", cpu_log_dict, "did not log an event")
self.assertEqual(
DecisionEvents.CPU_SCALE_UP,
cpu_log_dict["event"],
"did not log correct event",
)
self.assertIn("previous_value", cpu_log_dict,
"did not log previous memory value")
self.assertEqual(
0.5,
cpu_log_dict["previous_value"],
"did not log correct previous memory value",
)
self.assertIn("new_value", cpu_log_dict,
"did not log new memory value")
self.assertEqual(
decisions[0].cpu,
cpu_log_dict["new_value"],
"did not log correct new memory value",
)
self.assertIn("appname", mem_log_dict, "did not log correct app id")
self.assertEqual(apps[0].id, mem_log_dict["appname"],
"did not log correct app id")
self.assertIn("event", mem_log_dict, "did not log an event")
self.assertEqual(
DecisionEvents.MEM_SCALE_DOWN,
mem_log_dict["event"],
"did not log correct event",
)
self.assertIn("previous_value", mem_log_dict,
"did not log previous memory value")
self.assertEqual(
apps[0].mem_allocated,
mem_log_dict["previous_value"],
"did not log correct previous memory value",
)
self.assertIn("new_value", mem_log_dict,
"did not log new memory value")
self.assertEqual(
decisions[0].mem,
mem_log_dict["new_value"],
"did not log correct new memory value",
)
async def test_decide_to_scale_all_apps(self):
cloud_interface = AsgardCloudInterface()
state_checker = PeriodicStateChecker(cloud_interface)
decision_maker = DecisionComponent()
with aioresponses() as rsps:
stats_fixture = {
"stats": {
"type": "ASGARD",
"errors": {},
"cpu_pct": "100",
"ram_pct": "100",
"cpu_thr_pct": "0",
}
}
apps_fixture = {
"apps": [
{
"id": "/test_app1",
"cpus": 3.5,
"mem": 1.0,
"labels": {
"asgard.autoscale.cpu": 0.3,
"asgard.autoscale.mem": 0.8,
"asgard.autoscale.ignore": "cpu",
},
},
{
"id": "/test_app2",
"cpus": 3.5,
"mem": 1.0,
"labels": {
"asgard.autoscale.cpu": 0.1,
"asgard.autoscale.mem": 0.6,
"asgard.autoscale.ignore": "mem",
},
},
]
}
rsps.get(
f"{settings.ASGARD_API_ADDRESS}/v2/apps",
status=200,
payload=apps_fixture,
)
for app in apps_fixture["apps"]:
rsps.get(
f"{settings.ASGARD_API_ADDRESS}/apps{app['id']}/stats/avg-1min",
status=200,
payload=stats_fixture,
)
rsps.put(
f"{settings.ASGARD_API_ADDRESS}/v2/apps",
status=200,
payload={
"deploymentId": "test",
"version": "1.0"
},
)
apps_stats = await state_checker.get_scalable_apps_stats()
scaling_decision = decision_maker.decide_scaling_actions(
apps_stats)
await cloud_interface.apply_decisions(scaling_decision)
scale_spy = rsps.requests.get(
("put", URL(f"{settings.ASGARD_API_ADDRESS}/v2/apps")))
self.assertEqual(len(apps_stats), len(scaling_decision))
self.assertEqual(1.25, scaling_decision[0].mem)
self.assertEqual("test_app1", scaling_decision[0].id)
self.assertEqual(None, scaling_decision[0].cpu)
self.assertEqual("test_app2", scaling_decision[1].id)
self.assertEqual(None, scaling_decision[1].mem)
self.assertEqual(35, scaling_decision[1].cpu)
self.assertIsNotNone(scale_spy)
async def test_scales_when_difference_more_than_5_percent(self):
cloud_interface = AsgardCloudInterface()
state_checker = PeriodicStateChecker(cloud_interface)
decision_maker = DecisionComponent()
with aioresponses() as rsps:
stats_fixture = {
"stats": {
"type": "ASGARD",
"errors": {},
"cpu_pct": "24.9",
"ram_pct": "85.1",
"cpu_thr_pct": "0",
}
}
apps_fixture = {
"apps": [{
"id": "/test_app1",
"cpus": 3.5,
"mem": 1.0,
"labels": {
"asgard.autoscale.cpu": 0.3,
"asgard.autoscale.mem": 0.8,
},
}]
}
rsps.get(
f"{settings.ASGARD_API_ADDRESS}/v2/apps",
status=200,
payload=apps_fixture,
)
rsps.put(
f"{settings.ASGARD_API_ADDRESS}/v2/apps",
status=200,
payload={
"deploymentId": "test",
"version": "1.0"
},
)
for app in apps_fixture["apps"]:
rsps.get(
f"{settings.ASGARD_API_ADDRESS}/apps{app['id']}/stats/avg-1min",
status=200,
payload=stats_fixture,
)
apps_stats = await state_checker.get_scalable_apps_stats()
scaling_decision = decision_maker.decide_scaling_actions(
apps_stats)
await cloud_interface.apply_decisions(scaling_decision)
scale_spy = rsps.requests.get(
("put", URL(f"{settings.ASGARD_API_ADDRESS}/v2/apps")))
self.assertEqual(1, len(apps_stats), "didn't fetch one app")
self.assertEqual(1, len(scaling_decision),
"didn't make scaling decision")
self.assertEqual("test_app1", scaling_decision[0].id,
"made decision for wrong app")
self.assertEqual(2.905, scaling_decision[0].cpu,
"scaled cpu to incorrect value")
self.assertEqual(1.06375, scaling_decision[0].mem,
"scaled memory to incorrect value")
self.assertIsNotNone(scale_spy)