async def test_get_all_apps_which_should_be_scaled_all_apps_should(self):
scaler = AsgardInterface()
with aioresponses() as rsps:
payload = {
"apps": [
{
"id": "/test_app1",
"cpus": "0.2",
"mem": "0.2",
"labels": {
"asgard.autoscale.cpu": 0.3,
"asgard.autoscale.mem": 0.8,
},
},
{
"id": "/test_app2",
"cpus": "0.2",
"mem": "0.2",
"labels": {
"asgard.autoscale.cpu": 0.1,
"asgard.autoscale.mem": 0.1,
},
},
{
"id": "/test_app3",
"cpus": "0.2",
"mem": "0.2",
"labels": {
"asgard.autoscale.cpu": 0.5,
"asgard.autoscale.mem": 0.7,
},
},
]
}
fixture = [
ScalableApp("test_app1", cpu_threshold=0.3, mem_threshold=0.8),
ScalableApp("test_app2", cpu_threshold=0.1, mem_threshold=0.1),
ScalableApp("test_app3", cpu_threshold=0.5, mem_threshold=0.7),
]
rsps.get(
f"{settings.ASGARD_API_ADDRESS}/v2/apps",
status=200,
payload=payload,
)
apps = await scaler.get_all_scalable_apps()
self.assertEqual(len(fixture), len(apps))
for i in range(len(fixture)):
self.assertEqual(fixture[i].id, apps[i].id)
self.assertEqual(
fixture[i].cpu_threshold, apps[i].cpu_threshold
)
self.assertEqual(
fixture[i].mem_threshold, apps[i].mem_threshold
)
async def test_tune_everything_in_multiple_apps(self):
interface = AsgardInterface()
app1 = ScalableApp("test1")
app2 = ScalableApp("test2")
app3 = ScalableApp("test3")
decisions = [
Decision(app1.id, cpu=0.2, mem=10),
Decision(app2.id, cpu=0.4, mem=20),
Decision(app3.id, cpu=0.1, mem=9),
]
with aioresponses() as rsps:
rsps.put(
f"{settings.ASGARD_API_ADDRESS}/v2/apps",
status=200,
payload={
"deploymentId": "test2",
"version": "1.0"
},
)
applied_decisions = await interface.apply_decisions(decisions)
self.assertEqual(len(applied_decisions), 3)
for i in range(len(decisions)):
self.assertEqual(applied_decisions[i]["id"], decisions[i].id)
self.assertEqual(applied_decisions[i]["cpus"], decisions[i].cpu)
self.assertEqual(applied_decisions[i]["mem"], decisions[i].mem)
def test_everything_should_be_scaled(self):
app = ScalableApp("test_app1", cpu_threshold=0.3, mem_threshold=0.8)
set_to_scale = app.is_set_to_scale()
set_to_scale_cpu = app.is_set_to_scale_cpu()
set_to_scale_mem = app.is_set_to_scale_mem()
self.assertEqual(True, set_to_scale)
self.assertEqual(True, set_to_scale_cpu)
self.assertEqual(True, set_to_scale_mem)
def test_application_should_not_be_scaled_missing_attributes(self):
app = ScalableApp("id")
set_to_scale = app.is_set_to_scale()
set_to_scale_cpu = app.is_set_to_scale_cpu()
set_to_scale_mem = app.is_set_to_scale_mem()
self.assertEqual(False, set_to_scale)
self.assertEqual(False, set_to_scale_cpu)
self.assertEqual(False, set_to_scale_mem)
def test_only_mem_should_be_scaled(self):
app = ScalableApp("test_app1", mem_threshold=0.8)
set_to_scale = app.is_set_to_scale()
set_to_scale_cpu = app.is_set_to_scale_cpu()
set_to_scale_mem = app.is_set_to_scale_mem()
self.assertEqual(True, set_to_scale)
self.assertEqual(False, set_to_scale_cpu)
self.assertEqual(True, set_to_scale_mem)
async def test_get_all_apps_data(self):
scaler = AsgardInterface()
with aioresponses() as rsps:
rsps.get(
f"{settings.ASGARD_API_ADDRESS}/v2/apps",
status=200,
payload={
"apps": [
{
"id": "/test_app",
"cpus": "0.1",
"mem": "0.2",
"labels": {},
},
{
"id": "/test_app2",
"cpus": "0.1",
"mem": "0.2",
"labels": {},
},
{
"id": "/test_app3",
"cpus": "0.1",
"mem": "0.2",
"labels": {},
},
{
"id": "/test_app4",
"cpus": "0.1",
"mem": "0.2",
"labels": {},
},
]
},
)
apps = await scaler.fetch_all_apps()
fixture = [
ScalableApp("test_app"),
ScalableApp("test_app2"),
ScalableApp("test_app3"),
ScalableApp("test_app4"),
]
self.assertEqual(len(fixture), len(apps))
for i in range(len(fixture)):
self.assertEqual(fixture[i].id, apps[i].id)
async def test_get_app_stats_existing_app_id(self):
scaler = AsgardInterface()
with aioresponses() as rsps:
payload = {
"stats": {
"type": "ASGARD",
"errors": {},
"cpu_pct": "0.93",
"ram_pct": "8.91",
"cpu_thr_pct": "0.06",
}
}
app = ScalableApp("app_test1")
rsps.get(
f"{settings.ASGARD_API_ADDRESS}/apps/{app.id}/stats/avg-1min",
status=200,
payload=payload,
)
fixture = AppStats(cpu_usage=0.93, mem_usage=8.91)
app_stats = await scaler.get_app_stats(app)
self.assertEqual(fixture.cpu_usage, app_stats.cpu_usage)
self.assertEqual(fixture.mem_usage, app_stats.mem_usage)
async def test_get_app_stats_non_existing_app_id(self):
scaler = AsgardInterface()
with aioresponses() as rsps:
fixture = {
"stats": {
"type": "ASGARD",
"errors": {},
"cpu_pct": "0",
"ram_pct": "0",
"cpu_thr_pct": "0",
}
}
app = ScalableApp("app_test1")
rsps.get(
f"{settings.ASGARD_API_ADDRESS}/apps/{app.id}/stats/avg-1min",
status=200,
payload=fixture,
)
stats = await scaler.get_app_stats(app)
self.assertEqual(None, stats)
self.assertEqual(None, app.app_stats)
async def test_scales_cpu_and_mem_to_correct_value(self):
apps = [
ScalableApp(
"test1",
cpu_allocated=3.5,
mem_allocated=1.0,
cpu_threshold=0.1,
mem_threshold=0.1,
app_stats=AppStats(cpu_usage=1.0, mem_usage=1.0),
),
ScalableApp(
"test2",
cpu_allocated=3.5,
mem_allocated=1.0,
cpu_threshold=0.5,
mem_threshold=0.7,
app_stats=AppStats(cpu_usage=1.0, mem_usage=1.0),
),
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(2, len(decisions), "makes decisions for both apps")
self.assertEqual("test1", decisions[0].id,
"returns correct apps in correct order")
self.assertEqual("test2", decisions[1].id,
"returns correct apps in correct order")
self.assertEqual(35, decisions[0].cpu,
"decides correct values for cpu")
self.assertEqual(10, decisions[0].mem,
"decides correct values for memory")
self.assertEqual(7, decisions[1].cpu, "decides correct values for cpu")
self.assertEqual(
1.4286,
round(decisions[1].mem, 4),
"decides correct values for memory",
)
async def test_tune_multiple_apps_with_different_params(self):
interface = AsgardInterface()
app1 = ScalableApp("test1")
app2 = ScalableApp("test2")
app3 = ScalableApp("test3")
decisions = [
Decision(app1.id, mem=10),
Decision(app2.id, cpu=0.4),
Decision(app3.id, cpu=0.1, mem=9),
]
with aioresponses() as rsps:
rsps.put(
f"{settings.ASGARD_API_ADDRESS}/v2/apps",
status=200,
payload={
"deploymentId": "test2",
"version": "1.0"
},
)
applied_decisions = await interface.apply_decisions(decisions)
self.assertEqual(len(applied_decisions), 3)
self.assertEqual(applied_decisions[0]["id"], decisions[0].id)
self.assertEqual(applied_decisions[0]["mem"], decisions[0].mem)
self.assertEqual("cpus" in applied_decisions[0], False)
self.assertEqual(applied_decisions[1]["id"], decisions[1].id)
self.assertEqual(applied_decisions[1]["cpus"], decisions[1].cpu)
self.assertEqual("mem" in applied_decisions[1], False)
self.assertEqual(applied_decisions[2]["id"], decisions[2].id)
self.assertEqual(applied_decisions[2]["mem"], decisions[2].mem)
self.assertEqual(applied_decisions[2]["cpus"], decisions[2].cpu)
async def test_does_not_make_decision_when_there_are_no_stats(self):
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
cpu_threshold=0.2,
app_stats=None,
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(0, len(decisions), "decision was made")
async def test_does_not_make_any_decision_when_everything_is_ignored(self):
apps = [
ScalableApp(
"test",
cpu_allocated=1.0,
mem_allocated=1.0,
cpu_threshold=None,
mem_threshold=None,
app_stats=AppStats(cpu_usage=0.8, mem_usage=0.3),
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(0, len(decisions), "does not return any decisions")
async def test_scales_memory_to_correct_value(self):
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
mem_threshold=0.2,
app_stats=AppStats(cpu_usage=0.4129, mem_usage=0.6262),
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(1, len(decisions), "did not return any decisions")
self.assertEqual(400.768, decisions[0].mem)
async def test_does_not_make_decision_when_app_is_using_max_mem_and_decision_would_upscale(
self):
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
mem_threshold=0.75,
app_stats=AppStats(cpu_usage=0.1, mem_usage=1.0),
max_mem_scale_limit=128,
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(0, len(decisions), "a decision was made")
async def test_does_not_scale_app_when_difference_less_than_5_percent(
self):
apps = [
ScalableApp(
"test",
cpu_allocated=1.0,
mem_allocated=1.0,
cpu_threshold=0.5,
mem_threshold=0.7,
app_stats=AppStats(cpu_usage=0.4501, mem_usage=0.7499),
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(0, len(decisions))
async def test_scales_app_when_difference_greater_than_5_percent(self):
apps = [
ScalableApp(
"test",
cpu_allocated=1.0,
mem_allocated=1.0,
cpu_threshold=0.5,
mem_threshold=0.7,
app_stats=AppStats(cpu_usage=0.4499, mem_usage=0.7501),
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(1, len(decisions))
self.assertEqual("test", decisions[0].id)
def test_logs_memory_downscaling_decisions(self):
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
mem_threshold=1,
app_stats=AppStats(cpu_usage=0.1, mem_usage=0.1),
)
]
mock_logger = NonCallableMock()
decider = DecisionComponent(logger=mock_logger)
decisions = decider.decide_scaling_actions(apps)
mock_logger.info.assert_called()
logged_dict = mock_logger.info.call_args[0][0]
self.assertIn("appname", logged_dict, "did not log correct app id")
self.assertEqual(apps[0].id, logged_dict["appname"],
"did not log correct app id")
self.assertIn("event", logged_dict, "did not log an event")
self.assertEqual(
DecisionEvents.MEM_SCALE_DOWN,
logged_dict["event"],
"did not log correct event",
)
self.assertIn("previous_value", logged_dict,
"did not log previous memory value")
self.assertEqual(
apps[0].mem_allocated,
logged_dict["previous_value"],
"did not log correct previous memory value",
)
self.assertIn("new_value", logged_dict, "did not log new memory value")
self.assertEqual(
decisions[0].mem,
logged_dict["new_value"],
"did not log correct new memory value",
)
async def test_logs_cpu_upscaling_decisions(self):
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
cpu_threshold=0.2,
app_stats=AppStats(cpu_usage=100, mem_usage=100),
)
]
mock_logger = NonCallableMock()
decider = DecisionComponent(logger=mock_logger)
decisions = decider.decide_scaling_actions(apps)
mock_logger.info.assert_called()
logged_dict = mock_logger.info.call_args[0][0]
self.assertIn("appname", logged_dict, "did not log correct app id")
self.assertEqual(apps[0].id, logged_dict["appname"],
"did not log correct app id")
self.assertIn("event", logged_dict, "did not log an event")
self.assertEqual(
DecisionEvents.CPU_SCALE_UP,
logged_dict["event"],
"did not log correct event",
)
self.assertIn("previous_value", logged_dict,
"did not log previous CPU value")
self.assertEqual(
logged_dict["previous_value"],
apps[0].cpu_allocated,
"did not log correct previous CPU value",
)
self.assertIn("new_value", logged_dict, "did not log new CPU value")
self.assertEqual(
logged_dict["new_value"],
decisions[0].cpu,
"did not log correct new CPU value",
)
async def test_does_not_make_memory_decision_when_memory_is_ignored(self):
apps = [
ScalableApp(
"test",
cpu_allocated=1.0,
mem_allocated=1.0,
cpu_threshold=0.7,
mem_threshold=None,
app_stats=AppStats(cpu_usage=0.8, mem_usage=0.3),
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(1, len(decisions), "makes decision for the app only")
self.assertEqual("test", decisions[0].id, "returns the correct app")
self.assertNotEqual(None, decisions[0].cpu, "returns cpu decision")
self.assertEqual(None, decisions[0].mem,
"does not return memory decision")
async def test_tune_one_thing_in_one_app(self):
interface = AsgardInterface()
app = ScalableApp("test")
decisions = [Decision(app.id, cpu=0.3)]
with aioresponses() as rsps:
rsps.put(
f"{settings.ASGARD_API_ADDRESS}/v2/apps",
status=200,
payload={
"deploymentId": "test1",
"version": "1.0"
},
)
applied_decisions = await interface.apply_decisions(decisions)
self.assertEqual(len(applied_decisions), 1)
self.assertEqual(applied_decisions[0]["id"], decisions[0].id)
self.assertEqual(applied_decisions[0]["cpus"], decisions[0].cpu)
self.assertEqual("mem" in applied_decisions[0], False)
async def test_does_not_scale_cpu_above_max_scale_limit(self):
max_cpu_limit = float("-inf")
min_cpu_limit = float("-inf")
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
cpu_threshold=0.2,
app_stats=AppStats(cpu_usage=0.4129, mem_usage=0.8),
max_cpu_scale_limit=max_cpu_limit,
min_cpu_scale_limit=min_cpu_limit,
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(1, len(decisions), "did not return any decisions")
self.assertLessEqual(
max_cpu_limit,
decisions[0].cpu,
"cpu value is greater than the max limit",
)
async def test_does_not_scale_mem_below_min_scale_limit(self):
min_mem_limit = float("inf")
max_mem_limit = float("inf")
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
mem_threshold=0.5,
app_stats=AppStats(cpu_usage=0.4129, mem_usage=0.35),
min_mem_scale_limit=min_mem_limit,
max_mem_scale_limit=max_mem_limit,
)
]
decider = DecisionComponent()
decisions = decider.decide_scaling_actions(apps)
self.assertEqual(1, len(decisions), "did not return any decisions")
self.assertGreaterEqual(
min_mem_limit,
decisions[0].mem,
"mem value is less than the min limit",
)
def test_logs_cpu_and_memory_scaling_decisions(self):
apps = [
ScalableApp(
"test",
cpu_allocated=0.5,
mem_allocated=128,
mem_threshold=1,
cpu_threshold=0.2,
app_stats=AppStats(cpu_usage=1.0, mem_usage=0.1),
)
]
mock_logger = NonCallableMock()
decider = DecisionComponent(logger=mock_logger)
decisions = decider.decide_scaling_actions(apps)
mock_logger.info.assert_called()
logger_calls = [call[0][0] for call in mock_logger.info.call_args_list]
self.assertEqual(len(logger_calls), 2, "did not call log.info 2 times")
logger_calls.sort(key=lambda call: call["event"])
cpu_log_dict = logger_calls[0]
mem_log_dict = logger_calls[1]
self.assertIn("appname", cpu_log_dict, "did not log correct app id")
self.assertEqual(mem_log_dict["appname"], apps[0].id,
"did not log correct app id")
self.assertIn("event", cpu_log_dict, "did not log an event")
self.assertEqual(
DecisionEvents.CPU_SCALE_UP,
cpu_log_dict["event"],
"did not log correct event",
)
self.assertIn("previous_value", cpu_log_dict,
"did not log previous memory value")
self.assertEqual(
0.5,
cpu_log_dict["previous_value"],
"did not log correct previous memory value",
)
self.assertIn("new_value", cpu_log_dict,
"did not log new memory value")
self.assertEqual(
decisions[0].cpu,
cpu_log_dict["new_value"],
"did not log correct new memory value",
)
self.assertIn("appname", mem_log_dict, "did not log correct app id")
self.assertEqual(apps[0].id, mem_log_dict["appname"],
"did not log correct app id")
self.assertIn("event", mem_log_dict, "did not log an event")
self.assertEqual(
DecisionEvents.MEM_SCALE_DOWN,
mem_log_dict["event"],
"did not log correct event",
)
self.assertIn("previous_value", mem_log_dict,
"did not log previous memory value")
self.assertEqual(
apps[0].mem_allocated,
mem_log_dict["previous_value"],
"did not log correct previous memory value",
)
self.assertIn("new_value", mem_log_dict,
"did not log new memory value")
self.assertEqual(
decisions[0].mem,
mem_log_dict["new_value"],
"did not log correct new memory value",
)
def to_model(cls, dto_object: AppDto) -> ScalableApp:
if dto_object.id[0] == "/":
appid = dto_object.id[1:]
else:
appid = dto_object.id
scalable_app = ScalableApp(appid)
scalable_app.cpu_allocated = dto_object.cpus
scalable_app.mem_allocated = dto_object.mem
if dto_object.labels is not None:
if "asgard.autoscale.cpu" in dto_object.labels:
scalable_app.cpu_threshold = float(
dto_object.labels["asgard.autoscale.cpu"])
if "asgard.autoscale.mem" in dto_object.labels:
scalable_app.mem_threshold = float(
dto_object.labels["asgard.autoscale.mem"])
if "asgard.autoscale.ignore" in dto_object.labels:
if "all" in dto_object.labels["asgard.autoscale.ignore"]:
scalable_app.cpu_threshold = None
scalable_app.mem_threshold = None
else:
if "cpu" in dto_object.labels["asgard.autoscale.ignore"]:
scalable_app.cpu_threshold = None
if "mem" in dto_object.labels["asgard.autoscale.ignore"]:
scalable_app.mem_threshold = None
if "asgard.autoscale.max_cpu_limit" in dto_object.labels:
scalable_app.max_cpu_scale_limit = float(
dto_object.labels["asgard.autoscale.max_cpu_limit"])
if "asgard.autoscale.min_cpu_limit" in dto_object.labels:
scalable_app.min_cpu_scale_limit = float(
dto_object.labels["asgard.autoscale.min_cpu_limit"])
if "asgard.autoscale.max_mem_limit" in dto_object.labels:
scalable_app.max_mem_scale_limit = float(
dto_object.labels["asgard.autoscale.max_mem_limit"])
if "asgard.autoscale.min_mem_limit" in dto_object.labels:
scalable_app.min_mem_scale_limit = float(
dto_object.labels["asgard.autoscale.min_mem_limit"])
return scalable_app