def get_total_physical_memory_for_containers():
total_mem_used = 0
sysloads = system_manager.get_system_loads()
live = cgroup_manager.get_cgroup_containers()
for item in live:
try:
sample = cgroup_manager.get_container_sample(item)
total_mem_used += sample["mem_page_sample"]
except:
pass
total_mem_used = (total_mem_used + (1<<20) - 1) >> 20
total_physical_memory_for_containers = sysloads['mem_free'] + total_mem_used
return {"Mbytes": total_physical_memory_for_containers}
def get_available_memsw():
total_mem_limit = 0
total_mem_used = 0
sysloads = system_manager.get_system_loads()
live = cgroup_manager.get_cgroup_containers()
for item in live:
try:
sample = cgroup_manager.get_container_sample(item)
limit = cgroup_manager.get_container_limit(item)
total_mem_limit += limit["mem_page_quota"]
total_mem_used += sample["mem_page_sample"]
except:
pass
total_mem_limit >>= 20
total_mem_used = (total_mem_used + (1<<20) - 1) >> 20
available_mem_resource = sysloads['mem_free'] + \
sysloads['swap_free'] - total_mem_limit + total_mem_used
return {"Mbytes": available_mem_resource, "physical": sysloads['mem_free'], "cpu_free": sysloads['cpu_free']}
def cgroup_container_sample(form, args): return cgroup_manager.get_container_sample(form['uuid'])
def smart_control_forever(interval):
last_live = []
while True:
time.sleep(interval)
try:
mem_usage_mapping = {}
live = cgroup_manager.get_cgroup_containers()
for item in live:
try:
last_live.remove(item)
except:
pass
try:
cgroup_manager.protect_container_oom(item)
sample = cgroup_manager.get_container_sample(item)
mem_usage_mapping[item] = math.ceil(
sample['mem_page_sample'] * 1e-6)
billing_manager.add_usage_sample(
item, sample, interval)
except:
pass
for item in last_live:
billing_manager.clean_dead_node(item)
last_live = live
is_ready = True
memory_available = system_manager.get_available_memsw()
if memory_available['Mbytes'] <= 0:
size_in_gb = int(
math.ceil(-memory_available['Mbytes'] / 1024 / 16) *
16)
print(
"[warning]",
'overloaded containers, auto-extending %d G memsw.' %
size_in_gb)
system_manager.extend_swap(size_in_gb)
total_score = 0.0
score_mapping = {}
for item in live:
score = max(
1e-8, smart_controller.policy.get_score_by_uuid(item))
score_mapping[item] = score
print(item, "(score/cpu)", score)
total_score += score
# CPU Scoring
for item in live:
ceof = score_mapping[item] / total_score
cgroup_manager.set_container_cpu_priority_limit(item, ceof)
# Iterative Memory Scoring
free_mem = system_manager.get_total_physical_memory_for_containers(
)['Mbytes']
local_nodes = live
mem_alloc = {}
for item in live:
mem_alloc[item] = 0
while free_mem > 0 and len(local_nodes) > 0:
excess_mem = 0
next_local_nodes = []
for item in local_nodes:
mem_alloc[item] += int(
math.floor(free_mem * score_mapping[item] /
total_score))
if mem_alloc[item] >= mem_usage_mapping[item]:
excess_mem += mem_alloc[item] - mem_usage_mapping[
item]
mem_alloc[item] = mem_usage_mapping[item]
else:
next_local_nodes.append(item)
free_mem = excess_mem
local_nodes = next_local_nodes
for item in live:
mem_alloc[item] += int(
math.floor(free_mem * score_mapping[item] /
total_score))
cgroup_manager.set_container_physical_memory_limit(
item, mem_alloc[item])
print(item, "(malloc:usage)", mem_alloc[item],
mem_usage_mapping[item])
if len(live) > 0:
print("-------------------------------")
except:
pass
def cgroup_container_sample(form, args):
return cgroup_manager.get_container_sample(form['uuid'])
def get_score_by_uuid(uuid): sample = cgroup_manager.get_container_sample(uuid) pid = cgroup_manager.get_container_pid(uuid) etime = system_manager.get_proc_etime(pid) return sample["cpu_sample"] / etime
def get_score_by_uuid(uuid): sample = cgroup_manager.get_container_sample(uuid) return 1024 * 1024 / (1.0 + sample["cpu_sample"])
def get_score_by_uuid(uuid): sample = cgroup_manager.get_container_sample(uuid) return sample["mem_page_sample"]
def smart_control_forever(interval):
last_live = []
while True:
time.sleep(interval)
try:
mem_usage_mapping = {}
live = cgroup_manager.get_cgroup_containers()
for item in live:
try:
last_live.remove(item)
except:
pass
try:
cgroup_manager.protect_container_oom(item)
sample = cgroup_manager.get_container_sample(item)
mem_usage_mapping[item] = math.ceil(sample['mem_page_sample'] * 1e-6)
billing_manager.add_usage_sample(item, sample, interval)
except:
pass
for item in last_live:
billing_manager.clean_dead_node(item)
last_live = live
is_ready = True
memory_available = system_manager.get_available_memsw()
if memory_available['Mbytes'] <= 0:
size_in_gb = int(math.ceil(-memory_available['Mbytes'] / 1024 / 16) * 16)
print("[warning]", 'overloaded containers, auto-extending %d G memsw.' % size_in_gb)
system_manager.extend_swap(size_in_gb)
total_score = 0.0
score_mapping = {}
for item in live:
score = max(1e-8, smart_controller.policy.get_score_by_uuid(item))
score_mapping[item] = score
print(item, "(score/cpu)", score)
total_score += score
# CPU Scoring
for item in live:
ceof = score_mapping[item] / total_score
cgroup_manager.set_container_cpu_priority_limit(item, ceof)
# Iterative Memory Scoring
free_mem = system_manager.get_total_physical_memory_for_containers()['Mbytes']
local_nodes = live
mem_alloc = {}
for item in live:
mem_alloc[item] = 0
while free_mem > 0 and len(local_nodes) > 0:
excess_mem = 0
next_local_nodes = []
for item in local_nodes:
mem_alloc[item] += int(math.floor(free_mem * score_mapping[item] / total_score))
if mem_alloc[item] >= mem_usage_mapping[item]:
excess_mem += mem_alloc[item] - mem_usage_mapping[item]
mem_alloc[item] = mem_usage_mapping[item]
else:
next_local_nodes.append(item)
free_mem = excess_mem
local_nodes = next_local_nodes
for item in live:
mem_alloc[item] += int(math.floor(free_mem * score_mapping[item] / total_score))
cgroup_manager.set_container_physical_memory_limit(item, mem_alloc[item])
print(item, "(malloc:usage)", mem_alloc[item], mem_usage_mapping[item])
if len(live) > 0:
print("-------------------------------")
except:
pass