def __get_oversubscribable_thread_count(response: AllocateResponse):
if response is None:
return 0
oversubscribable_thread_count = 0
free_thread_ids = response.get_metadata().get(FREE_THREAD_IDS, [])
for t in response.get_cpu().get_threads():
if t.get_id() in free_thread_ids and len(t.get_workload_ids()) > 0:
oversubscribable_thread_count += 1
return oversubscribable_thread_count
def __apply_isolation(self, response: AllocateResponse):
last_w_responses = self.__get_workload_allocation_dict(
self.__last_response)
for w_alloc in response.get_workload_allocations():
last_w_alloc = last_w_responses.get(w_alloc.get_workload_id(),
None)
if w_alloc == last_w_alloc:
log.info("Skipping update of workload: {}".format(
w_alloc.get_workload_id()))
continue
workload_id = w_alloc.get_workload_id()
thread_ids = w_alloc.get_thread_ids()
quota = w_alloc.get_cpu_quota()
shares = w_alloc.get_cpu_shares()
log.info(
"updating workload: '{}' cpuset: '{}', quota: '{}', shares: '{}'"
.format(workload_id, thread_ids, quota, shares))
# This ordering is important for reporting whether a workload is isolated.
# We must always set the "cpuset" first.
self.__cgroup_manager.set_cpuset(workload_id, thread_ids)
self.__cgroup_manager.set_quota(workload_id, quota)
self.__cgroup_manager.set_shares(workload_id, shares)
def __deser(self, response: pb.IsolationResponse) -> AllocateResponse:
new_cpu = copy.deepcopy(self.__empty_cpu)
id2workloads = defaultdict(list)
wa_responses = []
for wid, cpuset in response.cpusets.items():
thread_ids = [
self.__natural2original_indexing[tid]
for tid in cpuset.thread_ids
]
war = WorkloadAllocateResponse(
wid, thread_ids, cpuset.cfs_tunables.shares,
cpuset.cfs_tunables.quota_us,
cpuset.cpuset_tunables.memory_migrate,
cpuset.cpuset_tunables.memory_spread_page,
cpuset.cpuset_tunables.memory_spread_slab)
wa_responses.append(war)
for tid in thread_ids:
id2workloads[tid].append(wid)
for package in new_cpu.get_packages():
for core in package.get_cores():
for thread in core.get_threads():
workloads = id2workloads.get(thread.get_id(), None)
if workloads is not None:
for wid in workloads:
thread.claim(wid)
return AllocateResponse(new_cpu, wa_responses, self.get_name(), {})
def rebalance(self, request: AllocateRequest) -> AllocateResponse:
log.info("Ignoring attempt to rebalance workloads: '{}'".format(
request.get_workloads()))
return AllocateResponse(
request.get_cpu(),
get_workload_allocations(request.get_cpu(),
list(request.get_workloads().values())),
self.get_name())
def free_threads(self,
request: AllocateThreadsRequest) -> AllocateResponse:
cpu = request.get_cpu()
workload = request.get_workloads()[request.get_workload_id()]
for t in cpu.get_threads():
t.free(workload.get_id())
return AllocateResponse(cpu, self.get_name())
def free_threads(self,
request: AllocateThreadsRequest) -> AllocateResponse:
log.info("Ignoring attempt to free threads for workload: '{}'".format(
request.get_workload_id()))
return AllocateResponse(
request.get_cpu(),
get_workload_allocations(request.get_cpu(),
list(request.get_workloads().values())),
self.get_name())
def rebalance(self, request: AllocateRequest) -> AllocateResponse:
cpu = request.get_cpu()
workloads = request.get_workloads()
metadata = {}
cpu = rebalance(cpu, workloads, self.__free_thread_provider, metadata)
return AllocateResponse(
cpu, get_workload_allocations(cpu, workloads.values()),
self.get_name(), metadata)
def assign_threads(self,
request: AllocateThreadsRequest) -> AllocateResponse:
cpu = request.get_cpu()
workload = request.get_workloads()[request.get_workload_id()]
threads = self._get_assign_threads(cpu, workload.get_thread_count())
for t in threads:
t.claim(workload.get_id())
return AllocateResponse(cpu, self.get_name())
def __get_workload_allocation_dict(response: AllocateResponse) -> Dict[str, WorkloadAllocateResponse]:
w_responses = {}
if response is None:
return w_responses
for w_alloc in response.get_workload_allocations():
w_responses[w_alloc.get_workload_id()] = w_alloc
return w_responses
def rebalance(self, request: AllocateRequest) -> AllocateResponse:
self.__call_meta = {}
cpu = request.get_cpu()
cpu_usage = request.get_cpu_usage()
workloads = request.get_workloads()
self.__cnt_rebalance_calls += 1
if len(workloads) == 0:
log.warning("Ignoring rebalance of empty CPU.")
self.__call_meta['rebalance_empty'] = 1
return AllocateResponse(cpu, self.get_name(), self.__call_meta)
log.info("Rebalancing with predictions...")
curr_ids_per_workload = cpu.get_workload_ids_to_thread_ids()
return AllocateResponse(
self.__compute_allocation(cpu, None, workloads, curr_ids_per_workload, cpu_usage, None),
self.get_name(),
self.__call_meta)
def assign_threads(self,
request: AllocateThreadsRequest) -> AllocateResponse:
thread_count = len(request.get_cpu().get_threads())
thread_ids = list(range(thread_count))
log.info(
"Setting cpuset.cpus to ALL cpus: '{}' for workload: '{}'".format(
thread_ids, request.get_workload_id()))
self.__cgroup_manager.set_cpuset(request.get_workload_id(), thread_ids)
return AllocateResponse(request.get_cpu(), self.get_name())
def assign_threads(self, request: AllocateThreadsRequest) -> AllocateResponse:
self.__call_meta = {}
cpu = request.get_cpu()
cpu_usage = request.get_cpu_usage()
workloads = request.get_workloads()
workload_id = request.get_workload_id()
curr_ids_per_workload = cpu.get_workload_ids_to_thread_ids()
return AllocateResponse(
self.__compute_allocation(cpu, workload_id, workloads, curr_ids_per_workload, cpu_usage, True),
self.get_name(),
self.__call_meta)
def assign_threads(self, request: AllocateThreadsRequest) -> AllocateResponse:
cpu = request.get_cpu()
workloads = request.get_workloads()
workload_id = request.get_workload_id()
burst_workloads = get_burst_workloads(workloads.values())
release_all_threads(cpu, burst_workloads)
if workloads[workload_id].get_type() == STATIC:
self.__assign_threads(cpu, workloads[workload_id])
update_burst_workloads(cpu, workloads, self.__free_thread_provider)
return AllocateResponse(cpu, self.get_name())
def free_threads(self, request: AllocateThreadsRequest) -> AllocateResponse:
cpu = request.get_cpu()
workloads = request.get_workloads()
workload_id = request.get_workload_id()
burst_workloads = get_burst_workloads(workloads.values())
release_all_threads(cpu, burst_workloads)
for t in cpu.get_threads():
if workload_id in t.get_workload_ids():
t.free(workload_id)
workloads.pop(workload_id)
update_burst_workloads(cpu, workloads, self.__free_thread_provider)
return AllocateResponse(cpu, self.get_name())
def free_threads(self, request: AllocateThreadsRequest) -> AllocateResponse:
self.__call_meta = {}
cpu = request.get_cpu()
cpu_usage = request.get_cpu_usage()
workloads = request.get_workloads()
workload_id = request.get_workload_id()
curr_ids_per_workload = cpu.get_workload_ids_to_thread_ids()
if workload_id not in curr_ids_per_workload:
raise Exception("workload_id=`%s` is not placed on the instance. Cannot free it." % (workload_id,))
return AllocateResponse(
self.__compute_allocation(cpu, workload_id, workloads, curr_ids_per_workload, cpu_usage, False),
self.get_name(),
self.__call_meta)
def __update_state(self, response: AllocateResponse, new_workloads):
start_time = time.time()
old_cpu = self.get_cpu_copy()
new_cpu = response.get_cpu()
self.__apply_isolation(response)
self.__cpu = new_cpu
self.__workloads = new_workloads
self.__last_response = response
if old_cpu != new_cpu:
self.__report_cpu_state(old_cpu, new_cpu)
stop_time = time.time()
if self.__reg is not None:
self.__reg.distribution_summary(UPDATE_STATE_DURATION, self.__tags).record(stop_time - start_time)
def free_threads(self,
request: AllocateThreadsRequest) -> AllocateResponse:
cpu = request.get_cpu()
workloads = request.get_workloads()
workload_id = request.get_workload_id()
burst_workloads = get_burst_workloads(workloads.values())
release_all_threads(cpu, burst_workloads)
if workloads[workload_id].get_type() == STATIC:
self.__free_threads(cpu, workload_id, workloads)
workloads.pop(workload_id)
metadata = {}
update_burst_workloads(cpu, workloads, self.__free_thread_provider,
metadata)
return AllocateResponse(
cpu, get_workload_allocations(cpu, workloads.values()),
self.get_name(), metadata)
def __apply_isolation(self, response: AllocateResponse):
last_w_responses = self.__get_workload_allocation_dict(
self.__last_response)
for w_alloc in response.get_workload_allocations():
last_w_alloc = last_w_responses.get(w_alloc.get_workload_id(),
None)
if w_alloc == last_w_alloc:
log.info("Skipping update of workload: {}".format(
w_alloc.get_workload_id()))
continue
workload_id = w_alloc.get_workload_id()
thread_ids = w_alloc.get_thread_ids()
quota = w_alloc.get_cpu_quota()
shares = w_alloc.get_cpu_shares()
memory_migrate = w_alloc.get_memory_migrate()
memory_spread_page = w_alloc.get_memory_spread_page()
memory_spread_slab = w_alloc.get_memory_spread_slab()
log.info(f'updating workload: {workload_id} '
f'cpuset: {thread_ids}, '
f'quota: {quota}, '
f'shares: {shares}, '
f'memory_migrate: {memory_migrate}, '
f'memory_spread_page: {memory_spread_page}, '
f'memory_spread_slab: {memory_spread_slab}')
# This ordering is important for reporting whether a workload is isolated.
# We must always set the "cpuset" first.
self.__cgroup_manager.set_cpuset(workload_id, thread_ids)
self.__cgroup_manager.set_quota(workload_id, quota)
self.__cgroup_manager.set_shares(workload_id, shares)
self.__cgroup_manager.set_memory_migrate(workload_id,
memory_migrate)
self.__cgroup_manager.set_memory_spread_page(
workload_id, memory_spread_page)
self.__cgroup_manager.set_memory_spread_slab(
workload_id, memory_spread_slab)
def rebalance(self, request: AllocateRequest) -> AllocateResponse:
return AllocateResponse(
request.get_cpu(),
get_workload_allocations(request.get_cpu(),
request.get_workloads().values()),
self.get_name())
def rebalance(self, request: AllocateRequest) -> AllocateResponse:
return AllocateResponse(request.get_cpu(), self.get_name())
def assign_threads(self,
request: AllocateThreadsRequest) -> AllocateResponse:
log.info("Ignoring attempt to assign threads to workload: '{}'".format(
request.get_workload_id()))
return AllocateResponse(request.get_cpu(), self.get_name())
def get_cpu_event(request: AllocateRequest, response: AllocateResponse):
return {
"request": request.to_dict(),
"response": response.to_dict(),
}
def rebalance(self, request: AllocateRequest) -> AllocateResponse:
cpu = request.get_cpu()
workloads = request.get_workloads()
cpu = rebalance(cpu, workloads, self.__free_thread_provider)
return AllocateResponse(cpu, self.get_name())
def __get_free_thread_count(response: AllocateResponse):
if response is None:
return 0
free_thread_ids = response.get_metadata().get(FREE_THREAD_IDS, [])
return len(free_thread_ids)