def test_shared_core_violation(self):
allocator = IntegerProgramCpuAllocator()
# Claim all thread but one
cpu = get_cpu()
w = get_test_workload(uuid.uuid4(), len(cpu.get_threads()) - 1, STATIC)
workloads = {w.get_id(): w}
request = AllocateThreadsRequest(cpu, w.get_id(), workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
log.info("{}".format(cpu))
violations = get_shared_core_violations(cpu)
log.info("shared core violations: {}".format(violations))
self.assertEqual(0, len(violations))
# Assign another workload which will force core sharing
w = get_test_workload(uuid.uuid4(), 1, STATIC)
workloads[w.get_id()] = w
request = AllocateThreadsRequest(cpu, w.get_id(), workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
log.info("{}".format(cpu))
violations = get_shared_core_violations(cpu)
log.info("shared core violations: {}".format(violations))
self.assertEqual(1, len(violations))
def test_forecast_threshold_no_usage(self):
allocator = ForecastIPCpuAllocator(
TestCpuUsagePredictorManager(),
ConfigManager(TestPropertyProvider({})),
OversubscribeFreeThreadProvider(0.1))
thread_count = DEFAULT_TOTAL_THREAD_COUNT / 2
cpu = get_cpu()
w0 = get_test_workload(uuid.uuid4(), thread_count, STATIC)
request = get_no_usage_threads_request(cpu, [w0])
cpu = allocator.assign_threads(request).get_cpu()
log.info(cpu)
# All cores should be occupied
for c in cpu.get_cores():
self.assertEqual(1, len(c.get_empty_threads()))
w1 = get_test_workload(uuid.uuid4(), thread_count, BURST)
request = get_no_usage_threads_request(cpu, [w0, w1])
cpu = allocator.assign_threads(request).get_cpu()
log.info(cpu)
# No threads should be shared
for c in cpu.get_cores():
self.assertEqual(c.get_threads()[0].get_workload_ids(),
c.get_threads()[1].get_workload_ids())
def test_assign_two_workloads_empty_cpu_ip(self):
"""
Workload 0: 2 threads --> (p:0 c:0 t:0) (p:0 c:1 t:0)
Workload 1: 1 thread --> (p:1 c:0 t:0)
"""
for allocator in [
IntegerProgramCpuAllocator(), forecast_ip_alloc_simple
]:
cpu = get_cpu()
w0 = get_test_workload(uuid.uuid4(), 2, STATIC)
w1 = get_test_workload(uuid.uuid4(), 1, STATIC)
request0 = get_no_usage_threads_request(cpu, [w0])
cpu = allocator.assign_threads(request0).get_cpu()
request1 = get_no_usage_threads_request(cpu, [w0, w1])
cpu = allocator.assign_threads(request1).get_cpu()
self.assertEqual(3, len(cpu.get_claimed_threads()))
ids_per_socket = []
for pid, p in enumerate(cpu.get_packages()):
r = []
for cid, c in enumerate(p.get_cores()):
for tid, t in enumerate(c.get_threads()):
if t.is_claimed():
self.assertEqual(1, len(t.get_workload_ids()))
r.append((t.get_workload_ids()[0], c.get_id()))
ids_per_socket.append(r)
for r in ids_per_socket:
# each workload should be on a different socket
self.assertEqual(1, len(set([e[0] for e in r])))
def test_override_previous_assignment(self):
"""
Workload 0: 1 thread --> (p:0 c:0 t:0)
"""
cpu = get_cpu()
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT,
len(cpu.get_empty_threads()))
w0 = get_test_workload(uuid.uuid4(), 1, STATIC)
w1 = get_test_workload(uuid.uuid4(), 2, STATIC)
greedy_allocator = GreedyCpuAllocator()
# Assign the first workload with Greedy
request = get_no_usage_threads_request(cpu, [w0])
cpu = greedy_allocator.assign_threads(request).get_cpu()
log.info(cpu)
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT - 1,
len(cpu.get_empty_threads()))
self.assertEqual(1, len(cpu.get_claimed_threads()))
# Assign the second workload with NoopReset
request = get_no_usage_threads_request(cpu, [w0, w1])
cpu = noop_reset_allocator.assign_threads(request).get_cpu()
log.info(cpu)
self.assertEqual(0, len(cpu.get_empty_threads()))
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT,
len(cpu.get_claimed_threads()))
for t in cpu.get_threads():
self.assertEqual(2, len(t.get_workload_ids()))
self.assertTrue(w0.get_id() in t.get_workload_ids())
self.assertTrue(w1.get_id() in t.get_workload_ids())
def test_fill_cpu(self):
"""
Workload 0: 8 cores
Workload 1: 4 cores
Workload 2: 2 cores
Workload 3: 1 core
Workload 4: 1 core
--------------------
Total: 16 cores
"""
for allocator in ALLOCATORS:
cpu = get_cpu()
workloads = [
get_test_workload("v", 8, STATIC),
get_test_workload("w", 4, STATIC),
get_test_workload("x", 2, STATIC),
get_test_workload("y", 1, STATIC),
get_test_workload("z", 1, STATIC)
]
tot_req = 0
__workloads = []
for w in workloads:
__workloads.append(w)
request = get_no_usage_threads_request(cpu, __workloads)
cpu = allocator.assign_threads(request).get_cpu()
log.info(cpu)
tot_req += w.get_thread_count()
self.assertEqual(tot_req, len(cpu.get_claimed_threads()))
def test_forecast_ip_burst_pool_with_usage(self):
class UsagePredictorWithBurst:
def __init__(self):
self.__model = TestPredictor()
def predict(self, workload: Workload,
cpu_usage_last_hour: np.array,
pred_env: PredEnvironment) -> float:
if workload.get_id() == 'static_a':
return workload.get_thread_count() * 0.8
elif workload.get_id() == 'static_b':
return workload.get_thread_count() * 0.01
elif workload.get_id() == 'burst_c':
return workload.get_thread_count() * 0.9
def get_model(self):
return self.__model
upm = TestCpuUsagePredictorManager(UsagePredictorWithBurst())
cm = ConfigManager(
TestPropertyProvider({BURST_CORE_COLLOC_USAGE_THRESH: 0.9}))
allocator = ForecastIPCpuAllocator(
upm, cm, OversubscribeFreeThreadProvider(0.1))
cpu = get_cpu(package_count=2, cores_per_package=16)
w_a = get_test_workload("static_a", 14, STATIC)
w_b = get_test_workload("static_b", 14, STATIC)
w_c = get_test_workload("burst_c", 2, BURST)
request = get_no_usage_threads_request(cpu, [w_a])
cpu = allocator.assign_threads(request).get_cpu()
request = get_no_usage_threads_request(cpu, [w_a, w_c])
cpu = allocator.assign_threads(request).get_cpu()
# with an aggressive burst pool expansion, burst should be collocated with static on cores:
self.assertLess(40, len(cpu.get_claimed_threads()))
num_burst_1 = len(cpu.get_workload_ids_to_thread_ids()[w_c.get_id()])
request = get_no_usage_threads_request(cpu, [w_a, w_c, w_b])
cpu = allocator.assign_threads(request).get_cpu()
# burst should retract, and prefer collocation with b over a:
num_burst_2 = len(cpu.get_workload_ids_to_thread_ids()[w_c.get_id()])
self.assertLessEqual(num_burst_2, num_burst_1)
colloc_a = 0
colloc_b = 0
for p in cpu.get_packages():
for c in p.get_cores():
t1 = c.get_threads()[0]
t2 = c.get_threads()[1]
if t1.is_claimed() and t2.is_claimed():
wt1 = t1.get_workload_ids()[0]
wt2 = t2.get_workload_ids()[0]
if (wt1 == w_a.get_id() and wt2 == w_c.get_id()) or (
wt1 == w_c.get_id() and wt2 == w_a.get_id()):
colloc_a += 1
elif (wt1 == w_b.get_id() and wt2 == w_c.get_id()) or (
wt1 == w_c.get_id() and wt2 == w_b.get_id()):
colloc_b += 1
self.assertLessEqual(colloc_a, colloc_b)
def test_assign_more_than_available_threads_with_two_workloads(self):
for allocator in OVER_ALLOCATORS:
cpu = get_cpu()
w_fill = get_test_workload("fill", DEFAULT_TOTAL_THREAD_COUNT,
STATIC)
w_extra = get_test_workload("extra",
DEFAULT_TOTAL_THREAD_COUNT * 1.5,
STATIC)
request = AllocateThreadsRequest(cpu, w_fill.get_id(),
{w_fill.get_id(): w_fill}, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
log.info(cpu)
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT,
len(cpu.get_claimed_threads()))
self.assertEqual([w_fill.get_id()],
list(cpu.get_workload_ids_to_thread_ids().keys()))
request = AllocateThreadsRequest(cpu, w_extra.get_id(), {
w_fill.get_id(): w_fill,
w_extra.get_id(): w_extra
}, {}, DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
log.info(cpu)
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT,
len(cpu.get_claimed_threads()))
self.assertEqual(
sorted([w_fill.get_id(), w_extra.get_id()]),
sorted(list(cpu.get_workload_ids_to_thread_ids().keys())))
def test_assign_two_workloads(self):
cpu = get_cpu()
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT,
len(cpu.get_empty_threads()))
w0 = get_test_workload(uuid.uuid4(), 1, STATIC)
w1 = get_test_workload(uuid.uuid4(), 2, STATIC)
# Assign the first workload
request = get_no_usage_threads_request(cpu, [w0])
cpu = noop_reset_allocator.assign_threads(request).get_cpu()
# Assign the second workload
request = get_no_usage_threads_request(cpu, [w0, w1])
cpu = noop_reset_allocator.assign_threads(request).get_cpu()
log.info(cpu)
self.assertEqual(0, len(cpu.get_empty_threads()))
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT,
len(cpu.get_claimed_threads()))
for t in cpu.get_threads():
self.assertEqual(2, len(t.get_workload_ids()))
self.assertTrue(w0.get_id() in t.get_workload_ids())
self.assertTrue(w1.get_id() in t.get_workload_ids())
def test_free_cpu_3_workloads(self):
# Add 3 workloads sequentially, and then remove the 2nd one added.
for allocator in ALLOCATORS:
cpu = get_cpu()
workloads = {}
w0 = get_test_workload(123, 3, STATIC)
w1 = get_test_workload(456, 2, STATIC)
w2 = get_test_workload(789, 4, STATIC)
request = get_no_usage_threads_request(cpu, [w0])
cpu = allocator.assign_threads(request).get_cpu()
request = get_no_usage_threads_request(cpu, [w0, w1])
cpu = allocator.assign_threads(request).get_cpu()
request = get_no_usage_threads_request(cpu, [w0, w1, w2])
cpu = allocator.assign_threads(request).get_cpu()
self.assertEqual(3 + 4 + 2, len(cpu.get_claimed_threads()))
request = get_no_usage_threads_request(cpu, [w0, w2, w1])
cpu = allocator.free_threads(request).get_cpu()
self.assertEqual(3 + 4, len(cpu.get_claimed_threads()))
workload_ids_left = set()
for t in cpu.get_threads():
if t.is_claimed():
for w_id in t.get_workload_ids():
workload_ids_left.add(w_id)
self.assertListEqual(sorted(list(workload_ids_left)), [123, 789])
def test_fill_cpu(self):
"""
Workload 0: 8 cores
Workload 1: 4 cores
Workload 2: 2 cores
Workload 3: 1 core
Workload 4: 1 core
--------------------
Total: 16 cores
"""
for allocator in ALLOCATORS:
cpu = get_cpu()
workloads = [
get_test_workload("a", 8, STATIC),
get_test_workload("b", 4, STATIC),
get_test_workload("c", 2, STATIC),
get_test_workload("d", 1, STATIC),
get_test_workload("e", 1, STATIC)
]
tot_req = 0
workload_map = {}
for w in workloads:
workload_map[w.get_id()] = w
request = AllocateThreadsRequest(
cpu, w.get_id(), workload_map, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
log.debug("{}".format(cpu))
tot_req += w.get_thread_count()
self.assertEqual(tot_req, len(cpu.get_claimed_threads()))
def test_assign_two_workloads_empty_cpu_greedy(self):
"""
Workload 0: 2 threads --> (p:0 c:0 t:0) (p:0 c:0 t:1)
Workload 1: 1 thread --> (p:1 c:0 t:0)
"""
cpu = get_cpu()
allocator = GreedyCpuAllocator()
w0 = get_test_workload(uuid.uuid4(), 2, STATIC)
w1 = get_test_workload(uuid.uuid4(), 1, STATIC)
request0 = get_no_usage_threads_request(cpu, [w0])
cpu = allocator.assign_threads(request0).get_cpu()
request1 = get_no_usage_threads_request(cpu, [w0, w1])
cpu = allocator.assign_threads(request1).get_cpu()
self.assertEqual(3, len(cpu.get_claimed_threads()))
packages = cpu.get_packages()
# WORKLOAD 0
core00 = packages[0].get_cores()[0]
thread0 = core00.get_threads()[0]
self.assertEqual(0, thread0.get_id())
self.assertTrue(thread0.is_claimed())
thread1 = core00.get_threads()[1]
self.assertEqual(8, thread1.get_id())
self.assertTrue(thread1.is_claimed())
# WORKLOAD 1
core00 = packages[1].get_cores()[0]
thread4 = core00.get_threads()[0]
self.assertEqual(4, thread4.get_id())
self.assertTrue(thread4.is_claimed())
def test_ip_fallback(self):
w_a = get_test_workload("a", 3, STATIC)
w_b = get_test_workload("b", 2, STATIC)
w_c = get_test_workload("c", 1, STATIC)
w_d = get_test_workload("d", 2, STATIC)
cpu = get_cpu(package_count=2, cores_per_package=2, threads_per_core=2)
allocator = FallbackCpuAllocator(CrashingAllocator(), IntegerProgramCpuAllocator())
request = get_no_usage_threads_request(cpu, [w_a])
cpu = allocator.assign_threads(request).get_cpu()
request = get_no_usage_threads_request(cpu, [w_a, w_b])
cpu = allocator.assign_threads(request).get_cpu()
request = get_no_usage_threads_request(cpu, [w_b, w_a])
cpu = allocator.free_threads(request).get_cpu()
request = get_no_usage_threads_request(cpu, [w_b, w_c])
cpu = allocator.assign_threads(request).get_cpu()
request = get_no_usage_threads_request(cpu, [w_c, w_b])
cpu = allocator.free_threads(request).get_cpu()
request = get_no_usage_threads_request(cpu, [w_c, w_d])
cpu = allocator.assign_threads(request).get_cpu()
self.assertEqual(3, len(cpu.get_claimed_threads()))
self.assertEqual(6, allocator.get_fallback_allocator_calls_count())
def test_balance_forecast_ip(self):
allocator = forecast_ip_alloc_simple
cpu = get_cpu()
w_a = get_test_workload("a", 2, STATIC)
w_b = get_test_workload("b", 4, BURST)
workloads = [w_a]
request = get_no_usage_threads_request(cpu, workloads)
cpu = allocator.assign_threads(request).get_cpu()
workloads = [w_a, w_b]
request = get_no_usage_threads_request(cpu, workloads)
cpu = allocator.assign_threads(request).get_cpu()
request = get_no_usage_rebalance_request(cpu, workloads)
cpu = allocator.rebalance(request).get_cpu()
self.assertLessEqual(2 + 4, len(cpu.get_claimed_threads()))
w2t = cpu.get_workload_ids_to_thread_ids()
self.assertEqual(2, len(w2t["a"]))
self.assertLessEqual(4, len(w2t["b"])) # burst got at least 4
for _ in range(20):
request = get_no_usage_rebalance_request(cpu, workloads)
cpu = allocator.rebalance(request).get_cpu()
w2t = cpu.get_workload_ids_to_thread_ids()
self.assertEqual(2, len(w2t["a"]))
self.assertLessEqual(4, len(w2t["b"]))
def test_occupies_entire_cpu(self):
cpu = get_cpu()
workload = get_test_workload("a", len(cpu.get_threads()), STATIC)
self.assertTrue(_occupies_entire_cpu(workload, cpu))
workload = get_test_workload("a", len(cpu.get_threads()) - 1, STATIC)
self.assertFalse(_occupies_entire_cpu(workload, cpu))
def test_thread_allocation_computation(self):
for allocator in [IntegerProgramCpuAllocator(), GreedyCpuAllocator()]:
static_thread_count = 2
burst_thread_count = 4
w_static = get_test_workload("s", static_thread_count, STATIC)
w_burst = get_test_workload("b", burst_thread_count, BURST)
cgroup_manager = MockCgroupManager()
registry = Registry()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
workload_manager.set_registry(registry, {})
workload_manager.add_workload(w_static)
workload_manager.add_workload(w_burst)
workload_manager.report_metrics({})
total_thread_count = len(workload_manager.get_cpu().get_threads())
expected_burst_allocation_size = total_thread_count - static_thread_count
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, STATIC_ALLOCATED_SIZE_KEY,
static_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_ALLOCATED_SIZE_KEY,
expected_burst_allocation_size))
self.assertTrue(
gauge_value_equals(registry, BURST_REQUESTED_SIZE_KEY,
burst_thread_count))
self.assertTrue(
gauge_value_equals(registry, OVERSUBSCRIBED_THREADS_KEY, 0))
# Claim every thread for the burst workload which will oversubscribe the static threads
for t in workload_manager.get_cpu().get_threads():
t.claim(w_burst.get_id())
workload_manager.report_metrics({})
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, STATIC_ALLOCATED_SIZE_KEY,
static_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_REQUESTED_SIZE_KEY,
burst_thread_count))
self.assertTrue(
gauge_value_equals(registry, OVERSUBSCRIBED_THREADS_KEY,
static_thread_count))
def test_filling_holes_ip(self):
"""
Initialize with fragmented placement, then fill the instance. Result should be
less fragmented, with the first workload completely filling a socket.
| a | | a | |
| | a | | a |
| ------------- |
| | a | | a |
| a | | a | |
"""
cpu = get_cpu()
allocator = IntegerProgramCpuAllocator()
# Initialize fragmented workload
wa = get_test_workload(uuid.uuid4(), 8, STATIC)
p0 = cpu.get_packages()[0]
p0.get_cores()[0].get_threads()[0].claim(wa.get_id())
p0.get_cores()[1].get_threads()[1].claim(wa.get_id())
p0.get_cores()[2].get_threads()[0].claim(wa.get_id())
p0.get_cores()[3].get_threads()[1].claim(wa.get_id())
p1 = cpu.get_packages()[1]
p1.get_cores()[0].get_threads()[1].claim(wa.get_id())
p1.get_cores()[1].get_threads()[0].claim(wa.get_id())
p1.get_cores()[2].get_threads()[1].claim(wa.get_id())
p1.get_cores()[3].get_threads()[0].claim(wa.get_id())
self.assertEqual(8, len(cpu.get_empty_threads()))
# Fill the rest of the CPU
w0 = get_test_workload(uuid.uuid4(), 2, STATIC)
w1 = get_test_workload(uuid.uuid4(), 3, STATIC)
w2 = get_test_workload(uuid.uuid4(), 1, STATIC)
w3 = get_test_workload(uuid.uuid4(), 2, STATIC)
workload_map = {wa.get_id(): wa}
workloads = [w0, w1, w2, w3]
__workloads = [wa]
for w in workloads:
__workloads.append(w)
workload_map[w.get_id()] = w
request = get_no_usage_threads_request(cpu, __workloads)
cpu = allocator.assign_threads(request).get_cpu()
self.assertEqual(0, len(cpu.get_empty_threads()))
# first workload should be filling completely a socket to avoid cross-socket job layout
for package in cpu.get_packages():
if package.get_cores()[0].get_threads()[0].get_workload_ids(
) != wa.get_id():
continue
ids = [
t.get_workload_ids() for core in package.get_cores()
for t in core.get_threads()
]
self.assertListEqual(ids, [wa.get_id()] * 8)
def test_get_sorted_workloads(self):
w_a = get_test_workload('a', 1, STATIC, 0)
w_b = get_test_workload('b', 1, STATIC, 1)
w_c = get_test_workload('c', 1, STATIC, 2)
expected_ids = ['a', 'b', 'c']
scrambled_workloads = [w_b, w_a, w_c]
sorted_workloads = get_sorted_workloads(scrambled_workloads)
actual_ids = [w.get_id() for w in sorted_workloads]
self.assertEqual(expected_ids, actual_ids)
def test_cache_ip(self):
"""
[add a=2, add b=2, remove b=2, add c=2, remove a=2, add d=2] should lead to the following cache entries:
(state=[], req=[2])
(state=[2], req=[2,2])
(state=[2,2], req=[2,0])
[cache hit]
[cache hit]
(state=[2,2], req=[2,2]) but different layout
"""
cpu = get_cpu()
allocator = IntegerProgramCpuAllocator()
workloads = {}
workload = get_test_workload("a", 2, STATIC)
workloads[workload.get_id()] = workload
request = AllocateThreadsRequest(cpu, workload.get_id(), workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
self.assertEqual(1, len(allocator._IntegerProgramCpuAllocator__cache))
workload = get_test_workload("b", 2, STATIC)
workloads[workload.get_id()] = workload
request = AllocateThreadsRequest(cpu, workload.get_id(), workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
self.assertEqual(2, len(allocator._IntegerProgramCpuAllocator__cache))
request = AllocateThreadsRequest(cpu, "b", workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.free_threads(request).get_cpu()
self.assertEqual(3, len(allocator._IntegerProgramCpuAllocator__cache))
workloads.pop("b")
workload = get_test_workload("c", 2, STATIC)
workloads[workload.get_id()] = workload
request = AllocateThreadsRequest(cpu, workload.get_id(), workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
self.assertEqual(3, len(allocator._IntegerProgramCpuAllocator__cache))
request = AllocateThreadsRequest(cpu, "a", workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.free_threads(request).get_cpu()
self.assertEqual(4, len(allocator._IntegerProgramCpuAllocator__cache))
workloads.pop("a")
workload = get_test_workload("d", 2, STATIC)
workloads[workload.get_id()] = workload
request = AllocateThreadsRequest(cpu, workload.get_id(), workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
allocator.assign_threads(request).get_cpu()
self.assertEqual(5, len(allocator._IntegerProgramCpuAllocator__cache))
def test_publish_window(self):
set_config_manager(ConfigManager(TestPropertyProvider({})))
set_workload_monitor_manager(TestWorkloadMonitorManager())
window_publisher = TestOpportunisticWindowPublisher(
get_current_end_func=lambda: datetime.utcnow() - timedelta(minutes=
1),
add_window_func=lambda: None,
)
w_id = str(uuid.uuid4())
workload = get_test_workload(w_id, 1, STATIC)
set_cpu_usage_predictor_manager(
TestCpuUsagePredictorManager(
TestSimpleCpuPredictor({w_id:
DEFAULT_TOTAL_THRESHOLD - 0.001})))
oeh = OversubscribeEventHandler(TestWorkloadManager([workload]),
window_publisher)
oeh._handle(json.loads(OVERSUBSCRIBE_EVENT.decode("utf-8")))
self.assertEqual(0, oeh.get_skip_count())
self.assertEqual(1, oeh.get_success_count())
self.assertEqual(1, window_publisher.get_current_end_count)
self.assertEqual(1, window_publisher.add_window_count)
def test_assign_ten_threads_empty_cpu_ip(self):
"""
Workload 0: 10 threads --> (p:0 c:[0-7] t:[0-9])
| 1 | 1 | 1 | 1 |
| 1 | 1 | | |
| ------------- |
| 1 | 1 | 1 | 1 |
| | | | |
"""
for allocator in [
IntegerProgramCpuAllocator(), forecast_ip_alloc_simple
]:
cpu = get_cpu()
w = get_test_workload(uuid.uuid4(), 10, STATIC)
request = get_no_usage_threads_request(cpu, [w])
cpu = allocator.assign_threads(request).get_cpu()
self.assertEqual(10, len(cpu.get_claimed_threads()))
threads_per_socket = []
for p in cpu.get_packages():
ths = []
for c in p.get_cores():
for t in c.get_threads():
if t.is_claimed():
ths.append(t)
threads_per_socket.append(len(ths))
self.assertEqual(5, threads_per_socket[0])
self.assertEqual(5, threads_per_socket[1])
def test_add_metrics(self):
test_context = TestContext()
registry = Registry()
reporter = test_context.get_workload_manager()
reporter.set_registry(registry, {})
workload = get_test_workload(uuid.uuid4(), 2, STATIC)
reporter.add_workload(workload)
reporter.report_metrics({})
self.assertTrue(gauge_value_equals(registry, RUNNING, 1))
self.assertTrue(counter_value_equals(registry, ADDED_KEY, 1))
self.assertTrue(counter_value_equals(registry, REMOVED_KEY, 0))
self.assertTrue(counter_value_equals(registry, SUCCEEDED_KEY, 1))
self.assertTrue(counter_value_equals(registry, FAILED_KEY, 0))
self.assertTrue(gauge_value_equals(registry, WORKLOAD_COUNT_KEY, 1))
self.assertTrue(gauge_value_equals(registry, PACKAGE_VIOLATIONS_KEY,
0))
self.assertTrue(gauge_value_equals(registry, CORE_VIOLATIONS_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, IP_ALLOCATOR_TIMEBOUND_COUNT, 0))
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
workload.get_thread_count()))
expected_unallocated_size = len(test_context.get_cpu().get_threads()
) - workload.get_thread_count()
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY,
expected_unallocated_size))
def test_single_burst_workload_lifecycle(self):
for allocator in ALLOCATORS:
requested_thread_count = 2
workload = get_test_workload(uuid.uuid4(), requested_thread_count,
BURST)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
# Add workload
workload_manager.add_workload(workload)
self.assertEqual(
1, cgroup_manager.container_update_counts[workload.get_id()])
# More than the requested threads should have been assigned to the only burst workload.
self.assertTrue(
len(cgroup_manager.container_update_map[workload.get_id()]) >
requested_thread_count)
# Remove workload
workload_manager.remove_workload(workload.get_id())
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
def test_remove_unknown_workload(self):
for allocator in ALLOCATORS:
unknown_workload_id = "unknown"
thread_count = 2
workload = get_test_workload(uuid.uuid4(), thread_count, STATIC)
workload_manager = WorkloadManager(get_cpu(), MockCgroupManager(),
allocator)
# Remove from empty set
workload_manager.remove_workload(unknown_workload_id)
# Add workload
workload_manager.add_workload(workload)
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT - thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Removal of an unknown workload should have no effect
workload_manager.remove_workload(unknown_workload_id)
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT - thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Remove workload with unknown workload, real workload should be removed
workload_manager.remove_workload(unknown_workload_id)
workload_manager.remove_workload(workload.get_id())
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
def test_single_sample(self):
workload = get_test_workload(uuid.uuid4(), 2, STATIC)
metrics_provider = CgroupMetricsProvider(workload)
perf_mon = WorkloadPerformanceMonitor(metrics_provider,
DEFAULT_SAMPLE_FREQUENCY_SEC)
# Initial state should just be an empty timestamps buffer
_, timestamps, buffers = perf_mon.get_buffers()
self.assertEqual(0, len(buffers))
self.assertEqual(0, len(timestamps))
# Expect no change because no workload is really running and we haven't started mocking anything
perf_mon.sample()
self.assertEqual(0, len(buffers))
# Mock reporting metrics on a single hardware thread
cpu_usage = CpuUsage(pu_id=0, user=100, system=50)
snapshot = CpuUsageSnapshot(timestamp=dt.now(), rows=[cpu_usage])
metrics_provider.get_cpu_usage = MagicMock(return_value=snapshot)
perf_mon.sample()
_, timestamps, buffers = perf_mon.get_buffers()
self.assertEqual(1, len(buffers))
self.assertEqual(1, len(timestamps))
buffer = buffers[cpu_usage.pu_id]
self.assertEqual(1, len(buffer))
self.assertEqual(cpu_usage.user + cpu_usage.system, buffer[0])
def test_external_cpu_manipulation(self):
cpu = get_cpu()
violations = get_shared_core_violations(cpu)
log.info("shared core violations: {}".format(violations))
self.assertEqual(0, len(violations))
# Claim 1 thread on every core
dummy_workload_id = uuid.uuid4()
for p in cpu.get_packages():
for c in p.get_cores():
c.get_threads()[0].claim(dummy_workload_id)
violations = get_shared_core_violations(cpu)
log.info("shared core violations: {}".format(violations))
self.assertEqual(0, len(violations))
# Assign another workload which will force core sharing
allocator = GreedyCpuAllocator()
w = get_test_workload(uuid.uuid4(), 2, STATIC)
workloads = {w.get_id(): w}
request = AllocateThreadsRequest(cpu, w.get_id(), workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
violations = get_shared_core_violations(cpu)
log.info("shared core violations: {}".format(violations))
self.assertEqual(2, len(violations))
def test_parse_workload(self):
w_in = get_test_workload("a", 2, STATIC)
log.info("w_in : {}".format(w_in))
w_out = parse_workload(w_in.to_dict())
log.info("w_out: {}".format(w_out))
self.assertEqual(w_in.to_dict(), w_out.to_dict())
def test_forecast_ip_big_burst_pool_if_empty_instance(self):
cpu = get_cpu()
allocator = forecast_ip_alloc_simple
w = get_test_workload("a", 1, BURST)
request = AllocateThreadsRequest(cpu, "a", {"a": w}, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
original_burst_claim_sz = len(cpu.get_claimed_threads())
# should at least consume all the cores:
self.assertLessEqual(
len(cpu.get_threads()) / 2, original_burst_claim_sz)
w2 = get_test_workload("b", 3, STATIC)
request = AllocateThreadsRequest(cpu, "b", {
"a": w,
"b": w2
}, {}, DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
new_burst_claim_sz = len(get_threads_with_workload(cpu, w2.get_id()))
self.assertLess(new_burst_claim_sz, original_burst_claim_sz)
total_claim_sz = len(cpu.get_claimed_threads())
self.assertLessEqual(3 + 1, total_claim_sz)
self.assertLessEqual(1, new_burst_claim_sz)
# there shouldn't be an empty core
for p in cpu.get_packages():
for c in p.get_cores():
self.assertLess(0,
sum(t.is_claimed() for t in c.get_threads()))
request = AllocateThreadsRequest(cpu, "b", {
"a": w,
"b": w2
}, {}, DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.free_threads(request).get_cpu()
request = AllocateRequest(cpu, {"a": w}, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.rebalance(request).get_cpu()
self.assertEqual(original_burst_claim_sz,
len(cpu.get_claimed_threads()))
def test_ip_fallback(self):
w_a = get_test_workload("a", 3, STATIC)
w_b = get_test_workload("b", 2, STATIC)
w_c = get_test_workload("c", 1, STATIC)
w_d = get_test_workload("d", 2, STATIC)
cpu = get_cpu(package_count=2, cores_per_package=2, threads_per_core=2)
allocator = FallbackCpuAllocator(CrashingAllocator(),
IntegerProgramCpuAllocator())
workloads = {}
workloads[w_a.get_id()] = w_a
request = AllocateThreadsRequest(cpu, w_a.get_id(), workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
workloads[w_b.get_id()] = w_b
request = AllocateThreadsRequest(cpu, w_b.get_id(), workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
request = AllocateThreadsRequest(cpu, "a", workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.free_threads(request).get_cpu()
workloads.pop("a")
workloads[w_c.get_id()] = w_c
request = AllocateThreadsRequest(cpu, w_c.get_id(), workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
request = AllocateThreadsRequest(cpu, "b", workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.free_threads(request).get_cpu()
workloads.pop("b")
workloads[w_d.get_id()] = w_d
request = AllocateThreadsRequest(cpu, w_d.get_id(), workloads, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
self.assertEqual(3, len(cpu.get_claimed_threads()))
self.assertEqual(6, allocator.get_fallback_allocator_calls_count())
def test_cache_ip(self):
"""
[add a=2, add b=2, remove b=2, add c=2, remove a=2, add d=2] should lead to the following cache entries:
(state=[], req=[2])
(state=[2], req=[2,2])
(state=[2,2], req=[2,0])
[cache hit]
[cache hit]
(state=[2,2], req=[2,2]) but different layout
"""
cpu = get_cpu()
allocator = IntegerProgramCpuAllocator()
w_a = get_test_workload("a", 2, STATIC)
w_b = get_test_workload("b", 2, STATIC)
w_c = get_test_workload("c", 2, STATIC)
w_d = get_test_workload("d", 2, STATIC)
workloads = [w_a]
request = get_no_usage_threads_request(cpu, workloads)
cpu = allocator.assign_threads(request).get_cpu()
self.assertEqual(1, len(allocator._IntegerProgramCpuAllocator__cache))
workloads = [w_a, w_b]
request = get_no_usage_threads_request(cpu, workloads)
allocator.assign_threads(request).get_cpu()
self.assertEqual(2, len(allocator._IntegerProgramCpuAllocator__cache))
cpu = allocator.free_threads(request).get_cpu()
self.assertEqual(3, len(allocator._IntegerProgramCpuAllocator__cache))
workloads = [w_a, w_c]
request = get_no_usage_threads_request(cpu, workloads)
cpu = allocator.assign_threads(request).get_cpu()
self.assertEqual(3, len(allocator._IntegerProgramCpuAllocator__cache))
workloads = [w_c, w_a]
request = get_no_usage_threads_request(cpu, workloads)
cpu = allocator.free_threads(request).get_cpu()
self.assertEqual(4, len(allocator._IntegerProgramCpuAllocator__cache))
workloads = [w_c, w_d]
request = get_no_usage_threads_request(cpu, workloads)
allocator.assign_threads(request).get_cpu()
self.assertEqual(5, len(allocator._IntegerProgramCpuAllocator__cache))
def test_one_cross_package_violation(self):
cpu = get_cpu()
allocator = IntegerProgramCpuAllocator()
w = get_test_workload(uuid.uuid4(), 9, STATIC)
request = get_no_usage_threads_request(cpu, [w])
cpu = allocator.assign_threads(request).get_cpu()
violations = get_cross_package_violations(cpu)
self.assertEqual(1, len(violations))