def test_remove_unknown_workload(self):
for allocator_class in [
GreedyCpuAllocator, IntegerProgramCpuAllocator
]:
unknown_workload_id = "unknown"
thread_count = 2
workload = Workload(uuid.uuid4(), thread_count, STATIC)
workload_manager = WorkloadManager(get_cpu(),
MockCgroupManager(),
allocator_class=allocator_class)
# Remove from empty set
workload_manager.remove_workload([unknown_workload_id])
# Add workload
workload_manager.add_workload(workload)
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT - thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Removal of an unknown workload should have no effect
workload_manager.remove_workload([unknown_workload_id])
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT - thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Remove workload with unknown workload, real workload should be removed
workload_manager.remove_workload(unknown_workload_id)
workload_manager.remove_workload(workload.get_id())
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
def test_remove_unknown_workload(self):
for allocator in ALLOCATORS:
unknown_workload_id = "unknown"
thread_count = 2
workload = get_test_workload(uuid.uuid4(), thread_count, STATIC)
workload_manager = WorkloadManager(get_cpu(), MockCgroupManager(),
allocator)
# Remove from empty set
workload_manager.remove_workload(unknown_workload_id)
# Add workload
workload_manager.add_workload(workload)
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT - thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Removal of an unknown workload should have no effect
workload_manager.remove_workload(unknown_workload_id)
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT - thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Remove workload with unknown workload, real workload should be removed
workload_manager.remove_workload(unknown_workload_id)
workload_manager.remove_workload(workload.get_id())
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
def test_single_burst_workload_lifecycle(self):
for allocator_class in [
GreedyCpuAllocator, IntegerProgramCpuAllocator
]:
thread_count = 2
workload = Workload(uuid.uuid4(), thread_count, BURST)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(),
cgroup_manager,
allocator_class=allocator_class)
# Add workload
workload_manager.add_workload(workload)
self.assertEqual(
2, cgroup_manager.container_update_counts[workload.get_id()])
# All threads should have been assigned to the only burst workload.
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(cgroup_manager.container_update_map[workload.get_id()]))
# No threads should have been consumed from the cpu model perspective.
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
# Remove workload
workload_manager.remove_workload(workload.get_id())
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
def test_single_static_workload_lifecycle(self):
for allocator_class in [
GreedyCpuAllocator, IntegerProgramCpuAllocator
]:
thread_count = 2
workload = Workload(uuid.uuid4(), thread_count, STATIC)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(),
cgroup_manager,
allocator_class=allocator_class)
# Add workload
workload_manager.add_workload(workload)
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT - thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
self.assertEqual(
1, cgroup_manager.container_update_counts[workload.get_id()])
# Remove workload
workload_manager.remove_workload(workload.get_id())
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
def test_thread_allocation_computation(self):
for allocator in [IntegerProgramCpuAllocator(), GreedyCpuAllocator()]:
static_thread_count = 2
burst_thread_count = 4
w_static = get_test_workload("s", static_thread_count, STATIC)
w_burst = get_test_workload("b", burst_thread_count, BURST)
cgroup_manager = MockCgroupManager()
registry = Registry()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
workload_manager.set_registry(registry, {})
workload_manager.add_workload(w_static)
workload_manager.add_workload(w_burst)
workload_manager.report_metrics({})
total_thread_count = len(workload_manager.get_cpu().get_threads())
expected_burst_allocation_size = total_thread_count - static_thread_count
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, STATIC_ALLOCATED_SIZE_KEY,
static_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_ALLOCATED_SIZE_KEY,
expected_burst_allocation_size))
self.assertTrue(
gauge_value_equals(registry, BURST_REQUESTED_SIZE_KEY,
burst_thread_count))
self.assertTrue(
gauge_value_equals(registry, OVERSUBSCRIBED_THREADS_KEY, 0))
# Claim every thread for the burst workload which will oversubscribe the static threads
for t in workload_manager.get_cpu().get_threads():
t.claim(w_burst.get_id())
workload_manager.report_metrics({})
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, STATIC_ALLOCATED_SIZE_KEY,
static_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_REQUESTED_SIZE_KEY,
burst_thread_count))
self.assertTrue(
gauge_value_equals(registry, OVERSUBSCRIBED_THREADS_KEY,
static_thread_count))
class TestContext:
def __init__(self, cpu=None, allocator=IntegerProgramCpuAllocator()):
if cpu is None:
cpu = get_cpu()
self.__workload_manager = WorkloadManager(cpu, MockCgroupManager(),
allocator)
self.__create_event_handler = CreateEventHandler(
self.__workload_manager)
self.__free_event_handler = FreeEventHandler(self.__workload_manager)
self.__rebalance_event_handler = RebalanceEventHandler(
self.__workload_manager)
def get_cpu(self):
return self.__workload_manager.get_cpu()
def get_workload_manager(self):
return self.__workload_manager
def get_create_event_handler(self):
return self.__create_event_handler
def get_free_event_handler(self):
return self.__free_event_handler
def get_rebalance_event_handler(self):
return self.__rebalance_event_handler
def get_event_handlers(self):
return [
self.__create_event_handler, self.__free_event_handler,
self.__rebalance_event_handler
]
def test_single_burst_workload_lifecycle(self):
for allocator in ALLOCATORS:
requested_thread_count = 2
workload = get_test_workload(uuid.uuid4(), requested_thread_count,
BURST)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
# Add workload
workload_manager.add_workload(workload)
self.assertEqual(
1, cgroup_manager.container_update_counts[workload.get_id()])
# More than the requested threads should have been assigned to the only burst workload.
self.assertTrue(
len(cgroup_manager.container_update_map[workload.get_id()]) >
requested_thread_count)
# Remove workload
workload_manager.remove_workload(workload.get_id())
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
class TestContext:
def __init__(self, docker_client=MockDockerClient(), cpu=None):
if cpu is None:
cpu = get_cpu()
self.__docker_client = docker_client
self.__workload_manager = WorkloadManager(cpu, MockCgroupManager())
self.__event_logger = EventLogger()
self.__create_event_handler = CreateEventHandler(
self.__workload_manager)
self.__free_event_handler = FreeEventHandler(self.__workload_manager)
def get_cpu(self):
return self.__workload_manager.get_cpu()
def get_docker_client(self):
return self.__docker_client
def get_workload_manager(self):
return self.__workload_manager
def get_create_event_handler(self):
return self.__create_event_handler
def get_free_event_handler(self):
return self.__free_event_handler
def get_event_handlers(self):
return [
self.__event_logger, self.__create_event_handler,
self.__free_event_handler
]
def test_no_cross_packages_placement_no_bad_affinity_ip(self):
w_a = get_test_workload("a", 3, STATIC)
w_b = get_test_workload("b", 2, STATIC)
w_c = get_test_workload("c", 1, STATIC)
w_d = get_test_workload("d", 2, STATIC)
cpu = get_cpu(package_count=2, cores_per_package=2, threads_per_core=2)
workload_manager = WorkloadManager(cpu, MockCgroupManager(),
IntegerProgramCpuAllocator())
workload_manager.add_workload(w_a)
workload_manager.add_workload(w_b)
workload_manager.add_workload(w_c)
workload_manager.add_workload(w_d)
self.assertEqual(
0, len(get_cross_package_violations(workload_manager.get_cpu())))
self.assertEqual(0,
len(workload_manager.get_cpu().get_empty_threads()))
def test_no_cross_packages_placement_no_bad_affinity_ip(self):
w_a = Workload("a", 3, STATIC)
w_b = Workload("b", 2, STATIC)
w_c = Workload("c", 1, STATIC)
w_d = Workload("d", 2, STATIC)
cpu = get_cpu(package_count=2, cores_per_package=2, threads_per_core=2)
workload_manager = WorkloadManager(cpu, MockCgroupManager())
workload_manager.add_workload(w_a)
workload_manager.add_workload(w_b)
workload_manager.add_workload(w_c)
workload_manager.add_workload(w_d)
self.assertEqual(
0, len(get_cross_package_violations(workload_manager.get_cpu())))
#self.assertEqual(1, len(get_shared_core_violations(workload_manager.get_cpu()))) # todo: fix me
self.assertEqual(0,
len(workload_manager.get_cpu().get_empty_threads()))
def test_assign_to_full_cpu_fails(self):
for allocator in LEGACY_ALLOCATORS:
# Fill the CPU
w0 = get_test_workload(uuid.uuid4(), DEFAULT_TOTAL_THREAD_COUNT,
STATIC)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
workload_manager.add_workload(w0)
self.assertTrue(is_cpu_full(workload_manager.get_cpu()))
# Fail to claim one more thread
error_count = workload_manager.get_error_count()
w1 = get_test_workload(uuid.uuid4(), 1, STATIC)
workload_manager.add_workload(w1)
self.assertEqual(error_count + 1,
workload_manager.get_error_count())
def test_ip_fallback(self):
w_a = Workload("a", 3, STATIC)
w_b = Workload("b", 2, STATIC)
w_c = Workload("c", 1, STATIC)
w_d = Workload("d", 2, STATIC)
cpu = get_cpu(package_count=2, cores_per_package=2, threads_per_core=2)
wm = WorkloadManager(cpu,
MockCgroupManager(),
allocator_class=CrashingAllocator)
wm.add_workload(w_a)
wm.add_workload(w_b)
wm.remove_workload("a")
wm.add_workload(w_c)
wm.remove_workload("b")
wm.add_workload(w_d)
self.assertEqual(3, len(wm.get_cpu().get_claimed_threads()))
self.assertEqual(
3, len(wm.get_allocator().get_cpu().get_claimed_threads()))
self.assertEqual(6, wm.get_fallback_allocator_calls_count())
def test_alternating_static_burst_workloads(self):
for allocator in ALLOCATORS:
thread_count = 2
burst0 = get_test_workload("burst0", thread_count, BURST)
burst1 = get_test_workload("burst1", thread_count, BURST)
static0 = get_test_workload("static0", thread_count, STATIC)
static1 = get_test_workload("static1", thread_count, STATIC)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
# Add static workload
log.info("ADDING STATIC0")
workload_manager.add_workload(static0)
self.__assert_container_thread_count(workload_manager.get_cpu(),
cgroup_manager, [static0])
self.__assert_cpu_thread_count(workload_manager.get_cpu(),
[static0])
# Add burst workload
log.info("ADDING BURST0")
workload_manager.add_workload(burst0)
self.__assert_container_thread_count(workload_manager.get_cpu(),
cgroup_manager,
[static0, burst0])
self.__assert_cpu_thread_count(workload_manager.get_cpu(),
[static0, burst0])
# Add static workload
log.info("ADDING STATIC1")
workload_manager.add_workload(static1)
self.__assert_container_thread_count(workload_manager.get_cpu(),
cgroup_manager,
[static0, burst0, static1])
self.__assert_cpu_thread_count(workload_manager.get_cpu(),
[static0, burst0, static1])
# Add burst workload
log.info("ADDING BURST1")
workload_manager.add_workload(burst1)
self.__assert_container_thread_count(
workload_manager.get_cpu(), cgroup_manager,
[static0, burst0, static1, burst1])
self.__assert_cpu_thread_count(workload_manager.get_cpu(),
[static0, burst0, static1, burst1])
# Remove static workload
log.info("REMOVING STATIC0")
workload_manager.remove_workload(static0.get_id())
self.__assert_container_thread_count(workload_manager.get_cpu(),
cgroup_manager,
[burst0, static1, burst1])
self.__assert_cpu_thread_count(workload_manager.get_cpu(),
[burst0, static1, burst1])
# Remove static workload
log.info("REMOVING BURST0")
workload_manager.remove_workload(burst0.get_id())
self.__assert_container_thread_count(workload_manager.get_cpu(),
cgroup_manager,
[static1, burst1])
self.__assert_cpu_thread_count(workload_manager.get_cpu(),
[static1, burst1])
def test_alternating_static_burst_workloads(self):
for allocator_class in [
GreedyCpuAllocator, IntegerProgramCpuAllocator
]:
thread_count = 2
burst0 = Workload("burst0", thread_count, BURST)
burst1 = Workload("burst1", thread_count, BURST)
static0 = Workload("static0", thread_count, STATIC)
static1 = Workload("static1", thread_count, STATIC)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(),
cgroup_manager,
allocator_class=allocator_class)
# Add static workload
log.info("ADDING STATIC0")
workload_manager.add_workload(static0)
self.assertTrue(
static0.get_id() in cgroup_manager.container_update_map)
self.assertEqual(
thread_count,
len(cgroup_manager.container_update_map[static0.get_id()]))
self.assertEqual(
1, cgroup_manager.container_update_counts[static0.get_id()])
expected_free_thread_count = DEFAULT_TOTAL_THREAD_COUNT - thread_count
self.assertEqual(
expected_free_thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Add burst workload
log.info("ADDING BURST0")
workload_manager.add_workload(burst0)
self.assertEqual(
expected_free_thread_count,
len(cgroup_manager.container_update_map[burst0.get_id()]))
self.assertEqual(
2, cgroup_manager.container_update_counts[burst0.get_id()])
# No change in empty threads expected
self.assertEqual(
expected_free_thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Add static workload
log.info("ADDING STATIC1")
workload_manager.add_workload(static1)
self.assertEqual(
thread_count,
len(cgroup_manager.container_update_map[static1.get_id()]))
self.assertEqual(
1, cgroup_manager.container_update_counts[static1.get_id()])
expected_free_thread_count = expected_free_thread_count - thread_count
self.assertEqual(
expected_free_thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# The burst0 container should be updated again because the burst footprint changed after the addition of a
# static workload
self.assertEqual(
3, cgroup_manager.container_update_counts[burst0.get_id()])
self.assertEqual(
expected_free_thread_count,
len(cgroup_manager.container_update_map[burst0.get_id()]))
# Add burst workload
log.info("ADDING BURST1")
workload_manager.add_workload(burst1)
self.assertEqual(
4, cgroup_manager.container_update_counts[burst0.get_id()])
self.assertEqual(
2, cgroup_manager.container_update_counts[burst1.get_id()])
self.assertEqual(
expected_free_thread_count,
len(cgroup_manager.container_update_map[burst1.get_id()]))
# No change in empty threads expected
self.assertEqual(
expected_free_thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Remove static workload
log.info("REMOVING STATIC0")
workload_manager.remove_workload(static0.get_id())
self.assertEqual(
5, cgroup_manager.container_update_counts[burst0.get_id()])
self.assertEqual(
3, cgroup_manager.container_update_counts[burst1.get_id()])
# Empty threads should have increased
expected_free_thread_count = expected_free_thread_count + thread_count
self.assertEqual(
expected_free_thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
self.assertEqual(
expected_free_thread_count,
len(cgroup_manager.container_update_map[burst0.get_id()]))
self.assertEqual(
expected_free_thread_count,
len(cgroup_manager.container_update_map[burst1.get_id()]))