def test_single_burst_workload_lifecycle(self):
for allocator_class in [
GreedyCpuAllocator, IntegerProgramCpuAllocator
]:
thread_count = 2
workload = Workload(uuid.uuid4(), thread_count, BURST)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(),
cgroup_manager,
allocator_class=allocator_class)
# Add workload
workload_manager.add_workload(workload)
self.assertEqual(
2, cgroup_manager.container_update_counts[workload.get_id()])
# All threads should have been assigned to the only burst workload.
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(cgroup_manager.container_update_map[workload.get_id()]))
# No threads should have been consumed from the cpu model perspective.
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
# Remove workload
workload_manager.remove_workload(workload.get_id())
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
def test_is_isolated(self):
real_allocators = [GreedyCpuAllocator(), IntegerProgramCpuAllocator()]
for allocator in real_allocators:
wm = WorkloadManager(get_cpu(), MockCgroupManager(), allocator)
self.assertFalse(wm.is_isolated(uuid.uuid4()))
for allocator in real_allocators:
workload = get_test_workload(uuid.uuid4(),
DEFAULT_TOTAL_THREAD_COUNT, STATIC)
wm = WorkloadManager(get_cpu(), MockCgroupManager(), allocator)
wm.add_workload(workload)
self.assertTrue(wm.is_isolated(workload.get_id()))
wm = WorkloadManager(get_cpu(), MockCgroupManager(),
NoopCpuAllocator())
self.assertTrue(wm.is_isolated(uuid.uuid4()))
def test_remove_unknown_workload(self):
for allocator_class in [
GreedyCpuAllocator, IntegerProgramCpuAllocator
]:
unknown_workload_id = "unknown"
thread_count = 2
workload = Workload(uuid.uuid4(), thread_count, STATIC)
workload_manager = WorkloadManager(get_cpu(),
MockCgroupManager(),
allocator_class=allocator_class)
# Remove from empty set
workload_manager.remove_workload([unknown_workload_id])
# Add workload
workload_manager.add_workload(workload)
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT - thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Removal of an unknown workload should have no effect
workload_manager.remove_workload([unknown_workload_id])
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT - thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Remove workload with unknown workload, real workload should be removed
workload_manager.remove_workload(unknown_workload_id)
workload_manager.remove_workload(workload.get_id())
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
def test_single_burst_workload_lifecycle(self):
for allocator in ALLOCATORS:
requested_thread_count = 2
workload = get_test_workload(uuid.uuid4(), requested_thread_count,
BURST)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
# Add workload
workload_manager.add_workload(workload)
self.assertEqual(
1, cgroup_manager.container_update_counts[workload.get_id()])
# More than the requested threads should have been assigned to the only burst workload.
self.assertTrue(
len(cgroup_manager.container_update_map[workload.get_id()]) >
requested_thread_count)
# Remove workload
workload_manager.remove_workload(workload.get_id())
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
def test_remove_unknown_workload(self):
for allocator in ALLOCATORS:
unknown_workload_id = "unknown"
thread_count = 2
workload = get_test_workload(uuid.uuid4(), thread_count, STATIC)
workload_manager = WorkloadManager(get_cpu(), MockCgroupManager(),
allocator)
# Remove from empty set
workload_manager.remove_workload(unknown_workload_id)
# Add workload
workload_manager.add_workload(workload)
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT - thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Removal of an unknown workload should have no effect
workload_manager.remove_workload(unknown_workload_id)
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT - thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Remove workload with unknown workload, real workload should be removed
workload_manager.remove_workload(unknown_workload_id)
workload_manager.remove_workload(workload.get_id())
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
def test_single_static_workload_lifecycle(self):
for allocator_class in [
GreedyCpuAllocator, IntegerProgramCpuAllocator
]:
thread_count = 2
workload = Workload(uuid.uuid4(), thread_count, STATIC)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(),
cgroup_manager,
allocator_class=allocator_class)
# Add workload
workload_manager.add_workload(workload)
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT - thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
self.assertEqual(
1, cgroup_manager.container_update_counts[workload.get_id()])
# Remove workload
workload_manager.remove_workload(workload.get_id())
self.assertEqual(
DEFAULT_TOTAL_THREAD_COUNT,
len(workload_manager.get_cpu().get_empty_threads()))
def __init__(self, cpu=None, allocator=IntegerProgramCpuAllocator()):
if cpu is None:
cpu = get_cpu()
self.__workload_manager = WorkloadManager(cpu, MockCgroupManager(), allocator)
self.__create_event_handler = CreateEventHandler(self.__workload_manager)
self.__free_event_handler = FreeEventHandler(self.__workload_manager)
self.__rebalance_event_handler = RebalanceEventHandler(self.__workload_manager)
def test_thread_allocation_computation(self):
for allocator in [IntegerProgramCpuAllocator(), GreedyCpuAllocator()]:
static_thread_count = 2
burst_thread_count = 4
w_static = get_test_workload("s", static_thread_count, STATIC)
w_burst = get_test_workload("b", burst_thread_count, BURST)
cgroup_manager = MockCgroupManager()
registry = Registry()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
workload_manager.set_registry(registry, {})
workload_manager.add_workload(w_static)
workload_manager.add_workload(w_burst)
workload_manager.report_metrics({})
total_thread_count = len(workload_manager.get_cpu().get_threads())
expected_burst_allocation_size = total_thread_count - static_thread_count
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, STATIC_ALLOCATED_SIZE_KEY,
static_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_ALLOCATED_SIZE_KEY,
expected_burst_allocation_size))
self.assertTrue(
gauge_value_equals(registry, BURST_REQUESTED_SIZE_KEY,
burst_thread_count))
self.assertTrue(
gauge_value_equals(registry, OVERSUBSCRIBED_THREADS_KEY, 0))
# Claim every thread for the burst workload which will oversubscribe the static threads
for t in workload_manager.get_cpu().get_threads():
t.claim(w_burst.get_id())
workload_manager.report_metrics({})
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, STATIC_ALLOCATED_SIZE_KEY,
static_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_REQUESTED_SIZE_KEY,
burst_thread_count))
self.assertTrue(
gauge_value_equals(registry, OVERSUBSCRIBED_THREADS_KEY,
static_thread_count))
def __init__(self, docker_client=MockDockerClient(), cpu=None):
if cpu is None:
cpu = get_cpu()
self.__docker_client = docker_client
self.__workload_manager = WorkloadManager(cpu, MockCgroupManager())
self.__event_logger = EventLogger()
self.__create_event_handler = CreateEventHandler(
self.__workload_manager)
self.__free_event_handler = FreeEventHandler(self.__workload_manager)
def test_no_fallback_should_error(self):
wm = WorkloadManager(get_cpu(2, 2, 2),
MockCgroupManager(),
allocator_class=CrashingAssignAllocator,
fallback_allocator_class=None)
wm.add_workload(Workload("foo", 1, STATIC))
self.assertEqual(1, wm.get_error_count())
def test_reconcile_no_workloads(self):
cpu = get_cpu()
reconciler = Reconciler(MockCgroupManager(), EXIT_HANDLER)
reconciler.reconcile(cpu)
self.__validate_state(reconciler,
EXIT_HANDLER,
exit_code=None,
expected_success_count=1,
expected_skip_count=0)
def test_assign_threads(self):
cpu = get_cpu()
cgroup_manager = MockCgroupManager()
cpu_allocator = NoopResetCpuAllocator(cpu)
cpu_allocator.set_cgroup_manager(cgroup_manager)
w = Workload(uuid.uuid4(), 1, STATIC)
cpu_allocator.assign_threads(w)
self.assertEqual(1, cgroup_manager.container_update_counts[w.get_id()])
self.assertEqual(len(cpu.get_threads()),
len(cgroup_manager.container_update_map[w.get_id()]))
def test_assign_threads(self):
cpu = get_cpu()
cgroup_manager = MockCgroupManager()
cpu_allocator = NoopResetCpuAllocator("", cgroup_manager)
w = get_test_workload(uuid.uuid4(), 1, STATIC)
request = AllocateThreadsRequest(cpu, w.get_id(), {w.get_id(): w}, {},
DEFAULT_TEST_REQUEST_METADATA)
cpu_allocator.assign_threads(request)
self.assertEqual(1, cgroup_manager.container_update_counts[w.get_id()])
self.assertEqual(len(cpu.get_threads()),
len(cgroup_manager.container_update_map[w.get_id()]))
def test_single_workload_memory_settings(self):
for allocator in ALLOCATORS:
thread_count = 2
workload = get_test_workload(uuid.uuid4(), thread_count, STATIC)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
# With an empty configuration we should expect default False behavior
# for all memory flags
set_config_manager(ConfigManager(TestPropertyProvider({})))
workload_manager.add_workload(workload)
self.assertFalse(
cgroup_manager.get_memory_migrate(workload.get_id()))
self.assertFalse(
cgroup_manager.get_memory_spread_page(workload.get_id()))
self.assertFalse(
cgroup_manager.get_memory_spread_slab(workload.get_id()))
workload_manager.remove_workload(workload.get_id())
# With all memory configuration options set to True we should expect all memory
# flags to be set to True
set_config_manager(
ConfigManager(
TestPropertyProvider({
TITUS_ISOLATE_MEMORY_MIGRATE:
True,
TITUS_ISOLATE_MEMORY_SPREAD_PAGE:
True,
TITUS_ISOLATE_MEMORY_SPREAD_SLAB:
True,
})))
workload_manager.add_workload(workload)
self.assertTrue(
cgroup_manager.get_memory_migrate(workload.get_id()))
self.assertTrue(
cgroup_manager.get_memory_spread_page(workload.get_id()))
self.assertTrue(
cgroup_manager.get_memory_spread_slab(workload.get_id()))
workload_manager.remove_workload(workload.get_id())
def test_get_workloads_endpoint(self):
override_config_manager(ConfigManager(TestPropertyProvider({})))
cpu = get_cpu()
thread_count = 2
workload_id = str(uuid.uuid4())
workload = Workload(workload_id, thread_count, STATIC)
workload_manager = WorkloadManager(cpu, MockCgroupManager())
set_wm(workload_manager)
workloads = json.loads(get_workloads())
self.assertEqual(0, len(workloads))
workload_manager.add_workload(workload)
workloads = json.loads(get_workloads())
self.assertEqual(workload_id, workloads[0]["id"])
self.assertEqual(STATIC, workloads[0]["type"])
self.assertEqual(thread_count, workloads[0]["thread_count"])
def test_assign_to_full_cpu_fails(self):
for allocator in LEGACY_ALLOCATORS:
# Fill the CPU
w0 = get_test_workload(uuid.uuid4(), DEFAULT_TOTAL_THREAD_COUNT,
STATIC)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
workload_manager.add_workload(w0)
self.assertTrue(is_cpu_full(workload_manager.get_cpu()))
# Fail to claim one more thread
error_count = workload_manager.get_error_count()
w1 = get_test_workload(uuid.uuid4(), 1, STATIC)
workload_manager.add_workload(w1)
self.assertEqual(error_count + 1,
workload_manager.get_error_count())
def test_no_cross_packages_placement_no_bad_affinity_ip(self):
w_a = get_test_workload("a", 3, STATIC)
w_b = get_test_workload("b", 2, STATIC)
w_c = get_test_workload("c", 1, STATIC)
w_d = get_test_workload("d", 2, STATIC)
cpu = get_cpu(package_count=2, cores_per_package=2, threads_per_core=2)
workload_manager = WorkloadManager(cpu, MockCgroupManager(),
IntegerProgramCpuAllocator())
workload_manager.add_workload(w_a)
workload_manager.add_workload(w_b)
workload_manager.add_workload(w_c)
workload_manager.add_workload(w_d)
self.assertEqual(
0, len(get_cross_package_violations(workload_manager.get_cpu())))
self.assertEqual(0,
len(workload_manager.get_cpu().get_empty_threads()))
def test_get_workloads(self):
reconciler = Reconciler(MockCgroupManager(), None)
cpu = get_cpu(1, 4, 1) # 1 socket, 4 cores, 1 thread per core
threads = cpu.get_threads()
threads[0].claim("a")
threads[0].claim("e")
threads[1].claim("b")
threads[1].claim("c")
threads[1].claim("d")
threads[2].claim("d")
threads[2].claim("e")
threads[3].claim("e")
workloads = reconciler.get_workloads(cpu)
self.assertEqual([0], workloads["a"])
self.assertEqual([1], workloads["b"])
self.assertEqual([1], workloads["c"])
self.assertEqual([1, 2], workloads["d"])
self.assertEqual([0, 2, 3], workloads["e"])
def test_no_cross_packages_placement_no_bad_affinity_ip(self):
w_a = Workload("a", 3, STATIC)
w_b = Workload("b", 2, STATIC)
w_c = Workload("c", 1, STATIC)
w_d = Workload("d", 2, STATIC)
cpu = get_cpu(package_count=2, cores_per_package=2, threads_per_core=2)
workload_manager = WorkloadManager(cpu, MockCgroupManager())
workload_manager.add_workload(w_a)
workload_manager.add_workload(w_b)
workload_manager.add_workload(w_c)
workload_manager.add_workload(w_d)
self.assertEqual(
0, len(get_cross_package_violations(workload_manager.get_cpu())))
#self.assertEqual(1, len(get_shared_core_violations(workload_manager.get_cpu()))) # todo: fix me
self.assertEqual(0,
len(workload_manager.get_cpu().get_empty_threads()))
def test_ip_fallback(self):
w_a = Workload("a", 3, STATIC)
w_b = Workload("b", 2, STATIC)
w_c = Workload("c", 1, STATIC)
w_d = Workload("d", 2, STATIC)
cpu = get_cpu(package_count=2, cores_per_package=2, threads_per_core=2)
wm = WorkloadManager(cpu,
MockCgroupManager(),
allocator_class=CrashingAllocator)
wm.add_workload(w_a)
wm.add_workload(w_b)
wm.remove_workload("a")
wm.add_workload(w_c)
wm.remove_workload("b")
wm.add_workload(w_d)
self.assertEqual(3, len(wm.get_cpu().get_claimed_threads()))
self.assertEqual(
3, len(wm.get_allocator().get_cpu().get_claimed_threads()))
self.assertEqual(6, wm.get_fallback_allocator_calls_count())
def __get_default_workload_manager():
cpu = get_cpu()
return WorkloadManager(cpu, MockCgroupManager(),
IntegerProgramCpuAllocator())
def __get_default_workload_manager():
cpu = get_cpu()
return WorkloadManager(cpu, MockCgroupManager())
def test_alternating_static_burst_workloads(self):
for allocator_class in [
GreedyCpuAllocator, IntegerProgramCpuAllocator
]:
thread_count = 2
burst0 = Workload("burst0", thread_count, BURST)
burst1 = Workload("burst1", thread_count, BURST)
static0 = Workload("static0", thread_count, STATIC)
static1 = Workload("static1", thread_count, STATIC)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(),
cgroup_manager,
allocator_class=allocator_class)
# Add static workload
log.info("ADDING STATIC0")
workload_manager.add_workload(static0)
self.assertTrue(
static0.get_id() in cgroup_manager.container_update_map)
self.assertEqual(
thread_count,
len(cgroup_manager.container_update_map[static0.get_id()]))
self.assertEqual(
1, cgroup_manager.container_update_counts[static0.get_id()])
expected_free_thread_count = DEFAULT_TOTAL_THREAD_COUNT - thread_count
self.assertEqual(
expected_free_thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Add burst workload
log.info("ADDING BURST0")
workload_manager.add_workload(burst0)
self.assertEqual(
expected_free_thread_count,
len(cgroup_manager.container_update_map[burst0.get_id()]))
self.assertEqual(
2, cgroup_manager.container_update_counts[burst0.get_id()])
# No change in empty threads expected
self.assertEqual(
expected_free_thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Add static workload
log.info("ADDING STATIC1")
workload_manager.add_workload(static1)
self.assertEqual(
thread_count,
len(cgroup_manager.container_update_map[static1.get_id()]))
self.assertEqual(
1, cgroup_manager.container_update_counts[static1.get_id()])
expected_free_thread_count = expected_free_thread_count - thread_count
self.assertEqual(
expected_free_thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# The burst0 container should be updated again because the burst footprint changed after the addition of a
# static workload
self.assertEqual(
3, cgroup_manager.container_update_counts[burst0.get_id()])
self.assertEqual(
expected_free_thread_count,
len(cgroup_manager.container_update_map[burst0.get_id()]))
# Add burst workload
log.info("ADDING BURST1")
workload_manager.add_workload(burst1)
self.assertEqual(
4, cgroup_manager.container_update_counts[burst0.get_id()])
self.assertEqual(
2, cgroup_manager.container_update_counts[burst1.get_id()])
self.assertEqual(
expected_free_thread_count,
len(cgroup_manager.container_update_map[burst1.get_id()]))
# No change in empty threads expected
self.assertEqual(
expected_free_thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
# Remove static workload
log.info("REMOVING STATIC0")
workload_manager.remove_workload(static0.get_id())
self.assertEqual(
5, cgroup_manager.container_update_counts[burst0.get_id()])
self.assertEqual(
3, cgroup_manager.container_update_counts[burst1.get_id()])
# Empty threads should have increased
expected_free_thread_count = expected_free_thread_count + thread_count
self.assertEqual(
expected_free_thread_count,
len(workload_manager.get_cpu().get_empty_threads()))
self.assertEqual(
expected_free_thread_count,
len(cgroup_manager.container_update_map[burst0.get_id()]))
self.assertEqual(
expected_free_thread_count,
len(cgroup_manager.container_update_map[burst1.get_id()]))
def test_alternating_static_burst_workloads(self):
for allocator in ALLOCATORS:
thread_count = 2
burst0 = get_test_workload("burst0", thread_count, BURST)
burst1 = get_test_workload("burst1", thread_count, BURST)
static0 = get_test_workload("static0", thread_count, STATIC)
static1 = get_test_workload("static1", thread_count, STATIC)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
# Add static workload
log.info("ADDING STATIC0")
workload_manager.add_workload(static0)
self.__assert_container_thread_count(workload_manager.get_cpu(),
cgroup_manager, [static0])
self.__assert_cpu_thread_count(workload_manager.get_cpu(),
[static0])
# Add burst workload
log.info("ADDING BURST0")
workload_manager.add_workload(burst0)
self.__assert_container_thread_count(workload_manager.get_cpu(),
cgroup_manager,
[static0, burst0])
self.__assert_cpu_thread_count(workload_manager.get_cpu(),
[static0, burst0])
# Add static workload
log.info("ADDING STATIC1")
workload_manager.add_workload(static1)
self.__assert_container_thread_count(workload_manager.get_cpu(),
cgroup_manager,
[static0, burst0, static1])
self.__assert_cpu_thread_count(workload_manager.get_cpu(),
[static0, burst0, static1])
# Add burst workload
log.info("ADDING BURST1")
workload_manager.add_workload(burst1)
self.__assert_container_thread_count(
workload_manager.get_cpu(), cgroup_manager,
[static0, burst0, static1, burst1])
self.__assert_cpu_thread_count(workload_manager.get_cpu(),
[static0, burst0, static1, burst1])
# Remove static workload
log.info("REMOVING STATIC0")
workload_manager.remove_workload(static0.get_id())
self.__assert_container_thread_count(workload_manager.get_cpu(),
cgroup_manager,
[burst0, static1, burst1])
self.__assert_cpu_thread_count(workload_manager.get_cpu(),
[burst0, static1, burst1])
# Remove static workload
log.info("REMOVING BURST0")
workload_manager.remove_workload(burst0.get_id())
self.__assert_container_thread_count(workload_manager.get_cpu(),
cgroup_manager,
[static1, burst1])
self.__assert_cpu_thread_count(workload_manager.get_cpu(),
[static1, burst1])