def test_unknown_workload_type_label(self):
registry = Registry()
test_context = TestContext()
unknown_event = get_event(
CONTAINER, CREATE, uuid.uuid4(), {
NAME: "container-name",
APP_NAME_LABEL_KEY: DEFAULT_TEST_APP_NAME,
CPU_LABEL_KEY: "1",
MEM_LABEL_KEY: str(DEFAULT_TEST_MEM),
DISK_LABEL_KEY: str(DEFAULT_TEST_DISK),
NETWORK_LABEL_KEY: str(DEFAULT_TEST_NETWORK),
JOB_TYPE_LABEL_KEY: DEFAULT_TEST_JOB_TYPE,
WORKLOAD_TYPE_LABEL_KEY: "unknown",
OWNER_EMAIL_LABEL_KEY: DEFAULT_TEST_OWNER_EMAIL,
IMAGE_LABEL_KEY: DEFAULT_TEST_IMAGE,
})
valid_event = get_container_create_event(1)
event_iterable = MockEventProvider([unknown_event, valid_event])
manager = EventManager(event_iterable,
test_context.get_event_handlers(),
DEFAULT_TEST_EVENT_TIMEOUT_SECS)
manager.set_registry(registry)
manager.start_processing_events()
wait_until(lambda: manager.get_error_count() == 1)
wait_until(lambda: manager.get_processed_count() == 2)
self.assertEqual(0, manager.get_queue_depth())
manager.stop_processing_events()
manager.report_metrics({})
self.assertTrue(gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
self.assertTrue(gauge_value_equals(registry, EVENT_SUCCEEDED_KEY, 5))
self.assertTrue(gauge_value_equals(registry, EVENT_FAILED_KEY, 1))
self.assertTrue(gauge_value_equals(registry, EVENT_PROCESSED_KEY, 2))
def test_absent_workload_type_label(self):
registry = Registry()
test_context = TestContext()
name = str(uuid.uuid4())
unknown_event = get_event(CONTAINER, CREATE, name, {
CPU_LABEL_KEY: "1",
NAME: name
})
event_handlers = test_context.get_event_handlers()
event_iterable = MockEventProvider([unknown_event])
manager = EventManager(event_iterable, event_handlers,
DEFAULT_TEST_EVENT_TIMEOUT_SECS)
manager.set_registry(registry)
manager.start_processing_events()
wait_until(lambda: test_context.get_create_event_handler().
get_ignored_event_count() == 1)
self.assertEqual(0, manager.get_queue_depth())
manager.stop_processing_events()
manager.report_metrics({})
self.assertTrue(gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, EVENT_SUCCEEDED_KEY,
len(test_context.get_event_handlers())))
self.assertTrue(gauge_value_equals(registry, EVENT_FAILED_KEY, 0))
self.assertTrue(gauge_value_equals(registry, EVENT_PROCESSED_KEY, 1))
def test_add_metrics(self):
test_context = TestContext()
registry = Registry()
reporter = test_context.get_workload_manager()
reporter.set_registry(registry, {})
workload = get_test_workload(uuid.uuid4(), 2, STATIC)
reporter.add_workload(workload)
reporter.report_metrics({})
self.assertTrue(gauge_value_equals(registry, RUNNING, 1))
self.assertTrue(counter_value_equals(registry, ADDED_KEY, 1))
self.assertTrue(counter_value_equals(registry, REMOVED_KEY, 0))
self.assertTrue(counter_value_equals(registry, SUCCEEDED_KEY, 1))
self.assertTrue(counter_value_equals(registry, FAILED_KEY, 0))
self.assertTrue(gauge_value_equals(registry, WORKLOAD_COUNT_KEY, 1))
self.assertTrue(gauge_value_equals(registry, PACKAGE_VIOLATIONS_KEY,
0))
self.assertTrue(gauge_value_equals(registry, CORE_VIOLATIONS_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, IP_ALLOCATOR_TIMEBOUND_COUNT, 0))
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
workload.get_thread_count()))
expected_unallocated_size = len(test_context.get_cpu().get_threads()
) - workload.get_thread_count()
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY,
expected_unallocated_size))
def test_free_cpu_on_container_die(self):
registry = Registry()
workload_name = str(uuid.uuid4())
events = [
get_container_create_event(DEFAULT_CPU_COUNT, STATIC,
workload_name, workload_name),
get_container_die_event(workload_name)
]
event_count = len(events)
event_iterable = MockEventProvider(
events, 1) # Force in order event processing for the test
test_context = TestContext()
manager = EventManager(event_iterable,
test_context.get_event_handlers(),
DEFAULT_TEST_EVENT_TIMEOUT_SECS)
manager.set_registry(registry)
manager.start_processing_events()
wait_until(lambda: event_count == manager.get_processed_count())
self.assertEqual(0, manager.get_queue_depth())
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT,
len(test_context.get_cpu().get_empty_threads()))
self.assertEqual(
1,
test_context.get_create_event_handler().get_handled_event_count())
self.assertEqual(
1,
test_context.get_free_event_handler().get_handled_event_count())
manager.stop_processing_events()
manager.report_metrics({})
self.assertTrue(gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
self.assertTrue(
gauge_value_equals(
registry, EVENT_SUCCEEDED_KEY,
event_count * len(test_context.get_event_handlers())))
self.assertTrue(gauge_value_equals(registry, EVENT_FAILED_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, EVENT_PROCESSED_KEY, event_count))
def test_rebalance(self):
registry = Registry()
events = [REBALANCE_EVENT]
event_count = len(events)
event_iterable = MockEventProvider(events)
test_context = TestContext()
manager = EventManager(event_iterable,
test_context.get_event_handlers(),
DEFAULT_TEST_EVENT_TIMEOUT_SECS)
manager.set_registry(registry)
manager.start_processing_events()
wait_until(lambda: event_count == manager.get_processed_count())
self.assertEqual(0, manager.get_queue_depth())
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT,
len(test_context.get_cpu().get_empty_threads()))
self.assertEqual(
0,
test_context.get_create_event_handler().get_handled_event_count())
self.assertEqual(
0,
test_context.get_free_event_handler().get_handled_event_count())
self.assertEqual(
1,
test_context.get_rebalance_event_handler().get_handled_event_count(
))
manager.stop_processing_events()
manager.report_metrics({})
self.assertTrue(gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
self.assertTrue(
gauge_value_equals(
registry, EVENT_SUCCEEDED_KEY,
event_count * len(test_context.get_event_handlers())))
self.assertTrue(gauge_value_equals(registry, EVENT_FAILED_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, EVENT_PROCESSED_KEY, event_count))
def test_unknown_action(self):
registry = Registry()
test_context = TestContext()
unknown_event = get_event(CONTAINER, "unknown", uuid.uuid4(), {})
event_iterable = MockEventProvider([unknown_event])
manager = EventManager(event_iterable,
test_context.get_event_handlers(),
DEFAULT_TEST_EVENT_TIMEOUT_SECS)
manager.set_registry(registry)
manager.start_processing_events()
wait_until(lambda: test_context.get_create_event_handler().
get_ignored_event_count() == 1)
self.assertEqual(0, manager.get_queue_depth())
manager.stop_processing_events()
manager.report_metrics({})
self.assertTrue(gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, EVENT_SUCCEEDED_KEY,
len(test_context.get_event_handlers())))
self.assertTrue(gauge_value_equals(registry, EVENT_FAILED_KEY, 0))
self.assertTrue(gauge_value_equals(registry, EVENT_PROCESSED_KEY, 1))
def test_update_mock_container(self):
registry = Registry()
test_pod = get_simple_test_pod()
get_pod_manager().set_pod(test_pod)
workload_name = test_pod.metadata.name
events = [
get_container_create_event(DEFAULT_CPU_COUNT, STATIC,
workload_name, workload_name)
]
event_count = len(events)
event_iterable = MockEventProvider(events)
test_context = TestContext()
manager = EventManager(event_iterable,
test_context.get_event_handlers(),
DEFAULT_TEST_EVENT_TIMEOUT_SECS)
manager.set_registry(registry, {})
manager.start_processing_events()
wait_until(lambda: event_count == manager.get_processed_count())
self.assertEqual(0, manager.get_queue_depth())
self.assertEqual(
event_count,
test_context.get_workload_manager().get_success_count())
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT - DEFAULT_CPU_COUNT,
len(test_context.get_cpu().get_empty_threads()))
self.assertEqual(
1,
test_context.get_create_event_handler().get_handled_event_count())
manager.stop_processing_events()
manager.report_metrics({})
self.assertTrue(gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
self.assertTrue(
counter_value_equals(
registry, EVENT_SUCCEEDED_KEY,
event_count * len(test_context.get_event_handlers())))
self.assertTrue(counter_value_equals(registry, EVENT_FAILED_KEY, 0))
self.assertTrue(
counter_value_equals(registry, EVENT_PROCESSED_KEY, event_count))
def test_empty_metrics(self):
test_context = TestContext()
registry = Registry()
reporter = test_context.get_workload_manager()
reporter.set_registry(registry, {})
reporter.report_metrics({})
self.assertTrue(gauge_value_equals(registry, RUNNING, 1))
self.assertTrue(counter_value_equals(registry, ADDED_KEY, 0))
self.assertTrue(counter_value_equals(registry, REMOVED_KEY, 0))
self.assertTrue(counter_value_equals(registry, SUCCEEDED_KEY, 0))
self.assertTrue(counter_value_equals(registry, FAILED_KEY, 0))
self.assertTrue(gauge_value_equals(registry, WORKLOAD_COUNT_KEY, 0))
self.assertTrue(gauge_value_equals(registry, PACKAGE_VIOLATIONS_KEY,
0))
self.assertTrue(gauge_value_equals(registry, CORE_VIOLATIONS_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, IP_ALLOCATOR_TIMEBOUND_COUNT, 0))
self.assertTrue(gauge_value_equals(registry, ALLOCATED_SIZE_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY,
len(test_context.get_cpu().get_threads())))
def test_thread_allocation_computation(self):
for allocator in [IntegerProgramCpuAllocator(), GreedyCpuAllocator()]:
static_thread_count = 2
burst_thread_count = 4
w_static = get_test_workload("s", static_thread_count, STATIC)
w_burst = get_test_workload("b", burst_thread_count, BURST)
cgroup_manager = MockCgroupManager()
registry = Registry()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
workload_manager.set_registry(registry, {})
workload_manager.add_workload(w_static)
workload_manager.add_workload(w_burst)
workload_manager.report_metrics({})
total_thread_count = len(workload_manager.get_cpu().get_threads())
expected_burst_allocation_size = total_thread_count - static_thread_count
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, STATIC_ALLOCATED_SIZE_KEY,
static_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_ALLOCATED_SIZE_KEY,
expected_burst_allocation_size))
self.assertTrue(
gauge_value_equals(registry, BURST_REQUESTED_SIZE_KEY,
burst_thread_count))
self.assertTrue(
gauge_value_equals(registry, OVERSUBSCRIBED_THREADS_KEY, 0))
# Claim every thread for the burst workload which will oversubscribe the static threads
for t in workload_manager.get_cpu().get_threads():
t.claim(w_burst.get_id())
workload_manager.report_metrics({})
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, STATIC_ALLOCATED_SIZE_KEY,
static_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_REQUESTED_SIZE_KEY,
burst_thread_count))
self.assertTrue(
gauge_value_equals(registry, OVERSUBSCRIBED_THREADS_KEY,
static_thread_count))
