def test_get_noop_cpu_allocator(self):
property_provider = TestPropertyProvider(
{
CPU_ALLOCATOR: NOOP
})
config_manager = ConfigManager(property_provider)
allocator = get_fallback_allocator(config_manager)
self.assertEqual(NoopCpuAllocator, allocator.get_primary_allocator().__class__)
def get_config_manager(property_provider=AgentPropertyProvider()):
global config_manager
with config_manager_lock:
if config_manager is None:
config_manager = ConfigManager(property_provider)
return config_manager
def test_nothing_to_no_change_update(self):
property_provider = TestPropertyProvider({})
exit_handler = TestExitHandler()
config_manager = ConfigManager(property_provider)
self.assertEqual(None, config_manager.get_str(CPU_ALLOCATOR))
watcher = RestartPropertyWatcher(config_manager, exit_handler, [CPU_ALLOCATOR])
watcher.detect_changes()
self.assertEqual(None, exit_handler.last_code)
def test_something_to_no_change_update(self):
property_provider = TestPropertyProvider({})
exit_handler = TestExitHandler()
config_manager = ConfigManager(property_provider, CONFIG_CHANGE_INTERVAL)
self.assertEqual(None, config_manager.get(ALLOCATOR_KEY))
watcher = CpuAllocatorWatcher(config_manager, exit_handler, CONFIG_CHANGE_INTERVAL)
watcher.detect_allocator_change()
self.assertEqual(None, exit_handler.last_code)
def test_ab_allocator_fallback(self):
property_provider = TestPropertyProvider({ALLOCATOR_KEY: AB_TEST})
config_manager = ConfigManager(property_provider)
allocator_class = get_allocator_class(config_manager)
self.assertEqual(NoopCpuAllocator, allocator_class)
allocator_class = get_allocator_class(config_manager)
self.assertEqual(NoopCpuAllocator, allocator_class)
def test_undefined_instance_id(self):
property_provider = TestPropertyProvider({
ALLOCATOR_KEY: AB_TEST,
CPU_ALLOCATOR_A: IP,
CPU_ALLOCATOR_B: GREEDY
})
config_manager = ConfigManager(property_provider)
allocator_class = get_allocator_class(config_manager)
self.assertEqual(NoopCpuAllocator, allocator_class)
def test_single_workload_memory_settings(self):
for allocator in ALLOCATORS:
thread_count = 2
workload = get_test_workload(uuid.uuid4(), thread_count, STATIC)
cgroup_manager = MockCgroupManager()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
# With an empty configuration we should expect default False behavior
# for all memory flags
set_config_manager(ConfigManager(TestPropertyProvider({})))
workload_manager.add_workload(workload)
self.assertFalse(
cgroup_manager.get_memory_migrate(workload.get_id()))
self.assertFalse(
cgroup_manager.get_memory_spread_page(workload.get_id()))
self.assertFalse(
cgroup_manager.get_memory_spread_slab(workload.get_id()))
workload_manager.remove_workload(workload.get_id())
# With all memory configuration options set to True we should expect all memory
# flags to be set to True
set_config_manager(
ConfigManager(
TestPropertyProvider({
TITUS_ISOLATE_MEMORY_MIGRATE:
True,
TITUS_ISOLATE_MEMORY_SPREAD_PAGE:
True,
TITUS_ISOLATE_MEMORY_SPREAD_SLAB:
True,
})))
workload_manager.add_workload(workload)
self.assertTrue(
cgroup_manager.get_memory_migrate(workload.get_id()))
self.assertTrue(
cgroup_manager.get_memory_spread_page(workload.get_id()))
self.assertTrue(
cgroup_manager.get_memory_spread_slab(workload.get_id()))
workload_manager.remove_workload(workload.get_id())
def test_none_to_something_update(self):
property_provider = TestPropertyProvider({})
exit_handler = TestExitHandler()
config_manager = ConfigManager(property_provider)
self.assertEqual(None, config_manager.get_str(CPU_ALLOCATOR))
watcher = RestartPropertyWatcher(config_manager, exit_handler, [CPU_ALLOCATOR])
property_provider.map[CPU_ALLOCATOR] = GREEDY
watcher.detect_changes()
self.assertEqual(GENERIC_PROPERTY_CHANGE_EXIT, exit_handler.last_code)
def test_get_ab_bucket(self):
even_instance_id = 'i-0cfefd19c9a8db976'
property_provider = TestPropertyProvider({
ALLOCATOR_KEY:
AB_TEST,
CPU_ALLOCATOR_A:
IP,
CPU_ALLOCATOR_B:
GREEDY,
EC2_INSTANCE_ID:
even_instance_id
})
config_manager = ConfigManager(property_provider)
self.assertEqual("A", get_ab_bucket(config_manager, 12))
odd_instance_id = 'i-0cfefd19c9a8db977'
property_provider = TestPropertyProvider({
ALLOCATOR_KEY:
AB_TEST,
CPU_ALLOCATOR_A:
IP,
CPU_ALLOCATOR_B:
GREEDY,
EC2_INSTANCE_ID:
odd_instance_id
})
config_manager = ConfigManager(property_provider)
self.assertEqual("B", get_ab_bucket(config_manager, 12))
letter_instance_id = 'i-0cfefd19c9a8db97x'
property_provider = TestPropertyProvider({
ALLOCATOR_KEY:
AB_TEST,
CPU_ALLOCATOR_A:
IP,
CPU_ALLOCATOR_B:
GREEDY,
EC2_INSTANCE_ID:
letter_instance_id
})
config_manager = ConfigManager(property_provider)
self.assertEqual("A", get_ab_bucket(config_manager, 12))
def test_skip_active_window(self):
set_config_manager(ConfigManager(TestPropertyProvider({})))
window_publisher = TestOpportunisticWindowPublisher(
is_window_active_func=lambda: True,
add_window_func=lambda: None,
cleanup_func=lambda: None)
oeh = OversubscribeEventHandler(TestWorkloadManager([]),
window_publisher)
oeh._handle(json.loads(OVERSUBSCRIBE_EVENT.decode("utf-8")))
self.assertEqual(1, oeh.get_skip_count())
self.assertEqual(1, window_publisher.is_window_active_count)
def test_skip_active_window(self):
set_config_manager(ConfigManager(TestPropertyProvider({})))
window_publisher = TestOpportunisticWindowPublisher(
get_current_end_func=lambda: datetime.utcnow() + timedelta(minutes=
5),
add_window_func=lambda: None,
)
oeh = OversubscribeEventHandler(TestWorkloadManager([]),
window_publisher)
oeh._handle(json.loads(OVERSUBSCRIBE_EVENT.decode("utf-8")))
self.assertEqual(1, oeh.get_skip_count())
self.assertEqual(1, window_publisher.get_current_end_count)
def test_noop_to_ip_update(self):
property_provider = TestPropertyProvider({CPU_ALLOCATOR: NOOP})
exit_handler = TestExitHandler()
config_manager = ConfigManager(property_provider)
watcher = RestartPropertyWatcher(config_manager, exit_handler,
[CPU_ALLOCATOR])
# No change yet
watcher.detect_changes()
self.assertEqual(None, exit_handler.last_code)
# titus-isolate should exit when the allocator changes
property_provider.map[CPU_ALLOCATOR] = IP
watcher.detect_changes()
self.assertEqual(ALLOCATOR_CONFIG_CHANGE_EXIT, exit_handler.last_code)
def test_noop_to_ip_update(self):
property_provider = TestPropertyProvider(
{
ALLOCATOR_KEY: NOOP
})
exit_handler = TestExitHandler()
config_manager = ConfigManager(property_provider, CONFIG_CHANGE_INTERVAL)
watcher = CpuAllocatorWatcher(config_manager, exit_handler, CONFIG_CHANGE_INTERVAL)
# No change yet
watcher.detect_allocator_change()
self.assertEqual(None, exit_handler.last_code)
# titus-isolate should exit when the allocator changes
property_provider.map[ALLOCATOR_KEY] = IP
watcher.detect_allocator_change()
self.assertEqual(ALLOCATOR_CONFIG_CHANGE_EXIT, exit_handler.last_code)
def test_forecast_threshold_usage(self):
allocator = ForecastIPCpuAllocator(
TestCpuUsagePredictorManager(TestCpuUsagePredictor(10)),
ConfigManager(TestPropertyProvider({})),
ThresholdFreeThreadProvider(0.05))
thread_count = DEFAULT_TOTAL_THREAD_COUNT / 4
cpu = get_cpu()
w0 = get_test_workload(uuid.uuid4(), thread_count, STATIC)
log.info(w0)
request = AllocateThreadsRequest(cpu, w0.get_id(), {w0.get_id(): w0},
{}, DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
log.info(cpu)
# All cores should be occupied
for c in cpu.get_cores():
self.assertTrue(
len(c.get_empty_threads()) == 1
or len(c.get_empty_threads()) == 2)
w1 = get_test_workload(uuid.uuid4(), thread_count, BURST)
log.info(w1)
request = AllocateThreadsRequest(cpu, w1.get_id(), {
w0.get_id(): w0,
w1.get_id(): w1
}, {}, DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
log.info(cpu)
for c in cpu.get_cores():
# Static workload should have unshared cores
if len(c.get_empty_threads()) == 1:
for t in c.get_threads():
if t.is_claimed():
self.assertEqual([w0.get_id()], t.get_workload_ids())
# Burst workload should have shared cores only with itself
if len(c.get_empty_threads()) == 0:
self.assertEqual(c.get_threads()[0].get_workload_ids(),
c.get_threads()[1].get_workload_ids())
self.assertEqual([w1.get_id()],
c.get_threads()[1].get_workload_ids())
def test_get_workloads_endpoint(self):
set_config_manager(ConfigManager(TestPropertyProvider({})))
thread_count = 2
workload_id = str(uuid.uuid4())
workload = get_test_workload(workload_id, thread_count, STATIC)
workload_manager = self.__get_default_workload_manager()
set_workload_manager(workload_manager)
workloads = json.loads(get_workloads())
self.assertEqual(0, len(workloads))
workload_manager.add_workload(workload)
workloads = json.loads(get_workloads())
self.assertEqual(workload_id, workloads[0]["id"])
self.assertEqual(STATIC, workloads[0]["type"])
self.assertEqual(thread_count, workloads[0]["thread_count"])
def test_get_workloads_endpoint(self):
override_config_manager(ConfigManager(TestPropertyProvider({})))
cpu = get_cpu()
thread_count = 2
workload_id = str(uuid.uuid4())
workload = Workload(workload_id, thread_count, STATIC)
workload_manager = WorkloadManager(cpu, MockCgroupManager())
set_wm(workload_manager)
workloads = json.loads(get_workloads())
self.assertEqual(0, len(workloads))
workload_manager.add_workload(workload)
workloads = json.loads(get_workloads())
self.assertEqual(workload_id, workloads[0]["id"])
self.assertEqual(STATIC, workloads[0]["type"])
self.assertEqual(thread_count, workloads[0]["thread_count"])
def test_ab_classification_swap(self):
even_instance_id = 'i-0cfefd19c9a8db976'
property_provider = TestPropertyProvider(
{
ALLOCATOR_KEY: AB_TEST,
CPU_ALLOCATOR_A: NOOP,
CPU_ALLOCATOR_B: IP,
EC2_INSTANCE_ID: even_instance_id
})
exit_handler = TestExitHandler()
config_manager = ConfigManager(property_provider, CONFIG_CHANGE_INTERVAL)
watcher = CpuAllocatorWatcher(config_manager, exit_handler, CONFIG_CHANGE_INTERVAL)
# No change yet
watcher.detect_allocator_change()
self.assertEqual(None, exit_handler.last_code)
# Swap A and B to simulate instance classification change
# N.B. the ALLOCATOR_KEY and EC_INSTANCE_ID do NOT change
property_provider.map[CPU_ALLOCATOR_A] = IP
property_provider.map[CPU_ALLOCATOR_B] = NOOP
watcher.detect_allocator_change()
self.assertEqual(ALLOCATOR_CONFIG_CHANGE_EXIT, exit_handler.last_code)
def test_publish_window(self):
set_config_manager(ConfigManager(TestPropertyProvider({})))
set_workload_monitor_manager(TestWorkloadMonitorManager())
window_publisher = TestOpportunisticWindowPublisher(
is_window_active_func=lambda: False,
add_window_func=lambda: None,
cleanup_func=lambda: None)
w_id = str(uuid.uuid4())
workload = get_test_workload(w_id, 1, STATIC)
set_cpu_usage_predictor_manager(
TestCpuUsagePredictorManager(
TestSimpleCpuPredictor({w_id:
DEFAULT_TOTAL_THRESHOLD - 0.001})))
oeh = OversubscribeEventHandler(TestWorkloadManager([workload]),
window_publisher)
oeh._handle(json.loads(OVERSUBSCRIBE_EVENT.decode("utf-8")))
self.assertEqual(0, oeh.get_skip_count())
self.assertEqual(1, oeh.get_success_count())
self.assertEqual(1, window_publisher.is_window_active_count)
self.assertEqual(1, window_publisher.add_window_count)
def test_forecast_ip_burst_pool_with_usage(self):
class UsagePredictorWithBurst:
def __init__(self):
self.__model = TestPredictor()
def predict(self, workload: Workload,
cpu_usage_last_hour: np.array,
pred_env: PredEnvironment) -> float:
if workload.get_id() == 'static_a':
return workload.get_thread_count() * 0.8
elif workload.get_id() == 'static_b':
return workload.get_thread_count() * 0.01
elif workload.get_id() == 'burst_c':
return workload.get_thread_count() * 0.9
def get_model(self):
return self.__model
upm = TestCpuUsagePredictorManager(UsagePredictorWithBurst())
cm = ConfigManager(
TestPropertyProvider({BURST_CORE_COLLOC_USAGE_THRESH: 0.9}))
allocator = ForecastIPCpuAllocator(upm, cm,
ThresholdFreeThreadProvider(0.1))
cpu = get_cpu(package_count=2, cores_per_package=16)
w_a = get_test_workload("static_a", 14, STATIC)
w_b = get_test_workload("static_b", 14, STATIC)
w_c = get_test_workload("burst_c", 2, BURST)
request = AllocateThreadsRequest(cpu, "static_a", {"static_a": w_a},
{}, DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
request = AllocateThreadsRequest(cpu, "burst_c", {
"static_a": w_a,
"burst_c": w_c
}, {}, DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
# with an aggressive burst pool expansion, burst should be collocated with static on cores:
self.assertLess(40, len(cpu.get_claimed_threads()))
num_burst_1 = len(cpu.get_workload_ids_to_thread_ids()["burst_c"])
request = AllocateThreadsRequest(cpu, "static_b", {
"static_a": w_a,
"static_b": w_b,
"burst_c": w_c
}, {}, DEFAULT_TEST_REQUEST_METADATA)
cpu = allocator.assign_threads(request).get_cpu()
# burst should retract, and prefer collocation with b over a:
num_burst_2 = len(cpu.get_workload_ids_to_thread_ids()["burst_c"])
self.assertLessEqual(num_burst_2, num_burst_1)
colloc_a = 0
colloc_b = 0
for p in cpu.get_packages():
for c in p.get_cores():
t1 = c.get_threads()[0]
t2 = c.get_threads()[1]
if t1.is_claimed() and t2.is_claimed():
wt1 = t1.get_workload_ids()[0]
wt2 = t2.get_workload_ids()[0]
if (wt1 == 'static_a'
and wt2 == 'burst_c') or (wt1 == 'burst_c'
and wt2 == 'static_a'):
colloc_a += 1
elif (wt1 == 'static_b'
and wt2 == 'burst_c') or (wt1 == 'burst_c'
and wt2 == 'static_b'):
colloc_b += 1
self.assertLessEqual(colloc_a, colloc_b)
def test_something_to_no_change_update(self):
property_provider = TestPropertyProvider({CPU_ALLOCATOR: IP})
config_manager = ConfigManager(property_provider)
self.assertEqual(IP, config_manager.get_str(CPU_ALLOCATOR))
self.assertEqual(IP, config_manager.get_str(CPU_ALLOCATOR))
DEFAULT_TEST_CPU = 1
DEFAULT_TEST_MEM = 256
DEFAULT_TEST_DISK = 512
DEFAULT_TEST_NETWORK = 1024
DEFAULT_TEST_APP_NAME = 'test_app_name'
DEFAULT_TEST_OWNER_EMAIL = '*****@*****.**'
DEFAULT_TEST_IMAGE = 'test_image'
DEFAULT_TEST_CMD = 'test_cmd'
DEFAULT_TEST_ENTRYPOINT = 'test_entrypoint'
DEFAULT_TEST_JOB_TYPE = 'SERVICE'
DEFAULT_TEST_WORKLOAD_TYPE = 'static'
DEFAULT_TEST_INSTANCE_ID = 'test_instance_id'
DEFAULT_TEST_REQUEST_METADATA = {INSTANCE_ID: DEFAULT_TEST_INSTANCE_ID}
DEFAULT_TEST_OPPORTUNISTIC_THREAD_COUNT = 0
set_config_manager(ConfigManager(TestPropertyProvider({})))
def wait_until(func, timeout=DEFAULT_TIMEOUT_SECONDS, period=0.01):
deadline = time.time() + timeout
while time.time() < deadline:
if func():
return
time.sleep(period)
raise TimeoutError(
"Function did not succeed within timeout: '{}'.".format(timeout))
def counter_value_equals(registry, key, expected_value, tags={}):
value = registry.counter(key, tags).count()
return {}
class TestPodManager:
def __init__(self):
self.pod = None
def set_pod(self, pod: V1Pod):
self.pod = pod
def get_pod(self, pod_name: str) -> Optional[V1Pod]:
return self.pod
forecast_ip_alloc_simple = ForecastIPCpuAllocator(
TestCpuUsagePredictorManager(), ConfigManager(TestPropertyProvider({})),
OversubscribeFreeThreadProvider(0.1))
ALLOCATORS = [
NaiveCpuAllocator(),
IntegerProgramCpuAllocator(),
GreedyCpuAllocator(), forecast_ip_alloc_simple
]
OVER_ALLOCATORS = [NaiveCpuAllocator(), forecast_ip_alloc_simple]
set_workload_monitor_manager(TestWorkloadMonitorManager())
class TestCpu(unittest.TestCase):
def test_assign_one_thread_empty_cpu(self):
"""
def test_parse_from_file(self):
override_config_manager(ConfigManager(TestPropertyProvider({})))
dir = os.path.dirname(os.path.abspath(__file__))
self.assertEqual(
expected_path,
get_cgroup_path_from_file(dir + "/test_cgroup_file", CPUSET))
def test_wait_for_file_to_exist(self):
override_config_manager(ConfigManager(TestPropertyProvider({})))
with self.assertRaises(TimeoutError):
_wait_for_file_to_exist("/tmp/foo", 0.1)
_wait_for_file_to_exist(__file__, 0.1)
def test_construction_without_properties(self):
property_provider = TestPropertyProvider({})
config_manager = ConfigManager(property_provider)
self.assertEqual(None, config_manager.get_str("foo"))
self.assertEqual(None, config_manager.get_str(CPU_ALLOCATOR))
log.info("Isolating currently running workloads...")
for workload in get_current_workloads(docker.from_env()):
try:
workload_manager.add_workload(workload)
except:
log.exception(
"Failed to add currently running workload: '{}', maybe it exited."
.format(workload.get_id()))
log.info("Isolated currently running workloads.")
# Start processing events after adding running workloads to avoid processing a die event before we add a workload
event_manager.start_processing_events()
if __name__ != '__main__' and not is_testing():
set_config_manager(ConfigManager(EnvPropertyProvider))
log.info("Configuring logging...")
gunicorn_logger = logging.getLogger('gunicorn.error')
app.logger.handlers = gunicorn_logger.handlers
app.logger.setLevel(gunicorn_logger.level)
# Set the schedule library's logging level higher so it doesn't spam messages every time it schedules a task
logging.getLogger('schedule').setLevel(logging.WARN)
exit_handler = RealExitHandler()
if is_kubernetes():
log.info("Setting pod manager...")
pod_manager = PodManager()
pod_manager.start()
set_pod_manager(pod_manager)
def test_get_noop_cpu_allocator(self):
property_provider = TestPropertyProvider({ALLOCATOR_KEY: NOOP})
config_manager = ConfigManager(property_provider)
allocator_class = get_allocator_class(config_manager)
self.assertEqual(NoopCpuAllocator, allocator_class)