def run_loop(root_cg_path, activity_path, sync_target_interval, restart_adapter, restart_grace_period): threading.current_thread().name = "index" job_queue = queue.Queue() activity_queue = queue.Queue() index = CgroupIndex(root_cg_path, job_queue, activity_queue) index.open() restarter = RestartEngine(restart_adapter, restart_grace_period, job_queue, activity_queue) restarter_thread = threading.Thread(target=restarter.run, name="restarter") restarter_thread.daemon = True activity = ActivityEngine(activity_path, activity_queue) activity_thread = threading.Thread(target=activity.run, name="activity") activity_thread.daemon = True # Now, fire an initial sync, then empty the activity queue (we don't want # to fire notifications for "new" containers if Captain Comeback is the one # that's starting), and start all worker threads. index.sync() while True: try: activity_queue.get_nowait() except queue.Empty: break restarter_thread.start() activity_thread.start() while True: index.sync() next_sync = time.time() + sync_target_interval while True: poll_timeout = next_sync - time.time() if poll_timeout <= 0: break logger.debug("poll with timeout: %s", poll_timeout) try: index.poll(poll_timeout) except IOError as e: if e.errno != errno.EINTR: raise logger.warning("interrupted") for thread in [activity_thread, restarter_thread]: if not thread.is_alive(): logger.critical("thread %s is dead", thread.name) return 1 return 0
def run_loop(root_cg_path, activity_path, sync_target_interval, restart_adapter, restart_grace_period): threading.current_thread().name = "index" job_queue = queue.Queue() activity_queue = queue.Queue() index = CgroupIndex(root_cg_path, job_queue, activity_queue) index.open() restarter = RestartEngine(restart_adapter, restart_grace_period, job_queue, activity_queue) restarter_thread = threading.Thread(target=restarter.run, name="restarter") restarter_thread.daemon = True activity = ActivityEngine(activity_path, activity_queue) activity_thread = threading.Thread(target=activity.run, name="activity") activity_thread.daemon = True # Now, fire an initial sync, then empty the activity queue (we don't want # to fire notifications for "new" containers if Captain Comeback is the one # that's starting), and start all worker threads. index.sync() while True: try: activity_queue.get_nowait() except queue.Empty: break restarter_thread.start() activity_thread.start() while True: index.sync() next_sync = time.time() + sync_target_interval while True: poll_timeout = next_sync - time.time() if poll_timeout <= 0: break logger.debug("poll with timeout: %s", poll_timeout) try: index.poll(poll_timeout) except IOError as e: if e.errno != errno.EINTR: raise logger.warning("interrupted") for thread in [activity_thread, restarter_thread]: if not thread.is_alive(): logger.critical("thread %s is dead", thread.name) return 1 return 0
def test_index_poll_close(self): cg_path = create_random_cg(self.parent_cg_path) job_q = queue.Queue() activity_q = queue.Queue() index = CgroupIndex(self.parent_cg_path, job_q, activity_q) index.open() self.assertHasNoMessages(activity_q) index.sync() self.assertHasMessageForCg(activity_q, NewCgroupMessage, cg_path) delete_cg(cg_path) index.sync() self.assertHasMessageForCg(activity_q, StaleCgroupMessage, cg_path) index.close()
def test_index_poll(self): cg_path = create_random_cg(self.parent_cg_path) q = queue.Queue() index = CgroupIndex(self.parent_cg_path, q) index.open() index.sync() self.assertRaises(queue.Empty, q.get_nowait) enable_memlimit_and_trigger_oom(cg_path) index.poll(10) msg = q.get_nowait() self.assertIsInstance(msg, RestartRequestedMessage) self.assertEqual(cg_path, msg.cg.path) index.close()
def test_index_poll(self): cg_path = create_random_cg(self.parent_cg_path) job_q = queue.Queue() activity_q = queue.Queue() index = CgroupIndex(self.parent_cg_path, job_q, activity_q) index.open() self.assertHasNoMessages(activity_q) index.sync() enable_memlimit_and_trigger_oom(cg_path) self.assertHasNoMessages(job_q) index.poll(10) self.assertHasMessageForCg(job_q, RestartRequestedMessage, cg_path) self.assertHasMessageForCg(activity_q, NewCgroupMessage, cg_path) index.close()
def test_index_sync_many(self): cg_paths = [create_random_cg(self.parent_cg_path) for _ in range(10)] index = CgroupIndex(self.parent_cg_path, queue.Queue()) index.open() index.sync() while cg_paths: self.assertEqual(len(cg_paths), len(index._path_hash)) path = cg_paths.pop() self.assertIn(path, index._path_hash) delete_cg(path) index.sync() self.assertEqual(len(cg_paths), len(index._path_hash)) self.assertNotIn(path, index._path_hash) index.close()
def test_index_poll(self): cg_path = create_random_cg(self.parent_cg_path) q = queue.Queue() index = CgroupIndex(self.parent_cg_path, q) index.open() index.sync() self.assertRaises(queue.Empty, q.get_nowait) enable_memlimit_and_trigger_oom(cg_path) index.poll(10) msg = q.get_nowait() self.assertIsInstance(msg, RestartRequestedMessage) self.assertEqual(cg_path, msg.cg.path) index.close()
def test_wakeup_on_sync(self): cg_path = create_random_cg(self.parent_cg_path) cg = Cgroup(cg_path) cg.open() cg.set_memory_limit_in_bytes(1024) self.assertEqual("0", cg.oom_control_status()["oom_kill_disable"]) index = CgroupIndex(self.parent_cg_path, queue.Queue()) index.open() index.sync() index.close() self.assertEqual("1", cg.oom_control_status()["oom_kill_disable"]) cg.close()
def test_index_sync_many(self): cg_paths = [create_random_cg(self.parent_cg_path) for _ in range(10)] index = CgroupIndex(self.parent_cg_path, queue.Queue()) index.open() index.sync() while cg_paths: self.assertEqual(len(cg_paths), len(index._path_hash)) path = cg_paths.pop() self.assertIn(path, index._path_hash) delete_cg(path) index.sync() self.assertEqual(len(cg_paths), len(index._path_hash)) self.assertNotIn(path, index._path_hash) index.close()
def test_wakeup_on_sync(self): cg_path = create_random_cg(self.parent_cg_path) cg = Cgroup(cg_path) cg.open() cg.set_memory_limit_in_bytes(1024) self.assertEqual("0", cg.oom_control_status()["oom_kill_disable"]) index = CgroupIndex(self.parent_cg_path, queue.Queue()) index.open() index.sync() index.close() self.assertEqual("1", cg.oom_control_status()["oom_kill_disable"]) cg.close()
def test_index_sync(self): cg_path = create_random_cg(self.parent_cg_path) q = queue.Queue() index = CgroupIndex(self.parent_cg_path, q, q) index.open() index.sync() # Check that the CG was added to the path hash self.assertEqual(1, len(index._path_hash)) self.assertEqual(1, len(index._efd_hash)) # Check that the CG was registered (adding it again will cause an # error) cg = index._path_hash[cg_path] self.assertRaises(EnvironmentError, index.epl.register, cg.event_fileno()) index.close()
def main(root_cg_path, sync_target_interval, restart_grace_period): threading.current_thread().name = "index" job_queue = queue.Queue() index = CgroupIndex(root_cg_path, job_queue) index.open() restarter = RestartEngine(job_queue, restart_grace_period) restarter_thread = threading.Thread(target=restarter.run, name="restarter") restarter_thread.daemon = True restarter_thread.start() while True: index.sync() next_sync = time.time() + sync_target_interval while True: poll_timeout = next_sync - time.time() if poll_timeout <= 0: break logger.debug("poll with timeout: %s", poll_timeout) index.poll(poll_timeout)
def test_index_sync(self): cg_path = create_random_cg(self.parent_cg_path) q = queue.Queue() index = CgroupIndex(self.parent_cg_path, q) index.open() index.sync() # Check that the CG was added to the path hash self.assertEqual(1, len(index._path_hash)) self.assertEqual(1, len(index._efd_hash)) # Check that the CG was registered (adding it again will cause an # error) cg = index._path_hash[cg_path] self.assertRaises(EnvironmentError, index.epl.register, cg.event_fileno()) index.close()
def main(root_cg_path, sync_target_interval, restart_grace_period): threading.current_thread().name = "index" job_queue = queue.Queue() index = CgroupIndex(root_cg_path, job_queue) index.open() restarter = RestartEngine(job_queue, restart_grace_period) restarter_thread = threading.Thread(target=restarter.run, name="restarter") restarter_thread.daemon = True restarter_thread.start() while True: index.sync() next_sync = time.time() + sync_target_interval while True: poll_timeout = next_sync - time.time() if poll_timeout <= 0: break logger.debug("poll with timeout: %s", poll_timeout) index.poll(poll_timeout)
def test_index_poll_many(self): for _ in range(10): create_random_cg(self.parent_cg_path) cg_path = create_random_cg(self.parent_cg_path) job_q = queue.Queue() activity_q = queue.Queue() index = CgroupIndex(self.parent_cg_path, job_q, activity_q) index.open() self.assertHasNoMessages(activity_q) index.sync() for _ in range(11): self.assertHasMessageForCg(activity_q, NewCgroupMessage, self.ANY_CG) self.assertHasNoMessages(activity_q) set_memlimit(cg_path) index.sync() self.assertHasNoMessages(job_q) trigger_oom(cg_path) index.poll(10) # We might receive a few pressure notifications before we finally get # the OOM event. So, wait for the message 100 times. for _ in self.assertEvnetuallyHasMessageForCg( job_q, RestartRequestedMessage, cg_path ): index.poll(1) index.close()
def test_index_leak(self): job_q = queue.Queue() activity_q = queue.Queue() index = CgroupIndex(self.parent_cg_path, job_q, activity_q) fd_dir = os.path.join('/proc', str(os.getpid()), 'fd') fd_initial = len(os.listdir(fd_dir)) logger.debug("fd_initial=%d", fd_initial) index.open() fd_intermediate = len(os.listdir(fd_dir)) logger.debug("fd_intermediate=%d", fd_intermediate) for _ in range(100): fd0 = len(os.listdir(fd_dir)) cgs = [ quick_create_cg(str(uuid.uuid4()), self.parent_cg_path) for _ in range(100) ] index.sync() fd1 = len(os.listdir(fd_dir)) index.sync() fd2 = len(os.listdir(fd_dir)) logger.debug("fd0=%d, fd1=%d, fd2=%d", fd0, fd1, fd2) self.assertEqual(fd0, fd_intermediate) self.assertEqual(fd1, fd2) for cg in cgs: quick_delete_cg(cg) index.sync() index.close() fd_final = len(os.listdir(fd_dir)) logger.debug("fd_final=%d", fd_final) self.assertEqual(fd_final, fd_initial)
def test_index_race(self): ready_q = queue.Queue() exit_q = queue.Queue() cg_count = 100 cg_cycle = 10 index_syncs = 50 file_limit = cg_count * 5 resource.setrlimit(resource.RLIMIT_NOFILE, (file_limit, file_limit)) def racer(): cgs = [ quick_create_cg(str(uuid.uuid4()), self.parent_cg_path) for _ in range(cg_count) ] ready_q.put(None) i = 0 while True: i += 1 try: exit_q.get_nowait() except queue.Empty: pass else: break logger.debug("racer: shuffle (%d)", i) random.shuffle(cgs) logger.debug("racer: split (%d)", i) del_cgs, keep_cgs = cgs[:cg_cycle], cgs[cg_cycle:] logger.debug("racer: delete (%d)", i) for cg in del_cgs: quick_delete_cg(cg) logger.debug("racer: remake (%d)", i) cgs = keep_cgs + [ quick_create_cg(str(uuid.uuid4()), self.parent_cg_path) for _ in range(cg_cycle) ] logger.debug("racer: done (%d)", i) for cg in cgs: quick_delete_cg(cg) t = threading.Thread(target=racer) t.start() job_q = queue.Queue() activity_q = queue.Queue() index = CgroupIndex(self.parent_cg_path, job_q, activity_q) index.open() ready_q.get(timeout=5) try: for _ in range(index_syncs): index.sync() index.close() except Exception: logger.error("sync errorred") # Make logs more usable raise finally: exit_q.put(None) t.join(5)