def test_reclose(self):
cg_path = create_random_cg(self.parent_cg_path)
cg = Cgroup(cg_path)
cg.open()
cg.close()
self.assertRaises(AssertionError, cg.close)
def test_trigger_restart(self):
cg_path = create_random_cg(self.parent_cg_path)
q = queue.Queue()
cg = Cgroup(cg_path)
cg.open()
cg.wakeup(q, None)
self.assertHasNoMessages(q)
set_memlimit(cg_path)
cg.wakeup(q, None)
self.assertHasNoMessages(q)
trigger_oom(cg_path)
# The test program should fill 128 MB rather fast; give it 10s
for _ in range(100):
cg.wakeup(q, None)
try:
msg = q.get_nowait()
except queue.Empty:
time.sleep(0.1)
continue
self.assertIsInstance(msg, RestartRequestedMessage)
self.assertEqual(cg, msg.cg)
break
else:
raise Exception("Queue never received a message!")
cg.close()
def test_trigger_restart(self):
cg_path = create_random_cg(self.parent_cg_path)
q = queue.Queue()
cg = Cgroup(cg_path)
cg.open()
cg.wakeup(q)
self.assertRaises(queue.Empty, q.get_nowait)
enable_memlimit_and_trigger_oom(cg_path)
# The test program should fill 128 MB rather fast; give it 10s
for _ in range(100):
cg.wakeup(q)
try:
msg = q.get_nowait()
except queue.Empty:
time.sleep(0.1)
continue
self.assertIsInstance(msg, RestartRequestedMessage)
self.assertEqual(cg, msg.cg)
break
else:
raise Exception("Queue never received a message!")
cg.close()
def test_reclose(self):
cg_path = create_random_cg(self.parent_cg_path)
cg = Cgroup(cg_path)
cg.open()
cg.close()
self.assertRaises(AssertionError, cg.close)
def enable_memlimit_and_trigger_oom(path):
cg = Cgroup(path)
cg.open()
# Set a memory limit then disable the OOM killer via a wakeup
cg.set_memory_limit_in_bytes(1024 * 1024 * 128) # 128 MB
cg.wakeup(queue.Queue())
cg.close()
test_program = 'l = []\nwhile True:\n l.append(object())'
subprocess.Popen(["sudo", "cgexec", "-g", _descriptor_from_cg_path(path),
"python", "-c", test_program])
def test_stale_cgroup(self):
cg_path = create_random_cg(self.parent_cg_path)
cg = Cgroup(cg_path)
cg.open()
delete_cg(cg_path)
q = queue.Queue()
cg.wakeup(q, None)
self.assertRaises(EnvironmentError, cg.wakeup, q, None, raise_for_stale=True)
cg.close()
def test_stale_cgroup(self):
cg_path = create_random_cg(self.parent_cg_path)
cg = Cgroup(cg_path)
cg.open()
delete_cg(cg_path)
q = queue.Queue()
cg.wakeup(q)
self.assertRaises(EnvironmentError, cg.wakeup, q, raise_for_stale=True)
cg.close()
def test_disable_oom_killer(self):
cg_path = create_random_cg(self.parent_cg_path)
cg = Cgroup(cg_path)
cg.open()
cg.wakeup(queue.Queue())
self.assertEqual("0", cg.oom_control_status()["oom_kill_disable"])
# The OOM Killer should be disabled if there is a task limit
cg.set_memory_limit_in_bytes(1024)
cg.wakeup(queue.Queue())
self.assertEqual("1", cg.oom_control_status()["oom_kill_disable"])
cg.close()
def test_disable_oom_killer(self):
cg_path = create_random_cg(self.parent_cg_path)
cg = Cgroup(cg_path)
cg.open()
cg.wakeup(queue.Queue(), None)
self.assertEqual("0", cg.oom_control_status()["oom_kill_disable"])
# The OOM Killer should be disabled if there is a task limit
cg.set_memory_limit_in_bytes(1024)
cg.wakeup(queue.Queue(), None)
self.assertEqual("1", cg.oom_control_status()["oom_kill_disable"])
cg.close()
def enable_memlimit_and_trigger_oom(path):
cg = Cgroup(path)
cg.open()
# Set a memory limit then disable the OOM killer via a wakeup
cg.set_memory_limit_in_bytes(1024 * 1024 * 128) # 128 MB
cg.wakeup(queue.Queue())
cg.close()
test_program = 'l = []\nwhile True:\n l.append(object())'
subprocess.Popen([
"sudo", "cgexec", "-g",
descriptor_from_cg_path(path), "python", "-c", test_program
])
def test_wakeup_on_sync(self):
cg_path = create_random_cg(self.parent_cg_path)
cg = Cgroup(cg_path)
cg.open()
cg.set_memory_limit_in_bytes(1024)
self.assertEqual("0", cg.oom_control_status()["oom_kill_disable"])
index = CgroupIndex(self.parent_cg_path, queue.Queue())
index.open()
index.sync()
index.close()
self.assertEqual("1", cg.oom_control_status()["oom_kill_disable"])
cg.close()
def test_wakeup_on_sync(self):
cg_path = create_random_cg(self.parent_cg_path)
cg = Cgroup(cg_path)
cg.open()
cg.set_memory_limit_in_bytes(1024)
self.assertEqual("0", cg.oom_control_status()["oom_kill_disable"])
index = CgroupIndex(self.parent_cg_path, queue.Queue())
index.open()
index.sync()
index.close()
self.assertEqual("1", cg.oom_control_status()["oom_kill_disable"])
cg.close()
def sync(self):
logger.debug("syncing cgroups")
# Sync all monitors with disk, and deregister stale ones. It's
# important to actually *wakeup* monitors here, so as to ensure we
# don't race with Docker when it creates a cgroup (which could result
# in us not seeing the memory limit and therefore not disabling the OOM
# killer).
for cg in list(self._path_hash.values()):
try:
cg.wakeup(self.job_queue, None, raise_for_stale=True)
except EnvironmentError:
self.deregister(cg)
cg.close()
for entry in os.listdir(self.root_cg_path):
path = os.path.join(self.root_cg_path, entry)
# Is this a CG or just a regular file?
if not os.path.isdir(path):
continue
# We're already tracking this CG. It *might* have changed between
# our check and now, but in that case we'll catch it at the next
# sync.
if path in self._path_hash:
continue
# This a new CG, Register and wake it up immediately after in case
# there already is some handling to do (typically: disabling the
# OOM killer).
cg = Cgroup(path)
try:
cg.open()
except EnvironmentError as e:
# CG exited before we had a chance to register it. That's OK.
logger.warning("%s: error opening new cg: %s", cg.name(), e)
else:
self.register(cg)
cg.wakeup(self.job_queue, None)
class CgroupTestUnit(unittest.TestCase, QueueAssertionHelper):
def setUp(self):
self.mock_cg = tempfile.mkdtemp()
self.write_oom_control()
with open(self.cg_path("memory.pressure_level"), "w") as f:
f.write('')
self.monitor = Cgroup(self.mock_cg)
self.queue = queue.Queue()
def tearDown(self):
shutil.rmtree(self.mock_cg)
# Helpers
def write_oom_control(self, oom_kill_disable="0", under_oom="0"):
control = ["oom_kill_disable {0}".format(oom_kill_disable),
"under_oom {0}".format(under_oom)]
with open(self.cg_path("memory.oom_control"), "w") as f:
f.write("\n".join(control))
f.write("\n")
def write_memory_usage(self, memory_usage=12345):
with open(self.cg_path("memory.usage_in_bytes"), "w") as f:
f.write(str(memory_usage))
f.write("\n")
def write_memory_limit(self, memory_limit=9223372036854771712):
with open(self.cg_path("memory.limit_in_bytes"), "w") as f:
f.write(str(memory_limit))
f.write("\n")
def cg_path(self, path):
return os.path.join(self.mock_cg, path)
# Tests
def test_open(self):
self.write_oom_control()
self.monitor.open()
expected = "{0} {1}\n{2} {3} {4}\n".format(
self.monitor.event_oom.fileno(),
self.monitor.oom_control.fileno(),
self.monitor.event_pressure.fileno(),
self.monitor.memory_pressure.fileno(),
"critical"
)
self.monitor.close()
with open(self.cg_path("cgroup.event_control")) as f:
self.assertEqual(expected, f.read())
def test_wakeup_with_limits_and_disable_oom_killer(self):
self.write_oom_control()
self.write_memory_limit(1024)
self.monitor.open()
self.monitor.wakeup(self.queue, None)
self.monitor.close()
with open(self.cg_path("memory.oom_control")) as f:
self.assertEqual("1\n", f.read())
def test_wakeup_with_oom_killer_disabled_is_noop(self):
self.write_oom_control(oom_kill_disable="1")
self.write_memory_limit(1024)
self.monitor.open()
self.monitor.wakeup(self.queue, None)
self.monitor.close()
# File shoud not have been touched
with open(self.cg_path("memory.oom_control")) as f:
self.assertEqual("oom_kill_disable 1\n", f.readline())
def test_wakeup_without_limits_is_noop(self):
self.write_oom_control(oom_kill_disable="0")
self.write_memory_limit()
self.monitor.open()
self.monitor.wakeup(self.queue, None)
self.monitor.close()
# File shoud not have been touched
with open(self.cg_path("memory.oom_control")) as f:
self.assertEqual("oom_kill_disable 0\n", f.readline())
def test_wakeup_with_stale_group_does_not_raise(self):
self.write_oom_control(oom_kill_disable="0")
self.monitor.open()
os.close(self.monitor.oom_control.fileno())
self.monitor.wakeup(self.queue, None)
self.assertRaises(EnvironmentError, self.monitor.wakeup, None,
self.queue, raise_for_stale=True)
# Close the other FD manually. We still need to attempt closing the
# wrapper to avoid a resource warning.
os.close(self.monitor.event_oom.fileno())
try:
self.monitor.oom_control.close()
except EnvironmentError:
pass
def test_wakeup_under_oom_requests_restart(self):
self.monitor.open()
self.write_oom_control(oom_kill_disable="1", under_oom="1")
self.monitor.wakeup(self.queue, None)
self.assertHasMessageForCg(
self.queue,
RestartRequestedMessage,
self.mock_cg
)
def test_wakeup_memory_pressure_notifies(self):
self.monitor.open()
self.write_memory_usage()
self.write_memory_limit()
self.monitor.wakeup(
self.queue,
self.monitor.event_pressure.fileno()
)
self.assertHasMessageForCg(
self.queue,
MemoryPressureMessage,
self.mock_cg
)
def test_pressure_wakeup_with_stale_group_does_not_raise(self):
self.monitor.open()
self.monitor.wakeup(
self.queue,
self.monitor.event_pressure.fileno()
)
self.assertHasMessageForCg(
self.queue,
MemoryPressureMessage,
self.mock_cg
)
def test_open_close(self):
cg_path = create_random_cg(self.parent_cg_path)
cg = Cgroup(cg_path)
cg.open()
cg.close()
class CgroupTestUnit(unittest.TestCase):
def setUp(self):
self.mock_cg = tempfile.mkdtemp()
self.monitor = Cgroup(self.mock_cg)
self.queue = queue.Queue()
def tearDown(self):
shutil.rmtree(self.mock_cg)
# Helpers
def write_oom_control(self, oom_kill_disable="0", under_oom="0"):
control = [
"oom_kill_disable {0}".format(oom_kill_disable),
"under_oom {0}".format(under_oom)
]
with open(self.cg_path("memory.oom_control"), "w") as f:
f.write("\n".join(control))
f.write("\n")
def write_memory_limit(self, memory_limit=9223372036854771712):
with open(self.cg_path("memory.limit_in_bytes"), "w") as f:
f.write(str(memory_limit))
f.write("\n")
def cg_path(self, path):
return os.path.join(self.mock_cg, path)
# Tests
def test_open(self):
self.write_oom_control()
self.monitor.open()
evt_fileno = self.monitor.event_fileno()
oom_control_fileno = self.monitor.oom_control.fileno()
self.monitor.close()
with open(self.cg_path("cgroup.event_control")) as f:
e = "{0} {1}\n".format(evt_fileno, oom_control_fileno)
self.assertEqual(e, f.read())
def test_wakeup_disable_oom_killer(self):
self.write_oom_control()
self.write_memory_limit(1024)
self.monitor.open()
self.monitor.wakeup(self.queue)
self.monitor.close()
with open(self.cg_path("memory.oom_control")) as f:
self.assertEqual("1\n", f.read())
def test_wakeup_oom_killer_is_disabled(self):
self.write_oom_control(oom_kill_disable="1")
self.write_memory_limit(1024)
self.monitor.open()
self.monitor.wakeup(self.queue)
self.monitor.close()
# File shoud not have been touched
with open(self.cg_path("memory.oom_control")) as f:
self.assertEqual("oom_kill_disable 1\n", f.readline())
def test_wakeup_no_memory_limit(self):
self.write_oom_control(oom_kill_disable="0")
self.write_memory_limit()
self.monitor.open()
self.monitor.wakeup(self.queue)
self.monitor.close()
# File shoud not have been touched
with open(self.cg_path("memory.oom_control")) as f:
self.assertEqual("oom_kill_disable 0\n", f.readline())
def test_wakeup_stale(self):
self.write_oom_control(oom_kill_disable="0")
self.monitor.open()
os.close(self.monitor.oom_control.fileno())
self.monitor.wakeup(self.queue)
self.assertRaises(EnvironmentError,
self.monitor.wakeup,
self.queue,
raise_for_stale=True)
# Close the other FD manually. We still need to attempt closing the
# wrapper to avoid a resource warning.
os.close(self.monitor.event_fileno())
try:
self.monitor.oom_control.close()
except EnvironmentError:
pass
def test_open_close(self):
cg_path = create_random_cg(self.parent_cg_path)
cg = Cgroup(cg_path)
cg.open()
cg.close()
