def test_k8s_cpu():
"""Test all the functions in dashboard/k8s_utils.py.
Also test ray._private.utils.get_num_cpus when running in a K8s pod.
Files were obtained from within a K8s pod with 2 CPU request, CPU limit
unset, with 1 CPU of stress applied.
"""
# Some experimentally-obtained K8S CPU usage files for use in test_k8s_cpu.
PROCSTAT1 = """cpu 2945022 98 3329420 148744854 39522 0 118587 0 0 0
cpu0 370299 14 413841 18589778 5304 0 15288 0 0 0
cpu1 378637 10 414414 18589275 5283 0 14731 0 0 0
cpu2 367328 8 420914 18590974 4844 0 14416 0 0 0
cpu3 368378 11 423720 18572899 4948 0 14394 0 0 0
cpu4 369051 13 414615 18607285 4736 0 14383 0 0 0
cpu5 362958 10 415984 18576655 4590 0 16614 0 0 0
cpu6 362536 13 414430 18605197 4785 0 14353 0 0 0
cpu7 365833 15 411499 18612787 5028 0 14405 0 0 0
intr 1000694027 125 0 0 39 154 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1028 0 2160913 0 2779605 8 0 3981333 3665198 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
ctxt 1574979439
btime 1615208601
processes 857411
procs_running 6
procs_blocked 0
softirq 524311775 0 230142964 27143 63542182 0 0 171 74042767 0 156556548
""" # noqa
PROCSTAT2 = """cpu 2945152 98 3329436 148745483 39522 0 118587 0 0 0
cpu0 370399 14 413841 18589778 5304 0 15288 0 0 0
cpu1 378647 10 414415 18589362 5283 0 14731 0 0 0
cpu2 367329 8 420916 18591067 4844 0 14416 0 0 0
cpu3 368381 11 423724 18572989 4948 0 14395 0 0 0
cpu4 369052 13 414618 18607374 4736 0 14383 0 0 0
cpu5 362968 10 415986 18576741 4590 0 16614 0 0 0
cpu6 362537 13 414432 18605290 4785 0 14353 0 0 0
cpu7 365836 15 411502 18612878 5028 0 14405 0 0 0
intr 1000700905 125 0 0 39 154 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1028 0 2160923 0 2779605 8 0 3981353 3665218 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
ctxt 1574988760
btime 1615208601
processes 857411
procs_running 4
procs_blocked 0
softirq 524317451 0 230145523 27143 63542930 0 0 171 74043232 0 156558452
""" # noqa
CPUACCTUSAGE1 = "2268980984108"
CPUACCTUSAGE2 = "2270120061999"
CPUSHARES = "2048"
shares_file, cpu_file, proc_stat_file = [
tempfile.NamedTemporaryFile("w+") for _ in range(3)
]
shares_file.write(CPUSHARES)
cpu_file.write(CPUACCTUSAGE1)
proc_stat_file.write(PROCSTAT1)
for file in shares_file, cpu_file, proc_stat_file:
file.flush()
with mock.patch("ray._private.utils.os.environ",
{"KUBERNETES_SERVICE_HOST"}), mock.patch(
"ray.dashboard.k8s_utils.CPU_USAGE_PATH",
cpu_file.name), mock.patch(
"ray.dashboard.k8s_utils.PROC_STAT_PATH",
proc_stat_file.name), mock.patch(
"ray._private.utils.get_k8s_cpus.__defaults__",
(shares_file.name, )):
# Test helpers
assert ray._private.utils.get_num_cpus() == 2
assert k8s_utils._cpu_usage() == 2268980984108
assert k8s_utils._system_usage() == 1551775030000000
assert k8s_utils._host_num_cpus() == 8
# No delta for first computation, return 0.
assert k8s_utils.cpu_percent() == 0.0
# Write new usage info obtained after 1 sec wait.
for file in cpu_file, proc_stat_file:
file.truncate(0)
file.seek(0)
cpu_file.write(CPUACCTUSAGE2)
proc_stat_file.write(PROCSTAT2)
for file in cpu_file, proc_stat_file:
file.flush()
# Files were extracted under 1 CPU of load on a 2 CPU pod
assert 50 < k8s_utils.cpu_percent() < 60
def test_k8s_cpu(use_cgroups_v2: bool):
"""Test all the functions in dashboard/k8s_utils.py.
Also test ray._private.utils.get_num_cpus when running in a K8s pod.
Files were obtained from within a K8s pod with 2 CPU request, CPU limit
unset, with 1 CPU of stress applied.
"""
# Some experimentally-obtained K8S CPU usage files for use in test_k8s_cpu.
PROCSTAT1 = """cpu 2945022 98 3329420 148744854 39522 0 118587 0 0 0
cpu0 370299 14 413841 18589778 5304 0 15288 0 0 0
cpu1 378637 10 414414 18589275 5283 0 14731 0 0 0
cpu2 367328 8 420914 18590974 4844 0 14416 0 0 0
cpu3 368378 11 423720 18572899 4948 0 14394 0 0 0
cpu4 369051 13 414615 18607285 4736 0 14383 0 0 0
cpu5 362958 10 415984 18576655 4590 0 16614 0 0 0
cpu6 362536 13 414430 18605197 4785 0 14353 0 0 0
cpu7 365833 15 411499 18612787 5028 0 14405 0 0 0
intr 1000694027 125 0 0 39 154 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1028 0 2160913 0 2779605 8 0 3981333 3665198 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
ctxt 1574979439
btime 1615208601
processes 857411
procs_running 6
procs_blocked 0
softirq 524311775 0 230142964 27143 63542182 0 0 171 74042767 0 156556548
""" # noqa
PROCSTAT2 = """cpu 2945152 98 3329436 148745483 39522 0 118587 0 0 0
cpu0 370399 14 413841 18589778 5304 0 15288 0 0 0
cpu1 378647 10 414415 18589362 5283 0 14731 0 0 0
cpu2 367329 8 420916 18591067 4844 0 14416 0 0 0
cpu3 368381 11 423724 18572989 4948 0 14395 0 0 0
cpu4 369052 13 414618 18607374 4736 0 14383 0 0 0
cpu5 362968 10 415986 18576741 4590 0 16614 0 0 0
cpu6 362537 13 414432 18605290 4785 0 14353 0 0 0
cpu7 365836 15 411502 18612878 5028 0 14405 0 0 0
intr 1000700905 125 0 0 39 154 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1028 0 2160923 0 2779605 8 0 3981353 3665218 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
ctxt 1574988760
btime 1615208601
processes 857411
procs_running 4
procs_blocked 0
softirq 524317451 0 230145523 27143 63542930 0 0 171 74043232 0 156558452
""" # noqa
CPUACCTUSAGE1 = "2268980984000"
CPUACCTUSAGE2 = "2270120061999"
CPU_STAT_1 = """usage_usec 2268980984
user_usec 5673216
system_usec 794353
nr_periods 168
nr_throttled 6
throttled_usec 638117
"""
CPU_STAT_2 = """usage_usec 2270120061
user_usec 5673216
system_usec 794353
nr_periods 168
nr_throttled 6
throttled_usec 638117
"""
cpu_file, cpu_v2_file, proc_stat_file = [
tempfile.NamedTemporaryFile("w+") for _ in range(3)
]
cpu_file.write(CPUACCTUSAGE1)
cpu_v2_file.write(CPU_STAT_1)
proc_stat_file.write(PROCSTAT1)
for file in cpu_file, cpu_v2_file, proc_stat_file:
file.flush()
if use_cgroups_v2:
# Should get a file not found for cpuacctusage if on cgroups v2
cpu_usage_file = "NO_SUCH_FILE"
else:
# If using cgroups v1, use the temp file we've just made
cpu_usage_file = cpu_file.name
with mock.patch(
"ray._private.utils.os.environ",
{"KUBERNETES_SERVICE_HOST": "host"}), mock.patch(
"ray.dashboard.k8s_utils.CPU_USAGE_PATH",
cpu_usage_file), mock.patch(
"ray.dashboard.k8s_utils.CPU_USAGE_PATH_V2",
cpu_v2_file.name), mock.patch(
"ray.dashboard.k8s_utils.PROC_STAT_PATH",
proc_stat_file.name), mock.patch(
# get_num_cpus is tested elsewhere
"ray.dashboard.k8s_utils.get_num_cpus",
mock.Mock(return_value=2),
), mock.patch(
# Reset this global variable between tests.
"ray.dashboard.k8s_utils.last_system_usage",
None,
):
# Validate mocks:
# Confirm CPU_USAGE_PATH is found with cgroups v2, but not with v2.
from ray.dashboard.k8s_utils import CPU_USAGE_PATH
if use_cgroups_v2:
with pytest.raises(FileNotFoundError):
print(open(CPU_USAGE_PATH).read())
else:
print(open(CPU_USAGE_PATH).read())
# Test helpers
assert k8s_utils._cpu_usage() == 2268980984000
assert k8s_utils._system_usage() == 1551775030000000
assert k8s_utils._host_num_cpus() == 8
# No delta for first computation, return 0.
assert k8s_utils.cpu_percent() == 0.0
# Write new usage info obtained after 1 sec wait.
for file in cpu_file, cpu_v2_file, proc_stat_file:
file.truncate(0)
file.seek(0)
cpu_file.write(CPUACCTUSAGE2)
cpu_v2_file.write(CPU_STAT_2)
proc_stat_file.write(PROCSTAT2)
for file in cpu_file, cpu_v2_file, proc_stat_file:
file.flush()
# Files were extracted under 1 CPU of load on a 2 CPU pod
assert 50 < k8s_utils.cpu_percent() < 60