def test_resgroup_add_pids(makedirs_mock, set_effective_root_uid_mock,
resgroup_name, pids, mongroup_name, expected_writes,
expected_setuid_calls_count, expected_makedirs):
"""Test that for ResGroup created with resgroup_name, when add_pids() is called with
pids and given mongroup_name, expected writes (filenames with expected bytes writes)
will happen together with number of setuid calls and makedirs calls.
"""
write_mocks = {filename: mock_open() for filename in expected_writes}
resgroup = ResGroup(name=resgroup_name)
# if expected_log:
with patch('builtins.open', new=create_open_mock(write_mocks)):
resgroup.add_pids(pids, mongroup_name)
for filename, write_mock in write_mocks.items():
expected_filename_writes = expected_writes[filename]
expected_write_calls = [
call().write(write_body) for write_body in expected_filename_writes
]
write_mock.assert_has_calls(expected_write_calls, any_order=True)
# makedirs used
makedirs_mock.assert_has_calls(expected_makedirs)
# setuid used (at least number of times)
expected_setuid_calls = [call.__enter__()] * expected_setuid_calls_count
set_effective_root_uid_mock.assert_has_calls(expected_setuid_calls,
any_order=True)
def test_get_measurements(*mocks):
mock_get_pids = Mock()
mock_get_pids.return_value = pids
with patch('builtins.open',
new=create_open_mock({
**smaps,
**clear_refs, '/dev/null': '0'
})) as files:
wss = WSS(get_pids=mock_get_pids, reset_interval=1)
# In megabytes: ( 1 + 2 + 3 + 4 + 5 ) * 1024 / 1024
assert wss.get_measurements() == {
MetricName.TASK_WSS_REFERENCED_BYTES: 15360000
}
# Check if gets info from smaps
for smap in smaps:
expected_calls = [
call(smap, 'rb'),
call().__enter__(),
call().readlines(),
call().__exit__(None, None, None)
]
assert files[smap].mock_calls == expected_calls
# Check if write '1' to clear_refs. It should happen on first run.
for ref in clear_refs:
expected_calls = [
call(ref, 'w'),
call().__enter__(),
call().write('1\n'),
call().__exit__(None, None, None)
]
assert files[ref].mock_calls == expected_calls
def test_create_open_mock_autocreated():
with patch('builtins.open', create_open_mock({
'/some/path.txt': 'foo',
})) as mocks:
assert open('/some/path.txt').read() == 'foo'
open('/some/path.txt', 'w').write('bar')
mocks['/some/path.txt'].assert_has_calls(
[call().write('bar')])
def test_parse_cpu_info(filename, expected_cpus, expected_cpu):
with patch('builtins.open',
new=create_open_mock(
{"/proc/cpuinfo": open(relative_module_path(__file__, filename)).read()})
):
got_data = _parse_cpuinfo()
assert len(got_data) == expected_cpus
assert _is_dict_match(got_data[0], expected_cpu), 'some keys do not match!'
def test_cgroup_write():
cgroup = Cgroup('/some/foo1', platform=platform_mock)
ctrl_file_mock = MagicMock()
full_path = '/sys/fs/cgroup/cpu/some/foo1/some_ctrl_file'
open_mock = create_open_mock({full_path: ctrl_file_mock})
with patch('builtins.open', open_mock):
cgroup._write('some_ctrl_file', 5, CgroupType.CPU)
ctrl_file_mock.assert_called_once_with(full_path, 'wb')
ctrl_file_mock.assert_has_calls([call().__enter__().write(b'5')])
def test_resgroup_remove(listdir_mock, set_effective_root_uid_mock, rmdir_mock, isdir_mock):
open_mock = create_open_mock({
"/sys/fs/resctrl": "0",
"/sys/fs/resctrl/best_efforts/mon_groups/some_container/tasks": "123\n124\n",
})
with patch('wca.resctrl.open', open_mock):
resgroup = ResGroup("best_efforts")
resgroup.remove('some-container')
rmdir_mock.assert_called_once_with('/sys/fs/resctrl/best_efforts/mon_groups/some-container')
def test_create_open_mock_manually():
my_own_mock_open = mock_open(read_data='foo')
with patch('builtins.open', create_open_mock({
'/some/path.txt': my_own_mock_open
})):
assert open('/some/path.txt').read() == 'foo'
open('/some/path.txt', 'w').write('bar')
my_own_mock_open.assert_has_calls(
[call().write('bar')])
def test_collect_topology_information(filename, expected_cpus, expected_cores,
expected_sockets):
with patch('builtins.open',
new=create_open_mock(
{"/proc/cpuinfo": open(relative_module_path(__file__, filename)).read()})
):
cpuinfo = _parse_cpuinfo()
got_cpus, got_cores, got_sockets, got_topology = collect_topology_information(cpuinfo)
assert got_cpus == expected_cpus
assert got_cores == expected_cores
assert got_sockets == expected_sockets
def test_resgroup_add_pids_invalid(makedirs_mock, set_effective_root_uid_mock,
side_effect, log_call):
resgroup = ResGroup(name='')
writes_mock = {
'/sys/fs/resctrl/tasks': Mock(return_value=Mock(write=Mock(side_effect=side_effect))),
'/sys/fs/resctrl/mon_groups/c1/tasks': MagicMock()
}
with patch('builtins.open', new=create_open_mock(writes_mock)), patch(
'wca.resctrl.log') as log_mock:
resgroup.add_pids(['123'], 'c1')
log_mock.assert_has_calls([log_call])
def test_clean_resctrl(exists_mock, isdir_mock, rmdir_mock, listdir_mock):
from wca.resctrl import cleanup_resctrl
schemata_mock = mock_open()
with patch('builtins.open', new=create_open_mock(
{"/sys/fs/resctrl/mesos-1/tasks": "1\n2\n",
# resctrl group to recycle - expected to be removed.
"/sys/fs/resctrl/mesos-2/tasks": "",
"/sys/fs/resctrl/mesos-3/tasks": "2",
"/sys/fs/resctrl/mon_groups/mesos-1/tasks": "1\n2\n",
# resctrl group to recycle - should be removed.
"/sys/fs/resctrl/mon_groups/mesos-2/tasks": "",
"/sys/fs/resctrl/mon_groups/mesos-3/tasks": "2",
# default values expected to be written
"/sys/fs/resctrl/schemata": schemata_mock})):
cleanup_resctrl(root_rdt_l3='L3:0=ff', root_rdt_mb='MB:0=100', reset_resctrl=True)
listdir_mock.assert_has_calls([
call('/sys/fs/resctrl/mon_groups'),
call('/sys/fs/resctrl/')
])
isdir_mock.assert_has_calls([
call('/sys/fs/resctrl/mon_groups/mesos-1'),
call('/sys/fs/resctrl/mon_groups/mesos-2'),
call('/sys/fs/resctrl/mon_groups/mesos-3'),
call('/sys/fs/resctrl/mesos-1'),
call('/sys/fs/resctrl/mesos-2'),
call('/sys/fs/resctrl/mesos-3'),
])
exists_mock.assert_has_calls([
call('/sys/fs/resctrl/mon_groups/mesos-1/tasks'),
call('/sys/fs/resctrl/mon_groups/mesos-2/tasks'),
call('/sys/fs/resctrl/mon_groups/mesos-3/tasks'),
call('/sys/fs/resctrl/mesos-1/tasks'),
call('/sys/fs/resctrl/mesos-2/tasks'),
call('/sys/fs/resctrl/mesos-3/tasks')
])
rmdir_mock.assert_has_calls([
call('/sys/fs/resctrl/mon_groups/mesos-1'),
call('/sys/fs/resctrl/mon_groups/mesos-2'),
call('/sys/fs/resctrl/mon_groups/mesos-3'),
call('/sys/fs/resctrl/mesos-1'),
call('/sys/fs/resctrl/mesos-2'),
call('/sys/fs/resctrl/mesos-3')
])
schemata_mock.assert_has_calls([
call().write(b'L3:0=ff\n'),
call().write(b'MB:0=100\n'),
], any_order=True)
def test_resgroup_write_schemata(resgroup_name, write_schemata_lines,
expected_writes: Dict[str, List[str]]):
write_mocks = {filename: mock_open() for filename in expected_writes}
resgroup = ResGroup(resgroup_name)
with patch('builtins.open', new=create_open_mock(write_mocks)):
resgroup.write_schemata(write_schemata_lines)
for filename, write_mock in write_mocks.items():
expected_filename_writes = expected_writes[filename]
expected_write_calls = [call().write(write_body) for write_body in expected_filename_writes]
assert expected_filename_writes
write_mock.assert_has_calls(expected_write_calls, any_order=True)
def test_clean_tasksless_resctrl_groups(rmdir_mock):
with patch('wca.resctrl.open', create_open_mock({
'/sys/fs/resctrl/mon_groups/c1/tasks': '', # empty
'/sys/fs/resctrl/mon_groups/c2/tasks': '1234',
'/sys/fs/resctrl/empty/mon_groups/c3/tasks': '',
'/sys/fs/resctrl/half_empty/mon_groups/c5/tasks': '1234',
'/sys/fs/resctrl/half_empty/mon_groups/c6/tasks': '',
})):
mon_groups_relation = {'': ['c1', 'c2'],
'empty': ['c3'],
'half_empty': ['c5', 'c6'],
}
clean_taskless_groups(mon_groups_relation)
rmdir_mock.assert_has_calls([
call('/sys/fs/resctrl/mon_groups/c1'),
call('/sys/fs/resctrl/empty'),
call('/sys/fs/resctrl/half_empty/mon_groups/c6')
])
"/sys/fs/resctrl/best_efforts/mon_groups/some_container/tasks":
"123\n124\n",
})
with patch('wca.resctrl.open', open_mock):
resgroup = ResGroup("best_efforts")
resgroup.remove('some-container')
rmdir_mock.assert_called_once_with(
'/sys/fs/resctrl/best_efforts/mon_groups/some-container')
@patch('wca.resctrl.log.warning')
@patch('os.path.exists', return_value=True)
@patch('os.makedirs', side_effect=OSError(errno.ENOSPC, "mock"))
@patch('builtins.open',
new=create_open_mock({
"/sys/fs/cgroup/cpu/ddd/tasks": "123",
}))
def test_resgroup_sync_no_space_left_on_device(makedirs_mock, exists_mock,
log_warning_mock):
with pytest.raises(Exception, match='Limit of workloads reached'):
ResGroup("best_efforts")._create_controlgroup_directory()
@patch(
'builtins.open',
new=create_open_mock({
"/sys/fs/resctrl/mon_groups/best_efforts/mon_data/1/mbm_total_bytes":
"1",
"/sys/fs/resctrl/mon_groups/best_efforts/mon_data/2/mbm_total_bytes":
"1",
"/sys/fs/resctrl/mon_groups/best_efforts/mon_data/1/llc_occupancy":
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import patch, mock_open, call
import pytest
from wca.resctrl import check_resctrl, get_max_rdt_values, read_mon_groups_relation, \
clean_taskless_groups
from wca.resctrl_allocations import check_cbm_mask
from tests.testing import create_open_mock
@patch('builtins.open', new=create_open_mock({
"/sys/fs/resctrl": "0",
"/sys/fs/resctrl/tasks": "0",
"/sys/fs/resctrl/mon_data/mon_L3_00/mbm_total_bytes": "0",
}))
def test_check_resctrl(*mock):
assert check_resctrl()
@patch('os.listdir', return_value=['mesos-1', 'mesos-2', 'mesos-3'])
@patch('os.rmdir')
@patch('os.path.isdir', return_value=True)
@patch('os.path.exists', return_value=True)
def test_clean_resctrl(exists_mock, isdir_mock, rmdir_mock, listdir_mock):
from wca.resctrl import cleanup_resctrl
schemata_mock = mock_open()
def test_get_measurements_not_found_cgroup(mock_file, mock_log_warning):
cgroup = Cgroup('/some/foo1', platform=platform_mock)
with pytest.raises(MissingMeasurementException):
cgroup.get_measurements()
@patch(
'builtins.open',
create_open_mock({
'/sys/fs/cgroup/cpu/some/foo1/cpuacct.usage':
'1000000000', # nanoseconds
'/sys/fs/cgroup/memory/some/foo1/memory.usage_in_bytes':
'101',
'/sys/fs/cgroup/memory/some/foo1/memory.max_usage_in_bytes':
'999',
'/sys/fs/cgroup/memory/some/foo1/memory.limit_in_bytes':
'2000',
'/sys/fs/cgroup/memory/some/foo1/memory.soft_limit_in_bytes':
'1500',
'/sys/fs/cgroup/memory/some/foo1/memory.numa_stat':
'hierarchical_total=1 N0=123 N1=234',
'/sys/fs/cgroup/memory/some/foo1/memory.stat':
'pgfault 2730362811\nfoo=1 N0=123 N1=234',
}))
def test_get_measurements():
cgroup = Cgroup('/some/foo1', platform=platform_mock)
measurements = cgroup.get_measurements()
assert measurements == {
MetricName.TASK_CPU_USAGE_SECONDS: 1,
MetricName.TASK_MEM_USAGE_BYTES: 101,
MetricName.TASK_MEM_MAX_USAGE_BYTES: 999,
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re
from unittest.mock import patch
from tests.testing import create_open_mock, relative_module_path
from wca.metrics import MetricName
from wca.zoneinfo import get_zoneinfo_measurements, DEFAULT_REGEXP
@patch('builtins.open',
new=create_open_mock({
"/proc/zoneinfo":
open(relative_module_path(__file__,
'fixtures/proc-zoneinfo.txt')).read(),
}))
def test_parse_proc_zoneinfo(*mocks):
assert get_zoneinfo_measurements(re.compile(DEFAULT_REGEXP)) == {
MetricName.PLATFORM_ZONEINFO: {
'0': {
'DMA': {
'high': 6.0,
'hmem_autonuma_promote_dst': 0.0,
'hmem_autonuma_promote_src': 0.0,
'hmem_reclaim_demote_dst': 0.0,
'hmem_reclaim_demote_src': 0.0,
'hmem_reclaim_promote_dst': 0.0,
'hmem_reclaim_promote_src': 0.0,
'hmem_swapcache_promote_dst': 0.0,
from wca.metrics import MetricName, MissingMeasurementException
from tests.testing import create_open_mock
@patch('wca.cgroups.log.warning')
@patch('builtins.open', side_effect=FileNotFoundError())
def test_get_measurements_not_found_cgroup(mock_file, mock_log_warning):
cgroup = Cgroup('/some/foo1', platform_cpus=1)
with pytest.raises(MissingMeasurementException):
cgroup.get_measurements()
@patch('builtins.open',
create_open_mock({
'/sys/fs/cgroup/cpu/some/foo1/cpuacct.usage':
'100',
'/sys/fs/cgroup/memory/some/foo1/memory.usage_in_bytes':
'100'
}))
def test_get_measurements():
cgroup = Cgroup('/some/foo1', platform_cpus=1)
measurements = cgroup.get_measurements()
assert measurements == {
MetricName.CPU_USAGE_PER_TASK: 100,
MetricName.MEM_USAGE_PER_TASK: 100
}
@patch('builtins.open', mock_open(read_data='100'))
def test_cgroup_read():
cgroup = Cgroup('/some/foo1', platform_cpus=1)
value = cgroup._read('some_ctr_file', CgroupType.CPU)
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import patch
from tests.testing import create_open_mock, relative_module_path
from wca import vmstats
@patch(
'builtins.open',
new=create_open_mock({
'/sys/devices/system/node/node0/vmstat':
open(relative_module_path(__file__,
'fixtures/proc-vmstat-simple.txt')).read(),
'/sys/devices/system/node/node1/vmstat':
open(relative_module_path(__file__,
'fixtures/proc-vmstat-simple.txt')).read()
}))
@patch('os.listdir', return_value=['node0', 'node1'])
def test_parse_node_meminfo(*mocks):
measurements = vmstats.parse_node_vmstat_keys(None)
assert measurements == {
'platform_node_vmstat': {
0: {
'nr_bar': 30,
'nr_foo': 20
},
1: {
'nr_bar': 30,
'nr_foo': 20
},
name='root/staging14/cassandra--9043',
task_id=
'root-staging14-cassandra--9043-0-9ee9fbf1-b51b-4bb3-9748-6a4327fd7e0e',
resources={
'mem': 2048.0 * 1000**2,
'cpus': 8.0,
'disk': 10240.0 * 1000**2
})
@patch('requests.post',
return_value=create_json_fixture_mock('mesos_get_state', __file__))
@patch('builtins.open',
new=create_open_mock({
"/proc/32620/cgroup": "2:cpuacct,cpu:/",
}))
def test_get_tasks_with_wrong_cgroup(post_mock):
node = MesosNode()
tasks = set(node.get_tasks()
) # Wrap with set to make sure that hash is implemented.
assert len(tasks) == 0
@pytest.mark.parametrize("json_mock", [
create_json_fixture_mock('missing_executor_pid_in_mesos_response',
__file__),
create_json_fixture_mock('missing_statuses_in_mesos_response', __file__),
create_json_fixture_mock('empty_statuses_in_mesos_response', __file__)
])
def test_not_enough_data_in_response(json_mock):
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re
from unittest.mock import patch
from tests.testing import create_open_mock, relative_module_path
from wca import sched_stats
from wca.metrics import MetricName
@patch('builtins.open',
new=create_open_mock({
"/proc/1/sched":
open(relative_module_path(__file__,
'fixtures/proc-1-sched.txt')).read(),
"/proc/2/sched":
open(relative_module_path(__file__,
'fixtures/proc-2-sched.txt')).read(),
}))
def test_parse_proc_vmstat_keys(*mocks):
pattern = re.compile('.*numa')
measurements = sched_stats.get_pids_sched_measurements([1, 2], pattern)
assert measurements == {
MetricName.TASK_SCHED_STAT: {
'mm->numa_scan_seq': 27,
'numa_pages_migrated': 10.5,
'numa_preferred_nid': -3.0,
'total_numa_faults': 0,
},
MetricName.TASK_SCHED_STAT_NUMA_FAULTS: {
'0': {
executor_pid=32620,
labels={'aurora_tier': 'preemptible',
'env_uniq_id': '14',
'name': 'cassandra--9043',
'workload_uniq_id': '9043',
'application': 'cassandra',
'load_generator': 'ycsb'},
name='root/staging14/cassandra--9043',
task_id='root-staging14-cassandra--9043-0-9ee9fbf1-b51b-4bb3-9748-6a4327fd7e0e',
resources={'mem': 2048.0, 'cpus': 8.0, 'disk': 10240.0}
)
@patch('requests.post', return_value=create_json_fixture_mock('mesos_get_state', __file__))
@patch('builtins.open', new=create_open_mock({
"/proc/32620/cgroup": "2:cpuacct,cpu:/",
}))
def test_get_tasks_with_wrong_cgroup(post_mock):
node = MesosNode()
tasks = set(node.get_tasks()) # Wrap with set to make sure that hash is implemented.
assert len(tasks) == 0
@pytest.mark.parametrize(
"json_mock", [create_json_fixture_mock('missing_executor_pid_in_mesos_response', __file__),
create_json_fixture_mock('missing_statuses_in_mesos_response', __file__),
create_json_fixture_mock('empty_statuses_in_mesos_response', __file__)])
def test_not_enough_data_in_response(json_mock):
"""MesosNode get_tasks should return none tasks as vital data in mesos response is missing."""
with patch('requests.post', return_value=json_mock):
node = MesosNode()
"MemFree: 18245956 kB\n"
"MemAvailable: 24963992 kB\n"
"Buffers: 1190812 kB\n"
"Cached: 6971960 kB\n"
"SwapCached: 0 kB\n"
"Active: 8808464 kB\n"
"Inactive: 4727816 kB\n"
"Active(anon): 5376088 kB\n", 6406972 * 1024)])
def test_parse_proc_meminfo(raw_meminfo_output, expected):
assert parse_proc_meminfo(raw_meminfo_output) == expected
@patch('builtins.open',
new=create_open_mock({
"/proc/vmstat":
open(relative_module_path(__file__,
'fixtures/proc-vmstat.txt')).read(),
}))
def test_parse_proc_vmstat(*mocks):
assert parse_proc_vmstat() == {
MetricName.PLATFORM_VMSTAT_NUMA_HINT_FAULTS: 97909,
MetricName.PLATFORM_VMSTAT_NUMA_HINT_FAULTS_LOCAL: 97909,
MetricName.PLATFORM_VMSTAT_NUMA_PAGES_MIGRATED: 14266,
MetricName.PLATFORM_VMSTAT_PGFAULTS: 2911936,
MetricName.PLATFORM_VMSTAT_PGMIGRATE_FAIL: 5633,
MetricName.PLATFORM_VMSTAT_PGMIGRATE_SUCCESS: 14266
}
@pytest.mark.parametrize("raw_proc_state_output,expected", [
("cpu 8202889 22275 2138696 483384497 138968 853793 184852 0 0 0\n"
@patch('wca.perf._get_cgroup_fd')
@patch('wca.perf.PerfCounters._open')
def test_read_events_zero_values_one_cpu(_open_mock, _get_cgroup_fd_mock):
prf = perf.PerfCounters('/mycgroup', [])
# File descriptor mock for single cpu
prf._group_event_leaders = {0: Mock()}
assert prf._read_events() == {}
@patch('wca.perf.SetEffectiveRootUid')
@patch('os.path.exists', Mock(return_value=True))
@patch('builtins.open',
new=create_open_mock({
"/dev/cpu/0/cpuid":
b"\x16\x00\x00\x00\x47\x65\x6e\x75\x6e\x74\x65\x6c\x69\x6e\x65\x49"
b"\x54\x06\x05\x00\x00\x08\x40\x00\xff\xfb\xfe\x7f\xff\xfb\xeb\xbf"
}))
def test_read_skylake_cpu_model(*args):
assert pc.CPUModel.SKYLAKE == perf._get_cpu_model()
@patch('wca.perf.SetEffectiveRootUid')
@patch('os.path.exists', Mock(return_value=True))
@patch('builtins.open',
new=create_open_mock({
"/dev/cpu/0/cpuid":
b"\x14\x00\x00\x00\x47\x65\x6e\x75\x6e\x74\x65\x6c\x69\x6e\x65\x49"
b"\xf1\x06\x04\x00\x00\x08\x20\x00\xff\xfb\xfe\x7f\xff\xfb\xeb\xbf"
}))
def test_read_broadwell_cpu_model(*args):
