def test_resgroup_add_pids(makedirs_mock, set_effective_root_uid_mock,
resgroup_name, pids, mongroup_name, expected_writes,
expected_setuid_calls_count, expected_makedirs):
"""Test that for ResGroup created with resgroup_name, when add_pids() is called with
pids and given mongroup_name, expected writes (filenames with expected bytes writes)
will happen together with number of setuid calls and makedirs calls.
"""
write_mocks = {filename: mock_open() for filename in expected_writes}
resgroup = ResGroup(name=resgroup_name)
# if expected_log:
with patch('builtins.open', new=create_open_mock(write_mocks)):
resgroup.add_pids(pids, mongroup_name)
for filename, write_mock in write_mocks.items():
expected_filename_writes = expected_writes[filename]
expected_write_calls = [
call().write(write_body) for write_body in expected_filename_writes
]
write_mock.assert_has_calls(expected_write_calls, any_order=True)
# makedirs used
makedirs_mock.assert_has_calls(expected_makedirs)
# setuid used (at least number of times)
expected_setuid_calls = [call.__enter__()] * expected_setuid_calls_count
set_effective_root_uid_mock.assert_has_calls(expected_setuid_calls,
any_order=True)
def unpatched():
if len(subcgroups_paths):
platform = Platform(sockets=1,
cores=1,
cpus=2,
numa_nodes=2,
topology={0: {
0: [1, 2]
}},
cpu_model='intel xeon',
cpu_model_number=0x5E,
cpu_codename=CPUCodeName.SKYLAKE,
timestamp=time.time(),
rdt_information=RDTInformation(
True, True, rdt_mb_control_enabled,
rdt_cache_control_enabled, '0', '0', 0, 0,
0),
node_cpus={0: {0, 1}},
node_distances={0: {
0: 10
}},
measurements={},
swap_enabled=False)
return ContainerSet(
cgroup_path=cgroup_path,
cgroup_paths=subcgroups_paths,
platform=platform,
allocation_configuration=AllocationConfiguration()
if with_config else None,
resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None,
event_names=['task_cycles'])
else:
platform = Platform(sockets=1,
cores=1,
cpus=2,
numa_nodes=2,
topology={0: {
0: [1, 2]
}},
cpu_model='intel xeon',
cpu_model_number=0x5E,
cpu_codename=CPUCodeName.SKYLAKE,
timestamp=time.time(),
rdt_information=RDTInformation(
True, True, True, True, '0', '0', 0, 0, 0),
node_cpus={0: {0, 1}},
node_distances={0: {
0: 10
}},
measurements={},
swap_enabled=False)
return Container(
cgroup_path=cgroup_path,
platform=platform,
allocation_configuration=AllocationConfiguration()
if with_config else None,
resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None,
event_names=['task_cycles'],
)
def test_get_measurements(*mock):
resgroup = ResGroup(name=RESCTRL_ROOT_NAME)
assert {'memory_bandwidth': 6,
'llc_occupancy': 2,
'memory_bandwidth_local': 4,
'memory_bandwidth_remote': 2} == \
resgroup.get_measurements('best_efforts', True, True)
def unpatched():
if len(subcgroups_paths):
return ContainerSet(
cgroup_path=cgroup_path,
cgroup_paths=subcgroups_paths,
platform_cpus=1,
platform_sockets=1,
allocation_configuration=AllocationConfiguration()
if with_config else None,
resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None,
rdt_information=RDTInformation(True, True,
rdt_mb_control_enabled,
rdt_cache_control_enabled, '0',
'0', 0, 0, 0),
event_names=DEFAULT_EVENTS)
else:
return Container(
cgroup_path=cgroup_path,
platform_cpus=1,
platform_sockets=1,
rdt_information=RDTInformation(True, True, True, True, '0',
'0', 0, 0, 0),
allocation_configuration=AllocationConfiguration()
if with_config else None,
resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None,
event_names=DEFAULT_EVENTS)
def test_get_measurements(*mock):
resgroup = ResGroup(name=RESCTRL_ROOT_NAME)
assert {'task_mem_bandwidth_bytes': 6,
'task_llc_occupancy_bytes': 2,
'task_mem_bandwidth_local_bytes': 4,
'task_mem_bandwidth_remote_bytes': 2} == \
resgroup.get_measurements('best_efforts', True, True)
def test_resgroup_remove(listdir_mock, set_effective_root_uid_mock, rmdir_mock, isdir_mock):
open_mock = create_open_mock({
"/sys/fs/resctrl": "0",
"/sys/fs/resctrl/best_efforts/mon_groups/some_container/tasks": "123\n124\n",
})
with patch('wca.resctrl.open', open_mock):
resgroup = ResGroup("best_efforts")
resgroup.remove('some-container')
rmdir_mock.assert_called_once_with('/sys/fs/resctrl/best_efforts/mon_groups/some-container')
def test_resgroup_add_pids_invalid(makedirs_mock, set_effective_root_uid_mock,
side_effect, log_call):
resgroup = ResGroup(name='')
writes_mock = {
'/sys/fs/resctrl/tasks': Mock(return_value=Mock(write=Mock(side_effect=side_effect))),
'/sys/fs/resctrl/mon_groups/c1/tasks': MagicMock()
}
with patch('builtins.open', new=create_open_mock(writes_mock)), patch(
'wca.resctrl.log') as log_mock:
resgroup.add_pids(['123'], 'c1')
log_mock.assert_has_calls([log_call])
def test_resgroup_write_schemata(resgroup_name, write_schemata_lines,
expected_writes: Dict[str, List[str]]):
write_mocks = {filename: mock_open() for filename in expected_writes}
resgroup = ResGroup(resgroup_name)
with patch('builtins.open', new=create_open_mock(write_mocks)):
resgroup.write_schemata(write_schemata_lines)
for filename, write_mock in write_mocks.items():
expected_filename_writes = expected_writes[filename]
expected_write_calls = [call().write(write_body) for write_body in expected_filename_writes]
assert expected_filename_writes
write_mock.assert_has_calls(expected_write_calls, any_order=True)
def unpatched():
if len(subcgroups_paths):
platform = Platform(sockets=1,
cores=1,
cpus=2,
cpu_model='intel xeon',
cpu_model_number=0x5E,
cpu_codename=CPUCodeName.SKYLAKE,
cpus_usage={
0: 10,
1: 10
},
total_memory_used=10,
timestamp=time.time(),
rdt_information=RDTInformation(
True, True, rdt_mb_control_enabled,
rdt_cache_control_enabled, '0', '0', 0, 0,
0))
return ContainerSet(
cgroup_path=cgroup_path,
cgroup_paths=subcgroups_paths,
platform=platform,
allocation_configuration=AllocationConfiguration()
if with_config else None,
resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None,
event_names=DEFAULT_EVENTS)
else:
platform = Platform(sockets=1,
cores=1,
cpus=2,
cpu_model='intel xeon',
cpu_model_number=0x5E,
cpu_codename=CPUCodeName.SKYLAKE,
cpus_usage={
0: 10,
1: 10
},
total_memory_used=10,
timestamp=time.time(),
rdt_information=RDTInformation(
True, True, True, True, '0', '0', 0, 0, 0))
return Container(
cgroup_path=cgroup_path,
platform=platform,
allocation_configuration=AllocationConfiguration()
if with_config else None,
resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None,
event_names=DEFAULT_EVENTS)
def test_rdt_allocations_changeset(
current, new,
expected_target, expected_changeset):
container_name = 'some_container-xx2'
resgroup = ResGroup(name=container_name)
rdt_groups = RDTGroups(16)
def convert(rdt_allocation):
rdt_information = RDTInformation(False, False, False, False, 'fffff', '1', 0, 0, 0)
if rdt_allocation is not None:
return RDTAllocationValue(container_name,
rdt_allocation,
resgroup,
lambda: ['1'],
platform_sockets=1,
rdt_information=rdt_information,
rdt_groups=rdt_groups,
common_labels={},
)
else:
return None
current_value = convert(current)
new_value = convert(new)
got_target, got_changeset = \
new_value.calculate_changeset(current_value)
assert got_target == convert(expected_target)
assert got_changeset == convert(expected_changeset)
def test_prepare_task_data_resgroup_not_found(*mocks):
containers = {
task('/t1', labels={'label_key': 'label_value'}, resources={'cpu': 3}):
Container('/t1', platform_mock, resgroup=ResGroup('/t1'))
}
tasks_measurements, tasks_resources, tasks_labels = \
_prepare_tasks_data(containers)
assert tasks_measurements == {}
def test_prepare_task_data_resgroup_not_found(*mocks):
containers = {
task('/t1', labels={'label_key': 'label_value'}, resources={'cpu': 3}):
Container('/t1', platform_mock, resgroup=ResGroup('/t1'))
}
with pytest.raises(MissingMeasurementException):
tasks_measurements, tasks_resources, tasks_labels = \
_prepare_tasks_data(containers)
def test_prepare_task_data_resgroup_not_found(*mocks):
rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0)
containers = {
task('/t1', labels={'label_key': 'label_value'}, resources={'cpu': 3}):
Container('/t1', 1, 1, rdt_information, resgroup=ResGroup('/t1'))
}
tasks_measurements, tasks_resources, tasks_labels = \
_prepare_tasks_data(containers)
assert tasks_measurements == {}
def test_rdt_allocation_generate_metrics(rdt_allocation: RDTAllocation,
extra_labels, expected_metrics):
rdt_information = RDTInformation(True, True, True, True, 'fff', '1', 0, 0, 0)
rdt_allocation_value = RDTAllocationValue(
'c1',
rdt_allocation, get_pids=lambda: [],
resgroup=ResGroup(name=rdt_allocation.name or ''),
platform_sockets=1, rdt_information=rdt_information,
common_labels=extra_labels,
rdt_groups=RDTGroups(10),
)
got_metrics = rdt_allocation_value.generate_metrics()
assert got_metrics == expected_metrics
def sync_containers_state(self, tasks: List[Task]) -> Dict[Task, ContainerInterface]:
"""Syncs state of ContainerManager with a system by removing orphaned containers,
and creating containers for newly arrived tasks, and synchronizing containers' state.
Function is responsible for cleaning and initializing stateful subsystems such as:
- perf counters: opens file descriptors for counters,
- resctrl (ResGroups): creates and manages directories in resctrl filesystem and scarce
"closid" hardware identifiers
Can throw OutOfClosidsException.
"""
# Find difference between discovered tasks and already watched containers.
new_tasks, containers_to_cleanup = _find_new_and_dead_tasks(
tasks, list(self.containers.values()))
if containers_to_cleanup:
log.debug('sync_containers_state: cleaning up %d containers',
len(containers_to_cleanup))
log.log(logger.TRACE, 'sync_containers_state: containers_to_cleanup=%r',
containers_to_cleanup)
# Clean up and remove orphaned containers.
for container_to_cleanup in containers_to_cleanup:
container_to_cleanup.cleanup()
# Recreate self.containers.
# mutated state e.g. labels for Kubernetes
containers = {}
for task in tasks:
for known_task, container in self.containers.items():
if task.cgroup_path == known_task.cgroup_path:
containers[task] = container
continue
self.containers = containers
if new_tasks:
log.debug('Found %d new tasks', len(new_tasks))
log.log(logger.TRACE, 'sync_containers_state: new_tasks=%r', new_tasks)
# Prepare state of currently assigned resgroups
# and remove some orphaned resgroups.
container_name_to_ctrl_group = {}
if self._rdt_information:
assert self._rdt_information.is_monitoring_enabled(), \
"rdt_enabled requires RDT monitoring for keeping groups relation."
mon_groups_relation = resctrl.read_mon_groups_relation()
log.log(TRACE, 'mon_groups_relation (before cleanup): %s',
pprint.pformat(mon_groups_relation))
resctrl.clean_taskless_groups(mon_groups_relation)
mon_groups_relation = resctrl.read_mon_groups_relation()
log.log(TRACE, 'mon_groups_relation (after cleanup): %s',
pprint.pformat(mon_groups_relation))
# Calculate inverse relation of container_name
# to res_group name based on mon_groups_relations.
for ctrl_group, container_names in mon_groups_relation.items():
for container_name in container_names:
container_name_to_ctrl_group[container_name] = ctrl_group
log.log(TRACE, 'container_name_to_ctrl_group: %s',
pprint.pformat(container_name_to_ctrl_group))
# Create new containers and store them.
for new_task in new_tasks:
self.containers[new_task] = self._create_container(new_task)
# Sync "state" of individual containers.
# Note: only pids are synchronized, not allocations.
for container in self.containers.values():
if self._rdt_information:
if container.get_name() in container_name_to_ctrl_group:
resgroup_name = container_name_to_ctrl_group[container.get_name()]
container.set_resgroup(ResGroup(name=resgroup_name))
else:
# Every newly detected container is first assigned to the root group.
container.set_resgroup(ResGroup(name=''))
container.sync()
log.log(logger.TRACE, 'sync_containers_state: containers=%r', self.containers)
return self.containers
def calculate_changeset(self, current: Optional['RDTAllocationValue']):
"""Merge with existing RDTAllocation objects and return
sum of the allocations (target_rdt_allocation)
and allocations that need to be updated (rdt_allocation_changeset).
current can be None - means we have just spotted the task, and we're moving
it from default root group.
current cannot have empty name in rdt_allocation.name !!!!
"""
assert isinstance(
current,
(type(None),
RDTAllocationValue)), 'type error on current=%r ' % current
# Any rdt_allocation that comes with current have to have rdt_allocation.name set)
assert current is None or (current.rdt_allocation is not None)
new: RDTAllocationValue = self
new_group_name = new.get_resgroup_name()
# new name, then new allocation will be used (overwrite) but no merge
if current is None:
# New tasks or is moved from root group.
log.debug(
'resctrl changeset: new name or no previous allocation exists (moving from root '
'group!)')
return new, new._copy(new.rdt_allocation,
resgroup=ResGroup(name=new_group_name),
source_resgroup=ResGroup(name=''))
current_group_name = current.get_resgroup_name()
if current_group_name != new_group_name:
# We need to move to another group.
log.debug('resctrl changeset: move to new group=%r from=%r',
current_group_name, new_group_name)
return new, new._copy(
new.rdt_allocation,
resgroup=ResGroup(name=new_group_name),
source_resgroup=ResGroup(name=current.get_resgroup_name()))
else:
log.debug(
'resctrl changeset: merging existing rdt allocation (the same resgroup name)'
)
# Prepare target first, overwrite current l3 & mb values with new
target_rdt_allocation = RDTAllocation(
name=current.rdt_allocation.name,
l3=new.rdt_allocation.l3 or current.rdt_allocation.l3,
mb=new.rdt_allocation.mb or current.rdt_allocation.mb,
)
target = current._copy(target_rdt_allocation)
# Prepare changeset
# Logic: if new value exists and is different from old one use the new.
if _is_rdt_suballocation_changed(current.rdt_allocation.l3,
new.rdt_allocation.l3):
new_l3 = new.rdt_allocation.l3
else:
log.debug('changeset l3: no change between: %r and %r' %
(current.rdt_allocation.l3, new.rdt_allocation.l3))
new_l3 = None
if _is_rdt_suballocation_changed(current.rdt_allocation.mb,
new.rdt_allocation.mb):
new_mb = new.rdt_allocation.mb
else:
log.debug('changeset l3: no change between: %r and %r' %
(current.rdt_allocation.mb, new.rdt_allocation.mb))
new_mb = None
if new_l3 or new_mb:
# Only return something if schemata resources differs.
rdt_allocation_changeset = RDTAllocation(
name=new.rdt_allocation.name,
l3=new_l3,
mb=new_mb,
)
changeset = current._copy(rdt_allocation_changeset)
return target, changeset
else:
return target, None
def test_get_measurements_race(*mock):
resgroup = ResGroup(name=RESCTRL_ROOT_NAME)
with pytest.raises(MissingMeasurementException):
resgroup.get_measurements('best_efforts', True, True)
def test_resgroup_sync_no_space_left_on_device(makedirs_mock, exists_mock,
log_warning_mock):
with pytest.raises(Exception, match='Limit of workloads reached'):
ResGroup("best_efforts")._create_controlgroup_directory()