def _build_allocations_from_rules(all_tasks_ids: Set[TaskId],
tasks_labels: Dict[TaskId, Dict[str, str]], rules):
tasks_allocations = {}
# Iterate over rules and apply one by one.
for rule_idx, rule in enumerate(rules):
if 'allocations' not in rule:
log.warning('StaticAllocator(%s): missing "allocations" - ignore!', rule_idx)
continue
log.debug('StaticAllocator(%s): processing %s rule.', rule_idx,
'(%s)' % rule['name'] if 'name' in rule else '')
new_task_allocations = rule['allocations']
if not new_task_allocations:
log.log(TRACE, 'StaticAllocator(%s): allocations are empty - ignore!', rule_idx)
continue
# Convert if nessesary.
if 'rdt' in new_task_allocations and isinstance(new_task_allocations['rdt'], dict):
new_task_allocations[AllocationType.RDT] = RDTAllocation(
**new_task_allocations['rdt'])
# Prepare match_task_ids filter:
if 'task_id' in rule:
# by task_id
task_id = rule['task_id']
match_task_ids = {task_id}
log.log(TRACE, 'StaticAllocator(%s): match by task_id=%r', rule_idx, rule['task_id'])
# Find all tasks that matches.
elif 'labels' in rule:
labels = rule['labels']
# by labels
match_task_ids = set()
for task_id, task_labels in tasks_labels.items():
matching_label_names = set(task_labels) & set(labels)
for label_name in matching_label_names:
if re.match(str(labels[label_name]), task_labels[label_name]):
match_task_ids.add(task_id)
log.log(TRACE, 'StaticAllocator(%s): match task %r by label=%s',
rule_idx, task_id, label_name)
else:
# match everything
log.log(TRACE, 'StaticAllocator(%s): match all tasks', rule_idx)
match_task_ids = all_tasks_ids
# for matching tasks calculate and remember target_tasks_allocations
log.log(TRACE, 'StaticAllocator(%s): applying allocations for %i tasks', rule_idx,
len(match_task_ids))
rule_tasks_allocations = {}
# Set rules for every matching task.
for match_task_id in match_task_ids:
rule_tasks_allocations[match_task_id] = new_task_allocations
# Merge rules with previous rules.
tasks_allocations = merge_rules(tasks_allocations, rule_tasks_allocations)
return tasks_allocations
def allocate(self, platform, tasks_data):
log.debug('allocated called with tasks_allocations: %r', tasks_data)
if tasks_data:
task_id = list(tasks_data.keys())[0]
tasks_data[task_id].allocations = {
AllocationType.QUOTA: 0.5,
AllocationType.RDT: RDTAllocation(l3='L3:0=ffff0;1=ffff0'),
}
return tasks_data, [], []
def allocate(self, platform, tasks_measurements, tasks_resources,
tasks_labels, tasks_allocations):
log.debug('allocated called with tasks_allocations: %r',
tasks_allocations)
if tasks_measurements:
task_id = list(tasks_measurements.keys())[0]
tasks_allocations[task_id] = {
AllocationType.QUOTA: 0.5,
AllocationType.RDT: RDTAllocation(l3='L3:0=ffff0;1=ffff0'),
}
return tasks_allocations, [], []
def set_alloc(self,
task_id,
alloc_type: AllocationType,
alloc: Union[float, str],
rdt_res: RDTResource = None,
name: str = None):
is_new = True
if task_id in self.cur_allocs and alloc_type in self.cur_allocs[
task_id]:
if alloc_type == AllocationType.RDT:
val = getattr(self.cur_allocs[task_id][alloc_type], rdt_res)
else:
val = self.cur_allocs[task_id][alloc_type]
if val == alloc:
is_new = False
if is_new:
if task_id in self.new_allocs:
task_allocs = self.new_allocs[task_id]
else:
task_allocs = dict()
self.new_allocs[task_id] = task_allocs
if alloc_type == AllocationType.RDT:
if rdt_res == RDTResource.L3:
old_mb = task_allocs.get(AllocationType.RDT,
RDTAllocation()).mb
task_allocs[alloc_type] = RDTAllocation(name=name,
l3=alloc,
mb=old_mb)
else:
old_l3 = task_allocs.get(AllocationType.RDT,
RDTAllocation()).l3
task_allocs[alloc_type] = RDTAllocation(name=name,
l3=old_l3,
mb=alloc)
else:
task_allocs[alloc_type] = alloc
def get_allocations(self) -> TaskAllocations:
"""Return TaskAllocations representing allocation for RDT resource."""
rdt_allocations_mb, rdt_allocations_l3 = None, None
try:
with open(os.path.join(self.fullpath, SCHEMATA)) as schemata:
for line in schemata:
if 'MB:' in line:
rdt_allocations_mb = line.strip()
elif 'L3:' in line:
rdt_allocations_l3 = line.strip()
except FileNotFoundError as e:
raise MissingAllocationException(
'File {} is missing. Allocation unavailable.'.format(e.filename))
rdt_allocations = RDTAllocation(
name=self.name,
l3=rdt_allocations_l3,
mb=rdt_allocations_mb,
)
return {AllocationType.RDT: rdt_allocations}
from wca.storage import Storage
@pytest.mark.parametrize('tasks_allocations, expected_metrics', (
({}, []),
({
't1_task_id': {
AllocationType.SHARES: 0.5
}
}, [
allocation_metric(
'cpu_shares', value=0.5, container_name='t1', task='t1_task_id')
]),
({
't1_task_id': {
AllocationType.RDT: RDTAllocation(mb='MB:0=20')
}
}, [
allocation_metric('rdt_mb',
20,
group_name='t1',
domain_id='0',
container_name='t1',
task='t1_task_id')
]),
({
't1_task_id': {
AllocationType.SHARES: 0.5,
AllocationType.RDT: RDTAllocation(mb='MB:0=20')
}
}, [
},
't2_task_id': {
'cpu_quota': 3
}
}),
# RDT are properly created for just first task
([{
'task_id': 't1_task_id',
'allocations': {
'rdt': {
'l3': 'somevalue'
}
}
}], {
't1_task_id': {
'rdt': RDTAllocation(l3='somevalue')
}
}),
# RDT are properly created for just first task with explicit name
([{
'task_id': 't1_task_id',
'allocations': {
'rdt': {
'name': 'foo',
'l3': 'somevalue'
}
}
}], {
't1_task_id': {
'rdt': RDTAllocation(name='foo', l3='somevalue')
}
def calculate_changeset(self, current: Optional['RDTAllocationValue']):
"""Merge with existing RDTAllocation objects and return
sum of the allocations (target_rdt_allocation)
and allocations that need to be updated (rdt_allocation_changeset).
current can be None - means we have just spotted the task, and we're moving
it from default root group.
current cannot have empty name in rdt_allocation.name !!!!
"""
assert isinstance(
current,
(type(None),
RDTAllocationValue)), 'type error on current=%r ' % current
# Any rdt_allocation that comes with current have to have rdt_allocation.name set)
assert current is None or (current.rdt_allocation is not None)
new: RDTAllocationValue = self
new_group_name = new.get_resgroup_name()
# new name, then new allocation will be used (overwrite) but no merge
if current is None:
# New tasks or is moved from root group.
log.debug(
'resctrl changeset: new name or no previous allocation exists (moving from root '
'group!)')
return new, new._copy(new.rdt_allocation,
resgroup=ResGroup(name=new_group_name),
source_resgroup=ResGroup(name=''))
current_group_name = current.get_resgroup_name()
if current_group_name != new_group_name:
# We need to move to another group.
log.debug('resctrl changeset: move to new group=%r from=%r',
current_group_name, new_group_name)
return new, new._copy(
new.rdt_allocation,
resgroup=ResGroup(name=new_group_name),
source_resgroup=ResGroup(name=current.get_resgroup_name()))
else:
log.debug(
'resctrl changeset: merging existing rdt allocation (the same resgroup name)'
)
# Prepare target first, overwrite current l3 & mb values with new
target_rdt_allocation = RDTAllocation(
name=current.rdt_allocation.name,
l3=new.rdt_allocation.l3 or current.rdt_allocation.l3,
mb=new.rdt_allocation.mb or current.rdt_allocation.mb,
)
target = current._copy(target_rdt_allocation)
# Prepare changeset
# Logic: if new value exists and is different from old one use the new.
if _is_rdt_suballocation_changed(current.rdt_allocation.l3,
new.rdt_allocation.l3):
new_l3 = new.rdt_allocation.l3
else:
log.debug('changeset l3: no change between: %r and %r' %
(current.rdt_allocation.l3, new.rdt_allocation.l3))
new_l3 = None
if _is_rdt_suballocation_changed(current.rdt_allocation.mb,
new.rdt_allocation.mb):
new_mb = new.rdt_allocation.mb
else:
log.debug('changeset l3: no change between: %r and %r' %
(current.rdt_allocation.mb, new.rdt_allocation.mb))
new_mb = None
if new_l3 or new_mb:
# Only return something if schemata resources differs.
rdt_allocation_changeset = RDTAllocation(
name=new.rdt_allocation.name,
l3=new_l3,
mb=new_mb,
)
changeset = current._copy(rdt_allocation_changeset)
return target, changeset
else:
return target, None
with patch('builtins.open', new=create_open_mock(write_mocks)):
resgroup.write_schemata(write_schemata_lines)
for filename, write_mock in write_mocks.items():
expected_filename_writes = expected_writes[filename]
expected_write_calls = [call().write(write_body) for write_body in expected_filename_writes]
assert expected_filename_writes
write_mock.assert_has_calls(expected_write_calls, any_order=True)
@pytest.mark.parametrize(
'current, new,'
'expected_target,expected_changeset', (
# --- Test cases withing the same group.
# Set l3 within the same auto group.
(RDTAllocation(l3='L3:0=0ff'), RDTAllocation(l3='L3:0=0f'),
RDTAllocation(l3='L3:0=0f'), RDTAllocation(l3='L3:0=0f')),
# Set mb within the same auto group to existing l3.
(RDTAllocation(l3='L3:0=0f', mb='MB:0=x'), RDTAllocation(mb='MB:0=y'),
RDTAllocation(l3='L3:0=0f', mb='MB:0=y'), RDTAllocation(mb='MB:0=y')),
# Set l3 within the same auto group to existing l3 and mb.
(RDTAllocation(l3='L3:0=0f', mb='MB:0=x'), RDTAllocation(l3='L3:0=0f', mb='MB:0=y'),
RDTAllocation(l3='L3:0=0f', mb='MB:0=y'), RDTAllocation(mb='MB:0=y')),
# Set l3 within the same auto group to existing l3 and mb.
(RDTAllocation(l3='L3:0=0f'), RDTAllocation(name='', l3='L3:0=0f'),
RDTAllocation(name='', l3='L3:0=0f'), RDTAllocation(name='', l3='L3:0=0f')),
# --- Move moving between groups.
# Initial put into auto-group.
(None, RDTAllocation(),
RDTAllocation(), RDTAllocation()),
# Move to auto-group from root group.
def test_allocation_runner(_get_allocations_mock, _get_allocations_mock_,
platform_mock, reset_counter_mock, subcgroups):
""" Low level system calls are not mocked - but higher level objects and functions:
Cgroup, Resgroup, Platform, etc. Thus the test do not cover the full usage scenario
(such tests would be much harder to write).
"""
# Tasks mock
t1 = redis_task_with_default_labels('t1', subcgroups)
t2 = redis_task_with_default_labels('t2', subcgroups)
# Allocator mock (lower the quota and number of cache ways in dedicated group).
# Patch some of the functions of AllocationRunner.
runner = AllocationRunner(measurement_runner=MeasurementRunner(
node=Mock(spec=MesosNode, get_tasks=Mock(return_value=[])),
metrics_storage=Mock(spec=storage.Storage, store=Mock()),
rdt_enabled=True,
gather_hw_mm_topology=False,
extra_labels=dict(extra_labels='extra_value'),
),
anomalies_storage=Mock(spec=storage.Storage,
store=Mock()),
allocations_storage=Mock(spec=storage.Storage,
store=Mock()),
rdt_mb_control_required=True,
rdt_cache_control_required=True,
allocator=Mock(spec=Allocator,
allocate=Mock(return_value=({},
[],
[]))))
runner._measurement_runner._wait = Mock()
runner._measurement_runner._initialize()
############
# First run (one task, one allocation).
runner._measurement_runner._node.get_tasks.return_value = [t1]
runner._allocator.allocate.return_value = ({
t1.task_id: {
AllocationType.QUOTA: .5,
AllocationType.RDT: RDTAllocation(name=None, l3='L3:0=0000f')
}
}, [], [])
runner._measurement_runner._iterate()
# Check that allocator.allocate was called with proper arguments.
assert runner._allocator.allocate.call_count == 1
(_, tasks_data) = runner._allocator.allocate.mock_calls[0][1]
assert_subdict(tasks_data[t1.task_id].allocations, _os_tasks_allocations)
# Check allocation metrics ...
got_allocations_metrics = runner._allocations_storage.store.call_args[0][0]
# ... generic allocation metrics ...
assert_metric(got_allocations_metrics,
'allocations_count',
dict(extra_labels='extra_value'),
expected_metric_value=1)
assert_metric(got_allocations_metrics,
'allocations_errors',
dict(extra_labels='extra_value'),
expected_metric_value=0)
assert_metric(got_allocations_metrics, 'allocation_duration',
dict(extra_labels='extra_value'))
# ... and allocation metrics for task t1.
assert_metric(got_allocations_metrics, 'allocation_cpu_quota',
dict(task=t1.task_id, extra_labels='extra_value'), 0.5)
assert_metric(got_allocations_metrics, 'allocation_rdt_l3_cache_ways',
dict(task=t1.task_id, extra_labels='extra_value'), 4)
assert_metric(got_allocations_metrics, 'allocation_rdt_l3_mask',
dict(task=t1.task_id, extra_labels='extra_value'), 15)
############################
# Second run (two tasks, one allocation)
runner._measurement_runner._node.get_tasks.return_value = [t1, t2]
first_run_t1_task_allocations = {
t1.task_id: {
AllocationType.QUOTA: .5,
AllocationType.RDT: RDTAllocation(name=None, l3='L3:0=0000f')
}
}
runner._allocator.allocate.return_value = (first_run_t1_task_allocations,
[], [])
runner._measurement_runner._iterate()
# Check allocation metrics...
got_allocations_metrics = runner._allocations_storage.store.call_args[0][0]
# ... generic allocation metrics ...
assert_metric(got_allocations_metrics,
'allocations_count',
expected_metric_value=2)
assert_metric(got_allocations_metrics,
'allocations_errors',
expected_metric_value=0)
assert_metric(got_allocations_metrics, 'allocation_duration')
# ... and metrics for task t1 ...
assert_metric(got_allocations_metrics, 'allocation_cpu_quota',
dict(task=t1.task_id), 0.5)
assert_metric(got_allocations_metrics, 'allocation_rdt_l3_cache_ways',
dict(task=t1.task_id), 4)
assert_metric(got_allocations_metrics, 'allocation_rdt_l3_mask',
dict(task=t1.task_id), 15)
# Check allocate call.
(_, tasks_data) = runner._allocator.allocate.mock_calls[1][1]
# (note: tasks_allocations are always read from filesystem)
assert_subdict(tasks_data[t1.task_id].allocations, _os_tasks_allocations)
assert_subdict(tasks_data[t2.task_id].allocations, _os_tasks_allocations)
############
# Third run (two tasks, two allocations) - modify L3 cache and put in the same group
runner._measurement_runner._node.get_tasks.return_value = [t1, t2]
runner._allocator.allocate.return_value = \
{
t1.task_id: {
AllocationType.QUOTA: 0.7,
AllocationType.RDT: RDTAllocation(name='one_group', l3='L3:0=00fff')
},
t2.task_id: {
AllocationType.QUOTA: 0.8,
AllocationType.RDT: RDTAllocation(name='one_group', l3='L3:0=00fff')
}
}, [], []
runner._measurement_runner._iterate()
got_allocations_metrics = runner._allocations_storage.store.call_args[0][0]
assert_metric(got_allocations_metrics,
'allocations_count',
expected_metric_value=4)
# ... and metrics for task t1 ...
assert_metric(got_allocations_metrics, 'allocation_cpu_quota',
dict(task=t1.task_id), 0.7)
assert_metric(got_allocations_metrics, 'allocation_cpu_quota',
dict(task=t2.task_id), 0.8)
assert_metric(got_allocations_metrics, 'allocation_rdt_l3_cache_ways',
dict(task=t1.task_id, group_name='one_group'),
12) # 00fff=12
assert_metric(got_allocations_metrics, 'allocation_rdt_l3_cache_ways',
dict(task=t1.task_id, group_name='one_group'),
12) # 00fff=12
from unittest.mock import Mock, patch
import pytest
from wca import storage
from wca.allocators import AllocationType, RDTAllocation, Allocator
from wca.mesos import MesosNode
from wca.runners.allocation import AllocationRunner
from wca.runners.measurement import MeasurementRunner
from tests.testing import redis_task_with_default_labels,\
prepare_runner_patches, assert_subdict, assert_metric,\
platform_mock
# Patch Container get_allocations (simulate allocations read from OS filesystem)
_os_tasks_allocations = {
AllocationType.QUOTA: 1.,
AllocationType.RDT: RDTAllocation(name='', l3='L3:0=fffff', mb='MB:0=50')
}
@prepare_runner_patches
@patch('wca.cgroups.Cgroup.reset_counters')
@patch('wca.containers.Container.get_allocations',
return_value=_os_tasks_allocations)
@patch('wca.containers.ContainerSet.get_allocations',
return_value=_os_tasks_allocations)
@patch('wca.platforms.collect_platform_information',
return_value=(platform_mock, [], {}))
@pytest.mark.parametrize('subcgroups', ([], ['/T/c1'], ['/T/c1', '/T/c2']))
def test_allocation_runner(_get_allocations_mock, _get_allocations_mock_,
platform_mock, reset_counter_mock, subcgroups):
""" Low level system calls are not mocked - but higher level objects and functions:
from wca.runners.allocation import (TasksAllocationsValues,
TaskAllocationsValues,
AllocationRunner,
validate_shares_allocation_for_kubernetes,
_get_tasks_allocations)
from tests.testing import allocation_metric, task, container
from tests.testing import platform_mock
@pytest.mark.parametrize('tasks_allocations, expected_metrics', (
({}, []),
({'t1_task_id': {AllocationType.SHARES: 0.5}}, [
allocation_metric('cpu_shares', value=0.5,
container_name='t1', task='t1_task_id')
]),
({'t1_task_id': {AllocationType.RDT: RDTAllocation(mb='MB:0=20')}}, [
allocation_metric('rdt_mb', 20, group_name='t1', domain_id='0', container_name='t1',
task='t1_task_id')
]),
({'t1_task_id': {AllocationType.SHARES: 0.5,
AllocationType.RDT: RDTAllocation(mb='MB:0=20')}}, [
allocation_metric('cpu_shares', value=0.5, container_name='t1', task='t1_task_id'),
allocation_metric('rdt_mb', 20, group_name='t1', domain_id='0', container_name='t1',
task='t1_task_id')
]),
({'t1_task_id': {
AllocationType.SHARES: 0.5, AllocationType.RDT: RDTAllocation(
mb='MB:0=30', l3='L3:0=f')
},
't2_task_id': {
AllocationType.QUOTA: 0.6,
