def task_data(cgroup_path, subcgroups_paths=None, labels=None,
resources=None, measurements=None, allocations=None):
"""Helper method to create task with default values."""
prefix = cgroup_path.replace('/', '')
return TaskData(
name=prefix + '_tasks_name',
task_id=prefix + '_task_id',
cgroup_path=cgroup_path,
subcgroups_paths=subcgroups_paths or [],
labels=labels or {},
resources=resources or {},
measurements=measurements or {},
allocations=allocations or {}
)
def test_prepare_tasks_data(*mocks):
t = task('/t1', labels={'label_key': 'label_value'}, resources={'cpu': 3})
containers = {t: Container('/t1', platform_mock)}
tasks_data = _prepare_tasks_data(containers)
assert tasks_data == {
't1_task_id':
TaskData(t.name, t.task_id, t.cgroup_path, t.subcgroups_paths,
t.labels, t.resources, {
'task_up': 1,
'task_last_seen': 12345.6,
'task_cpu_usage_seconds': 13
})
}
def _prepare_tasks_data(containers: Dict[Task, Container]) -> TasksData:
"""Prepare all resource usage and resource allocation information and
creates container-specific labels for all the generated metrics.
"""
# Prepare empty structure for return all the information.
tasks_data: TasksData = {}
now = time.time()
for task, container in containers.items():
# Task measurements and measurements based metrics.
try:
task_measurements = container.get_measurements()
except MissingMeasurementException as e:
log.warning('One or more measurements are missing '
'for container {} - ignoring! '
'(because {})'.format(container, e))
raise
# Extra internal metrics
task_measurements[MetricName.TASK_UP.value] = 1
task_measurements[MetricName.TASK_LAST_SEEN.value] = now
task_measurements[MetricName.TASK_SUBCONTAINERS.value] = len(task.subcgroups_paths)
# Extra metrics from orchestrator about resources
if TaskResource.CPUS in task.resources:
task_measurements[MetricName.TASK_REQUESTED_CPUS.value] = task.resources[
TaskResource.CPUS.value]
if TaskResource.MEM in task.resources:
task_measurements[MetricName.TASK_REQUESTED_MEM_BYTES.value] = task.resources[
TaskResource.MEM.value]
tasks_data[task.task_id] = TaskData(
name=task.name,
task_id=task.task_id,
cgroup_path=task.cgroup_path,
subcgroups_paths=task.subcgroups_paths,
labels=task.labels,
resources=task.resources,
measurements=task_measurements
)
return tasks_data
def prepare_input(tasks: Dict[TaskId, Dict[NumaNodeId, PercentageMemUsage]],
numa_nodes: int) -> Tuple[Platform, TasksData]:
"""if len(tasks[task]) == 1, it means that task is pinned to that single numa node"""
assert numa_nodes > 1, 'numa nodes must be greater than 1'
print_structures: bool = False
node_size = 96 * GB
node_cpu = 10
node_size_pages = node_size / get_page_size()
cp_memory_per_node_percentage = 0.04
tasks_data: TasksData = dict()
for task_name, numa_memory in tasks.items():
measurements = dict()
measurements[MetricName.TASK_MEM_NUMA_PAGES] = \
{numa_id: int(v * node_size_pages) for numa_id, v in numa_memory.items()}
data = TaskData(
name=task_name,
task_id=task_name,
cgroup_path='',
subcgroups_paths=[''],
labels={'uid': task_name},
resources={'mem': int(sum(numa_memory.values()) * node_size)},
measurements=measurements)
tasks_data[task_name] = data
if print_structures:
pprint(tasks_data)
def node_cpus(numa_nodes):
return {
i: set(range(i * node_cpu, (i + 1) * node_cpu))
for i in range(numa_nodes)
}
platform_mock = Mock(spec=Platform,
cpus=2 * node_cpu,
sockets=numa_nodes,
node_cpus=node_cpus(numa_nodes),
topology={},
numa_nodes=numa_nodes,
cpu_codename=None)
if print_structures:
pprint(platform_mock.topology)
def empty_measurements():
return {v: {} for v in range(numa_nodes)}
platform_mock.measurements = {
MetricName.PLATFORM_MEM_NUMA_FREE_BYTES: empty_measurements(),
MetricName.PLATFORM_MEM_NUMA_USED_BYTES: empty_measurements()
}
# Only percentage first
for numa_node in range(numa_nodes):
platform_mock.measurements[MetricName.PLATFORM_MEM_NUMA_FREE_BYTES][numa_node] = \
(1.0 - cp_memory_per_node_percentage - sum(
[memory.get(numa_node, 0) for memory in tasks.values()]))
if print_structures:
pprint(platform_mock.measurements)
# Multiply by node_size
for numa_node in range(numa_nodes):
platform_mock.measurements[MetricName.PLATFORM_MEM_NUMA_FREE_BYTES][numa_node] = \
int(platform_mock.measurements[MetricName.PLATFORM_MEM_NUMA_FREE_BYTES][
numa_node] * node_size)
platform_mock.measurements[MetricName.PLATFORM_MEM_NUMA_USED_BYTES][numa_node] = \
node_size - platform_mock.measurements[MetricName.PLATFORM_MEM_NUMA_FREE_BYTES][
numa_node]
if print_structures:
pprint(platform_mock.measurements)
# Add allocations to tasks_data[task]
for task_name, numa_memory in tasks.items():
# if len(..) == 1, then pinned, just to shorten notation
if len(numa_memory.keys()) == 1:
tasks_data[task_name].allocations = \
{AllocationType.CPUSET_CPUS: ','.join(
map(str, platform_mock.node_cpus[list(numa_memory.keys())[0]]))}
# cpuset_cpus no pinned, means pinned to all available cpus
else:
tasks_data[task_name].allocations = \
{AllocationType.CPUSET_CPUS: ','.join(map(str, range(numa_nodes * node_cpu)))}
if print_structures:
pprint(tasks_data.allocations)
return platform_mock, tasks_data