Exemple #1
0
    def unpatched():

        if len(subcgroups_paths):
            platform = Platform(sockets=1,
                                cores=1,
                                cpus=2,
                                numa_nodes=2,
                                topology={0: {
                                    0: [1, 2]
                                }},
                                cpu_model='intel xeon',
                                cpu_model_number=0x5E,
                                cpu_codename=CPUCodeName.SKYLAKE,
                                timestamp=time.time(),
                                rdt_information=RDTInformation(
                                    True, True, rdt_mb_control_enabled,
                                    rdt_cache_control_enabled, '0', '0', 0, 0,
                                    0),
                                node_cpus={0: {0, 1}},
                                node_distances={0: {
                                    0: 10
                                }},
                                measurements={},
                                swap_enabled=False)
            return ContainerSet(
                cgroup_path=cgroup_path,
                cgroup_paths=subcgroups_paths,
                platform=platform,
                allocation_configuration=AllocationConfiguration()
                if with_config else None,
                resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None,
                event_names=['task_cycles'])
        else:
            platform = Platform(sockets=1,
                                cores=1,
                                cpus=2,
                                numa_nodes=2,
                                topology={0: {
                                    0: [1, 2]
                                }},
                                cpu_model='intel xeon',
                                cpu_model_number=0x5E,
                                cpu_codename=CPUCodeName.SKYLAKE,
                                timestamp=time.time(),
                                rdt_information=RDTInformation(
                                    True, True, True, True, '0', '0', 0, 0, 0),
                                node_cpus={0: {0, 1}},
                                node_distances={0: {
                                    0: 10
                                }},
                                measurements={},
                                swap_enabled=False)
            return Container(
                cgroup_path=cgroup_path,
                platform=platform,
                allocation_configuration=AllocationConfiguration()
                if with_config else None,
                resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None,
                event_names=['task_cycles'],
            )
 def unpatched():
     if len(subcgroups_paths):
         return ContainerSet(
             cgroup_path=cgroup_path,
             cgroup_paths=subcgroups_paths,
             platform_cpus=1,
             platform_sockets=1,
             allocation_configuration=AllocationConfiguration()
             if with_config else None,
             resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None,
             rdt_information=RDTInformation(True, True,
                                            rdt_mb_control_enabled,
                                            rdt_cache_control_enabled, '0',
                                            '0', 0, 0, 0),
             event_names=DEFAULT_EVENTS)
     else:
         return Container(
             cgroup_path=cgroup_path,
             platform_cpus=1,
             platform_sockets=1,
             rdt_information=RDTInformation(True, True, True, True, '0',
                                            '0', 0, 0, 0),
             allocation_configuration=AllocationConfiguration()
             if with_config else None,
             resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None,
             event_names=DEFAULT_EVENTS)
Exemple #3
0
    def unpatched():

        if len(subcgroups_paths):
            platform = Platform(sockets=1,
                                cores=1,
                                cpus=2,
                                cpu_model='intel xeon',
                                cpu_model_number=0x5E,
                                cpu_codename=CPUCodeName.SKYLAKE,
                                cpus_usage={
                                    0: 10,
                                    1: 10
                                },
                                total_memory_used=10,
                                timestamp=time.time(),
                                rdt_information=RDTInformation(
                                    True, True, rdt_mb_control_enabled,
                                    rdt_cache_control_enabled, '0', '0', 0, 0,
                                    0))
            return ContainerSet(
                cgroup_path=cgroup_path,
                cgroup_paths=subcgroups_paths,
                platform=platform,
                allocation_configuration=AllocationConfiguration()
                if with_config else None,
                resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None,
                event_names=DEFAULT_EVENTS)
        else:
            platform = Platform(sockets=1,
                                cores=1,
                                cpus=2,
                                cpu_model='intel xeon',
                                cpu_model_number=0x5E,
                                cpu_codename=CPUCodeName.SKYLAKE,
                                cpus_usage={
                                    0: 10,
                                    1: 10
                                },
                                total_memory_used=10,
                                timestamp=time.time(),
                                rdt_information=RDTInformation(
                                    True, True, True, True, '0', '0', 0, 0, 0))
            return Container(
                cgroup_path=cgroup_path,
                platform=platform,
                allocation_configuration=AllocationConfiguration()
                if with_config else None,
                resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None,
                event_names=DEFAULT_EVENTS)
def test_migrate_pages_raise_exception_when_swap_is_enabled(*mocks):
    rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0)

    platform_mock = Mock(
        spec=Platform,
        cpus=10,
        sockets=1,
        rdt_information=rdt_information,
        node_cpus={
            0: [0, 1],
            1: [2, 3]
        },
        numa_nodes=2,
        swap_enabled=is_swap_enabled(),
    )

    foo_container = Container('/somepath', platform=platform_mock)

    foo_container._cgroup.platform = platform_mock

    migrate_pages = MigratePagesAllocationValue(0, foo_container,
                                                dict(foo='bar'))

    with pytest.raises(
            InvalidAllocations,
            match=
            "Swap should be disabled due to possibility of OOM killer occurrence!"
    ):
        migrate_pages.validate()
def test_collect_platform_information(*mocks):
    got_platform, got_metrics, got_labels = collect_platform_information(
        include_optional_labels=True
    )

    assert got_platform == Platform(
        sockets=1,
        cores=1,
        cpus=2,
        numa_nodes=1,
        topology={},
        cpu_model='intel xeon',
        cpu_model_number=0x5E,
        cpu_codename=CPUCodeName.SKYLAKE,
        timestamp=1536071557.123456,  # timestamp,
        node_cpus={},
        node_distances={},
        rdt_information=RDTInformation(True, True, True, True, 'fffff', '2', 8, 10, 20),
        measurements={MetricName.PLATFORM_CPU_USAGE: {0: 100, 1: 200},
                      MetricName.PLATFORM_MEM_USAGE_BYTES: 1337,
                      MetricName.PLATFORM_MEM_NUMA_FREE_BYTES: {0: 1},
                      MetricName.PLATFORM_MEM_NUMA_USED_BYTES: {0: 2},
                      MetricName.PLATFORM_VMSTAT_NUMA_PAGES_MIGRATED: 5,
                      },
        swap_enabled=False
    )

    assert_metric(got_metrics, MetricName.PLATFORM_MEM_USAGE_BYTES, expected_metric_value=1337)
    assert_metric(got_metrics, MetricName.PLATFORM_CPU_USAGE, {'cpu': '0'},
                  expected_metric_value=100)
    assert_metric(got_metrics, MetricName.PLATFORM_TOPOLOGY_CORES, expected_metric_value=1)
    assert_metric(got_metrics, MetricName.PLATFORM_VMSTAT_NUMA_PAGES_MIGRATED,
                  expected_metric_value=5)
    assert got_labels == {"sockets": "1", "cores": "1", "cpus": "2", "host": "test_host",
                          "wca_version": "0.1", "cpu_model": "intel xeon"}
def test_prepare_task_data_cgroup_not_found(*mocks):
    rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0)
    containers = {
        task('/t1', labels={'label_key': 'label_value'}, resources={'cpu': 3}):
        Container('/t1', 1, 1, rdt_information)
    }
    tasks_measurements, tasks_resources, tasks_labels = \
        _prepare_tasks_data(containers)
    assert tasks_measurements == {}
def test_get_tasks_allocations_fail(*mock):
    rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0)
    containers = {
        task('/t1', labels={'label_key': 'label_value'}, resources={'cpu': 3}):
            Container('/t1', 1, 1, rdt_information,
                      allocation_configuration=AllocationConfiguration(
                          cpu_quota_period=1000))
    }
    assert {} == _get_tasks_allocations(containers)
 def rdt_allocation_value_constructor(allocation_value, container, common_labels):
     rdt_information = RDTInformation(
         False, False, False, False, 'fff', '1', 0, 0, 0)
     return RDTAllocationValue('c1', allocation_value, CgroupMock(), ResGroupMock(),
                               platform_sockets=1,
                               rdt_information=rdt_information,
                               rdt_groups=rdt_groups,
                               common_labels=common_labels,
                               )
def test_rdt_allocation_generate_metrics(rdt_allocation: RDTAllocation,
                                         extra_labels, expected_metrics):
    rdt_information = RDTInformation(True, True, True, True, 'fff', '1', 0, 0, 0)
    rdt_allocation_value = RDTAllocationValue(
        'c1',
        rdt_allocation, get_pids=lambda: [],
        resgroup=ResGroup(name=rdt_allocation.name or ''),
        platform_sockets=1, rdt_information=rdt_information,
        common_labels=extra_labels,
        rdt_groups=RDTGroups(10),
    )
    got_metrics = rdt_allocation_value.generate_metrics()
    assert got_metrics == expected_metrics
def test_prepare_tasks_data(*mocks):
    rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0)
    containers = {
        task('/t1', labels={'label_key': 'label_value'}, resources={'cpu': 3}):
        Container('/t1', 1, 1, rdt_information)
    }

    tasks_measurements, tasks_resources, tasks_labels = _prepare_tasks_data(
        containers)

    assert tasks_measurements == {'t1_task_id': {'cpu_usage': 13}}
    assert tasks_resources == {'t1_task_id': {'cpu': 3}}
    assert tasks_labels == {'t1_task_id': {'label_key': 'label_value'}}
 def convert(rdt_allocation):
     rdt_information = RDTInformation(False, False, False, False, 'fffff', '1', 0, 0, 0)
     if rdt_allocation is not None:
         return RDTAllocationValue(container_name,
                                   rdt_allocation,
                                   resgroup,
                                   lambda: ['1'],
                                   platform_sockets=1,
                                   rdt_information=rdt_information,
                                   rdt_groups=rdt_groups,
                                   common_labels={},
                                   )
     else:
         return None
Exemple #12
0
def test_cpuset_for_container_set():
    rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0)

    platform_mock = Mock(
        spec=Platform,
        cpus=10,
        sockets=1,
        rdt_information=rdt_information,
        node_cpus={
            0: [0, 1],
            1: [2, 3]
        },
        numa_nodes=2,
        swap_enabled=False,
    )

    foo_container_set = ContainerSet(
        cgroup_path='/foo',
        cgroup_paths=['/foo/bar-1', '/foo/bar-2', '/foo/bar-3'],
        platform=platform_mock,
    )

    cgroup = foo_container_set.get_cgroup()
    cgroup.set_cpuset_cpus = Mock()
    cgroup.set_cpuset_mems = Mock()

    for subcgroup in foo_container_set.get_subcgroups():
        subcgroup.set_cpuset_cpus = Mock()
        subcgroup.set_cpuset_mems = Mock()

    cpuset_cpus = CPUSetCPUSAllocationValue('0-2', foo_container_set, {})
    cpuset_cpus.perform_allocations()

    cpuset_mems = CPUSetMEMSAllocationValue('0-1', foo_container_set, {})
    cpuset_mems.perform_allocations()

    # Cgroup shouldn't be affected.
    cgroup.set_cpuset_cpus.assert_not_called()
    cgroup.set_cpuset_mems.assert_not_called()

    # Subcgroups should change cpuset param.
    for subcgroup in foo_container_set.get_subcgroups():
        subcgroup.set_cpuset_cpus.assert_called_once_with({0, 1, 2})
        subcgroup.set_cpuset_mems.assert_called_once_with({0, 1})
def test_cgroup_allocations(Cgroup_mock, PerfCounters_mock):
    rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0)
    platform_mock = Mock(spec=Platform,
                         sockets=1,
                         cpus=1,
                         rdt_information=rdt_information)

    foo_container = Container('/somepath', platform=platform_mock)
    foo_container._cgroup.allocation_configuration = AllocationConfiguration()
    foo_container._cgroup.platform_cpus = 10
    foo_container._cgroup.platform_sockets = 1

    quota_allocation_value = QuotaAllocationValue(0.2, foo_container,
                                                  dict(foo='bar'))
    quota_allocation_value.perform_allocations()
    assert quota_allocation_value.generate_metrics() == [
        allocation_metric('cpu_quota', 0.2, foo='bar')
    ]

    shares_allocation_value = SharesAllocationValue(0.5, foo_container,
                                                    dict(foo='bar'))
    shares_allocation_value.perform_allocations()

    assert shares_allocation_value.generate_metrics() == [
        allocation_metric('cpu_shares', 0.5, foo='bar')
    ]

    cpuset_allocation_value = CPUSetAllocationValue('0-2,4,6-8', foo_container,
                                                    dict(foo='bar'))
    cpuset_allocation_value.perform_allocations()

    assert cpuset_allocation_value.generate_metrics() == [
        allocation_metric('cpuset', [0, 1, 2, 4, 6, 7, 8], foo='bar')
    ]

    Cgroup_mock.assert_has_calls([
        call().set_quota(0.2),
        call().set_shares(0.5),
        call().set_cpuset('0,1,2,4,6,7,8', '0')
    ], True)
Exemple #14
0
platform_mock = Mock(spec=Platform,
                     cpus=4,
                     sockets=1,
                     topology={0: {
                         0: [0, 1],
                         1: [2, 3]
                     }},
                     numa_nodes=2,
                     cpu_codename=CPUCodeName.SKYLAKE,
                     rdt_information=RDTInformation(
                         cbm_mask='fffff',
                         min_cbm_bits='1',
                         rdt_cache_monitoring_enabled=True,
                         rdt_cache_control_enabled=True,
                         rdt_mb_monitoring_enabled=True,
                         rdt_mb_control_enabled=True,
                         num_closids=2,
                         mb_bandwidth_gran=0,
                         mb_min_bandwidth=0,
                     ))


def redis_task_with_default_labels(task_id, subcgroups_paths=None):
    """Returns task instance and its labels."""
    if subcgroups_paths is None:
        subcgroups_paths = []
    task_labels = {
        'load_generator': 'rpc-perf-%s' % task_id,
        'name': 'redis-6792-%s' % task_id,
        'task_name': 'example/root/staging/redis-6792',
def test_cgroup_allocations(Cgroup_mock, PerfCounters_mock):
    rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0)

    platform_mock = Mock(
        spec=Platform,
        cpus=10,
        sockets=1,
        rdt_information=rdt_information,
        node_cpus={
            0: [0, 1],
            1: [2, 3]
        },
    )

    foo_container = Container('/somepath', platform=platform_mock)
    foo_container._cgroup.allocation_configuration = AllocationConfiguration()
    foo_container._cgroup.platform = platform_mock

    quota_allocation_value = QuotaAllocationValue(0.2, foo_container,
                                                  dict(foo='bar'))
    quota_allocation_value.perform_allocations()
    assert quota_allocation_value.generate_metrics() == [
        allocation_metric('cpu_quota', 0.2, foo='bar')
    ]

    shares_allocation_value = SharesAllocationValue(0.5, foo_container,
                                                    dict(foo='bar'))
    shares_allocation_value.perform_allocations()

    assert shares_allocation_value.generate_metrics() == [
        allocation_metric('cpu_shares', 0.5, foo='bar')
    ]

    cpuset_cpus_allocation_value = CPUSetCPUSAllocationValue(
        '0-2,4,6-8', foo_container, dict(foo='bar'))
    cpuset_cpus_allocation_value.perform_allocations()

    cpuset_mems_allocation_value = CPUSetMEMSAllocationValue(
        '0-1', foo_container, dict(foo='bar'))
    cpuset_mems_allocation_value.perform_allocations()

    assert cpuset_cpus_allocation_value.generate_metrics() == [
        Metric(name='allocation_cpuset_cpus_number_of_cpus',
               value=7,
               labels={
                   'allocation_type': AllocationType.CPUSET_CPUS,
                   'foo': 'bar'
               },
               type=MetricType.GAUGE)
    ]

    assert cpuset_mems_allocation_value.generate_metrics() == [
        Metric(name='allocation_cpuset_mems_number_of_mems',
               value=2,
               labels={
                   'allocation_type': AllocationType.CPUSET_MEMS,
                   'foo': 'bar'
               },
               type=MetricType.GAUGE)
    ]

    Cgroup_mock.assert_has_calls([
        call().set_quota(0.2),
        call().set_shares(0.5),
        call().set_cpuset_cpus({0, 1, 2, 4, 6, 7, 8}),
        call().set_cpuset_mems({0, 1})
    ], True)
Exemple #16
0
def test_sync_containers_state(_, get_pids_mock, sync_mock, perf_counters_mock,
                               add_pids_mock, clean_taskless_groups_mock,
                               subcgroups,
                               tasks_, pre_running_containers_,
                               mon_groups_relation, expected_running_containers_,
                               labels_relation_, pre_running_labels_relation_):
    """Tests both Container and ContainerSet classes.

        Note: the input arguments tasks_, existing_containers_, expected_running_containers_
        contain in their names underscore at the end to distinguish them from the ones
        created inside the function body to emphasize the relationship: the input arguments
        is used to create real objects. We cannot pass already created objects, as to
        create them we need another argument from first of two paramatrize decorators:
        subcgroups.

        Note: we have three variables names with the same postfix:
        * pre_running_containers - state of ContainerManager before (pre) call sync_containers_state
        * expected_running_containers - similar as above but state expected after the call,
        * got_running_containers - similar as above but state which we got after the call.
        All of three are of the same type Dict[Task, ContainerInterface].
        """
    # Create Task and Container/ContainerSet objects from input arguments.
    #   This is done to both test Container and ContainerSet classes (to pass
    #   subcgroups argument into the constructing function >>container<<.
    tasks = [task(t, subcgroups_paths=subcgroups, labels=labels_relation_.get(t)) for t in tasks_]
    pre_running_containers = \
        {task(t, subcgroups_paths=subcgroups,
              labels=pre_running_labels_relation_.get(t)): container(c, subcgroups)
         for t, c in pre_running_containers_.items()}

    expected_running_containers = \
        {task(t, subcgroups_paths=subcgroups,
              labels=labels_relation_.get(t)): container(c, subcgroups)
         for t, c in expected_running_containers_.items()}

    rdt_information = RDTInformation(True, True, True, True, 'fff', '1', 0, 0, 0)
    platform_mock = Mock(
        spec=Platform,
        sockets=1,
        cores=1,
        cpus=1,
        rdt_information=rdt_information)

    containers_manager = ContainerManager(platform=platform_mock,
                                          allocation_configuration=AllocationConfiguration(),
                                          event_names=[],
                                          )
    # Put in into ContainerManager our input dict of containers.
    containers_manager.containers = dict(pre_running_containers)

    # Call sync_containers_state
    with patch('wca.resctrl.read_mon_groups_relation', return_value=mon_groups_relation):
        got_running_containers = containers_manager.sync_containers_state(tasks)

    # -----------------------
    # Assert that two sets of keys of two dictionaries got_containers and
    # expected_running_containers are equal.
    assert len(got_running_containers) == len(expected_running_containers)
    assert all([expected_task in got_running_containers
                for expected_task in expected_running_containers.keys()])
    for t in expected_running_containers.keys():
        assert_equal_containers(expected_running_containers[t], got_running_containers[t])

    # Check container objects has proper resgroup assigned.
    got_container_resgroup_names = {c.get_name():
                                    c.get_resgroup().name for c in got_running_containers.values()}
    for expected_resgroup_name, container_names in mon_groups_relation.items():
        for container_name in container_names:
            if container_name in got_container_resgroup_names:
                got_resgroup_name = got_container_resgroup_names.get(container_name)
                assert got_resgroup_name == expected_resgroup_name