def unpatched(): if len(subcgroups_paths): platform = Platform(sockets=1, cores=1, cpus=2, numa_nodes=2, topology={0: { 0: [1, 2] }}, cpu_model='intel xeon', cpu_model_number=0x5E, cpu_codename=CPUCodeName.SKYLAKE, timestamp=time.time(), rdt_information=RDTInformation( True, True, rdt_mb_control_enabled, rdt_cache_control_enabled, '0', '0', 0, 0, 0), node_cpus={0: {0, 1}}, node_distances={0: { 0: 10 }}, measurements={}, swap_enabled=False) return ContainerSet( cgroup_path=cgroup_path, cgroup_paths=subcgroups_paths, platform=platform, allocation_configuration=AllocationConfiguration() if with_config else None, resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None, event_names=['task_cycles']) else: platform = Platform(sockets=1, cores=1, cpus=2, numa_nodes=2, topology={0: { 0: [1, 2] }}, cpu_model='intel xeon', cpu_model_number=0x5E, cpu_codename=CPUCodeName.SKYLAKE, timestamp=time.time(), rdt_information=RDTInformation( True, True, True, True, '0', '0', 0, 0, 0), node_cpus={0: {0, 1}}, node_distances={0: { 0: 10 }}, measurements={}, swap_enabled=False) return Container( cgroup_path=cgroup_path, platform=platform, allocation_configuration=AllocationConfiguration() if with_config else None, resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None, event_names=['task_cycles'], )
def unpatched(): if len(subcgroups_paths): return ContainerSet( cgroup_path=cgroup_path, cgroup_paths=subcgroups_paths, platform_cpus=1, platform_sockets=1, allocation_configuration=AllocationConfiguration() if with_config else None, resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None, rdt_information=RDTInformation(True, True, rdt_mb_control_enabled, rdt_cache_control_enabled, '0', '0', 0, 0, 0), event_names=DEFAULT_EVENTS) else: return Container( cgroup_path=cgroup_path, platform_cpus=1, platform_sockets=1, rdt_information=RDTInformation(True, True, True, True, '0', '0', 0, 0, 0), allocation_configuration=AllocationConfiguration() if with_config else None, resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None, event_names=DEFAULT_EVENTS)
def unpatched(): if len(subcgroups_paths): platform = Platform(sockets=1, cores=1, cpus=2, cpu_model='intel xeon', cpu_model_number=0x5E, cpu_codename=CPUCodeName.SKYLAKE, cpus_usage={ 0: 10, 1: 10 }, total_memory_used=10, timestamp=time.time(), rdt_information=RDTInformation( True, True, rdt_mb_control_enabled, rdt_cache_control_enabled, '0', '0', 0, 0, 0)) return ContainerSet( cgroup_path=cgroup_path, cgroup_paths=subcgroups_paths, platform=platform, allocation_configuration=AllocationConfiguration() if with_config else None, resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None, event_names=DEFAULT_EVENTS) else: platform = Platform(sockets=1, cores=1, cpus=2, cpu_model='intel xeon', cpu_model_number=0x5E, cpu_codename=CPUCodeName.SKYLAKE, cpus_usage={ 0: 10, 1: 10 }, total_memory_used=10, timestamp=time.time(), rdt_information=RDTInformation( True, True, True, True, '0', '0', 0, 0, 0)) return Container( cgroup_path=cgroup_path, platform=platform, allocation_configuration=AllocationConfiguration() if with_config else None, resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None, event_names=DEFAULT_EVENTS)
def test_migrate_pages_raise_exception_when_swap_is_enabled(*mocks): rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0) platform_mock = Mock( spec=Platform, cpus=10, sockets=1, rdt_information=rdt_information, node_cpus={ 0: [0, 1], 1: [2, 3] }, numa_nodes=2, swap_enabled=is_swap_enabled(), ) foo_container = Container('/somepath', platform=platform_mock) foo_container._cgroup.platform = platform_mock migrate_pages = MigratePagesAllocationValue(0, foo_container, dict(foo='bar')) with pytest.raises( InvalidAllocations, match= "Swap should be disabled due to possibility of OOM killer occurrence!" ): migrate_pages.validate()
def test_collect_platform_information(*mocks): got_platform, got_metrics, got_labels = collect_platform_information( include_optional_labels=True ) assert got_platform == Platform( sockets=1, cores=1, cpus=2, numa_nodes=1, topology={}, cpu_model='intel xeon', cpu_model_number=0x5E, cpu_codename=CPUCodeName.SKYLAKE, timestamp=1536071557.123456, # timestamp, node_cpus={}, node_distances={}, rdt_information=RDTInformation(True, True, True, True, 'fffff', '2', 8, 10, 20), measurements={MetricName.PLATFORM_CPU_USAGE: {0: 100, 1: 200}, MetricName.PLATFORM_MEM_USAGE_BYTES: 1337, MetricName.PLATFORM_MEM_NUMA_FREE_BYTES: {0: 1}, MetricName.PLATFORM_MEM_NUMA_USED_BYTES: {0: 2}, MetricName.PLATFORM_VMSTAT_NUMA_PAGES_MIGRATED: 5, }, swap_enabled=False ) assert_metric(got_metrics, MetricName.PLATFORM_MEM_USAGE_BYTES, expected_metric_value=1337) assert_metric(got_metrics, MetricName.PLATFORM_CPU_USAGE, {'cpu': '0'}, expected_metric_value=100) assert_metric(got_metrics, MetricName.PLATFORM_TOPOLOGY_CORES, expected_metric_value=1) assert_metric(got_metrics, MetricName.PLATFORM_VMSTAT_NUMA_PAGES_MIGRATED, expected_metric_value=5) assert got_labels == {"sockets": "1", "cores": "1", "cpus": "2", "host": "test_host", "wca_version": "0.1", "cpu_model": "intel xeon"}
def test_prepare_task_data_cgroup_not_found(*mocks): rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0) containers = { task('/t1', labels={'label_key': 'label_value'}, resources={'cpu': 3}): Container('/t1', 1, 1, rdt_information) } tasks_measurements, tasks_resources, tasks_labels = \ _prepare_tasks_data(containers) assert tasks_measurements == {}
def test_get_tasks_allocations_fail(*mock): rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0) containers = { task('/t1', labels={'label_key': 'label_value'}, resources={'cpu': 3}): Container('/t1', 1, 1, rdt_information, allocation_configuration=AllocationConfiguration( cpu_quota_period=1000)) } assert {} == _get_tasks_allocations(containers)
def rdt_allocation_value_constructor(allocation_value, container, common_labels): rdt_information = RDTInformation( False, False, False, False, 'fff', '1', 0, 0, 0) return RDTAllocationValue('c1', allocation_value, CgroupMock(), ResGroupMock(), platform_sockets=1, rdt_information=rdt_information, rdt_groups=rdt_groups, common_labels=common_labels, )
def test_rdt_allocation_generate_metrics(rdt_allocation: RDTAllocation, extra_labels, expected_metrics): rdt_information = RDTInformation(True, True, True, True, 'fff', '1', 0, 0, 0) rdt_allocation_value = RDTAllocationValue( 'c1', rdt_allocation, get_pids=lambda: [], resgroup=ResGroup(name=rdt_allocation.name or ''), platform_sockets=1, rdt_information=rdt_information, common_labels=extra_labels, rdt_groups=RDTGroups(10), ) got_metrics = rdt_allocation_value.generate_metrics() assert got_metrics == expected_metrics
def test_prepare_tasks_data(*mocks): rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0) containers = { task('/t1', labels={'label_key': 'label_value'}, resources={'cpu': 3}): Container('/t1', 1, 1, rdt_information) } tasks_measurements, tasks_resources, tasks_labels = _prepare_tasks_data( containers) assert tasks_measurements == {'t1_task_id': {'cpu_usage': 13}} assert tasks_resources == {'t1_task_id': {'cpu': 3}} assert tasks_labels == {'t1_task_id': {'label_key': 'label_value'}}
def convert(rdt_allocation): rdt_information = RDTInformation(False, False, False, False, 'fffff', '1', 0, 0, 0) if rdt_allocation is not None: return RDTAllocationValue(container_name, rdt_allocation, resgroup, lambda: ['1'], platform_sockets=1, rdt_information=rdt_information, rdt_groups=rdt_groups, common_labels={}, ) else: return None
def test_cpuset_for_container_set(): rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0) platform_mock = Mock( spec=Platform, cpus=10, sockets=1, rdt_information=rdt_information, node_cpus={ 0: [0, 1], 1: [2, 3] }, numa_nodes=2, swap_enabled=False, ) foo_container_set = ContainerSet( cgroup_path='/foo', cgroup_paths=['/foo/bar-1', '/foo/bar-2', '/foo/bar-3'], platform=platform_mock, ) cgroup = foo_container_set.get_cgroup() cgroup.set_cpuset_cpus = Mock() cgroup.set_cpuset_mems = Mock() for subcgroup in foo_container_set.get_subcgroups(): subcgroup.set_cpuset_cpus = Mock() subcgroup.set_cpuset_mems = Mock() cpuset_cpus = CPUSetCPUSAllocationValue('0-2', foo_container_set, {}) cpuset_cpus.perform_allocations() cpuset_mems = CPUSetMEMSAllocationValue('0-1', foo_container_set, {}) cpuset_mems.perform_allocations() # Cgroup shouldn't be affected. cgroup.set_cpuset_cpus.assert_not_called() cgroup.set_cpuset_mems.assert_not_called() # Subcgroups should change cpuset param. for subcgroup in foo_container_set.get_subcgroups(): subcgroup.set_cpuset_cpus.assert_called_once_with({0, 1, 2}) subcgroup.set_cpuset_mems.assert_called_once_with({0, 1})
def test_cgroup_allocations(Cgroup_mock, PerfCounters_mock): rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0) platform_mock = Mock(spec=Platform, sockets=1, cpus=1, rdt_information=rdt_information) foo_container = Container('/somepath', platform=platform_mock) foo_container._cgroup.allocation_configuration = AllocationConfiguration() foo_container._cgroup.platform_cpus = 10 foo_container._cgroup.platform_sockets = 1 quota_allocation_value = QuotaAllocationValue(0.2, foo_container, dict(foo='bar')) quota_allocation_value.perform_allocations() assert quota_allocation_value.generate_metrics() == [ allocation_metric('cpu_quota', 0.2, foo='bar') ] shares_allocation_value = SharesAllocationValue(0.5, foo_container, dict(foo='bar')) shares_allocation_value.perform_allocations() assert shares_allocation_value.generate_metrics() == [ allocation_metric('cpu_shares', 0.5, foo='bar') ] cpuset_allocation_value = CPUSetAllocationValue('0-2,4,6-8', foo_container, dict(foo='bar')) cpuset_allocation_value.perform_allocations() assert cpuset_allocation_value.generate_metrics() == [ allocation_metric('cpuset', [0, 1, 2, 4, 6, 7, 8], foo='bar') ] Cgroup_mock.assert_has_calls([ call().set_quota(0.2), call().set_shares(0.5), call().set_cpuset('0,1,2,4,6,7,8', '0') ], True)
platform_mock = Mock(spec=Platform, cpus=4, sockets=1, topology={0: { 0: [0, 1], 1: [2, 3] }}, numa_nodes=2, cpu_codename=CPUCodeName.SKYLAKE, rdt_information=RDTInformation( cbm_mask='fffff', min_cbm_bits='1', rdt_cache_monitoring_enabled=True, rdt_cache_control_enabled=True, rdt_mb_monitoring_enabled=True, rdt_mb_control_enabled=True, num_closids=2, mb_bandwidth_gran=0, mb_min_bandwidth=0, )) def redis_task_with_default_labels(task_id, subcgroups_paths=None): """Returns task instance and its labels.""" if subcgroups_paths is None: subcgroups_paths = [] task_labels = { 'load_generator': 'rpc-perf-%s' % task_id, 'name': 'redis-6792-%s' % task_id, 'task_name': 'example/root/staging/redis-6792',
def test_cgroup_allocations(Cgroup_mock, PerfCounters_mock): rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0) platform_mock = Mock( spec=Platform, cpus=10, sockets=1, rdt_information=rdt_information, node_cpus={ 0: [0, 1], 1: [2, 3] }, ) foo_container = Container('/somepath', platform=platform_mock) foo_container._cgroup.allocation_configuration = AllocationConfiguration() foo_container._cgroup.platform = platform_mock quota_allocation_value = QuotaAllocationValue(0.2, foo_container, dict(foo='bar')) quota_allocation_value.perform_allocations() assert quota_allocation_value.generate_metrics() == [ allocation_metric('cpu_quota', 0.2, foo='bar') ] shares_allocation_value = SharesAllocationValue(0.5, foo_container, dict(foo='bar')) shares_allocation_value.perform_allocations() assert shares_allocation_value.generate_metrics() == [ allocation_metric('cpu_shares', 0.5, foo='bar') ] cpuset_cpus_allocation_value = CPUSetCPUSAllocationValue( '0-2,4,6-8', foo_container, dict(foo='bar')) cpuset_cpus_allocation_value.perform_allocations() cpuset_mems_allocation_value = CPUSetMEMSAllocationValue( '0-1', foo_container, dict(foo='bar')) cpuset_mems_allocation_value.perform_allocations() assert cpuset_cpus_allocation_value.generate_metrics() == [ Metric(name='allocation_cpuset_cpus_number_of_cpus', value=7, labels={ 'allocation_type': AllocationType.CPUSET_CPUS, 'foo': 'bar' }, type=MetricType.GAUGE) ] assert cpuset_mems_allocation_value.generate_metrics() == [ Metric(name='allocation_cpuset_mems_number_of_mems', value=2, labels={ 'allocation_type': AllocationType.CPUSET_MEMS, 'foo': 'bar' }, type=MetricType.GAUGE) ] Cgroup_mock.assert_has_calls([ call().set_quota(0.2), call().set_shares(0.5), call().set_cpuset_cpus({0, 1, 2, 4, 6, 7, 8}), call().set_cpuset_mems({0, 1}) ], True)
def test_sync_containers_state(_, get_pids_mock, sync_mock, perf_counters_mock, add_pids_mock, clean_taskless_groups_mock, subcgroups, tasks_, pre_running_containers_, mon_groups_relation, expected_running_containers_, labels_relation_, pre_running_labels_relation_): """Tests both Container and ContainerSet classes. Note: the input arguments tasks_, existing_containers_, expected_running_containers_ contain in their names underscore at the end to distinguish them from the ones created inside the function body to emphasize the relationship: the input arguments is used to create real objects. We cannot pass already created objects, as to create them we need another argument from first of two paramatrize decorators: subcgroups. Note: we have three variables names with the same postfix: * pre_running_containers - state of ContainerManager before (pre) call sync_containers_state * expected_running_containers - similar as above but state expected after the call, * got_running_containers - similar as above but state which we got after the call. All of three are of the same type Dict[Task, ContainerInterface]. """ # Create Task and Container/ContainerSet objects from input arguments. # This is done to both test Container and ContainerSet classes (to pass # subcgroups argument into the constructing function >>container<<. tasks = [task(t, subcgroups_paths=subcgroups, labels=labels_relation_.get(t)) for t in tasks_] pre_running_containers = \ {task(t, subcgroups_paths=subcgroups, labels=pre_running_labels_relation_.get(t)): container(c, subcgroups) for t, c in pre_running_containers_.items()} expected_running_containers = \ {task(t, subcgroups_paths=subcgroups, labels=labels_relation_.get(t)): container(c, subcgroups) for t, c in expected_running_containers_.items()} rdt_information = RDTInformation(True, True, True, True, 'fff', '1', 0, 0, 0) platform_mock = Mock( spec=Platform, sockets=1, cores=1, cpus=1, rdt_information=rdt_information) containers_manager = ContainerManager(platform=platform_mock, allocation_configuration=AllocationConfiguration(), event_names=[], ) # Put in into ContainerManager our input dict of containers. containers_manager.containers = dict(pre_running_containers) # Call sync_containers_state with patch('wca.resctrl.read_mon_groups_relation', return_value=mon_groups_relation): got_running_containers = containers_manager.sync_containers_state(tasks) # ----------------------- # Assert that two sets of keys of two dictionaries got_containers and # expected_running_containers are equal. assert len(got_running_containers) == len(expected_running_containers) assert all([expected_task in got_running_containers for expected_task in expected_running_containers.keys()]) for t in expected_running_containers.keys(): assert_equal_containers(expected_running_containers[t], got_running_containers[t]) # Check container objects has proper resgroup assigned. got_container_resgroup_names = {c.get_name(): c.get_resgroup().name for c in got_running_containers.values()} for expected_resgroup_name, container_names in mon_groups_relation.items(): for container_name in container_names: if container_name in got_container_resgroup_names: got_resgroup_name = got_container_resgroup_names.get(container_name) assert got_resgroup_name == expected_resgroup_name