def unpatched(): if len(subcgroups_paths): platform = Platform(sockets=1, cores=1, cpus=2, numa_nodes=2, topology={0: { 0: [1, 2] }}, cpu_model='intel xeon', cpu_model_number=0x5E, cpu_codename=CPUCodeName.SKYLAKE, timestamp=time.time(), rdt_information=RDTInformation( True, True, rdt_mb_control_enabled, rdt_cache_control_enabled, '0', '0', 0, 0, 0), node_cpus={0: {0, 1}}, node_distances={0: { 0: 10 }}, measurements={}, swap_enabled=False) return ContainerSet( cgroup_path=cgroup_path, cgroup_paths=subcgroups_paths, platform=platform, allocation_configuration=AllocationConfiguration() if with_config else None, resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None, event_names=['task_cycles']) else: platform = Platform(sockets=1, cores=1, cpus=2, numa_nodes=2, topology={0: { 0: [1, 2] }}, cpu_model='intel xeon', cpu_model_number=0x5E, cpu_codename=CPUCodeName.SKYLAKE, timestamp=time.time(), rdt_information=RDTInformation( True, True, True, True, '0', '0', 0, 0, 0), node_cpus={0: {0, 1}}, node_distances={0: { 0: 10 }}, measurements={}, swap_enabled=False) return Container( cgroup_path=cgroup_path, platform=platform, allocation_configuration=AllocationConfiguration() if with_config else None, resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None, event_names=['task_cycles'], )
def unpatched(): if len(subcgroups_paths): return ContainerSet( cgroup_path=cgroup_path, cgroup_paths=subcgroups_paths, platform_cpus=1, platform_sockets=1, allocation_configuration=AllocationConfiguration() if with_config else None, resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None, rdt_information=RDTInformation(True, True, rdt_mb_control_enabled, rdt_cache_control_enabled, '0', '0', 0, 0, 0), event_names=DEFAULT_EVENTS) else: return Container( cgroup_path=cgroup_path, platform_cpus=1, platform_sockets=1, rdt_information=RDTInformation(True, True, True, True, '0', '0', 0, 0, 0), allocation_configuration=AllocationConfiguration() if with_config else None, resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None, event_names=DEFAULT_EVENTS)
def unpatched(): if len(subcgroups_paths): platform = Platform(sockets=1, cores=1, cpus=2, cpu_model='intel xeon', cpu_model_number=0x5E, cpu_codename=CPUCodeName.SKYLAKE, cpus_usage={ 0: 10, 1: 10 }, total_memory_used=10, timestamp=time.time(), rdt_information=RDTInformation( True, True, rdt_mb_control_enabled, rdt_cache_control_enabled, '0', '0', 0, 0, 0)) return ContainerSet( cgroup_path=cgroup_path, cgroup_paths=subcgroups_paths, platform=platform, allocation_configuration=AllocationConfiguration() if with_config else None, resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None, event_names=DEFAULT_EVENTS) else: platform = Platform(sockets=1, cores=1, cpus=2, cpu_model='intel xeon', cpu_model_number=0x5E, cpu_codename=CPUCodeName.SKYLAKE, cpus_usage={ 0: 10, 1: 10 }, total_memory_used=10, timestamp=time.time(), rdt_information=RDTInformation( True, True, True, True, '0', '0', 0, 0, 0)) return Container( cgroup_path=cgroup_path, platform=platform, allocation_configuration=AllocationConfiguration() if with_config else None, resgroup=ResGroup(name=resgroup_name) if rdt_enabled else None, event_names=DEFAULT_EVENTS)
def __init__(self, measurement_runner: MeasurementRunner, allocator: Allocator, allocations_storage: Storage = DEFAULT_STORAGE, anomalies_storage: Storage = DEFAULT_STORAGE, rdt_mb_control_required: bool = False, rdt_cache_control_required: bool = False, remove_all_resctrl_groups: bool = False): if not measurement_runner._allocation_configuration: measurement_runner._allocation_configuration = AllocationConfiguration( ) self._measurement_runner = measurement_runner # Allocation specific. self._allocator = allocator self._allocations_storage = allocations_storage self._rdt_mb_control_required = rdt_mb_control_required self._rdt_cache_control_required = rdt_cache_control_required # Anomaly. self._anomalies_storage = anomalies_storage self._anomalies_statistics = AnomalyStatistics() # Internal allocation statistics self._allocations_counter = 0 self._allocations_errors = 0 self._remove_all_resctrl_groups = remove_all_resctrl_groups self._measurement_runner._set_iterate_body_callback(self._iterate_body) self._measurement_runner._set_initialize_rdt_callback( self._initialize_rdt)
def test_get_tasks_allocations_fail(*mock): containers = { task('/t1', labels={'label_key': 'label_value'}, resources={'cpu': 3}): Container('/t1', platform_mock, allocation_configuration=AllocationConfiguration( cpu_quota_period=1000)) } assert {} == _get_tasks_allocations(containers)
def test_get_tasks_allocations_fail(*mock): rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0) containers = { task('/t1', labels={'label_key': 'label_value'}, resources={'cpu': 3}): Container('/t1', 1, 1, rdt_information, allocation_configuration=AllocationConfiguration( cpu_quota_period=1000)) } assert {} == _get_tasks_allocations(containers)
def __init__( self, node: nodes.Node, allocator: Allocator, metrics_storage: storage.Storage = DEFAULT_STORAGE, anomalies_storage: storage.Storage = DEFAULT_STORAGE, allocations_storage: storage.Storage = DEFAULT_STORAGE, action_delay: Numeric(0, 60) = 1., # [s] rdt_enabled: Optional[ bool] = None, # Defaults(None) - auto configuration. rdt_mb_control_required: bool = False, rdt_cache_control_required: bool = False, extra_labels: Dict[Str, Str] = None, allocation_configuration: Optional[AllocationConfiguration] = None, remove_all_resctrl_groups: bool = False, event_names: Optional[List[str]] = None, enable_derived_metrics: bool = False, task_label_generators: Dict[str, TaskLabelGenerator] = None, ): self._allocation_configuration = allocation_configuration or AllocationConfiguration( ) super().__init__( node, metrics_storage, action_delay, rdt_enabled, extra_labels, _allocation_configuration=self._allocation_configuration, event_names=event_names, enable_derived_metrics=enable_derived_metrics, task_label_generators=task_label_generators) # Allocation specific. self._allocator = allocator self._allocations_storage = allocations_storage self._rdt_mb_control_required = rdt_mb_control_required # Override False from superclass. self._rdt_cache_control_required = rdt_cache_control_required # Anomaly. self._anomalies_storage = anomalies_storage self._anomalies_statistics = AnomalyStatistics() # Internal allocation statistics self._allocations_counter = 0 self._allocations_errors = 0 self._remove_all_resctrl_groups = remove_all_resctrl_groups
def test_rdt_initialize( rdt_max_values_mock, cleanup_resctrl_mock, default_rdt_l3, default_rdt_mb, config_rdt_mb_control_enabled, config_rdt_cache_control_enabled, platform_rdt_mb_control_enabled, platform_rdt_cache_control_enabled, expected_error, expected_final_rdt_mb_control_enabled_with_value, expected_cleanup_arguments, ): allocation_configuration = AllocationConfiguration( default_rdt_mb=default_rdt_mb, default_rdt_l3=default_rdt_l3) runner = AllocationRunner( measurement_runner=MeasurementRunner( node=Mock(spec=Node), interval=1, rdt_enabled=True, metrics_storage=Mock(spec=Storage), allocation_configuration=allocation_configuration, ), allocator=Mock(spec=Allocator), anomalies_storage=Mock(spec=Storage), allocations_storage=Mock(spec=Storage), rdt_mb_control_required=config_rdt_mb_control_enabled, rdt_cache_control_required=config_rdt_cache_control_enabled) with patch( 'tests.testing.platform_mock.rdt_information', Mock( spec=RDTInformation, cbm_mask='fff', min_cbm_bits='2', mb_min_bandwidth=10, mb_bandwidth_gran=10, rdt_mb_control_enabled=platform_rdt_mb_control_enabled, rdt_cache_control_enabled=platform_rdt_cache_control_enabled)): assert runner._initialize_rdt() is not expected_error if expected_final_rdt_mb_control_enabled_with_value: assert runner._rdt_mb_control_required == expected_final_rdt_mb_control_enabled_with_value if expected_cleanup_arguments: cleanup_resctrl_mock.assert_called_with(*expected_cleanup_arguments) else: assert cleanup_resctrl_mock.call_count == 0
def test_set_normalized_quota(normalized_quota, cpu_quota_period, platforms_cpu, initial_period_value, expected_period_write, expected_quota_write): with patch('wca.cgroups.Cgroup._read', return_value=initial_period_value): with patch('wca.cgroups.Cgroup._write') as write_mock: cgroup = Cgroup('/some/foo1', platform_cpus=platforms_cpu, allocation_configuration=AllocationConfiguration( cpu_quota_period=cpu_quota_period)) cgroup.set_quota(normalized_quota) write_mock.assert_has_calls([ call(CgroupResource.CPU_QUOTA, expected_quota_write, CgroupType.CPU) ]) if expected_period_write: write_mock.assert_has_calls([ call(CgroupResource.CPU_PERIOD, expected_period_write, CgroupType.CPU) ])
def test_cgroup_allocations(Cgroup_mock, PerfCounters_mock): rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0) platform_mock = Mock(spec=Platform, sockets=1, cpus=1, rdt_information=rdt_information) foo_container = Container('/somepath', platform=platform_mock) foo_container._cgroup.allocation_configuration = AllocationConfiguration() foo_container._cgroup.platform_cpus = 10 foo_container._cgroup.platform_sockets = 1 quota_allocation_value = QuotaAllocationValue(0.2, foo_container, dict(foo='bar')) quota_allocation_value.perform_allocations() assert quota_allocation_value.generate_metrics() == [ allocation_metric('cpu_quota', 0.2, foo='bar') ] shares_allocation_value = SharesAllocationValue(0.5, foo_container, dict(foo='bar')) shares_allocation_value.perform_allocations() assert shares_allocation_value.generate_metrics() == [ allocation_metric('cpu_shares', 0.5, foo='bar') ] cpuset_allocation_value = CPUSetAllocationValue('0-2,4,6-8', foo_container, dict(foo='bar')) cpuset_allocation_value.perform_allocations() assert cpuset_allocation_value.generate_metrics() == [ allocation_metric('cpuset', [0, 1, 2, 4, 6, 7, 8], foo='bar') ] Cgroup_mock.assert_has_calls([ call().set_quota(0.2), call().set_shares(0.5), call().set_cpuset('0,1,2,4,6,7,8', '0') ], True)
def test_cgroup_allocations(Cgroup_mock, PerfCounters_mock): rdt_information = RDTInformation(True, True, True, True, '0', '0', 0, 0, 0) platform_mock = Mock( spec=Platform, cpus=10, sockets=1, rdt_information=rdt_information, node_cpus={ 0: [0, 1], 1: [2, 3] }, ) foo_container = Container('/somepath', platform=platform_mock) foo_container._cgroup.allocation_configuration = AllocationConfiguration() foo_container._cgroup.platform = platform_mock quota_allocation_value = QuotaAllocationValue(0.2, foo_container, dict(foo='bar')) quota_allocation_value.perform_allocations() assert quota_allocation_value.generate_metrics() == [ allocation_metric('cpu_quota', 0.2, foo='bar') ] shares_allocation_value = SharesAllocationValue(0.5, foo_container, dict(foo='bar')) shares_allocation_value.perform_allocations() assert shares_allocation_value.generate_metrics() == [ allocation_metric('cpu_shares', 0.5, foo='bar') ] cpuset_cpus_allocation_value = CPUSetCPUSAllocationValue( '0-2,4,6-8', foo_container, dict(foo='bar')) cpuset_cpus_allocation_value.perform_allocations() cpuset_mems_allocation_value = CPUSetMEMSAllocationValue( '0-1', foo_container, dict(foo='bar')) cpuset_mems_allocation_value.perform_allocations() assert cpuset_cpus_allocation_value.generate_metrics() == [ Metric(name='allocation_cpuset_cpus_number_of_cpus', value=7, labels={ 'allocation_type': AllocationType.CPUSET_CPUS, 'foo': 'bar' }, type=MetricType.GAUGE) ] assert cpuset_mems_allocation_value.generate_metrics() == [ Metric(name='allocation_cpuset_mems_number_of_mems', value=2, labels={ 'allocation_type': AllocationType.CPUSET_MEMS, 'foo': 'bar' }, type=MetricType.GAUGE) ] Cgroup_mock.assert_has_calls([ call().set_quota(0.2), call().set_shares(0.5), call().set_cpuset_cpus({0, 1, 2, 4, 6, 7, 8}), call().set_cpuset_mems({0, 1}) ], True)
def test_sync_containers_state(_, get_pids_mock, sync_mock, perf_counters_mock, add_pids_mock, clean_taskless_groups_mock, subcgroups, tasks_, pre_running_containers_, mon_groups_relation, expected_running_containers_, labels_relation_, pre_running_labels_relation_): """Tests both Container and ContainerSet classes. Note: the input arguments tasks_, existing_containers_, expected_running_containers_ contain in their names underscore at the end to distinguish them from the ones created inside the function body to emphasize the relationship: the input arguments is used to create real objects. We cannot pass already created objects, as to create them we need another argument from first of two paramatrize decorators: subcgroups. Note: we have three variables names with the same postfix: * pre_running_containers - state of ContainerManager before (pre) call sync_containers_state * expected_running_containers - similar as above but state expected after the call, * got_running_containers - similar as above but state which we got after the call. All of three are of the same type Dict[Task, ContainerInterface]. """ # Create Task and Container/ContainerSet objects from input arguments. # This is done to both test Container and ContainerSet classes (to pass # subcgroups argument into the constructing function >>container<<. tasks = [task(t, subcgroups_paths=subcgroups, labels=labels_relation_.get(t)) for t in tasks_] pre_running_containers = \ {task(t, subcgroups_paths=subcgroups, labels=pre_running_labels_relation_.get(t)): container(c, subcgroups) for t, c in pre_running_containers_.items()} expected_running_containers = \ {task(t, subcgroups_paths=subcgroups, labels=labels_relation_.get(t)): container(c, subcgroups) for t, c in expected_running_containers_.items()} rdt_information = RDTInformation(True, True, True, True, 'fff', '1', 0, 0, 0) platform_mock = Mock( spec=Platform, sockets=1, cores=1, cpus=1, rdt_information=rdt_information) containers_manager = ContainerManager(platform=platform_mock, allocation_configuration=AllocationConfiguration(), event_names=[], ) # Put in into ContainerManager our input dict of containers. containers_manager.containers = dict(pre_running_containers) # Call sync_containers_state with patch('wca.resctrl.read_mon_groups_relation', return_value=mon_groups_relation): got_running_containers = containers_manager.sync_containers_state(tasks) # ----------------------- # Assert that two sets of keys of two dictionaries got_containers and # expected_running_containers are equal. assert len(got_running_containers) == len(expected_running_containers) assert all([expected_task in got_running_containers for expected_task in expected_running_containers.keys()]) for t in expected_running_containers.keys(): assert_equal_containers(expected_running_containers[t], got_running_containers[t]) # Check container objects has proper resgroup assigned. got_container_resgroup_names = {c.get_name(): c.get_resgroup().name for c in got_running_containers.values()} for expected_resgroup_name, container_names in mon_groups_relation.items(): for container_name in container_names: if container_name in got_container_resgroup_names: got_resgroup_name = got_container_resgroup_names.get(container_name) assert got_resgroup_name == expected_resgroup_name
def test_get_normalized_shares(_read_mock): cgroup = Cgroup('/some/foo1', platform=platform_mock, allocation_configuration=AllocationConfiguration()) assert cgroup._get_normalized_shares() == pytest.approx(1, 0.01)
def test_get_normalized_quota(): cgroup = Cgroup('/some/foo1', platform=platform_mock, allocation_configuration=AllocationConfiguration()) assert cgroup._get_normalized_quota() == 1.0
@patch('builtins.open', create_open_mock({ '/sys/fs/cgroup/cpu/some/foo1/tasks': '101\n102', '/sys/fs/cgroup/cpu/foo2/tasks': '', })) def test_cgroup_get_pids(): assert Cgroup('/some/foo1', platform=platform_mock).get_pids() == ['101', '102'] assert Cgroup('/foo2', platform=platform_mock).get_pids() == [] @pytest.mark.parametrize( 'normalized_shares, allocation_configuration, expected_shares_write', [ (0., AllocationConfiguration(), 2), (1., AllocationConfiguration(), 1000), # based on cpu_shares_unit (default 1000) (1., AllocationConfiguration(cpu_shares_unit=10000), 10000), (2., AllocationConfiguration(cpu_shares_unit=10000), 20000), ]) def test_set_normalized_shares(normalized_shares, allocation_configuration, expected_shares_write): with patch('wca.cgroups.Cgroup._write') as write_mock: cgroup = Cgroup('/some/foo1', platform=platform_mock, allocation_configuration=allocation_configuration) cgroup.set_shares(normalized_shares) write_mock.assert_called_with(CgroupResource.CPU_SHARES, expected_shares_write, CgroupType.CPU)