def _get_fake_host_state_with_networks(self): network_a = objects.NetworkMetadata(physnets=set(['foo', 'bar']), tunneled=False) network_b = objects.NetworkMetadata(physnets=set(), tunneled=True) host_topology = objects.NUMATopology(cells=[ objects.NUMACell(id=1, cpuset=set([1, 2]), pcpuset=set(), memory=2048, cpu_usage=2, memory_usage=2048, mempages=[], siblings=[set([1]), set([2])], pinned_cpus=set(), network_metadata=network_a), objects.NUMACell(id=2, cpuset=set([3, 4]), pcpuset=set(), memory=2048, cpu_usage=2, memory_usage=2048, mempages=[], siblings=[set([3]), set([4])], pinned_cpus=set(), network_metadata=network_b) ]) return fakes.FakeHostState( 'host1', 'node1', { 'numa_topology': host_topology, 'pci_stats': None, 'cpu_allocation_ratio': 16.0, 'ram_allocation_ratio': 1.5 })
def test_support_requests_numa(self): cells = [ objects.NUMACell(id=0, cpuset=set(), memory=0), objects.NUMACell(id=1, cpuset=set(), memory=0) ] self.assertEqual(True, self.pci_stats.support_requests(pci_requests, cells))
def test_consume_requests_numa(self): cells = [objects.NUMACell(id=0, cpuset=set(), memory=0), objects.NUMACell(id=1, cpuset=set(), memory=0)] devs = self.pci_stats.consume_requests(pci_requests, cells) self.assertEqual(2, len(devs)) self.assertEqual(set(['v1', 'v2']), set([dev['vendor_id'] for dev in devs]))
def test_numa_cell_not_equivalent_missing_b(self): cell1 = objects.NUMACell(id=1, cpuset=set([1, 2]), memory=32, cpu_usage=10, pinned_cpus=set([3, 4]), siblings=[set([5, 6])]) cell2 = objects.NUMACell(id=2, cpuset=set([1, 2]), memory=32, pinned_cpus=set([3, 4]), siblings=[set([5, 6])]) self.assertNotEqual(cell1, cell2)
def test_numa_cell_not_equivalent_different_pages(self): pt1 = objects.NUMAPagesTopology(size_kb=1024, total=32, used=0) pt2 = objects.NUMAPagesTopology(size_kb=1024, total=32, used=1) cell1 = objects.NUMACell(id=1, cpuset=set([1, 2]), memory=32, cpu_usage=10, pinned_cpus=set([3, 4]), siblings=[set([5, 6])], mempages=[pt1]) cell2 = objects.NUMACell(id=1, cpuset=set([1, 2]), memory=32, cpu_usage=10, pinned_cpus=set([3, 4]), siblings=[set([5, 6])], mempages=[pt2]) self.assertNotEqual(cell1, cell2)
def _claim_topology(self, mem, cpus=1): if self.tracker.driver.numa_topology is None: return None mem = mem * 1024 return objects.NUMATopology( cells=[objects.NUMACell( id=0, cpuset=set([1, 2]), memory=3072, cpu_usage=cpus, memory_usage=mem, mempages=[], siblings=[], pinned_cpus=set([])), objects.NUMACell( id=1, cpuset=set([3, 4]), memory=3072, cpu_usage=cpus, memory_usage=mem, mempages=[], siblings=[], pinned_cpus=set([]))])
def test_free_cpus(self): obj = objects.NUMATopology(cells=[ objects.NUMACell( id=0, cpuset=set([1, 2]), memory=512, cpu_usage=2, memory_usage=256, pinned_cpus=set([1])), objects.NUMACell( id=1, cpuset=set([3, 4]), memory=512, cpu_usage=1, memory_usage=128, pinned_cpus=set([])) ] ) self.assertEqual(set([2]), obj.cells[0].free_cpus) self.assertEqual(set([3, 4]), obj.cells[1].free_cpus)
def test_obj_cell_relationships(self): obj = objects.NUMATopology(cells=[objects.NUMACell()]) rel = objects.NUMATopology.obj_relationships['cells'] for topo_ver, cell_ver in rel: prim = obj.obj_to_primitive(target_version=topo_ver) cell = objects.NUMATopology.obj_from_primitive(prim).cells[0] self.assertEqual(cell_ver, cell.VERSION)
def test_can_fit_pagesize_oversubscription(self): """Validate behavior when using page oversubscription. While hugepages aren't themselves oversubscribable, we also track small pages which are. """ # NOTE(stephenfin): '**' is Python's "power of" symbol cell = objects.NUMACell( id=0, cpuset=set([1, 2]), pcpuset=set(), memory=1024, siblings=[set([1]), set([2])], pinned_cpus=set(), mempages=[ # 1 GiB total, all used objects.NUMAPagesTopology(size_kb=4, total=2**18, used=2**18), ]) pagesize = 4 # request 2^20 KiB (so 1 GiB) self.assertTrue(cell.can_fit_pagesize(pagesize, 2**20, use_free=False)) # request 2^20 + 1 KiB (so # > 1 GiB) self.assertFalse( cell.can_fit_pagesize(pagesize, 2**20 + 1, use_free=False))
def test_pinning_logic(self): numacell = objects.NUMACell(id=0, cpuset=set([1, 2, 3, 4]), memory=512, cpu_usage=2, memory_usage=256, pinned_cpus=set([1]), siblings=[], mempages=[]) numacell.pin_cpus(set([2, 3])) self.assertEqual(set([4]), numacell.free_cpus) expect_msg = ( exception.CPUPinningUnknown.msg_fmt % {'requested': "\[1, 55\]", 'cpuset': "\[1, 2, 3, 4\]"}) with testtools.ExpectedException(exception.CPUPinningUnknown, expect_msg): numacell.pin_cpus(set([1, 55])) self.assertRaises(exception.CPUPinningInvalid, numacell.pin_cpus, set([1, 4])) expect_msg = (exception.CPUUnpinningUnknown.msg_fmt % {'requested': "\[1, 55\]", 'cpuset': "\[1, 2, 3, 4\]"}) with testtools.ExpectedException(exception.CPUUnpinningUnknown, expect_msg): numacell.unpin_cpus(set([1, 55])) self.assertRaises(exception.CPUUnpinningInvalid, numacell.unpin_cpus, set([1, 4])) numacell.unpin_cpus(set([1, 2, 3])) self.assertEqual(set([1, 2, 3, 4]), numacell.free_cpus)
def test_filter_pools_for_socket_affinity_no_socket(self): self._setup_pci_stats( objects.NUMATopology(cells=[objects.NUMACell(socket=None)])) self.assertEqual([], self.pci_stats._filter_pools_for_socket_affinity( self.pci_stats.pools, [objects.InstanceNUMACell()]))
def test_obj_make_compatible(self): network_metadata = objects.NetworkMetadata(physnets=set(['foo', 'bar']), tunneled=True) cell = objects.NUMACell(id=0, socket=0, cpuset=set([1, 2]), pcpuset=set([3, 4]), memory=32, cpu_usage=10, pinned_cpus=set([3, 4]), siblings=[set([5, 6])], network_metadata=network_metadata) versions = ovo_base.obj_tree_get_versions('NUMACell') primitive = cell.obj_to_primitive(target_version='1.5', version_manifest=versions) self.assertIn('socket', primitive['nova_object.data']) primitive = cell.obj_to_primitive(target_version='1.4', version_manifest=versions) self.assertIn('pcpuset', primitive['nova_object.data']) self.assertNotIn('socket', primitive['nova_object.data']) primitive = cell.obj_to_primitive(target_version='1.3', version_manifest=versions) self.assertNotIn('pcpuset', primitive['nova_object.data']) self.assertIn('network_metadata', primitive['nova_object.data']) primitive = cell.obj_to_primitive(target_version='1.2', version_manifest=versions) self.assertNotIn('network_metadata', primitive['nova_object.data'])
def test_pinning_with_siblings(self): numacell = objects.NUMACell(id=0, cpuset=set(), pcpuset=set([1, 2, 3, 4]), memory=512, cpu_usage=2, memory_usage=256, pinned_cpus=set(), siblings=[set([1, 3]), set([2, 4])], mempages=[]) numacell.pin_cpus_with_siblings(set([1, 2])) self.assertEqual(set(), numacell.free_pcpus) numacell.unpin_cpus_with_siblings(set([1])) self.assertEqual(set([1, 3]), numacell.free_pcpus) self.assertRaises(exception.CPUUnpinningInvalid, numacell.unpin_cpus_with_siblings, set([3])) self.assertRaises(exception.CPUPinningInvalid, numacell.pin_cpus_with_siblings, set([4])) self.assertRaises(exception.CPUUnpinningInvalid, numacell.unpin_cpus_with_siblings, set([3, 4])) self.assertEqual(set([1, 3]), numacell.free_pcpus) numacell.unpin_cpus_with_siblings(set([4])) self.assertEqual(set([1, 2, 3, 4]), numacell.free_pcpus)
def test_can_fit_hugepages(self): cell = objects.NUMACell( id=0, cpuset=set([1, 2]), memory=1024, siblings=[], pinned_cpus=set([]), mempages=[ objects.NUMAPagesTopology( size_kb=4, total=1548736, used=0), objects.NUMAPagesTopology( size_kb=2048, total=513, used=0), objects.NUMAPagesTopology( size_kb=1048576, total=4, used=1, reserved=1)]) pagesize = 2048 self.assertTrue(cell.can_fit_hugepages(pagesize, 2 ** 20)) self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 21)) self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 19 + 1)) pagesize = 1048576 self.assertTrue(cell.can_fit_hugepages(pagesize, 2 ** 20)) self.assertTrue(cell.can_fit_hugepages(pagesize, 2 ** 20 * 2)) self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 20 * 3)) self.assertRaises( exception.MemoryPageSizeNotSupported, cell.can_fit_hugepages, 12345, 2 ** 20)
def test_can_fit_pagesize(self): # NOTE(stephenfin): '**' is Python's "power of" symbol cell = objects.NUMACell(id=0, cpuset=set([1, 2]), pcpuset=set(), memory=1024, siblings=[set([1]), set([2])], pinned_cpus=set(), mempages=[ objects.NUMAPagesTopology(size_kb=4, total=1548736, used=0), objects.NUMAPagesTopology(size_kb=2048, total=513, used=0), objects.NUMAPagesTopology(size_kb=1048576, total=4, used=1, reserved=1) ]) pagesize = 2048 self.assertTrue(cell.can_fit_pagesize(pagesize, 2**20)) self.assertFalse(cell.can_fit_pagesize(pagesize, 2**21)) self.assertFalse(cell.can_fit_pagesize(pagesize, 2**19 + 1)) pagesize = 1048576 self.assertTrue(cell.can_fit_pagesize(pagesize, 2**20)) self.assertTrue(cell.can_fit_pagesize(pagesize, 2**20 * 2)) self.assertFalse(cell.can_fit_pagesize(pagesize, 2**20 * 3)) self.assertRaises(exception.MemoryPageSizeNotSupported, cell.can_fit_pagesize, 12345, 2**20)
def test_equivalent(self): cell1 = objects.NUMACell(id=1, cpuset=set([1, 2]), pcpuset=set(), memory=32, cpu_usage=10, pinned_cpus=set([3, 4]), siblings=[set([5, 6])]) cell2 = objects.NUMACell(id=1, cpuset=set([1, 2]), pcpuset=set(), memory=32, cpu_usage=10, pinned_cpus=set([3, 4]), siblings=[set([5, 6])]) self.assertEqual(cell1, cell2)
def _fake_resources(self, values=None): resources = { 'hypervisor_type': 'fake', 'memory_mb': 2048, 'memory_mb_used': 0, 'free_ram_mb': 2048, 'local_gb': 20, 'local_gb_used': 0, 'free_disk_gb': 20, 'vcpus': 2, 'vcpus_used': 0, 'l3_closids': 16, 'l3_closids_used': 1, 'numa_topology': objects.NUMATopology(cells=[ objects.NUMACell(id=1, cpuset=set([1, 2]), memory=512, memory_usage=0, cpu_usage=0, mempages=[], siblings=[], pinned_cpus=set([])), objects.NUMACell(id=2, cpuset=set([3, 4]), memory=512, memory_usage=0, cpu_usage=0, mempages=[], siblings=[], pinned_cpus=set([])) ])._to_json() } if values: resources.update(values) return objects.ComputeNode(**resources)
def test_filter_pools_for_socket_affinity(self): self._setup_pci_stats( objects.NUMATopology(cells=[objects.NUMACell(id=1, socket=1)])) pools = self.pci_stats._filter_pools_for_socket_affinity( self.pci_stats.pools, [objects.InstanceNUMACell(id=1)]) self.assertEqual(1, len(pools)) self.assertEqual('p2', pools[0]['product_id']) self.assertEqual('v2', pools[0]['vendor_id'])
def test_support_requests_no_numa_info(self): cells = [objects.NUMACell(id=0, cpuset=set(), memory=0)] pci_request = [ objects.InstancePCIRequest(count=1, spec=[{ 'vendor_id': 'v3' }]) ] self.assertTrue(self.pci_stats.support_requests(pci_request, cells))
def test_consume_requests_no_numa_info(self): cells = [objects.NUMACell(id=0, cpuset=set(), memory=0)] pci_request = [objects.InstancePCIRequest(count=1, spec=[{'vendor_id': 'v3'}])] devs = self.pci_stats.consume_requests(pci_request, cells) self.assertEqual(1, len(devs)) self.assertEqual(set(['v3']), set([dev['vendor_id'] for dev in devs]))
def _fake_compute_node(self, values=None): compute_node = { 'memory_mb': 2048, 'memory_mb_used': 0, 'free_ram_mb': 2048, 'local_gb': 20, 'local_gb_used': 0, 'free_disk_gb': 20, 'vcpus': 2, 'vcpus_used': 0, 'numa_topology': objects.NUMATopology(cells=[ objects.NUMACell(id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512, memory_usage=0, cpu_usage=0, mempages=[], siblings=[set([1]), set([2])], pinned_cpus=set()), objects.NUMACell(id=2, cpuset=set([3, 4]), pcpuset=set(), memory=512, memory_usage=0, cpu_usage=0, mempages=[], siblings=[set([3]), set([4])], pinned_cpus=set()) ])._to_json() } if values: compute_node.update(values) return objects.ComputeNode(**compute_node)
def _get_host_numa_topology(self): numa_nodes = self._hostutils.get_numa_nodes() cells = [] for numa_node in numa_nodes: numa_node['pinned_cpus'] = set([]) numa_node['mempages'] = [] numa_node['siblings'] = [] cell = objects.NUMACell(**numa_node) cells.append(cell) return objects.NUMATopology(cells=cells)
def test_free_cpus(self): cell_a = objects.NUMACell(id=0, cpuset=set([1, 2]), memory=512, cpu_usage=2, memory_usage=256, pinned_cpus=set([1]), siblings=[set([1]), set([2])], mempages=[]) cell_b = objects.NUMACell(id=1, cpuset=set([3, 4]), memory=512, cpu_usage=1, memory_usage=128, pinned_cpus=set(), siblings=[set([3]), set([4])], mempages=[]) self.assertEqual(set([2]), cell_a.free_cpus) self.assertEqual(set([3, 4]), cell_b.free_cpus)
def _get_host_numa_topology(self): numa_nodes = self._hostutils.get_numa_nodes() cells = [] for numa_node in numa_nodes: # Hyper-V does not support CPU pinning / mempages. # initializing the rest of the fields. numa_node.update(pinned_cpus=set(), mempages=[], siblings=[]) cell = objects.NUMACell(**numa_node) cells.append(cell) return objects.NUMATopology(cells=cells)
def numa_usage_from_instances(host, instances, free=False): """Get host topology usage :param host: objects.NUMATopology with usage information :param instances: list of objects.InstanceNUMATopology :param free: If True usage of the host will be decreased Sum the usage from all @instances to report the overall host topology usage :returns: objects.NUMATopology including usage information """ if host is None: return instances = instances or [] cells = [] sign = -1 if free else 1 for hostcell in host.cells: memory_usage = hostcell.memory_usage cpu_usage = hostcell.cpu_usage newcell = objects.NUMACell(id=hostcell.id, cpuset=hostcell.cpuset, memory=hostcell.memory, cpu_usage=0, memory_usage=0, mempages=hostcell.mempages, pinned_cpus=hostcell.pinned_cpus, siblings=hostcell.siblings) for instance in instances: for instancecell in instance.cells: if instancecell.id == hostcell.id: memory_usage = (memory_usage + sign * instancecell.memory) cpu_usage = cpu_usage + sign * len(instancecell.cpuset) if instancecell.pagesize and instancecell.pagesize > 0: newcell.mempages = _numa_pagesize_usage_from_cell( hostcell, instancecell, sign) if instance.cpu_pinning_requested: pinned_cpus = set(instancecell.cpu_pinning.values()) if free: newcell.unpin_cpus(pinned_cpus) else: newcell.pin_cpus(pinned_cpus) newcell.cpu_usage = max(0, cpu_usage) newcell.memory_usage = max(0, memory_usage) cells.append(newcell) return objects.NUMATopology(cells=cells)
def _fake_resources(self, values=None): resources = { 'memory_mb': 2048, 'memory_mb_used': 0, 'free_ram_mb': 2048, 'local_gb': 20, 'local_gb_used': 0, 'free_disk_gb': 20, 'vcpus': 2, 'vcpus_used': 0, 'numa_topology': objects.NUMATopology( cells=[objects.NUMACell(id=1, cpuset=set([1, 2]), memory=512, memory_usage=0, cpu_usage=0, mempages=[]), objects.NUMACell(id=2, cpuset=set([3, 4]), memory=512, memory_usage=0, cpu_usage=0, mempages=[])] )._to_json() } if values: resources.update(values) return resources
def test_pinning_logic(self): numacell = objects.NUMACell(id=0, cpuset=set([1, 2, 3, 4]), memory=512, cpu_usage=2, memory_usage=256, pinned_cpus=set([1])) numacell.pin_cpus(set([2, 3])) self.assertEqual(set([4]), numacell.free_cpus) self.assertRaises(exception.CPUPinningInvalid, numacell.pin_cpus, set([1, 4])) self.assertRaises(exception.CPUPinningInvalid, numacell.pin_cpus, set([1, 6])) self.assertRaises(exception.CPUPinningInvalid, numacell.unpin_cpus, set([1, 4])) numacell.unpin_cpus(set([1, 2, 3])) self.assertEqual(set([1, 2, 3, 4]), numacell.free_cpus)
def test_to_legacy_limits(self): limits = objects.NUMATopologyLimits( cpu_allocation_ratio=16, ram_allocation_ratio=2) host_topo = objects.NUMATopology(cells=[ objects.NUMACell(id=0, cpuset=set([1, 2]), memory=1024) ]) old_style = {'cells': [ {'mem': {'total': 1024, 'limit': 2048.0}, 'id': 0, 'cpus': '1,2', 'cpu_limit': 32.0}]} self.assertEqual(old_style, limits.to_dict_legacy(host_topo))
def test_can_fit_hugepages(self): cell = objects.NUMACell( id=0, cpuset=set([1, 2]), memory=1024, mempages=[ objects.NUMAPagesTopology( size_kb=4, total=1548736, used=0), objects.NUMAPagesTopology( size_kb=2048, total=513, used=0)]) # 1,002G pagesize = 2048 self.assertTrue(cell.can_fit_hugepages(pagesize, 2 ** 20)) self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 21)) self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 19 + 1)) self.assertRaises( exception.MemoryPageSizeNotSupported, cell.can_fit_hugepages, 12345, 2 ** 20)
def numa_usage_from_instances(host, instances, free=False): """Get host topology usage :param host: objects.NUMATopology with usage information :param instances: list of objects.InstanceNUMATopology :param free: If True usage of the host will be decreased Sum the usage from all @instances to report the overall host topology usage :returns: objects.NUMATopology including usage information """ if host is None: return instances = instances or [] cells = [] sign = -1 if free else 1 for hostcell in host.cells: memory_usage = hostcell.memory_usage cpu_usage = hostcell.cpu_usage for instance in instances: for instancecell in instance.cells: if instancecell.id == hostcell.id: memory_usage = ( memory_usage + sign * instancecell.memory) cpu_usage = cpu_usage + sign * len(instancecell.cpuset) cell = objects.NUMACell( id=hostcell.id, cpuset=hostcell.cpuset, memory=hostcell.memory, cpu_usage=max(0, cpu_usage), memory_usage=max(0, memory_usage)) cells.append(cell) return objects.NUMATopology(cells=cells)