def _get_fake_host_state_with_networks(self):
network_a = objects.NetworkMetadata(physnets=set(['foo', 'bar']),
tunneled=False)
network_b = objects.NetworkMetadata(physnets=set(), tunneled=True)
host_topology = objects.NUMATopology(cells=[
objects.NUMACell(id=1,
cpuset=set([1, 2]),
pcpuset=set(),
memory=2048,
cpu_usage=2,
memory_usage=2048,
mempages=[],
siblings=[set([1]), set([2])],
pinned_cpus=set(),
network_metadata=network_a),
objects.NUMACell(id=2,
cpuset=set([3, 4]),
pcpuset=set(),
memory=2048,
cpu_usage=2,
memory_usage=2048,
mempages=[],
siblings=[set([3]), set([4])],
pinned_cpus=set(),
network_metadata=network_b)
])
return fakes.FakeHostState(
'host1', 'node1', {
'numa_topology': host_topology,
'pci_stats': None,
'cpu_allocation_ratio': 16.0,
'ram_allocation_ratio': 1.5
})
def test_support_requests_numa(self):
cells = [
objects.NUMACell(id=0, cpuset=set(), memory=0),
objects.NUMACell(id=1, cpuset=set(), memory=0)
]
self.assertEqual(True,
self.pci_stats.support_requests(pci_requests, cells))
def test_consume_requests_numa(self):
cells = [objects.NUMACell(id=0, cpuset=set(), memory=0),
objects.NUMACell(id=1, cpuset=set(), memory=0)]
devs = self.pci_stats.consume_requests(pci_requests, cells)
self.assertEqual(2, len(devs))
self.assertEqual(set(['v1', 'v2']),
set([dev['vendor_id'] for dev in devs]))
def test_numa_cell_not_equivalent_missing_b(self):
cell1 = objects.NUMACell(id=1, cpuset=set([1, 2]), memory=32,
cpu_usage=10, pinned_cpus=set([3, 4]),
siblings=[set([5, 6])])
cell2 = objects.NUMACell(id=2, cpuset=set([1, 2]), memory=32,
pinned_cpus=set([3, 4]),
siblings=[set([5, 6])])
self.assertNotEqual(cell1, cell2)
def test_numa_cell_not_equivalent_different_pages(self):
pt1 = objects.NUMAPagesTopology(size_kb=1024, total=32, used=0)
pt2 = objects.NUMAPagesTopology(size_kb=1024, total=32, used=1)
cell1 = objects.NUMACell(id=1, cpuset=set([1, 2]), memory=32,
cpu_usage=10, pinned_cpus=set([3, 4]),
siblings=[set([5, 6])],
mempages=[pt1])
cell2 = objects.NUMACell(id=1, cpuset=set([1, 2]), memory=32,
cpu_usage=10, pinned_cpus=set([3, 4]),
siblings=[set([5, 6])],
mempages=[pt2])
self.assertNotEqual(cell1, cell2)
def _claim_topology(self, mem, cpus=1):
if self.tracker.driver.numa_topology is None:
return None
mem = mem * 1024
return objects.NUMATopology(
cells=[objects.NUMACell(
id=0, cpuset=set([1, 2]), memory=3072, cpu_usage=cpus,
memory_usage=mem, mempages=[], siblings=[],
pinned_cpus=set([])),
objects.NUMACell(
id=1, cpuset=set([3, 4]), memory=3072, cpu_usage=cpus,
memory_usage=mem, mempages=[], siblings=[],
pinned_cpus=set([]))])
def test_free_cpus(self):
obj = objects.NUMATopology(cells=[
objects.NUMACell(
id=0, cpuset=set([1, 2]), memory=512,
cpu_usage=2, memory_usage=256,
pinned_cpus=set([1])),
objects.NUMACell(
id=1, cpuset=set([3, 4]), memory=512,
cpu_usage=1, memory_usage=128,
pinned_cpus=set([]))
]
)
self.assertEqual(set([2]), obj.cells[0].free_cpus)
self.assertEqual(set([3, 4]), obj.cells[1].free_cpus)
def test_obj_cell_relationships(self):
obj = objects.NUMATopology(cells=[objects.NUMACell()])
rel = objects.NUMATopology.obj_relationships['cells']
for topo_ver, cell_ver in rel:
prim = obj.obj_to_primitive(target_version=topo_ver)
cell = objects.NUMATopology.obj_from_primitive(prim).cells[0]
self.assertEqual(cell_ver, cell.VERSION)
def test_can_fit_pagesize_oversubscription(self):
"""Validate behavior when using page oversubscription.
While hugepages aren't themselves oversubscribable, we also track small
pages which are.
"""
# NOTE(stephenfin): '**' is Python's "power of" symbol
cell = objects.NUMACell(
id=0,
cpuset=set([1, 2]),
pcpuset=set(),
memory=1024,
siblings=[set([1]), set([2])],
pinned_cpus=set(),
mempages=[
# 1 GiB total, all used
objects.NUMAPagesTopology(size_kb=4, total=2**18, used=2**18),
])
pagesize = 4
# request 2^20 KiB (so 1 GiB)
self.assertTrue(cell.can_fit_pagesize(pagesize, 2**20, use_free=False))
# request 2^20 + 1 KiB (so # > 1 GiB)
self.assertFalse(
cell.can_fit_pagesize(pagesize, 2**20 + 1, use_free=False))
def test_pinning_logic(self):
numacell = objects.NUMACell(id=0, cpuset=set([1, 2, 3, 4]), memory=512,
cpu_usage=2, memory_usage=256,
pinned_cpus=set([1]), siblings=[],
mempages=[])
numacell.pin_cpus(set([2, 3]))
self.assertEqual(set([4]), numacell.free_cpus)
expect_msg = (
exception.CPUPinningUnknown.msg_fmt % {'requested': "\[1, 55\]",
'cpuset': "\[1, 2, 3, 4\]"})
with testtools.ExpectedException(exception.CPUPinningUnknown,
expect_msg):
numacell.pin_cpus(set([1, 55]))
self.assertRaises(exception.CPUPinningInvalid,
numacell.pin_cpus, set([1, 4]))
expect_msg = (exception.CPUUnpinningUnknown.msg_fmt %
{'requested': "\[1, 55\]", 'cpuset': "\[1, 2, 3, 4\]"})
with testtools.ExpectedException(exception.CPUUnpinningUnknown,
expect_msg):
numacell.unpin_cpus(set([1, 55]))
self.assertRaises(exception.CPUUnpinningInvalid,
numacell.unpin_cpus, set([1, 4]))
numacell.unpin_cpus(set([1, 2, 3]))
self.assertEqual(set([1, 2, 3, 4]), numacell.free_cpus)
def test_filter_pools_for_socket_affinity_no_socket(self):
self._setup_pci_stats(
objects.NUMATopology(cells=[objects.NUMACell(socket=None)]))
self.assertEqual([],
self.pci_stats._filter_pools_for_socket_affinity(
self.pci_stats.pools,
[objects.InstanceNUMACell()]))
def test_obj_make_compatible(self):
network_metadata = objects.NetworkMetadata(physnets=set(['foo',
'bar']),
tunneled=True)
cell = objects.NUMACell(id=0,
socket=0,
cpuset=set([1, 2]),
pcpuset=set([3, 4]),
memory=32,
cpu_usage=10,
pinned_cpus=set([3, 4]),
siblings=[set([5, 6])],
network_metadata=network_metadata)
versions = ovo_base.obj_tree_get_versions('NUMACell')
primitive = cell.obj_to_primitive(target_version='1.5',
version_manifest=versions)
self.assertIn('socket', primitive['nova_object.data'])
primitive = cell.obj_to_primitive(target_version='1.4',
version_manifest=versions)
self.assertIn('pcpuset', primitive['nova_object.data'])
self.assertNotIn('socket', primitive['nova_object.data'])
primitive = cell.obj_to_primitive(target_version='1.3',
version_manifest=versions)
self.assertNotIn('pcpuset', primitive['nova_object.data'])
self.assertIn('network_metadata', primitive['nova_object.data'])
primitive = cell.obj_to_primitive(target_version='1.2',
version_manifest=versions)
self.assertNotIn('network_metadata', primitive['nova_object.data'])
def test_pinning_with_siblings(self):
numacell = objects.NUMACell(id=0,
cpuset=set(),
pcpuset=set([1, 2, 3, 4]),
memory=512,
cpu_usage=2,
memory_usage=256,
pinned_cpus=set(),
siblings=[set([1, 3]),
set([2, 4])],
mempages=[])
numacell.pin_cpus_with_siblings(set([1, 2]))
self.assertEqual(set(), numacell.free_pcpus)
numacell.unpin_cpus_with_siblings(set([1]))
self.assertEqual(set([1, 3]), numacell.free_pcpus)
self.assertRaises(exception.CPUUnpinningInvalid,
numacell.unpin_cpus_with_siblings, set([3]))
self.assertRaises(exception.CPUPinningInvalid,
numacell.pin_cpus_with_siblings, set([4]))
self.assertRaises(exception.CPUUnpinningInvalid,
numacell.unpin_cpus_with_siblings, set([3, 4]))
self.assertEqual(set([1, 3]), numacell.free_pcpus)
numacell.unpin_cpus_with_siblings(set([4]))
self.assertEqual(set([1, 2, 3, 4]), numacell.free_pcpus)
def test_can_fit_hugepages(self):
cell = objects.NUMACell(
id=0, cpuset=set([1, 2]), memory=1024,
siblings=[], pinned_cpus=set([]),
mempages=[
objects.NUMAPagesTopology(
size_kb=4, total=1548736, used=0),
objects.NUMAPagesTopology(
size_kb=2048, total=513, used=0),
objects.NUMAPagesTopology(
size_kb=1048576, total=4, used=1, reserved=1)])
pagesize = 2048
self.assertTrue(cell.can_fit_hugepages(pagesize, 2 ** 20))
self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 21))
self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 19 + 1))
pagesize = 1048576
self.assertTrue(cell.can_fit_hugepages(pagesize, 2 ** 20))
self.assertTrue(cell.can_fit_hugepages(pagesize, 2 ** 20 * 2))
self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 20 * 3))
self.assertRaises(
exception.MemoryPageSizeNotSupported,
cell.can_fit_hugepages, 12345, 2 ** 20)
def test_can_fit_pagesize(self):
# NOTE(stephenfin): '**' is Python's "power of" symbol
cell = objects.NUMACell(id=0,
cpuset=set([1, 2]),
pcpuset=set(),
memory=1024,
siblings=[set([1]), set([2])],
pinned_cpus=set(),
mempages=[
objects.NUMAPagesTopology(size_kb=4,
total=1548736,
used=0),
objects.NUMAPagesTopology(size_kb=2048,
total=513,
used=0),
objects.NUMAPagesTopology(size_kb=1048576,
total=4,
used=1,
reserved=1)
])
pagesize = 2048
self.assertTrue(cell.can_fit_pagesize(pagesize, 2**20))
self.assertFalse(cell.can_fit_pagesize(pagesize, 2**21))
self.assertFalse(cell.can_fit_pagesize(pagesize, 2**19 + 1))
pagesize = 1048576
self.assertTrue(cell.can_fit_pagesize(pagesize, 2**20))
self.assertTrue(cell.can_fit_pagesize(pagesize, 2**20 * 2))
self.assertFalse(cell.can_fit_pagesize(pagesize, 2**20 * 3))
self.assertRaises(exception.MemoryPageSizeNotSupported,
cell.can_fit_pagesize, 12345, 2**20)
def test_equivalent(self):
cell1 = objects.NUMACell(id=1,
cpuset=set([1, 2]),
pcpuset=set(),
memory=32,
cpu_usage=10,
pinned_cpus=set([3, 4]),
siblings=[set([5, 6])])
cell2 = objects.NUMACell(id=1,
cpuset=set([1, 2]),
pcpuset=set(),
memory=32,
cpu_usage=10,
pinned_cpus=set([3, 4]),
siblings=[set([5, 6])])
self.assertEqual(cell1, cell2)
def _fake_resources(self, values=None):
resources = {
'hypervisor_type':
'fake',
'memory_mb':
2048,
'memory_mb_used':
0,
'free_ram_mb':
2048,
'local_gb':
20,
'local_gb_used':
0,
'free_disk_gb':
20,
'vcpus':
2,
'vcpus_used':
0,
'l3_closids':
16,
'l3_closids_used':
1,
'numa_topology':
objects.NUMATopology(cells=[
objects.NUMACell(id=1,
cpuset=set([1, 2]),
memory=512,
memory_usage=0,
cpu_usage=0,
mempages=[],
siblings=[],
pinned_cpus=set([])),
objects.NUMACell(id=2,
cpuset=set([3, 4]),
memory=512,
memory_usage=0,
cpu_usage=0,
mempages=[],
siblings=[],
pinned_cpus=set([]))
])._to_json()
}
if values:
resources.update(values)
return objects.ComputeNode(**resources)
def test_filter_pools_for_socket_affinity(self):
self._setup_pci_stats(
objects.NUMATopology(cells=[objects.NUMACell(id=1, socket=1)]))
pools = self.pci_stats._filter_pools_for_socket_affinity(
self.pci_stats.pools, [objects.InstanceNUMACell(id=1)])
self.assertEqual(1, len(pools))
self.assertEqual('p2', pools[0]['product_id'])
self.assertEqual('v2', pools[0]['vendor_id'])
def test_support_requests_no_numa_info(self):
cells = [objects.NUMACell(id=0, cpuset=set(), memory=0)]
pci_request = [
objects.InstancePCIRequest(count=1, spec=[{
'vendor_id': 'v3'
}])
]
self.assertTrue(self.pci_stats.support_requests(pci_request, cells))
def test_consume_requests_no_numa_info(self):
cells = [objects.NUMACell(id=0, cpuset=set(), memory=0)]
pci_request = [objects.InstancePCIRequest(count=1,
spec=[{'vendor_id': 'v3'}])]
devs = self.pci_stats.consume_requests(pci_request, cells)
self.assertEqual(1, len(devs))
self.assertEqual(set(['v3']),
set([dev['vendor_id'] for dev in devs]))
def _fake_compute_node(self, values=None):
compute_node = {
'memory_mb':
2048,
'memory_mb_used':
0,
'free_ram_mb':
2048,
'local_gb':
20,
'local_gb_used':
0,
'free_disk_gb':
20,
'vcpus':
2,
'vcpus_used':
0,
'numa_topology':
objects.NUMATopology(cells=[
objects.NUMACell(id=1,
cpuset=set([1, 2]),
pcpuset=set(),
memory=512,
memory_usage=0,
cpu_usage=0,
mempages=[],
siblings=[set([1]), set([2])],
pinned_cpus=set()),
objects.NUMACell(id=2,
cpuset=set([3, 4]),
pcpuset=set(),
memory=512,
memory_usage=0,
cpu_usage=0,
mempages=[],
siblings=[set([3]), set([4])],
pinned_cpus=set())
])._to_json()
}
if values:
compute_node.update(values)
return objects.ComputeNode(**compute_node)
def _get_host_numa_topology(self):
numa_nodes = self._hostutils.get_numa_nodes()
cells = []
for numa_node in numa_nodes:
numa_node['pinned_cpus'] = set([])
numa_node['mempages'] = []
numa_node['siblings'] = []
cell = objects.NUMACell(**numa_node)
cells.append(cell)
return objects.NUMATopology(cells=cells)
def test_free_cpus(self):
cell_a = objects.NUMACell(id=0,
cpuset=set([1, 2]),
memory=512,
cpu_usage=2,
memory_usage=256,
pinned_cpus=set([1]),
siblings=[set([1]), set([2])],
mempages=[])
cell_b = objects.NUMACell(id=1,
cpuset=set([3, 4]),
memory=512,
cpu_usage=1,
memory_usage=128,
pinned_cpus=set(),
siblings=[set([3]), set([4])],
mempages=[])
self.assertEqual(set([2]), cell_a.free_cpus)
self.assertEqual(set([3, 4]), cell_b.free_cpus)
def _get_host_numa_topology(self):
numa_nodes = self._hostutils.get_numa_nodes()
cells = []
for numa_node in numa_nodes:
# Hyper-V does not support CPU pinning / mempages.
# initializing the rest of the fields.
numa_node.update(pinned_cpus=set(), mempages=[], siblings=[])
cell = objects.NUMACell(**numa_node)
cells.append(cell)
return objects.NUMATopology(cells=cells)
def numa_usage_from_instances(host, instances, free=False):
"""Get host topology usage
:param host: objects.NUMATopology with usage information
:param instances: list of objects.InstanceNUMATopology
:param free: If True usage of the host will be decreased
Sum the usage from all @instances to report the overall
host topology usage
:returns: objects.NUMATopology including usage information
"""
if host is None:
return
instances = instances or []
cells = []
sign = -1 if free else 1
for hostcell in host.cells:
memory_usage = hostcell.memory_usage
cpu_usage = hostcell.cpu_usage
newcell = objects.NUMACell(id=hostcell.id,
cpuset=hostcell.cpuset,
memory=hostcell.memory,
cpu_usage=0,
memory_usage=0,
mempages=hostcell.mempages,
pinned_cpus=hostcell.pinned_cpus,
siblings=hostcell.siblings)
for instance in instances:
for instancecell in instance.cells:
if instancecell.id == hostcell.id:
memory_usage = (memory_usage + sign * instancecell.memory)
cpu_usage = cpu_usage + sign * len(instancecell.cpuset)
if instancecell.pagesize and instancecell.pagesize > 0:
newcell.mempages = _numa_pagesize_usage_from_cell(
hostcell, instancecell, sign)
if instance.cpu_pinning_requested:
pinned_cpus = set(instancecell.cpu_pinning.values())
if free:
newcell.unpin_cpus(pinned_cpus)
else:
newcell.pin_cpus(pinned_cpus)
newcell.cpu_usage = max(0, cpu_usage)
newcell.memory_usage = max(0, memory_usage)
cells.append(newcell)
return objects.NUMATopology(cells=cells)
def _fake_resources(self, values=None):
resources = {
'memory_mb': 2048,
'memory_mb_used': 0,
'free_ram_mb': 2048,
'local_gb': 20,
'local_gb_used': 0,
'free_disk_gb': 20,
'vcpus': 2,
'vcpus_used': 0,
'numa_topology': objects.NUMATopology(
cells=[objects.NUMACell(id=1, cpuset=set([1, 2]), memory=512,
memory_usage=0, cpu_usage=0,
mempages=[]),
objects.NUMACell(id=2, cpuset=set([3, 4]), memory=512,
memory_usage=0, cpu_usage=0,
mempages=[])]
)._to_json()
}
if values:
resources.update(values)
return resources
def test_pinning_logic(self):
numacell = objects.NUMACell(id=0, cpuset=set([1, 2, 3, 4]), memory=512,
cpu_usage=2, memory_usage=256,
pinned_cpus=set([1]))
numacell.pin_cpus(set([2, 3]))
self.assertEqual(set([4]), numacell.free_cpus)
self.assertRaises(exception.CPUPinningInvalid,
numacell.pin_cpus, set([1, 4]))
self.assertRaises(exception.CPUPinningInvalid,
numacell.pin_cpus, set([1, 6]))
self.assertRaises(exception.CPUPinningInvalid,
numacell.unpin_cpus, set([1, 4]))
numacell.unpin_cpus(set([1, 2, 3]))
self.assertEqual(set([1, 2, 3, 4]), numacell.free_cpus)
def test_to_legacy_limits(self):
limits = objects.NUMATopologyLimits(
cpu_allocation_ratio=16,
ram_allocation_ratio=2)
host_topo = objects.NUMATopology(cells=[
objects.NUMACell(id=0, cpuset=set([1, 2]), memory=1024)
])
old_style = {'cells': [
{'mem': {'total': 1024,
'limit': 2048.0},
'id': 0,
'cpus': '1,2',
'cpu_limit': 32.0}]}
self.assertEqual(old_style, limits.to_dict_legacy(host_topo))
def test_can_fit_hugepages(self):
cell = objects.NUMACell(
id=0, cpuset=set([1, 2]), memory=1024,
mempages=[
objects.NUMAPagesTopology(
size_kb=4, total=1548736, used=0),
objects.NUMAPagesTopology(
size_kb=2048, total=513, used=0)]) # 1,002G
pagesize = 2048
self.assertTrue(cell.can_fit_hugepages(pagesize, 2 ** 20))
self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 21))
self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 19 + 1))
self.assertRaises(
exception.MemoryPageSizeNotSupported,
cell.can_fit_hugepages, 12345, 2 ** 20)
def numa_usage_from_instances(host, instances, free=False):
"""Get host topology usage
:param host: objects.NUMATopology with usage information
:param instances: list of objects.InstanceNUMATopology
:param free: If True usage of the host will be decreased
Sum the usage from all @instances to report the overall
host topology usage
:returns: objects.NUMATopology including usage information
"""
if host is None:
return
instances = instances or []
cells = []
sign = -1 if free else 1
for hostcell in host.cells:
memory_usage = hostcell.memory_usage
cpu_usage = hostcell.cpu_usage
for instance in instances:
for instancecell in instance.cells:
if instancecell.id == hostcell.id:
memory_usage = (
memory_usage + sign * instancecell.memory)
cpu_usage = cpu_usage + sign * len(instancecell.cpuset)
cell = objects.NUMACell(
id=hostcell.id, cpuset=hostcell.cpuset, memory=hostcell.memory,
cpu_usage=max(0, cpu_usage), memory_usage=max(0, memory_usage))
cells.append(cell)
return objects.NUMATopology(cells=cells)
