def get_instance(self, uuid):
try:
nova = self.get_nova()
server = nova.servers.get(uuid)
flavor_info = nova.flavors.get(server.flavor["id"])
hostname = server.__dict__['OS-EXT-SRV-ATTR:host']
except Exception as e:
LOG.warning("Could not get instance=%s from Nova! error=%s" %
(uuid, e))
return None
LOG.debug('GET VM:%s in node:%s' % (server.name, hostname))
if hostname == self._hostname:
inst = instance.instance(uuid, server.name, flavor_info.get_keys())
# get numa topology and pci info from libvirt
try:
domain = guest.get_guest_domain_by_uuid(self._conn, uuid)
if domain:
inst.update(domain)
except Exception as e:
LOG.warning("Failed to access libvirt! error=%s" % e)
return inst
else:
LOG.debug('The VM is not in current host!')
return None
def reset_irq_affinity(self, uuid, irqs=None, msi_irqs=None):
"""Reset irq affinity for instance
The instance has already been deleted or
related PCI not used by it anymore.
"""
if irqs or msi_irqs:
# reset irq affinity for specified irqs
_irqs = irqs
_msi_irqs = msi_irqs
elif uuid in self.inst_dict:
# reset all irq affinity for deleted instance
_irqs = self.inst_dict[uuid][0]
_msi_irqs = self.inst_dict[uuid][1]
else:
LOG.debug("No pci affinity need to be reset for instance=%s!" %
uuid)
return
try:
with open('%s/default_smp_affinity' % COMPUTE_IRQ) as f:
cpulist = f.readline().strip()
LOG.debug("default smp affinity bitmap:%s" % cpulist)
for x in [_irqs, _msi_irqs]:
if len(x) > 0:
pci_utils.set_irq_affinity(True, x, cpulist)
except Exception as e:
LOG.error("Failed to reset smp affinity! error=%s" % e)
LOG.info("Reset smp affinity done for instance=%s!" % uuid)
def get_inst(instance_uuid, callback):
# get instance info from nova
nova_client = nova_provider.get_nova_client()
inst = nova_client.get_instance(instance_uuid)
if inst is not None:
LOG.debug("inst:%s" % inst)
callback(inst)
def affine_pci_dev_instance(self, instance, wait_for_irqs=True):
if instance is not None:
if instance.get_cpu_policy(
) == 'dedicated' and instance.get_pci_devices():
LOG.debug("Instance=%s use dedicated cpu policy!!!" %
instance.uuid)
irqs, msi_irqs, cpulist = \
self.affinePciIrqDriver.affine_pci_dev_irqs(instance, wait_for_irqs)
# record instance on which pci affinity has been applied
self.instance_irq_pcpulist_update(instance.uuid, irqs,
msi_irqs, cpulist)
return
def instance_irq_pcpulist_update(self, uuid, irqs, msi_irqs, cpulist):
if uuid in self.inst_dict:
_prev = self.inst_dict[uuid]
# get irqs that not appear anymore.
_irqs = _prev[0].difference(irqs)
_msi_irqs = _prev[1].difference(msi_irqs)
# reset pci affinity for those pcis not used by intance anymore
if (len(_irqs) + len(_msi_irqs)) > 0:
self.reset_irq_affinity(uuid, _irqs, _msi_irqs)
self.inst_dict[uuid] = [irqs, msi_irqs, cpulist]
LOG.debug(self.inst_dict)
def info(self, ctxt, publisher_id, event_type, payload, metadata):
instance_host = self.payload_decoder.decode_instance_host(payload)
current_host = os.getenv("COMPUTE_HOSTNAME",
default=socket.gethostname())
if instance_host is not None and instance_host != current_host:
LOG.debug(
"Requeue notification: instance_host=%s != current_host=%s" %
(instance_host, current_host))
return oslo_messaging.NotificationResult.REQUEUE
instance_uuid = self.payload_decoder.decode_instance_uuid(payload)
if instance_uuid:
LOG.info(
"Instance offline: uuid=%s, instance_host=%s, event_type=%s" %
(instance_uuid, instance_host, event_type))
affinity.pci_irq_affinity.reset_irq_affinity(instance_uuid)
def find_and_fill_pci_addrs(dom_xml, device_type):
LOG.debug("Finding pci_addrs for %s devices" % device_type)
def parse_pci_addr(tag):
return "%04x:%02x:%02x.%01x" % (int(
tag.get('domain'), base=16), int(
tag.get('bus'), base=16), int(tag.get('slot'), base=16),
int(tag.get('function'), base=16))
for node in dom_xml.findall('./devices/' + device_type):
for driver in node.findall('driver'):
if driver.get('name').startswith('vfio'):
addr_tag = node.find('source/address')
if (addr_tag.get('type') == 'pci'
or node.get('type') == 'pci'):
pci_addr = parse_pci_addr(addr_tag)
LOG.debug("Add pci device: %s" % pci_addr)
pci_addrs.update([pci_addr])
def update(self, domain):
cells = set()
for node_id in domain['nodelist']:
cell = numa_cell(node_id, list(range(domain['nr_vcpus'])),
domain['cpu_pinning'])
LOG.debug("cell_id=%s, vcpuset=%s, cpu_pinning=%s" %
(node_id, list(range(
domain['nr_vcpus'])), domain['cpu_pinning']))
cells.update([cell])
self.numa_topology = numa_topology(self.uuid, cells)
if domain['IsCpuPinned']:
self.cpu_policy = 'dedicated'
else:
self.cpu_policy = 'shared'
for pci_addr in domain['pci_addrs']:
pci_dev = pci_device(pci_addr)
self.pci_devices.update([pci_dev])
def get_pci_irqs_pinned_cpuset(extra_spec=None,
numa_topology=None,
pci_numa_node=None):
"""Get pinned cpuset where pci irq are affined.
:param extra_spec: extra_spec
:param pci_numa_node: numa node of a specific PCI device
:param numa_topology: instance numa topology
:return: cpuset, cpulist
"""
cpuset = set()
cpulist = ''
LOG.debug("extra_spec:%s, topo:%s, numa_node:%s" %
(extra_spec, numa_topology, pci_numa_node))
if numa_topology is None or pci_numa_node is None or pci_numa_node < 0:
return (cpuset, cpulist)
# Determine full affinity cpuset, but restrict to pci's numa node
for cell in numa_topology.cells:
if cell.id == pci_numa_node and cell.cpu_pinning is not None:
cpuset.update(set(cell.cpu_pinning.values()))
LOG.info("pinning pcpu list:%s" % cpuset)
# Use extra-spec hw:pci_irq_affinity_mask only when the instance is pinned.
if cpuset:
pci_cpuset = _get_pci_irq_affinity_mask(extra_spec)
if pci_cpuset:
cpuset = set()
for cell in numa_topology.cells:
if cell.cpu_pinning is not None:
for vcpu in cell.cpuset:
if vcpu in pci_cpuset:
vcpu_cell, pcpu = numa_topology.vcpu_to_pcpu(vcpu)
cpuset.update(set([pcpu]))
cpulist = list_to_range(input_list=list(cpuset))
return (cpuset, cpulist)
def query_instance_callback(inst):
LOG.debug("query inst:%s" % inst)
affinity.pci_irq_affinity.affine_pci_dev_instance(inst)
def set_irqs_affinity_by_pci_address(pci_addr,
extra_spec=None,
numa_topology=None):
"""Set cpu affinity for list of PCI IRQs with a VF's pci address,
Restrict cpuset to the numa node of the PCI.
Return list
Raises PciDeviceNotFoundById in case the pci device is not found,
or when there is an underlying problem getting associated irqs.
:param pci_addr: PCI address
:param extra_spec: extra_spec
:param numa_topology: instance numa topology
:return: irqs, msi_irqs, numa_node, cpulist
"""
irqs = set()
msi_irqs = set()
numa_node = None
cpulist = ''
if numa_topology is None:
return (irqs, msi_irqs, numa_node, cpulist)
# Get the irqs associated with pci addr
_irqs, _msi_irqs = get_irqs_by_pci_address(pci_addr)
LOG.debug("pci: %s, irqs: %s, msi_irqs: %s" % (pci_addr, _irqs, _msi_irqs))
# Obtain physical numa_node for this pci addr
numa_path = "%s/%s/numa_node" % (COMPUTE_PCI_DEVICES, pci_addr)
try:
with open(numa_path) as f:
numa_node = [int(x) for x in f.readline().split()][0]
except Exception as e:
LOG.error(
'set_irqs_affinity_by_pci_address: '
'pci_addr=%(A)s: numa_path=%(P)s; error=%(E)s', {
'A': pci_addr,
'P': numa_path,
'E': e
})
raise Exception("PciDeviceNotFoundById id = %r" % pci_addr)
# Skip irq configuration if there is no associated numa node
if numa_node is None or numa_node < 0:
return (irqs, msi_irqs, numa_node, cpulist)
# Determine the pinned cpuset where irqs are to be affined
cpuset, cpulist = get_pci_irqs_pinned_cpuset(extra_spec, numa_topology,
numa_node)
LOG.debug("cpuset where irqs are to be affined:%s or %s" %
(cpuset, cpulist))
# Skip irq configuration if there are no pinned cpus
if not cpuset:
return (irqs, msi_irqs, numa_node, cpulist)
# Set IRQ affinity, but do not treat errors as fatal.
LOG.debug("Setting affinity %s for irqs: %s and msi_irqs: %s" %
(cpulist, _irqs, _msi_irqs))
irqs = set_irq_affinity(False, _irqs, cpulist)
msi_irqs = set_irq_affinity(False, _msi_irqs, cpulist)
return (irqs, msi_irqs, numa_node, cpulist)