def _update_host_cpu_maps(self, host, compute_config):
host_cpus = self._get_host_cpu_list(host, threads=True)
if host_cpus:
# "Applicaton" CPUs on the platform are used for regular Openstack
# VMs
vm_cpus = self._get_host_cpu_list(
host, function=constants.APPLICATION_FUNCTION, threads=True)
vm_cpu_list = [c.cpu for c in vm_cpus]
vm_cpu_fmt = "\"%s\"" % utils.format_range_set(vm_cpu_list)
compute_config.update({'cpu_shared_set': vm_cpu_fmt})
# "Application-isolated" CPUs are completely isolated from the host
# process scheduler and are used on Openstack VMs that require
# dedicated CPUs
isol_cpus = self._get_host_cpu_list(
host, function=constants.ISOLATED_FUNCTION, threads=True)
isol_cpu_list = [c.cpu for c in isol_cpus]
isol_cpu_fmt = "\"%s\"" % utils.format_range_set(isol_cpu_list)
compute_config.update({'cpu_dedicated_set': isol_cpu_fmt})
def _update_host_cpu_maps(self, host, default_config):
host_cpus = self._get_host_cpu_list(host, threads=True)
if host_cpus:
vm_cpus = self._get_host_cpu_list(
host, function=constants.APPLICATION_FUNCTION, threads=True)
vm_cpu_list = [c.cpu for c in vm_cpus]
vm_cpu_fmt = "\"%s\"" % utils.format_range_set(vm_cpu_list)
default_config.update({'vcpu_pin_set': vm_cpu_fmt})
shared_cpus = self._get_host_cpu_list(
host, function=constants.SHARED_FUNCTION, threads=True)
shared_cpu_map = {c.numa_node: c.cpu for c in shared_cpus}
shared_cpu_fmt = "\"%s\"" % ','.join(
"%r:%r" % (node, cpu) for node, cpu in shared_cpu_map.items())
default_config.update({'shared_pcpu_map': shared_cpu_fmt})
def _get_host_k8s_cgroup_config(self, host):
config = {}
# determine set of all logical cpus and nodes
host_cpus = self._get_host_cpu_list(host, threads=True)
host_cpuset = set([c.cpu for c in host_cpus])
host_nodeset = set([c.numa_node for c in host_cpus])
# determine set of platform logical cpus and nodes
platform_cpus = self._get_host_cpu_list(
host, function=constants.PLATFORM_FUNCTION, threads=True)
platform_cpuset = set([c.cpu for c in platform_cpus])
platform_nodeset = set([c.numa_node for c in platform_cpus])
# determine set of nonplatform logical cpus and nodes
nonplatform_cpuset = host_cpuset - platform_cpuset
nonplatform_nodeset = set()
for c in host_cpus:
if c.cpu not in platform_cpuset:
nonplatform_nodeset.update([c.numa_node])
if constants.WORKER in utils.get_personalities(host):
if self.is_openstack_compute(host):
k8s_cpuset = utils.format_range_set(platform_cpuset)
k8s_nodeset = utils.format_range_set(platform_nodeset)
else:
k8s_cpuset = utils.format_range_set(nonplatform_cpuset)
k8s_nodeset = utils.format_range_set(nonplatform_nodeset)
else:
k8s_cpuset = utils.format_range_set(host_cpuset)
k8s_nodeset = utils.format_range_set(host_nodeset)
LOG.debug('host:%s, k8s_cpuset:%s, k8s_nodeset:%s', host.hostname,
k8s_cpuset, k8s_nodeset)
config.update({
'platform::kubernetes::params::k8s_cpuset':
"\"%s\"" % k8s_cpuset,
'platform::kubernetes::params::k8s_nodeset':
"\"%s\"" % k8s_nodeset,
})
return config
def _get_host_k8s_cgroup_config(self, host):
config = {}
# determine set of all logical cpus and nodes
host_cpus = self._get_host_cpu_list(host, threads=True)
host_cpuset = set([c.cpu for c in host_cpus])
host_nodeset = set([c.numa_node for c in host_cpus])
# determine set of platform logical cpus and nodes
platform_cpus = self._get_host_cpu_list(
host, function=constants.PLATFORM_FUNCTION, threads=True)
platform_cpuset = set([c.cpu for c in platform_cpus])
platform_nodeset = set([c.numa_node for c in platform_cpus])
vswitch_cpus = self._get_host_cpu_list(
host, function=constants.VSWITCH_FUNCTION, threads=True)
vswitch_cpuset = set([c.cpu for c in vswitch_cpus])
# determine set of isolcpus logical cpus and nodes
isol_cpus = self._get_host_cpu_list(
host, function=constants.ISOLATED_FUNCTION, threads=True)
isol_cpuset = set([c.cpu for c in isol_cpus])
# determine reserved sets of logical cpus in a string range set format
# to pass as options to kubelet
k8s_platform_cpuset = utils.format_range_set(platform_cpuset)
k8s_all_reserved_cpuset = utils.format_range_set(platform_cpuset
| vswitch_cpuset
| isol_cpuset)
# determine platform reserved memory
k8s_reserved_mem = 0
host_memory = self.dbapi.imemory_get_by_ihost(host.id)
numa_memory = utils.get_numa_index_list(host_memory)
for node, memory in numa_memory.items():
reserved_mib = memory[0].platform_reserved_mib
if reserved_mib is not None:
k8s_reserved_mem += reserved_mib
# determine set of nonplatform logical cpus
# TODO(jgauld): Commented out for now, using host_cpuset instead.
# nonplatform_cpuset = host_cpuset - platform_cpuset
if constants.WORKER in utils.get_personalities(host):
if self.is_openstack_compute(host):
k8s_cpuset = utils.format_range_set(platform_cpuset)
k8s_nodeset = utils.format_range_set(platform_nodeset)
else:
# kubelet cpumanager is configured with static policy.
# The resulting DefaultCPUSet excludes reserved cpus
# based on topology, and that also happens to correspond
# to the platform_cpuset. kubepods are allowed to
# span all host numa nodes.
# TODO(jgauld): Temporary workaround until we have a version
# of kubelet that excludes reserved cpus from DefaultCPUSet.
# The intent is to base k8s_cpuset on nonplatform_cpuset.
# Commented out for now, using host_cpuset instead.
# k8s_cpuset = utils.format_range_set(nonplatform_cpuset)
k8s_cpuset = utils.format_range_set(host_cpuset)
k8s_nodeset = utils.format_range_set(host_nodeset)
else:
k8s_cpuset = utils.format_range_set(host_cpuset)
k8s_nodeset = utils.format_range_set(host_nodeset)
LOG.debug('host:%s, k8s_cpuset:%s, k8s_nodeset:%s', host.hostname,
k8s_cpuset, k8s_nodeset)
# determine cpu/topology mgr policies
labels = self.dbapi.label_get_by_host(host.uuid)
for label in labels:
if label.label_key == constants.KUBE_TOPOLOGY_MANAGER_LABEL:
config.update({
'platform::kubernetes::params::k8s_topology_mgr_policy':
label.label_value
})
elif label.label_key == constants.KUBE_CPU_MANAGER_LABEL:
config.update({
'platform::kubernetes::params::k8s_cpu_mgr_policy':
label.label_value
})
config.update({
'platform::kubernetes::params::k8s_cpuset':
"\"%s\"" % k8s_cpuset,
'platform::kubernetes::params::k8s_nodeset':
"\"%s\"" % k8s_nodeset,
'platform::kubernetes::params::k8s_platform_cpuset':
"\"%s\"" % k8s_platform_cpuset,
'platform::kubernetes::params::k8s_all_reserved_cpuset':
"\"%s\"" % k8s_all_reserved_cpuset,
'platform::kubernetes::params::k8s_reserved_mem':
k8s_reserved_mem,
})
return config
def _get_host_cpu_config(self, host):
config = {}
if constants.WORKER in utils.get_personalities(host):
host_cpus = self._get_host_cpu_list(host, threads=True)
if not host_cpus:
return config
platform_cpus_no_threads = self._get_platform_cpu_list(host)
vswitch_cpus_no_threads = self._get_vswitch_cpu_list(host)
platform_numa_cpus = utils.get_numa_index_list(platform_cpus_no_threads)
vswitch_numa_cpus = utils.get_numa_index_list(vswitch_cpus_no_threads)
# build a list of platform reserved cpus per numa node
platform_cores = []
for node, cpus in platform_numa_cpus.items():
cpu_list = ','.join([str(c.cpu) for c in cpus])
platform_node = "\"node%d:%s\"" % (node, cpu_list)
platform_cores.append(platform_node)
# build a list of vswitch reserved cpu counts per numa node
vswitch_cores = []
for node, cpus in vswitch_numa_cpus.items():
cpu_count = len(cpus)
vswitch_node = "\"node%d:%d\"" % (node, cpu_count)
vswitch_cores.append(vswitch_node)
reserved_platform_cores = "(%s)" % ' '.join(platform_cores)
reserved_vswitch_cores = "(%s)" % ' '.join(vswitch_cores)
# all logical cpus
host_cpus = self._get_host_cpu_list(host, threads=True)
host_cpuset = set([c.cpu for c in host_cpus])
host_ranges = utils.format_range_set(host_cpuset)
n_cpus = len(host_cpuset)
# platform logical cpus
platform_cpus = self._get_host_cpu_list(
host, function=constants.PLATFORM_FUNCTION, threads=True)
platform_cpuset = set([c.cpu for c in platform_cpus])
platform_ranges = utils.format_range_set(platform_cpuset)
# vswitch logical cpus
vswitch_cpus = self._get_host_cpu_list(
host, constants.VSWITCH_FUNCTION, threads=True)
vswitch_cpuset = set([c.cpu for c in vswitch_cpus])
vswitch_ranges = utils.format_range_set(vswitch_cpuset)
# non-platform logical cpus
rcu_nocbs_cpuset = host_cpuset - platform_cpuset
rcu_nocbs_ranges = utils.format_range_set(rcu_nocbs_cpuset)
# non-vswitch logical cpus
non_vswitch_cpuset = host_cpuset - vswitch_cpuset
non_vswitch_ranges = utils.format_range_set(non_vswitch_cpuset)
cpu_options = ""
if constants.LOWLATENCY in host.subfunctions:
config.update({
'platform::compute::pmqos::low_wakeup_cpus':
"\"%s\"" % vswitch_ranges,
'platform::compute::pmqos::hight_wakeup_cpus':
"\"%s\"" % non_vswitch_ranges,
})
vswitch_ranges = rcu_nocbs_ranges
cpu_options += "nohz_full=%s " % vswitch_ranges
cpu_options += "isolcpus=%s rcu_nocbs=%s kthread_cpus=%s " \
"irqaffinity=%s" % (vswitch_ranges,
rcu_nocbs_ranges,
platform_ranges,
platform_ranges)
config.update({
'platform::compute::params::worker_cpu_list':
"\"%s\"" % host_ranges,
'platform::compute::params::platform_cpu_list':
"\"%s\"" % platform_ranges,
'platform::compute::params::reserved_vswitch_cores':
reserved_vswitch_cores,
'platform::compute::params::reserved_platform_cores':
reserved_platform_cores,
'platform::compute::grub::params::n_cpus': n_cpus,
'platform::compute::grub::params::cpu_options': cpu_options,
})
return config
def _get_vcpu_pin_set(self, host):
vm_cpus = self._get_host_cpu_list(
host, function=constants.APPLICATION_FUNCTION, threads=True)
cpu_list = [c.cpu for c in vm_cpus]
return "\"%s\"" % utils.format_range_set(cpu_list)
def _get_host_k8s_cgroup_config(self, host):
config = {}
# determine set of all logical cpus and nodes
host_cpus = self._get_host_cpu_list(host, threads=True)
host_cpuset = set([c.cpu for c in host_cpus])
host_nodeset = set([c.numa_node for c in host_cpus])
# determine set of platform logical cpus and nodes
platform_cpus = self._get_host_cpu_list(
host, function=constants.PLATFORM_FUNCTION, threads=True)
platform_cpuset = set([c.cpu for c in platform_cpus])
platform_nodeset = set([c.numa_node for c in platform_cpus])
# determine platform reserved number of logical cpus
k8s_reserved_cpus = len(platform_cpuset)
# determine platform reserved memory
k8s_reserved_mem = 0
host_memory = self.dbapi.imemory_get_by_ihost(host.id)
numa_memory = utils.get_numa_index_list(host_memory)
for node, memory in numa_memory.items():
reserved_mib = memory[0].platform_reserved_mib
if reserved_mib is not None:
k8s_reserved_mem += reserved_mib
# determine set of nonplatform logical cpus
# TODO(jgauld): Commented out for now, using host_cpuset instead.
# nonplatform_cpuset = host_cpuset - platform_cpuset
if constants.WORKER in utils.get_personalities(host):
if self.is_openstack_compute(host):
k8s_cpuset = utils.format_range_set(platform_cpuset)
k8s_nodeset = utils.format_range_set(platform_nodeset)
else:
# kubelet cpumanager is configured with static policy.
# The resulting DefaultCPUSet excludes reserved cpus
# based on topology, and that also happens to correspond
# to the platform_cpuset. kubepods are allowed to
# span all host numa nodes.
# TODO(jgauld): Temporary workaround until we have a version
# of kubelet that excludes reserved cpus from DefaultCPUSet.
# The intent is to base k8s_cpuset on nonplatform_cpuset.
# Commented out for now, using host_cpuset instead.
# k8s_cpuset = utils.format_range_set(nonplatform_cpuset)
k8s_cpuset = utils.format_range_set(host_cpuset)
k8s_nodeset = utils.format_range_set(host_nodeset)
else:
k8s_cpuset = utils.format_range_set(host_cpuset)
k8s_nodeset = utils.format_range_set(host_nodeset)
LOG.debug('host:%s, k8s_cpuset:%s, k8s_nodeset:%s', host.hostname,
k8s_cpuset, k8s_nodeset)
config.update({
'platform::kubernetes::params::k8s_cpuset':
"\"%s\"" % k8s_cpuset,
'platform::kubernetes::params::k8s_nodeset':
"\"%s\"" % k8s_nodeset,
'platform::kubernetes::params::k8s_reserved_cpus':
k8s_reserved_cpus,
'platform::kubernetes::params::k8s_reserved_mem':
k8s_reserved_mem,
})
return config
def _get_host_cpu_config(self, host):
config = {}
if constants.WORKER in utils.get_personalities(host):
host_cpus = self._get_host_cpu_list(host, threads=True)
if not host_cpus:
return config
platform_cpus_no_threads = self._get_platform_cpu_list(host)
vswitch_cpus_no_threads = self._get_vswitch_cpu_list(host)
platform_numa_cpus = utils.get_numa_index_list(
platform_cpus_no_threads)
vswitch_numa_cpus = utils.get_numa_index_list(
vswitch_cpus_no_threads)
# build a list of platform reserved cpus per numa node
platform_cores = []
for node, cpus in platform_numa_cpus.items():
cpu_list = ','.join([str(c.cpu) for c in cpus])
platform_node = "\"node%d:%s\"" % (node, cpu_list)
platform_cores.append(platform_node)
# build a list of vswitch reserved cpu counts per numa node
vswitch_cores = []
for node, cpus in vswitch_numa_cpus.items():
cpu_count = len(cpus)
vswitch_node = "\"node%d:%d\"" % (node, cpu_count)
vswitch_cores.append(vswitch_node)
reserved_platform_cores = "(%s)" % ' '.join(platform_cores)
reserved_vswitch_cores = "(%s)" % ' '.join(vswitch_cores)
# all logical cpus
host_cpus = self._get_host_cpu_list(host, threads=True)
host_cpuset = set([c.cpu for c in host_cpus])
host_ranges = utils.format_range_set(host_cpuset)
n_cpus = len(host_cpuset)
# platform logical cpus
platform_cpus = self._get_host_cpu_list(
host, function=constants.PLATFORM_FUNCTION, threads=True)
platform_cpuset = set([c.cpu for c in platform_cpus])
platform_ranges = utils.format_range_set(platform_cpuset)
# vswitch logical cpus
vswitch_cpus = self._get_host_cpu_list(host,
constants.VSWITCH_FUNCTION,
threads=True)
vswitch_cpuset = set([c.cpu for c in vswitch_cpus])
# non-platform logical cpus
rcu_nocbs_cpuset = host_cpuset - platform_cpuset
rcu_nocbs_ranges = utils.format_range_set(rcu_nocbs_cpuset)
# isolated logical cpus
app_isolated_cpus = self._get_host_cpu_list(
host, constants.ISOLATED_FUNCTION, threads=True)
app_isolated_cpuset = set([c.cpu for c in app_isolated_cpus])
# application cpus
app_cpus = self._get_host_cpu_list(host,
constants.APPLICATION_FUNCTION,
threads=True)
app_cpuset = set([c.cpu for c in app_cpus])
app_ranges = utils.format_range_set(app_cpuset)
cpu_options = ""
cpu_ranges = {}
if constants.LOWLATENCY in host.subfunctions:
# set PM QoS latency that achieves C1 state for all cpus
config.update({
'platform::compute::pmqos::low_wakeup_cpus':
"\"%s\"" % host_ranges,
'platform::compute::pmqos::hight_wakeup_cpus':
"\"%s\"" % "",
})
cpu_ranges.update({"nohz_full": rcu_nocbs_ranges})
isolcpus_ranges = utils.format_range_set(
vswitch_cpuset.union(app_isolated_cpuset))
cpu_ranges.update({
"isolcpus": isolcpus_ranges,
"rcu_nocbs": rcu_nocbs_ranges,
"kthread_cpus": platform_ranges
})
# Put IRQs on application cores if they are configured.
# Note that PCI IRQs for platform interfaces are reaffined to
# platform cores at runtime.
if app_cpuset:
cpu_ranges.update({"irqaffinity": app_ranges})
else:
cpu_ranges.update({"irqaffinity": platform_ranges})
for key, value in cpu_ranges.items():
if str(value).strip() != "":
cpu_options += "%s=%s " % (key, value)
config.update({
'platform::compute::params::worker_cpu_list':
"\"%s\"" % host_ranges,
'platform::compute::params::platform_cpu_list':
"\"%s\"" % platform_ranges,
'platform::compute::params::reserved_vswitch_cores':
reserved_vswitch_cores,
'platform::compute::params::reserved_platform_cores':
reserved_platform_cores,
'platform::compute::grub::params::n_cpus':
n_cpus,
'platform::compute::grub::params::cpu_options':
cpu_options,
})
return config