def _get_host_k8s_cgroup_config(self, host): config = {} # determine set of all logical cpus and nodes host_cpus = self._get_host_cpu_list(host, threads=True) host_cpuset = set([c.cpu for c in host_cpus]) host_nodeset = set([c.numa_node for c in host_cpus]) # determine set of platform logical cpus and nodes platform_cpus = self._get_host_cpu_list( host, function=constants.PLATFORM_FUNCTION, threads=True) platform_cpuset = set([c.cpu for c in platform_cpus]) platform_nodeset = set([c.numa_node for c in platform_cpus]) vswitch_cpus = self._get_host_cpu_list( host, function=constants.VSWITCH_FUNCTION, threads=True) vswitch_cpuset = set([c.cpu for c in vswitch_cpus]) # determine set of isolcpus logical cpus and nodes isol_cpus = self._get_host_cpu_list( host, function=constants.ISOLATED_FUNCTION, threads=True) isol_cpuset = set([c.cpu for c in isol_cpus]) # determine reserved sets of logical cpus in a string range set format # to pass as options to kubelet k8s_platform_cpuset = utils.format_range_set(platform_cpuset) k8s_all_reserved_cpuset = utils.format_range_set(platform_cpuset | vswitch_cpuset | isol_cpuset) # determine platform reserved memory k8s_reserved_mem = 0 host_memory = self.dbapi.imemory_get_by_ihost(host.id) numa_memory = utils.get_numa_index_list(host_memory) for node, memory in numa_memory.items(): reserved_mib = memory[0].platform_reserved_mib if reserved_mib is not None: k8s_reserved_mem += reserved_mib # determine set of nonplatform logical cpus # TODO(jgauld): Commented out for now, using host_cpuset instead. # nonplatform_cpuset = host_cpuset - platform_cpuset if constants.WORKER in utils.get_personalities(host): if self.is_openstack_compute(host): k8s_cpuset = utils.format_range_set(platform_cpuset) k8s_nodeset = utils.format_range_set(platform_nodeset) else: # kubelet cpumanager is configured with static policy. # The resulting DefaultCPUSet excludes reserved cpus # based on topology, and that also happens to correspond # to the platform_cpuset. kubepods are allowed to # span all host numa nodes. # TODO(jgauld): Temporary workaround until we have a version # of kubelet that excludes reserved cpus from DefaultCPUSet. # The intent is to base k8s_cpuset on nonplatform_cpuset. # Commented out for now, using host_cpuset instead. # k8s_cpuset = utils.format_range_set(nonplatform_cpuset) k8s_cpuset = utils.format_range_set(host_cpuset) k8s_nodeset = utils.format_range_set(host_nodeset) else: k8s_cpuset = utils.format_range_set(host_cpuset) k8s_nodeset = utils.format_range_set(host_nodeset) LOG.debug('host:%s, k8s_cpuset:%s, k8s_nodeset:%s', host.hostname, k8s_cpuset, k8s_nodeset) # determine cpu/topology mgr policies labels = self.dbapi.label_get_by_host(host.uuid) for label in labels: if label.label_key == constants.KUBE_TOPOLOGY_MANAGER_LABEL: config.update({ 'platform::kubernetes::params::k8s_topology_mgr_policy': label.label_value }) elif label.label_key == constants.KUBE_CPU_MANAGER_LABEL: config.update({ 'platform::kubernetes::params::k8s_cpu_mgr_policy': label.label_value }) config.update({ 'platform::kubernetes::params::k8s_cpuset': "\"%s\"" % k8s_cpuset, 'platform::kubernetes::params::k8s_nodeset': "\"%s\"" % k8s_nodeset, 'platform::kubernetes::params::k8s_platform_cpuset': "\"%s\"" % k8s_platform_cpuset, 'platform::kubernetes::params::k8s_all_reserved_cpuset': "\"%s\"" % k8s_all_reserved_cpuset, 'platform::kubernetes::params::k8s_reserved_mem': k8s_reserved_mem, }) return config
def _get_host_memory_config(self, host): config = {} if constants.WORKER in utils.get_personalities(host): host_memory = self.dbapi.imemory_get_by_ihost(host.id) memory_numa_list = utils.get_numa_index_list(host_memory) platform_cpus_no_threads = self._get_platform_cpu_list(host) platform_core_count = len(platform_cpus_no_threads) platform_nodes = [] vswitch_nodes = [] hugepages_2Ms = [] hugepages_1Gs = [] vswitch_2M_pages = [] vswitch_1G_pages = [] vm_4K_pages = [] vm_2M_pages = [] vm_1G_pages = [] vs_pages_updated = False for node, memory_list in memory_numa_list.items(): memory = memory_list[0] vswitch_2M_page = 0 vswitch_1G_page = 0 platform_size = memory.platform_reserved_mib platform_node = "\"node%d:%dMB:%d\"" % ( node, platform_size, platform_core_count) platform_nodes.append(platform_node) vswitch_size = memory.vswitch_hugepages_size_mib vswitch_pages = memory.vswitch_hugepages_reqd \ if memory.vswitch_hugepages_reqd is not None \ else memory.vswitch_hugepages_nr if vswitch_pages == 0: vswitch_pages = memory.vswitch_hugepages_nr vswitch_node = "\"node%d:%dkB:%d\"" % ( node, vswitch_size * 1024, vswitch_pages) vswitch_nodes.append(vswitch_node) vm_hugepages_nr_2M = memory.vm_hugepages_nr_2M_pending \ if memory.vm_hugepages_nr_2M_pending is not None \ else memory.vm_hugepages_nr_2M vm_hugepages_nr_1G = memory.vm_hugepages_nr_1G_pending \ if memory.vm_hugepages_nr_1G_pending is not None \ else memory.vm_hugepages_nr_1G vm_hugepages_nr_4K = memory.vm_hugepages_nr_4K \ if memory.vm_hugepages_nr_4K is not None else 0 total_hugepages_2M = vm_hugepages_nr_2M total_hugepages_1G = vm_hugepages_nr_1G if memory.vswitch_hugepages_size_mib == constants.MIB_2M: total_hugepages_2M += vswitch_pages vswitch_2M_page += vswitch_pages elif memory.vswitch_hugepages_size_mib == constants.MIB_1G: total_hugepages_1G += vswitch_pages vswitch_1G_page += vswitch_pages vswitch_2M_pages.append(vswitch_2M_page) vswitch_1G_pages.append(vswitch_1G_page) hugepages_2M = "\"node%d:%dkB:%d\"" % ( node, constants.MIB_2M * 1024, total_hugepages_2M) hugepages_1G = "\"node%d:%dkB:%d\"" % ( node, constants.MIB_1G * 1024, total_hugepages_1G) hugepages_2Ms.append(hugepages_2M) hugepages_1Gs.append(hugepages_1G) vm_4K_pages.append(vm_hugepages_nr_4K) vm_2M_pages.append(vm_hugepages_nr_2M) vm_1G_pages.append(vm_hugepages_nr_1G) if (memory.vswitch_hugepages_reqd and vswitch_pages != memory.vswitch_hugepages_nr): vs_pages_updated = True platform_reserved_memory = "(%s)" % ' '.join(platform_nodes) vswitch_reserved_memory = "(%s)" % ' '.join(vswitch_nodes) nr_hugepages_2Ms = "(%s)" % ' '.join(hugepages_2Ms) nr_hugepages_1Gs = "(%s)" % ' '.join(hugepages_1Gs) vswitch_2M = "\"%s\"" % ','.join([str(i) for i in vswitch_2M_pages]) vswitch_1G = "\"%s\"" % ','.join([str(i) for i in vswitch_1G_pages]) vm_4K = "\"%s\"" % ','.join([str(i) for i in vm_4K_pages]) vm_2M = "\"%s\"" % ','.join([str(i) for i in vm_2M_pages]) vm_1G = "\"%s\"" % ','.join([str(i) for i in vm_1G_pages]) config.update({ 'platform::compute::params::worker_base_reserved': platform_reserved_memory, 'platform::compute::params::compute_vswitch_reserved': vswitch_reserved_memory, 'platform::compute::hugepage::params::nr_hugepages_2M': nr_hugepages_2Ms, 'platform::compute::hugepage::params::nr_hugepages_1G': nr_hugepages_1Gs, 'platform::compute::hugepage::params::vswitch_2M_pages': vswitch_2M, 'platform::compute::hugepage::params::vswitch_1G_pages': vswitch_1G, 'platform::compute::hugepage::params::vm_4K_pages': vm_4K, 'platform::compute::hugepage::params::vm_2M_pages': vm_2M, 'platform::compute::hugepage::params::vm_1G_pages': vm_1G, }) if vs_pages_updated: grub_hugepages_1G = "hugepagesz=1G hugepages=%d" % ( sum(vswitch_1G_pages) + sum(vm_1G_pages)) config.update({ 'platform::compute::grub::params::g_hugepages': grub_hugepages_1G, }) if sum(vswitch_2M_pages) > 0: config.update({ 'platform::vswitch::params::hugepage_dir': '/mnt/huge-2048kB' }) return config
def _get_host_cpu_config(self, host): config = {} if constants.WORKER in utils.get_personalities(host): host_cpus = self._get_host_cpu_list(host, threads=True) if not host_cpus: return config platform_cpus_no_threads = self._get_platform_cpu_list(host) vswitch_cpus_no_threads = self._get_vswitch_cpu_list(host) platform_numa_cpus = utils.get_numa_index_list(platform_cpus_no_threads) vswitch_numa_cpus = utils.get_numa_index_list(vswitch_cpus_no_threads) # build a list of platform reserved cpus per numa node platform_cores = [] for node, cpus in platform_numa_cpus.items(): cpu_list = ','.join([str(c.cpu) for c in cpus]) platform_node = "\"node%d:%s\"" % (node, cpu_list) platform_cores.append(platform_node) # build a list of vswitch reserved cpu counts per numa node vswitch_cores = [] for node, cpus in vswitch_numa_cpus.items(): cpu_count = len(cpus) vswitch_node = "\"node%d:%d\"" % (node, cpu_count) vswitch_cores.append(vswitch_node) reserved_platform_cores = "(%s)" % ' '.join(platform_cores) reserved_vswitch_cores = "(%s)" % ' '.join(vswitch_cores) # all logical cpus host_cpus = self._get_host_cpu_list(host, threads=True) host_cpuset = set([c.cpu for c in host_cpus]) host_ranges = utils.format_range_set(host_cpuset) n_cpus = len(host_cpuset) # platform logical cpus platform_cpus = self._get_host_cpu_list( host, function=constants.PLATFORM_FUNCTION, threads=True) platform_cpuset = set([c.cpu for c in platform_cpus]) platform_ranges = utils.format_range_set(platform_cpuset) # vswitch logical cpus vswitch_cpus = self._get_host_cpu_list( host, constants.VSWITCH_FUNCTION, threads=True) vswitch_cpuset = set([c.cpu for c in vswitch_cpus]) vswitch_ranges = utils.format_range_set(vswitch_cpuset) # non-platform logical cpus rcu_nocbs_cpuset = host_cpuset - platform_cpuset rcu_nocbs_ranges = utils.format_range_set(rcu_nocbs_cpuset) # non-vswitch logical cpus non_vswitch_cpuset = host_cpuset - vswitch_cpuset non_vswitch_ranges = utils.format_range_set(non_vswitch_cpuset) cpu_options = "" if constants.LOWLATENCY in host.subfunctions: config.update({ 'platform::compute::pmqos::low_wakeup_cpus': "\"%s\"" % vswitch_ranges, 'platform::compute::pmqos::hight_wakeup_cpus': "\"%s\"" % non_vswitch_ranges, }) vswitch_ranges = rcu_nocbs_ranges cpu_options += "nohz_full=%s " % vswitch_ranges cpu_options += "isolcpus=%s rcu_nocbs=%s kthread_cpus=%s " \ "irqaffinity=%s" % (vswitch_ranges, rcu_nocbs_ranges, platform_ranges, platform_ranges) config.update({ 'platform::compute::params::worker_cpu_list': "\"%s\"" % host_ranges, 'platform::compute::params::platform_cpu_list': "\"%s\"" % platform_ranges, 'platform::compute::params::reserved_vswitch_cores': reserved_vswitch_cores, 'platform::compute::params::reserved_platform_cores': reserved_platform_cores, 'platform::compute::grub::params::n_cpus': n_cpus, 'platform::compute::grub::params::cpu_options': cpu_options, }) return config
def _get_host_k8s_cgroup_config(self, host): config = {} # determine set of all logical cpus and nodes host_cpus = self._get_host_cpu_list(host, threads=True) host_cpuset = set([c.cpu for c in host_cpus]) host_nodeset = set([c.numa_node for c in host_cpus]) # determine set of platform logical cpus and nodes platform_cpus = self._get_host_cpu_list( host, function=constants.PLATFORM_FUNCTION, threads=True) platform_cpuset = set([c.cpu for c in platform_cpus]) platform_nodeset = set([c.numa_node for c in platform_cpus]) # determine platform reserved number of logical cpus k8s_reserved_cpus = len(platform_cpuset) # determine platform reserved memory k8s_reserved_mem = 0 host_memory = self.dbapi.imemory_get_by_ihost(host.id) numa_memory = utils.get_numa_index_list(host_memory) for node, memory in numa_memory.items(): reserved_mib = memory[0].platform_reserved_mib if reserved_mib is not None: k8s_reserved_mem += reserved_mib # determine set of nonplatform logical cpus # TODO(jgauld): Commented out for now, using host_cpuset instead. # nonplatform_cpuset = host_cpuset - platform_cpuset if constants.WORKER in utils.get_personalities(host): if self.is_openstack_compute(host): k8s_cpuset = utils.format_range_set(platform_cpuset) k8s_nodeset = utils.format_range_set(platform_nodeset) else: # kubelet cpumanager is configured with static policy. # The resulting DefaultCPUSet excludes reserved cpus # based on topology, and that also happens to correspond # to the platform_cpuset. kubepods are allowed to # span all host numa nodes. # TODO(jgauld): Temporary workaround until we have a version # of kubelet that excludes reserved cpus from DefaultCPUSet. # The intent is to base k8s_cpuset on nonplatform_cpuset. # Commented out for now, using host_cpuset instead. # k8s_cpuset = utils.format_range_set(nonplatform_cpuset) k8s_cpuset = utils.format_range_set(host_cpuset) k8s_nodeset = utils.format_range_set(host_nodeset) else: k8s_cpuset = utils.format_range_set(host_cpuset) k8s_nodeset = utils.format_range_set(host_nodeset) LOG.debug('host:%s, k8s_cpuset:%s, k8s_nodeset:%s', host.hostname, k8s_cpuset, k8s_nodeset) config.update({ 'platform::kubernetes::params::k8s_cpuset': "\"%s\"" % k8s_cpuset, 'platform::kubernetes::params::k8s_nodeset': "\"%s\"" % k8s_nodeset, 'platform::kubernetes::params::k8s_reserved_cpus': k8s_reserved_cpus, 'platform::kubernetes::params::k8s_reserved_mem': k8s_reserved_mem, }) return config
def _get_host_cpu_config(self, host): config = {} if constants.WORKER in utils.get_personalities(host): host_cpus = self._get_host_cpu_list(host, threads=True) if not host_cpus: return config platform_cpus_no_threads = self._get_platform_cpu_list(host) vswitch_cpus_no_threads = self._get_vswitch_cpu_list(host) platform_numa_cpus = utils.get_numa_index_list( platform_cpus_no_threads) vswitch_numa_cpus = utils.get_numa_index_list( vswitch_cpus_no_threads) # build a list of platform reserved cpus per numa node platform_cores = [] for node, cpus in platform_numa_cpus.items(): cpu_list = ','.join([str(c.cpu) for c in cpus]) platform_node = "\"node%d:%s\"" % (node, cpu_list) platform_cores.append(platform_node) # build a list of vswitch reserved cpu counts per numa node vswitch_cores = [] for node, cpus in vswitch_numa_cpus.items(): cpu_count = len(cpus) vswitch_node = "\"node%d:%d\"" % (node, cpu_count) vswitch_cores.append(vswitch_node) reserved_platform_cores = "(%s)" % ' '.join(platform_cores) reserved_vswitch_cores = "(%s)" % ' '.join(vswitch_cores) # all logical cpus host_cpus = self._get_host_cpu_list(host, threads=True) host_cpuset = set([c.cpu for c in host_cpus]) host_ranges = utils.format_range_set(host_cpuset) n_cpus = len(host_cpuset) # platform logical cpus platform_cpus = self._get_host_cpu_list( host, function=constants.PLATFORM_FUNCTION, threads=True) platform_cpuset = set([c.cpu for c in platform_cpus]) platform_ranges = utils.format_range_set(platform_cpuset) # vswitch logical cpus vswitch_cpus = self._get_host_cpu_list(host, constants.VSWITCH_FUNCTION, threads=True) vswitch_cpuset = set([c.cpu for c in vswitch_cpus]) # non-platform logical cpus rcu_nocbs_cpuset = host_cpuset - platform_cpuset rcu_nocbs_ranges = utils.format_range_set(rcu_nocbs_cpuset) # isolated logical cpus app_isolated_cpus = self._get_host_cpu_list( host, constants.ISOLATED_FUNCTION, threads=True) app_isolated_cpuset = set([c.cpu for c in app_isolated_cpus]) # application cpus app_cpus = self._get_host_cpu_list(host, constants.APPLICATION_FUNCTION, threads=True) app_cpuset = set([c.cpu for c in app_cpus]) app_ranges = utils.format_range_set(app_cpuset) cpu_options = "" cpu_ranges = {} if constants.LOWLATENCY in host.subfunctions: # set PM QoS latency that achieves C1 state for all cpus config.update({ 'platform::compute::pmqos::low_wakeup_cpus': "\"%s\"" % host_ranges, 'platform::compute::pmqos::hight_wakeup_cpus': "\"%s\"" % "", }) cpu_ranges.update({"nohz_full": rcu_nocbs_ranges}) isolcpus_ranges = utils.format_range_set( vswitch_cpuset.union(app_isolated_cpuset)) cpu_ranges.update({ "isolcpus": isolcpus_ranges, "rcu_nocbs": rcu_nocbs_ranges, "kthread_cpus": platform_ranges }) # Put IRQs on application cores if they are configured. # Note that PCI IRQs for platform interfaces are reaffined to # platform cores at runtime. if app_cpuset: cpu_ranges.update({"irqaffinity": app_ranges}) else: cpu_ranges.update({"irqaffinity": platform_ranges}) for key, value in cpu_ranges.items(): if str(value).strip() != "": cpu_options += "%s=%s " % (key, value) config.update({ 'platform::compute::params::worker_cpu_list': "\"%s\"" % host_ranges, 'platform::compute::params::platform_cpu_list': "\"%s\"" % platform_ranges, 'platform::compute::params::reserved_vswitch_cores': reserved_vswitch_cores, 'platform::compute::params::reserved_platform_cores': reserved_platform_cores, 'platform::compute::grub::params::n_cpus': n_cpus, 'platform::compute::grub::params::cpu_options': cpu_options, }) return config
def _get_host_cpu_config(self, host): config = {} if constants.WORKER in utils.get_personalities(host): host_cpus = self._get_host_cpu_list(host, threads=True) if not host_cpus: return config # Define the full range of CPUs for the compute host max_cpu = max(host_cpus, key=operator.attrgetter('cpu')) worker_cpu_list = "\"0-%d\"" % max_cpu.cpu platform_cpus_no_threads = self._get_platform_cpu_list(host) vswitch_cpus_no_threads = self._get_vswitch_cpu_list(host) platform_cpu_list_with_quotes = \ "\"%s\"" % ','.join([str(c.cpu) for c in platform_cpus_no_threads]) platform_numa_cpus = utils.get_numa_index_list( platform_cpus_no_threads) vswitch_numa_cpus = utils.get_numa_index_list( vswitch_cpus_no_threads) # build a list of platform reserved cpus per numa node platform_cores = [] for node, cpus in platform_numa_cpus.items(): cpu_list = ','.join([str(c.cpu) for c in cpus]) platform_node = "\"node%d:%s\"" % (node, cpu_list) platform_cores.append(platform_node) # build a list of vswitch reserved cpu counts per numa node vswitch_cores = [] for node, cpus in vswitch_numa_cpus.items(): cpu_count = len(cpus) vswitch_node = "\"node%d:%d\"" % (node, cpu_count) vswitch_cores.append(vswitch_node) reserved_platform_cores = "(%s)" % ' '.join(platform_cores) reserved_vswitch_cores = "(%s)" % ' '.join(vswitch_cores) host_cpus = sorted(host_cpus, key=lambda c: c.cpu) n_cpus = len(host_cpus) host_cpu_list = [c.cpu for c in host_cpus] platform_cpus = self._get_host_cpu_list( host, function=constants.PLATFORM_FUNCTION, threads=True) platform_cpus = sorted(platform_cpus, key=lambda c: c.cpu) platform_cpu_list = \ "%s" % ','.join([str(c.cpu) for c in platform_cpus]) vswitch_cpus = self._get_host_cpu_list(host, constants.VSWITCH_FUNCTION, threads=True) vswitch_cpus = sorted(vswitch_cpus, key=lambda c: c.cpu) vswitch_cpu_list = \ "%s" % ','.join([str(c.cpu) for c in vswitch_cpus]) # rcu_nocbs = all cores - platform cores rcu_nocbs = copy.deepcopy(host_cpu_list) for i in [int(s) for s in platform_cpu_list.split(',')]: rcu_nocbs.remove(i) # change the CPU list to ranges rcu_nocbs_ranges = "" for key, group in itertools.groupby(enumerate(rcu_nocbs), lambda xy: xy[1] - xy[0]): group = list(group) rcu_nocbs_ranges += "%s-%s," % (group[0][1], group[-1][1]) rcu_nocbs_ranges = rcu_nocbs_ranges.rstrip(',') # non-vswitch CPUs = all cores - vswitch cores non_vswitch_cpus = host_cpu_list for i in [c.cpu for c in vswitch_cpus]: non_vswitch_cpus.remove(i) # change the CPU list to ranges non_vswitch_cpus_ranges = "" for key, group in itertools.groupby(enumerate(non_vswitch_cpus), lambda xy: xy[1] - xy[0]): group = list(group) non_vswitch_cpus_ranges += "\"%s-%s\"," % (group[0][1], group[-1][1]) cpu_options = "" if constants.LOWLATENCY in host.subfunctions: vswitch_cpu_list_with_quotes = \ "\"%s\"" % ','.join([str(c.cpu) for c in vswitch_cpus]) config.update({ 'platform::compute::pmqos::low_wakeup_cpus': vswitch_cpu_list_with_quotes, 'platform::compute::pmqos::hight_wakeup_cpus': non_vswitch_cpus_ranges.rstrip(',') }) vswitch_cpu_list = rcu_nocbs_ranges cpu_options += "nohz_full=%s " % vswitch_cpu_list cpu_options += "isolcpus=%s rcu_nocbs=%s kthread_cpus=%s " \ "irqaffinity=%s" % (vswitch_cpu_list, rcu_nocbs_ranges, platform_cpu_list, platform_cpu_list) config.update({ 'platform::compute::params::worker_cpu_list': worker_cpu_list, 'platform::compute::params::platform_cpu_list': platform_cpu_list_with_quotes, 'platform::compute::params::reserved_vswitch_cores': reserved_vswitch_cores, 'platform::compute::params::reserved_platform_cores': reserved_platform_cores, 'platform::compute::grub::params::n_cpus': n_cpus, 'platform::compute::grub::params::cpu_options': cpu_options, }) return config