def test_lwkcpu_lwkmem_auto(self): v = '3' if ARGS.test_debug else '0' # Partition CPUs in following ratio between Linux and LWK lwkcpus_spec = 'lwkcpus=auto' lwkmem_spec = 'lwkmem=auto' s = '{} {}'.format(lwkcpus_spec, lwkmem_spec) logging.debug('Testing LWK partition spec: {}'.format(s)) create_and_verify(self, s, CpuSet(0), CpuSet(0), v, True)
def test_valid_no_syscall_cpus(self): v = '3' if ARGS.test_debug else '0' # Partition CPUs in following ratio between Linux and LWK spec = Spec() lwkcpus_spec = spec.create_lwkcpu_spec(0.90) self.assertNotEqual(lwkcpus_spec, 'lwkcpus=', 'Failed to create LWKCPU partition spec') lwkmem_spec = spec.create_lwkmem_spec(0.90) self.assertNotEqual(lwkmem_spec, 'lwkmem=', 'Failed to create LWKMEM partition spec') # No utility CPUs specified # Extract the utility CPUs from the lwkcpus specification utilcpus = list() regex = re.compile("[=:](\d+\.)") for m in regex.finditer(lwkcpus_spec): lwkcpus_spec = lwkcpus_spec.replace(m.group(1), '', 1) utilcpus.append(m.group(1).rstrip('.')) s = '{} {}'.format(lwkcpus_spec, lwkmem_spec) logging.debug('Testing LWK partition spec: {}'.format(s)) create_and_verify(self, s, CpuSet(0), spec.lwkcpus, v) # Utility CPUs exist but are are not syscall targets # Append the Utility CPUs to end of the lwkcpus specification utils = ','.join(utilcpus) s = '{}:{}. {}'.format(lwkcpus_spec, utils, lwkmem_spec) logging.debug('Testing LWK partition spec: {}'.format(s)) create_and_verify(self, s, spec.utilcpus, spec.lwkcpus, v)
def clearLastNCores(cs, N): cpumask = CpuSet(0) nthCore = cs.countBy(self.topology.cores) for i in range(N): cpumask += cs.selectNthBy(nthCore, self.topology.cores) nthCore -= 1 return cs - cpumask
def create_lwkcpu_spec(self, ratio): lwkcpuspec = 'lwkcpus=' self.lwkcpus = CpuSet(0) self.utilcpus = CpuSet(0) self.n_cpus = self.topology.allcpus.countCpus() assert (self.n_cpus > 0), "Invalid topology" if ratio >= 1: lwkcpuspec += '0.{}'.format(self.topology.allcpus.toList()) return lwkcpuspec elif ratio <= 0: return lwkcpuspec self.n_lwkcpus = int(ratio * self.n_cpus) self.n_linuxcpus = self.n_cpus - self.n_lwkcpus lwkcpus_per_sc_cpu = math.ceil(self.n_lwkcpus / self.n_linuxcpus) assert (self.n_lwkcpus > 0), 'Invalid no. of LWKCPUs' assert (self.n_linuxcpus > 0), 'Invalid no. of Linux CPUs' assert (lwkcpus_per_sc_cpu > 0), 'Invalid no. of LWKCPUs per utility CPU' logging.debug('Total CPUs : {}'.format(self.n_cpus)) logging.debug('Total LWK CPUs : {}'.format(self.n_lwkcpus)) logging.debug('Total Linux CPUs: {}'.format(self.n_linuxcpus)) logging.debug( 'LWK CPUs per utility CPU: {}'.format(lwkcpus_per_sc_cpu)) sc = 0 lwkcpus_count = 0 mask = CpuSet(0) for i in range(self.n_linuxcpus, self.n_cpus): if lwkcpus_count >= lwkcpus_per_sc_cpu: if lwkcpuspec != 'lwkcpus=': lwkcpuspec += ':' lwkcpuspec += '{}.{}'.format(sc, mask.toList()) self.lwkcpus += mask self.utilcpus += self.topology.allcpus.nthCpu(sc + 1) sc += 1 mask = CpuSet(0) lwkcpus_count = 0 mask += self.topology.allcpus.nthCpu(i + 1) lwkcpus_count += 1 if lwkcpus_count != 0: if lwkcpuspec != 'lwkcpus=': lwkcpuspec += ':' lwkcpuspec += '{}.{}'.format(sc, mask.toList()) self.lwkcpus += mask self.utilcpus += self.topology.allcpus.nthCpu(sc + 1) return lwkcpuspec
def get_cpus(): utilcpus = CpuSet(0) lwkcpus = CpuSet(0) op, rc = run(['lwkctl', '-v', v, '-s']) lines = op.splitlines() for l in lines: if l.startswith('Utility CPU(s):'): f, m, s = l.partition('Utility CPU(s):') s = s.split('[')[0].strip() if s != '': utilcpus.fromList(s) if l.startswith('LWK CPU(s):'): f, m, s = l.partition('LWK CPU(s):') s = s.split('[')[0].strip() if s != '': lwkcpus.fromList(s) return utilcpus, lwkcpus
def create_and_verify(obj, lwk_spec, utilcpus_req, lwkcpus_req, v='0', autogen=False): def get_cpus(): utilcpus = CpuSet(0) lwkcpus = CpuSet(0) op, rc = run(['lwkctl', '-v', v, '-s']) lines = op.splitlines() for l in lines: if l.startswith('Utility CPU(s):'): f, m, s = l.partition('Utility CPU(s):') s = s.split('[')[0].strip() if s != '': utilcpus.fromList(s) if l.startswith('LWK CPU(s):'): f, m, s = l.partition('LWK CPU(s):') s = s.split('[')[0].strip() if s != '': lwkcpus.fromList(s) return utilcpus, lwkcpus def get_profile(spec=None): if spec == None: op, rc = run(['lwkctl', '-v', v, '-s', '-r']) else: op, rc = spec, 0 for token in op.split(): if token.startswith('lwkcpu_profile='): f, m, l = token.partition('lwkcpu_profile=') return l return '' def get_lwkmem(spec=None): if spec == None: op, rc = run(['lwkctl', '-v', v, '-s', '-r']) else: op, rc = spec, 0 units = ['K', 'M', 'G', 'T', 'P', 'E'] total_size = 0 for token in op.split(): if token.startswith('lwkmem='): f, m, l = token.partition('=') l = l.strip() if l != '': for node_spec in l.split(','): node, delimiter, size = node_spec.partition(':') if delimiter == '': size = node size = size.strip() if size != '': multiplier = 1 if size.endswith(('K', 'M', 'G', 'T', 'P', 'E')): for u in units: index = size.find(u) if index != -1: multiplier = 1024**(1 + units.index(u)) break size = size.strip('KMGTPE') total_size += int(size) * multiplier return total_size def lwkcpus_auto(): with open('/sys/kernel/mOS/lwk_config', 'r') as f: data = f.read().rstrip('\n') m = re.search("auto=\S*cpu\S*", data) if m: return True return False def lwkmem_auto(): with open('/sys/kernel/mOS/lwk_config', 'r') as f: data = f.read().rstrip('\n') m = re.search("auto=\S*mem\S*", data) if m: return True return False # Create LWK partition lwk_spec = lwk_spec.strip() out, rc = run(['lwkctl', '-v', v, '-c', lwk_spec], requiresRoot=True) # Read and verify LWK partition using lwkctl -s # Verify CPUs lwkcpus_auto_set = lwkcpus_auto() if not autogen: utilcpus, lwkcpus = get_cpus() if utilcpus != utilcpus_req: logging.error('Mismatch : Syscall CPUs') logging.error('Requested: {}'.format(utilcpus_req.toList())) logging.error('Created : {}'.format(utilcpus.toList())) if lwkcpus != lwkcpus_req: logging.error('Mismatch : LWK CPUs') logging.error('Requested: {}'.format(lwkcpus_req.toList())) logging.error('Created : {}'.format(lwkcpus.toList())) if lwkcpus_auto_set: logging.error('The lwkcpus_auto indicator is set') assert (utilcpus == utilcpus_req) assert (lwkcpus == lwkcpus_req) assert (lwkcpus_auto_set == False) else: if not lwkcpus_auto_set: logging.error('The lwkcpus_auto indicator is not set') assert (lwkcpus_auto == True) # Verify LWK CPU profile profile_req = get_profile(lwk_spec) profile_set = get_profile() msg = 'LWKCPU profile requested: {} set: {}'.format( profile_req, profile_set) logging.debug(msg) if (profile_req == 'debug'): assert (profile_set == profile_req), 'Mismatch: ' + msg else: assert (profile_set == 'normal'), 'Mismatch: ' + msg # Verify LWK Memory if not lwkmem_static and not autogen: lwkmem_req = get_lwkmem(lwk_spec) lwkmem_set = get_lwkmem() msg = 'LWKMEM requested {} allocated {}'.format(lwkmem_req, lwkmem_set) logging.debug(msg) if lwkmem_req != 0: assert (lwkmem_set > 0), 'LWK memory partition was not created' assert (lwkmem_set <= lwkmem_req), 'Mismatch ' + msg lwkmem_auto_set = lwkmem_auto() if not autogen: if lwkmem_auto_set: logging.error('The lwkmem_auto indicator is set') assert (lwkmem_auto_set == False) else: if not lwkmem_auto_set: logging.error('The lwkmem_auto indicator is not set') assert (lwkmem_auto_set == True) # Run tests on LWK yod(obj, '-u', 0, '../lwksched/aff_scan', '-efm') if not lwkmem_static: yod(obj, '../lwkmem/maptest', '--verbose', '--type', 'anonymous', '--num', 10, '--size', 4096, '--iterations', 10) # Delete partition run(['lwkctl', '-v', v, '-d'], requiresRoot=True) # Read and verify using lwkctl -s utilcpus, lwkcpus = get_cpus() profile_set = get_profile() assert (utilcpus == CpuSet(0)) assert (lwkcpus == CpuSet(0)) assert (profile_set == '') if not lwkmem_static: lwkmem_set = get_lwkmem() assert (lwkmem_set == 0), 'Failed to delete LWKMEM partition' # Run tests on Linux run_bin(obj, '../lwksched/aff_scan')
def clearFirstNCores(cs, N): cpumask = CpuSet(0) for i in range(N): cpumask += cs.selectNthBy(i + 1, self.topology.cores) return cs - cpumask
def create_lwkcpu_spec(self, ratio, noutility_cpus=False): KEY_FMT = 'ratio_{}_noutility_cpus_{}' def clearFirstNCores(cs, N): cpumask = CpuSet(0) for i in range(N): cpumask += cs.selectNthBy(i + 1, self.topology.cores) return cs - cpumask def clearLastNCores(cs, N): cpumask = CpuSet(0) nthCore = cs.countBy(self.topology.cores) for i in range(N): cpumask += cs.selectNthBy(nthCore, self.topology.cores) nthCore -= 1 return cs - cpumask def update_cache_lwkcpuspec(ratio, noutility_cpus, value): key = KEY_FMT.format(ratio, noutility_cpus) self.cache[key] = value # Try to look up for a cached spec before computing from scratch key = KEY_FMT.format(ratio, noutility_cpus) if key in self.cache: return self.cache[key] # Manufacture one and store in the cache lwkcpuspec = 'lwkcpus=' utilcpus = CpuSet(0) lwkcpus = CpuSet(0) if ratio <= 0: return lwkcpuspec, utilcpus, lwkcpus if ratio > 1: ratio = 1 # Consider core 0 as Linux core irrespective of topology. core0_cpus = self.topology.nodes[0].selectNthBy(1, self.topology.cores) numa_nodes = len(self.topology.nodes) node_utilcpus = self.topology.nodes.copy() node_lwkcpus = self.topology.nodes.copy() node_lwkcpus[0] -= core0_cpus # Find base minimum utility CPUs for the given hardware. These are the # number of Linux CPUs left after balancing number of LWK CPUs across # NUMA nodes. We do not scale the number of utility CPUs with ratio. # Instead we scale only the number of LWK CPUs using the given ratio. lwkcorespn_max = min( *[node.countBy(self.topology.cores) for node in node_lwkcpus]) lwkcorespn = math.ceil(lwkcorespn_max * ratio) logging.debug('LWK cores per node: {} max x {} ratio = {}'.format( lwkcorespn_max, ratio, lwkcorespn)) # Create maps of Linux and LWK cpus for every node total_util_cpus = 0 for n in range(numa_nodes): node_utilcpus[n] = clearLastNCores(node_utilcpus[n], lwkcorespn_max) node_lwkcpus[n] -= node_utilcpus[n] if lwkcorespn_max != lwkcorespn: node_lwkcpus[n] = clearFirstNCores(node_lwkcpus[n], lwkcorespn_max - lwkcorespn) lwkcpus += node_lwkcpus[n] utilcpus += node_utilcpus[n] total_util_cpus += node_utilcpus[n].countCpus() logging.debug('Node[{}] Linux CPUs : {}'.format( n, node_utilcpus[n].toList())) logging.debug('Node[{}] LWK CPUs : {}'.format( n, node_lwkcpus[n].toList())) if noutility_cpus == True: if lwkcpuspec != 'lwkcpus=': lwkcpuspec += ':' lwkcpuspec += '{}'.format(node_lwkcpus[n].toList()) # If there are no utility cpus requested then we are done here if noutility_cpus == True: rval = (lwkcpuspec, CpuSet(0), lwkcpus) update_cache_lwkcpuspec(ratio, True, rval) return rval # Assign LWKCPUs to Utility CPU mapping utilitycpuspn = int(total_util_cpus / numa_nodes) assert (utilitycpuspn >= 1), 'Utility CPUs {} lesser than NUMA nodes {}'.format( total_util_cpus, numa_nodes) lwkcores_per_utilitycpu = int(lwkcorespn / utilitycpuspn) if lwkcores_per_utilitycpu < 1: lwkcores_per_utilitycpu = 1 # Compute LWKCPU specification for each node logging.debug('Utility cpus per node : {}'.format(utilitycpuspn)) logging.debug( 'LWK cores per utility cpus: {}'.format(lwkcores_per_utilitycpu)) for n in range(numa_nodes): for i in range(1, utilitycpuspn + 1): subgroup_cpumask = CpuSet(0) for j in range(lwkcores_per_utilitycpu): cpumask = node_lwkcpus[n].selectNthBy( 1, self.topology.cores) subgroup_cpumask += cpumask node_lwkcpus[n] -= cpumask # Add any remaining LWK CPUs due to the truncation # from integer division to the last subgroup if i == utilitycpuspn and node_lwkcpus[n].isEmpty() == False: subgroup_cpumask += node_lwkcpus[n] node_lwkcpus[n] -= node_lwkcpus[n] # Pick a target utility CPU # Try current node first utilcpumask = node_utilcpus[n].nthCpu(1) if utilcpumask.isEmpty() == False: node_utilcpus[n] -= utilcpumask else: # If there is no utility CPU on this node # then try to get one from other NUMA nodes for m in range(numa_nodes): utilcpumask = node_utilcpus[m].nthCpu(1) if utilcpumask.isEmpty() == False: node_utilcpus[m] -= utilcpumask break assert ( utilcpumask.isEmpty() == False ), 'Node[{}]: Ran out of utility cpus to assign'.format(n) assert ( utilcpumask.countCpus() == 1 ), 'Node[{}]: More than 1 utility cpus selected: {}'.format(n) if lwkcpuspec != 'lwkcpus=': lwkcpuspec += ':' node_lwkcpuspec = '{}.{}'.format(utilcpumask.toList(), subgroup_cpumask.toList()) lwkcpuspec += node_lwkcpuspec logging.debug('Node[{}] LWKCPU spec: {}'.format( n, node_lwkcpuspec)) assert ( node_lwkcpus[n].countCpus() == 0 ), 'Node[{}]: LWKCPUs {} are not assigned a utility CPU'.format( n, node_lwkcpus[n].toList()) logging.debug('LWKCPU spec: {}'.format(lwkcpuspec)) rval = (lwkcpuspec, utilcpus, lwkcpus) update_cache_lwkcpuspec(ratio, False, rval) return rval
def create_and_verify(obj, lwk_spec, sccpus_req, lwkcpus_req, v='0'): def get_cpus(): sccpus = CpuSet(0) lwkcpus = CpuSet(0) op, rc = run(['lwkctl', '-v', v, '-s']) lines = op.splitlines() for l in lines: if l.startswith('Syscall CPU(s):'): f,m,s = l.partition('Syscall CPU(s):') s = s.strip() if s != '': sccpus.fromList(s) if l.startswith('LWK CPU(s):'): f,m,s = l.partition('LWK CPU(s):') s = s.strip() if s != '': lwkcpus.fromList(s) return sccpus, lwkcpus def get_profile(spec=None): if spec == None: op, rc = run(['lwkctl', '-v', v, '-s', '-r']) else: op, rc = spec, 0 for token in op.split(): if token.startswith('lwkcpu_profile='): f,m,l = token.partition('lwkcpu_profile=') return l return '' def get_lwkmem(spec=None): if spec == None: op, rc = run(['lwkctl', '-v', v, '-s', '-r']) else: op, rc = spec, 0 units = ['K', 'M', 'G', 'T', 'P', 'E'] total_size = 0 for token in op.split(): if token.startswith('lwkmem='): f,m,l = token.partition('=') l = l.strip() if l != '': for node_spec in l.split(','): node,delimiter,size = node_spec.partition(':') if delimiter == '': size = node size = size.strip() if size != '': multiplier = 1 if size.endswith(('K', 'M', 'G', 'T', 'P', 'E')): for u in units: index = size.find(u) if index != -1: multiplier = 1024**(1+units.index(u)) break size = size.strip('KMGTPE') total_size += int(size) * multiplier return total_size # Create LWK partition lwk_spec = lwk_spec.strip() out, rc = run(['sudo', 'lwkctl', '-v', v, '-c', lwk_spec]) # Read and verify LWK partition using lwkctl -s # Verify CPUs sccpus, lwkcpus = get_cpus() if sccpus != sccpus_req or lwkcpus != lwkcpus_req: if sccpus != sccpus_req: logging.error('Mismatch : Syscall CPUs') logging.error('Requested: {}'.format(sccpus_req.toList())) logging.error('Created : {}'.format(sccpus.toList())) if lwkcpus != lwkcpus_req: logging.error('Mismatch : LWK CPUs') logging.error('Requested: {}'.format(lwkcpus_req.toList())) logging.error('Created : {}'.format(lwkcpus.toList())) assert(sccpus == sccpus_req) assert(lwkcpus == lwkcpus_req) # Verify LWK CPU profile profile_req = get_profile(lwk_spec) profile_set = get_profile() msg = 'LWKCPU profile requested: {} set: {}'.format(profile_req, profile_set) logging.debug(msg) if (profile_req == 'debug'): assert(profile_set == profile_req), 'Mismatch: ' + msg else: assert(profile_set == 'normal'), 'Mismatch: ' + msg # Verify LWK Memory if not lwkmem_static: lwkmem_req = get_lwkmem(lwk_spec) lwkmem_set = get_lwkmem() msg = 'LWKMEM requested {} allocated {}'.format(lwkmem_req, lwkmem_set) logging.debug(msg) if lwkmem_req != 0: assert(lwkmem_set > 0), 'LWK memory partition was not created' assert(lwkmem_set <= lwkmem_req), 'Mismatch ' + msg # Run tests on LWK yod(obj, '-u', 0, '../lwksched/aff_scan', '-efm') if not lwkmem_static: yod(obj, '../lwkmem/maptest', '--verbose', '--type', 'anonymous', '--num', 10, '--size', 4096, '--iterations', 10) # Delete partition run(['sudo', 'lwkctl', '-v', v, '-d']) # Read and verify using lwkctl -s sccpus, lwkcpus = get_cpus() profile_set = get_profile() assert(sccpus == CpuSet(0)) assert(lwkcpus == CpuSet(0)) assert(profile_set == '') if not lwkmem_static: lwkmem_set = get_lwkmem() assert(lwkmem_set == 0), 'Failed to delete LWKMEM partition' # Run tests on Linux run_bin(obj, '../lwksched/aff_scan')