def test_lwkcpu_lwkmem_auto(self):
v = '3' if ARGS.test_debug else '0'
# Partition CPUs in following ratio between Linux and LWK
lwkcpus_spec = 'lwkcpus=auto'
lwkmem_spec = 'lwkmem=auto'
s = '{} {}'.format(lwkcpus_spec, lwkmem_spec)
logging.debug('Testing LWK partition spec: {}'.format(s))
create_and_verify(self, s, CpuSet(0), CpuSet(0), v, True)
def test_valid_no_syscall_cpus(self):
v = '3' if ARGS.test_debug else '0'
# Partition CPUs in following ratio between Linux and LWK
spec = Spec()
lwkcpus_spec = spec.create_lwkcpu_spec(0.90)
self.assertNotEqual(lwkcpus_spec, 'lwkcpus=',
'Failed to create LWKCPU partition spec')
lwkmem_spec = spec.create_lwkmem_spec(0.90)
self.assertNotEqual(lwkmem_spec, 'lwkmem=',
'Failed to create LWKMEM partition spec')
# No utility CPUs specified
# Extract the utility CPUs from the lwkcpus specification
utilcpus = list()
regex = re.compile("[=:](\d+\.)")
for m in regex.finditer(lwkcpus_spec):
lwkcpus_spec = lwkcpus_spec.replace(m.group(1), '', 1)
utilcpus.append(m.group(1).rstrip('.'))
s = '{} {}'.format(lwkcpus_spec, lwkmem_spec)
logging.debug('Testing LWK partition spec: {}'.format(s))
create_and_verify(self, s, CpuSet(0), spec.lwkcpus, v)
# Utility CPUs exist but are are not syscall targets
# Append the Utility CPUs to end of the lwkcpus specification
utils = ','.join(utilcpus)
s = '{}:{}. {}'.format(lwkcpus_spec, utils, lwkmem_spec)
logging.debug('Testing LWK partition spec: {}'.format(s))
create_and_verify(self, s, spec.utilcpus, spec.lwkcpus, v)
def clearLastNCores(cs, N):
cpumask = CpuSet(0)
nthCore = cs.countBy(self.topology.cores)
for i in range(N):
cpumask += cs.selectNthBy(nthCore, self.topology.cores)
nthCore -= 1
return cs - cpumask
def create_lwkcpu_spec(self, ratio):
lwkcpuspec = 'lwkcpus='
self.lwkcpus = CpuSet(0)
self.utilcpus = CpuSet(0)
self.n_cpus = self.topology.allcpus.countCpus()
assert (self.n_cpus > 0), "Invalid topology"
if ratio >= 1:
lwkcpuspec += '0.{}'.format(self.topology.allcpus.toList())
return lwkcpuspec
elif ratio <= 0:
return lwkcpuspec
self.n_lwkcpus = int(ratio * self.n_cpus)
self.n_linuxcpus = self.n_cpus - self.n_lwkcpus
lwkcpus_per_sc_cpu = math.ceil(self.n_lwkcpus / self.n_linuxcpus)
assert (self.n_lwkcpus > 0), 'Invalid no. of LWKCPUs'
assert (self.n_linuxcpus > 0), 'Invalid no. of Linux CPUs'
assert (lwkcpus_per_sc_cpu >
0), 'Invalid no. of LWKCPUs per utility CPU'
logging.debug('Total CPUs : {}'.format(self.n_cpus))
logging.debug('Total LWK CPUs : {}'.format(self.n_lwkcpus))
logging.debug('Total Linux CPUs: {}'.format(self.n_linuxcpus))
logging.debug(
'LWK CPUs per utility CPU: {}'.format(lwkcpus_per_sc_cpu))
sc = 0
lwkcpus_count = 0
mask = CpuSet(0)
for i in range(self.n_linuxcpus, self.n_cpus):
if lwkcpus_count >= lwkcpus_per_sc_cpu:
if lwkcpuspec != 'lwkcpus=':
lwkcpuspec += ':'
lwkcpuspec += '{}.{}'.format(sc, mask.toList())
self.lwkcpus += mask
self.utilcpus += self.topology.allcpus.nthCpu(sc + 1)
sc += 1
mask = CpuSet(0)
lwkcpus_count = 0
mask += self.topology.allcpus.nthCpu(i + 1)
lwkcpus_count += 1
if lwkcpus_count != 0:
if lwkcpuspec != 'lwkcpus=':
lwkcpuspec += ':'
lwkcpuspec += '{}.{}'.format(sc, mask.toList())
self.lwkcpus += mask
self.utilcpus += self.topology.allcpus.nthCpu(sc + 1)
return lwkcpuspec
def get_cpus():
utilcpus = CpuSet(0)
lwkcpus = CpuSet(0)
op, rc = run(['lwkctl', '-v', v, '-s'])
lines = op.splitlines()
for l in lines:
if l.startswith('Utility CPU(s):'):
f, m, s = l.partition('Utility CPU(s):')
s = s.split('[')[0].strip()
if s != '':
utilcpus.fromList(s)
if l.startswith('LWK CPU(s):'):
f, m, s = l.partition('LWK CPU(s):')
s = s.split('[')[0].strip()
if s != '':
lwkcpus.fromList(s)
return utilcpus, lwkcpus
def create_and_verify(obj,
lwk_spec,
utilcpus_req,
lwkcpus_req,
v='0',
autogen=False):
def get_cpus():
utilcpus = CpuSet(0)
lwkcpus = CpuSet(0)
op, rc = run(['lwkctl', '-v', v, '-s'])
lines = op.splitlines()
for l in lines:
if l.startswith('Utility CPU(s):'):
f, m, s = l.partition('Utility CPU(s):')
s = s.split('[')[0].strip()
if s != '':
utilcpus.fromList(s)
if l.startswith('LWK CPU(s):'):
f, m, s = l.partition('LWK CPU(s):')
s = s.split('[')[0].strip()
if s != '':
lwkcpus.fromList(s)
return utilcpus, lwkcpus
def get_profile(spec=None):
if spec == None:
op, rc = run(['lwkctl', '-v', v, '-s', '-r'])
else:
op, rc = spec, 0
for token in op.split():
if token.startswith('lwkcpu_profile='):
f, m, l = token.partition('lwkcpu_profile=')
return l
return ''
def get_lwkmem(spec=None):
if spec == None:
op, rc = run(['lwkctl', '-v', v, '-s', '-r'])
else:
op, rc = spec, 0
units = ['K', 'M', 'G', 'T', 'P', 'E']
total_size = 0
for token in op.split():
if token.startswith('lwkmem='):
f, m, l = token.partition('=')
l = l.strip()
if l != '':
for node_spec in l.split(','):
node, delimiter, size = node_spec.partition(':')
if delimiter == '':
size = node
size = size.strip()
if size != '':
multiplier = 1
if size.endswith(('K', 'M', 'G', 'T', 'P', 'E')):
for u in units:
index = size.find(u)
if index != -1:
multiplier = 1024**(1 + units.index(u))
break
size = size.strip('KMGTPE')
total_size += int(size) * multiplier
return total_size
def lwkcpus_auto():
with open('/sys/kernel/mOS/lwk_config', 'r') as f:
data = f.read().rstrip('\n')
m = re.search("auto=\S*cpu\S*", data)
if m:
return True
return False
def lwkmem_auto():
with open('/sys/kernel/mOS/lwk_config', 'r') as f:
data = f.read().rstrip('\n')
m = re.search("auto=\S*mem\S*", data)
if m:
return True
return False
# Create LWK partition
lwk_spec = lwk_spec.strip()
out, rc = run(['lwkctl', '-v', v, '-c', lwk_spec], requiresRoot=True)
# Read and verify LWK partition using lwkctl -s
# Verify CPUs
lwkcpus_auto_set = lwkcpus_auto()
if not autogen:
utilcpus, lwkcpus = get_cpus()
if utilcpus != utilcpus_req:
logging.error('Mismatch : Syscall CPUs')
logging.error('Requested: {}'.format(utilcpus_req.toList()))
logging.error('Created : {}'.format(utilcpus.toList()))
if lwkcpus != lwkcpus_req:
logging.error('Mismatch : LWK CPUs')
logging.error('Requested: {}'.format(lwkcpus_req.toList()))
logging.error('Created : {}'.format(lwkcpus.toList()))
if lwkcpus_auto_set:
logging.error('The lwkcpus_auto indicator is set')
assert (utilcpus == utilcpus_req)
assert (lwkcpus == lwkcpus_req)
assert (lwkcpus_auto_set == False)
else:
if not lwkcpus_auto_set:
logging.error('The lwkcpus_auto indicator is not set')
assert (lwkcpus_auto == True)
# Verify LWK CPU profile
profile_req = get_profile(lwk_spec)
profile_set = get_profile()
msg = 'LWKCPU profile requested: {} set: {}'.format(
profile_req, profile_set)
logging.debug(msg)
if (profile_req == 'debug'):
assert (profile_set == profile_req), 'Mismatch: ' + msg
else:
assert (profile_set == 'normal'), 'Mismatch: ' + msg
# Verify LWK Memory
if not lwkmem_static and not autogen:
lwkmem_req = get_lwkmem(lwk_spec)
lwkmem_set = get_lwkmem()
msg = 'LWKMEM requested {} allocated {}'.format(lwkmem_req, lwkmem_set)
logging.debug(msg)
if lwkmem_req != 0:
assert (lwkmem_set > 0), 'LWK memory partition was not created'
assert (lwkmem_set <= lwkmem_req), 'Mismatch ' + msg
lwkmem_auto_set = lwkmem_auto()
if not autogen:
if lwkmem_auto_set:
logging.error('The lwkmem_auto indicator is set')
assert (lwkmem_auto_set == False)
else:
if not lwkmem_auto_set:
logging.error('The lwkmem_auto indicator is not set')
assert (lwkmem_auto_set == True)
# Run tests on LWK
yod(obj, '-u', 0, '../lwksched/aff_scan', '-efm')
if not lwkmem_static:
yod(obj, '../lwkmem/maptest', '--verbose', '--type', 'anonymous',
'--num', 10, '--size', 4096, '--iterations', 10)
# Delete partition
run(['lwkctl', '-v', v, '-d'], requiresRoot=True)
# Read and verify using lwkctl -s
utilcpus, lwkcpus = get_cpus()
profile_set = get_profile()
assert (utilcpus == CpuSet(0))
assert (lwkcpus == CpuSet(0))
assert (profile_set == '')
if not lwkmem_static:
lwkmem_set = get_lwkmem()
assert (lwkmem_set == 0), 'Failed to delete LWKMEM partition'
# Run tests on Linux
run_bin(obj, '../lwksched/aff_scan')
def clearFirstNCores(cs, N):
cpumask = CpuSet(0)
for i in range(N):
cpumask += cs.selectNthBy(i + 1, self.topology.cores)
return cs - cpumask
def create_lwkcpu_spec(self, ratio, noutility_cpus=False):
KEY_FMT = 'ratio_{}_noutility_cpus_{}'
def clearFirstNCores(cs, N):
cpumask = CpuSet(0)
for i in range(N):
cpumask += cs.selectNthBy(i + 1, self.topology.cores)
return cs - cpumask
def clearLastNCores(cs, N):
cpumask = CpuSet(0)
nthCore = cs.countBy(self.topology.cores)
for i in range(N):
cpumask += cs.selectNthBy(nthCore, self.topology.cores)
nthCore -= 1
return cs - cpumask
def update_cache_lwkcpuspec(ratio, noutility_cpus, value):
key = KEY_FMT.format(ratio, noutility_cpus)
self.cache[key] = value
# Try to look up for a cached spec before computing from scratch
key = KEY_FMT.format(ratio, noutility_cpus)
if key in self.cache:
return self.cache[key]
# Manufacture one and store in the cache
lwkcpuspec = 'lwkcpus='
utilcpus = CpuSet(0)
lwkcpus = CpuSet(0)
if ratio <= 0:
return lwkcpuspec, utilcpus, lwkcpus
if ratio > 1:
ratio = 1
# Consider core 0 as Linux core irrespective of topology.
core0_cpus = self.topology.nodes[0].selectNthBy(1, self.topology.cores)
numa_nodes = len(self.topology.nodes)
node_utilcpus = self.topology.nodes.copy()
node_lwkcpus = self.topology.nodes.copy()
node_lwkcpus[0] -= core0_cpus
# Find base minimum utility CPUs for the given hardware. These are the
# number of Linux CPUs left after balancing number of LWK CPUs across
# NUMA nodes. We do not scale the number of utility CPUs with ratio.
# Instead we scale only the number of LWK CPUs using the given ratio.
lwkcorespn_max = min(
*[node.countBy(self.topology.cores) for node in node_lwkcpus])
lwkcorespn = math.ceil(lwkcorespn_max * ratio)
logging.debug('LWK cores per node: {} max x {} ratio = {}'.format(
lwkcorespn_max, ratio, lwkcorespn))
# Create maps of Linux and LWK cpus for every node
total_util_cpus = 0
for n in range(numa_nodes):
node_utilcpus[n] = clearLastNCores(node_utilcpus[n],
lwkcorespn_max)
node_lwkcpus[n] -= node_utilcpus[n]
if lwkcorespn_max != lwkcorespn:
node_lwkcpus[n] = clearFirstNCores(node_lwkcpus[n],
lwkcorespn_max - lwkcorespn)
lwkcpus += node_lwkcpus[n]
utilcpus += node_utilcpus[n]
total_util_cpus += node_utilcpus[n].countCpus()
logging.debug('Node[{}] Linux CPUs : {}'.format(
n, node_utilcpus[n].toList()))
logging.debug('Node[{}] LWK CPUs : {}'.format(
n, node_lwkcpus[n].toList()))
if noutility_cpus == True:
if lwkcpuspec != 'lwkcpus=':
lwkcpuspec += ':'
lwkcpuspec += '{}'.format(node_lwkcpus[n].toList())
# If there are no utility cpus requested then we are done here
if noutility_cpus == True:
rval = (lwkcpuspec, CpuSet(0), lwkcpus)
update_cache_lwkcpuspec(ratio, True, rval)
return rval
# Assign LWKCPUs to Utility CPU mapping
utilitycpuspn = int(total_util_cpus / numa_nodes)
assert (utilitycpuspn >=
1), 'Utility CPUs {} lesser than NUMA nodes {}'.format(
total_util_cpus, numa_nodes)
lwkcores_per_utilitycpu = int(lwkcorespn / utilitycpuspn)
if lwkcores_per_utilitycpu < 1:
lwkcores_per_utilitycpu = 1
# Compute LWKCPU specification for each node
logging.debug('Utility cpus per node : {}'.format(utilitycpuspn))
logging.debug(
'LWK cores per utility cpus: {}'.format(lwkcores_per_utilitycpu))
for n in range(numa_nodes):
for i in range(1, utilitycpuspn + 1):
subgroup_cpumask = CpuSet(0)
for j in range(lwkcores_per_utilitycpu):
cpumask = node_lwkcpus[n].selectNthBy(
1, self.topology.cores)
subgroup_cpumask += cpumask
node_lwkcpus[n] -= cpumask
# Add any remaining LWK CPUs due to the truncation
# from integer division to the last subgroup
if i == utilitycpuspn and node_lwkcpus[n].isEmpty() == False:
subgroup_cpumask += node_lwkcpus[n]
node_lwkcpus[n] -= node_lwkcpus[n]
# Pick a target utility CPU
# Try current node first
utilcpumask = node_utilcpus[n].nthCpu(1)
if utilcpumask.isEmpty() == False:
node_utilcpus[n] -= utilcpumask
else:
# If there is no utility CPU on this node
# then try to get one from other NUMA nodes
for m in range(numa_nodes):
utilcpumask = node_utilcpus[m].nthCpu(1)
if utilcpumask.isEmpty() == False:
node_utilcpus[m] -= utilcpumask
break
assert (
utilcpumask.isEmpty() == False
), 'Node[{}]: Ran out of utility cpus to assign'.format(n)
assert (
utilcpumask.countCpus() == 1
), 'Node[{}]: More than 1 utility cpus selected: {}'.format(n)
if lwkcpuspec != 'lwkcpus=':
lwkcpuspec += ':'
node_lwkcpuspec = '{}.{}'.format(utilcpumask.toList(),
subgroup_cpumask.toList())
lwkcpuspec += node_lwkcpuspec
logging.debug('Node[{}] LWKCPU spec: {}'.format(
n, node_lwkcpuspec))
assert (
node_lwkcpus[n].countCpus() == 0
), 'Node[{}]: LWKCPUs {} are not assigned a utility CPU'.format(
n, node_lwkcpus[n].toList())
logging.debug('LWKCPU spec: {}'.format(lwkcpuspec))
rval = (lwkcpuspec, utilcpus, lwkcpus)
update_cache_lwkcpuspec(ratio, False, rval)
return rval
def create_and_verify(obj, lwk_spec, sccpus_req, lwkcpus_req, v='0'):
def get_cpus():
sccpus = CpuSet(0)
lwkcpus = CpuSet(0)
op, rc = run(['lwkctl', '-v', v, '-s'])
lines = op.splitlines()
for l in lines:
if l.startswith('Syscall CPU(s):'):
f,m,s = l.partition('Syscall CPU(s):')
s = s.strip()
if s != '':
sccpus.fromList(s)
if l.startswith('LWK CPU(s):'):
f,m,s = l.partition('LWK CPU(s):')
s = s.strip()
if s != '':
lwkcpus.fromList(s)
return sccpus, lwkcpus
def get_profile(spec=None):
if spec == None:
op, rc = run(['lwkctl', '-v', v, '-s', '-r'])
else:
op, rc = spec, 0
for token in op.split():
if token.startswith('lwkcpu_profile='):
f,m,l = token.partition('lwkcpu_profile=')
return l
return ''
def get_lwkmem(spec=None):
if spec == None:
op, rc = run(['lwkctl', '-v', v, '-s', '-r'])
else:
op, rc = spec, 0
units = ['K', 'M', 'G', 'T', 'P', 'E']
total_size = 0
for token in op.split():
if token.startswith('lwkmem='):
f,m,l = token.partition('=')
l = l.strip()
if l != '':
for node_spec in l.split(','):
node,delimiter,size = node_spec.partition(':')
if delimiter == '':
size = node
size = size.strip()
if size != '':
multiplier = 1
if size.endswith(('K', 'M', 'G', 'T', 'P', 'E')):
for u in units:
index = size.find(u)
if index != -1:
multiplier = 1024**(1+units.index(u))
break
size = size.strip('KMGTPE')
total_size += int(size) * multiplier
return total_size
# Create LWK partition
lwk_spec = lwk_spec.strip()
out, rc = run(['sudo', 'lwkctl', '-v', v, '-c', lwk_spec])
# Read and verify LWK partition using lwkctl -s
# Verify CPUs
sccpus, lwkcpus = get_cpus()
if sccpus != sccpus_req or lwkcpus != lwkcpus_req:
if sccpus != sccpus_req:
logging.error('Mismatch : Syscall CPUs')
logging.error('Requested: {}'.format(sccpus_req.toList()))
logging.error('Created : {}'.format(sccpus.toList()))
if lwkcpus != lwkcpus_req:
logging.error('Mismatch : LWK CPUs')
logging.error('Requested: {}'.format(lwkcpus_req.toList()))
logging.error('Created : {}'.format(lwkcpus.toList()))
assert(sccpus == sccpus_req)
assert(lwkcpus == lwkcpus_req)
# Verify LWK CPU profile
profile_req = get_profile(lwk_spec)
profile_set = get_profile()
msg = 'LWKCPU profile requested: {} set: {}'.format(profile_req, profile_set)
logging.debug(msg)
if (profile_req == 'debug'):
assert(profile_set == profile_req), 'Mismatch: ' + msg
else:
assert(profile_set == 'normal'), 'Mismatch: ' + msg
# Verify LWK Memory
if not lwkmem_static:
lwkmem_req = get_lwkmem(lwk_spec)
lwkmem_set = get_lwkmem()
msg = 'LWKMEM requested {} allocated {}'.format(lwkmem_req, lwkmem_set)
logging.debug(msg)
if lwkmem_req != 0:
assert(lwkmem_set > 0), 'LWK memory partition was not created'
assert(lwkmem_set <= lwkmem_req), 'Mismatch ' + msg
# Run tests on LWK
yod(obj, '-u', 0, '../lwksched/aff_scan', '-efm')
if not lwkmem_static:
yod(obj, '../lwkmem/maptest', '--verbose', '--type', 'anonymous', '--num', 10, '--size', 4096, '--iterations', 10)
# Delete partition
run(['sudo', 'lwkctl', '-v', v, '-d'])
# Read and verify using lwkctl -s
sccpus, lwkcpus = get_cpus()
profile_set = get_profile()
assert(sccpus == CpuSet(0))
assert(lwkcpus == CpuSet(0))
assert(profile_set == '')
if not lwkmem_static:
lwkmem_set = get_lwkmem()
assert(lwkmem_set == 0), 'Failed to delete LWKMEM partition'
# Run tests on Linux
run_bin(obj, '../lwksched/aff_scan')
