def test_numa_pack_domain(self):
# Pre-reserve a portion of NUMA domain N and launch a job that
# will logically fit into the remainder of domain N. Ensure that
# it is, indeed, packed into the remaining space.
# Construct a list of designated CPUs by domain
nodes = []
for nid in range(len(self.topology.nodes)):
nodes.append(self.designated & self.topology.allcpus.selectNthBy(
nid + 1, self.topology.nodes))
# Pre-reserve the first half of domain 1, and select the second half
# for this request.
n_cores = nodes[1].countBy(self.topology.cores)
n_half = n_cores // 2
lwkcpus_reserved = yod.CpuSet(0)
self.lwkcpus_request = yod.CpuSet(0)
for n in range(1, n_half + 1):
lwkcpus_reserved += nodes[1].selectNthBy(n, self.topology.cores)
self.lwkcpus_request += nodes[1].selectNthBy(
n_half + n, self.topology.cores)
lwkcpus_reserved += nodes[0]
# Now pre-reserve the memory from domain 0 and half of the memory
# from domain 1. Note that n_cores might be odd and therefore
# 2 * n_half might not equal n_cores.
self.domain_info = ''
prefix = ''
for g in range(self.n_mem_groups):
n = self.nearest(0, g)
self.lwkmem_reserved[n] = self.lwkmem[n]
n = self.nearest(1, g)
self.lwkmem_reserved[n] = self.lwkmem[n] // 2
self.lwkmem_request[n] = self.lwkmem[n] // 2
self.domain_info += '{}{}={}'.format(prefix,
self.mem_group_names[g], n)
prefix = ' '
self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved)
# The reserved memory (as seen by the launched process) is the total
# of the pre-reserved and requested memory:
lwkmem_reserved_after = list(
a + b for a, b in zip(self.lwkmem_reserved, self.lwkmem_request))
cmd = [
'-v', 2, '%RESOURCES%', '.125', '%AFFINITY_TEST%',
'--lwkcpus_reserved',
str(lwkcpus_reserved + self.lwkcpus_request), '--lwkmem_reserved',
strseq(lwkmem_reserved_after, ',')
]
self.expand_and_run(cmd, 0, postrun=[self.check_lwkmem_domain_info])
def test_numa_half_node_scattered(self):
# Pre-reserve portions of all four NUMA domains and launch a
# half-node job.
# Construct a list of designted CPUs by domain
nodes = []
lwkcpus_reserved = yod.CpuSet(0)
self.lwkcpus_request = yod.CpuSet(0)
cores_remaining = self.designated.countBy(self.topology.cores) // 2
dom_info = {'dram': [], 'hbm': []}
for nid in range(len(self.topology.nodes)):
nodes.append(self.designated & self.topology.allcpus.selectNthBy(
nid + 1, self.topology.nodes))
remaining = nodes[nid]
n_cores = remaining.countBy(self.topology.cores)
n_half = n_cores // 2
for core in range(n_half):
selected = remaining.selectNthBy(1, self.topology.cores)
lwkcpus_reserved += selected
remaining -= selected
while not remaining.isEmpty() and cores_remaining > 0:
selected = remaining.selectNthBy(1, self.topology.cores)
self.lwkcpus_request += selected
remaining -= selected
cores_remaining -= 1
for group in range(self.n_mem_groups):
nearest = self.nearest(nid, group)
self.lwkmem_reserved[nearest] = self.lwkmem[nearest] // 2
self.lwkmem_request[nearest] = self.lwkmem[nearest] // 2
dom_info[self.mem_group_names[group]].append(str(nearest))
self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved)
self.domain_info = 'dram={} hbm={}'.format(','.join(dom_info['dram']),
','.join(dom_info['hbm']))
# The lwkmem_reserved status (as seen by the launched process) is the total
# of the pre-reserved and the requested memory:
lwkmem_reserved_after = list(
a + b for a, b in zip(self.lwkmem_reserved, self.lwkmem_request))
cmd = [
'-v', 2, '%RESOURCES%', '.5', '%AFFINITY_TEST%',
'--lwkcpus_reserved',
str(lwkcpus_reserved + self.lwkcpus_request), '--lwkmem_reserved',
strseq(lwkmem_reserved_after, ',')
]
self.expand_and_run(cmd, 0, postrun=[self.check_lwkmem_domain_info])
def test_numa_half_node_scattered(self):
# Pre-reserve portions of all four NUMA domains and launch a
# half-node job.
# Construct a list of designted CPUs by domain
nodes = []
lwkcpus_reserved = yod.CpuSet(0)
lwkcpus_request = yod.CpuSet(0)
lwkmem_reserved_before = [0] * self.n_mem_nodes
lwkmem_reserved_after = [0] * self.n_mem_nodes
cores_remaining = self.designated.countBy(self.topology.cores) // 2
dom_info = {'dram': [], 'hbm': []}
for nid in range(len(self.topology.nodes)):
nodes.append(self.designated & self.topology.allcpus.selectNthBy(
nid + 1, self.topology.nodes))
remaining = nodes[nid]
n_cores = remaining.countBy(self.topology.cores)
n_half = n_cores // 2
for core in range(n_half):
selected = remaining.selectNthBy(1, self.topology.cores)
lwkcpus_reserved += selected
remaining -= selected
while not remaining.isEmpty() and cores_remaining > 0:
selected = remaining.selectNthBy(1, self.topology.cores)
lwkcpus_request += selected
remaining -= selected
cores_remaining -= 1
for group in range(self.n_mem_groups):
nearest = self.nearest(nid, group)
lwkmem_reserved_before[nearest] = self.lwkmem[nearest] // 2
lwkmem_reserved_after[nearest] = int(self.lwkmem[nearest])
dom_info[self.mem_group_names[group]].append(str(nearest))
self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved)
self.var['I_LWKMEM_RESERVED'] = strseq(lwkmem_reserved_before)
self.domain_info = 'dram={} hbm={}'.format(','.join(dom_info['dram']),
','.join(dom_info['hbm']))
cmd = [
'-v', 2, '%RESOURCES%', '.5', '--resource_algorithm', 'numa',
'%AFFINITY_TEST%', '--lwkcpus_reserved',
str(lwkcpus_reserved + lwkcpus_request), '--lwkmem_reserved',
strseq(lwkmem_reserved_after, ',')
]
self.expand_and_run(cmd, 0, postrun=[self.check_lwkmem_domain_info])
def test_cores(self):
lwkcpus_mask = int(LWK_CPUS)
lwkcpus = yod.CpuSet(lwkcpus_mask)
n_lwk_cores = lwkcpus.countBy(self.topology.cores)
ncores = len(self.topology.cores) + 1
mask = yod.CpuSet(0)
for n in range(-1, ncores+1):
with self.subTest(n=n):
should_work = 1 <= n <= n_lwk_cores
mask += lwkcpus.selectNthBy(n, self.topology.cores)
cmd = ['-C', n, '-u', 0, '-M', 'all', '--resource_algorithm', 'simple',
'./affinity_test', '--affinity', str(mask)]
out, rc = yod.launch(self, cmd, self.test_env)
self.assertCommand(rc, should_work)
def test_fraction(self):
designated = self.get_designated_lwkcpus()
n_cores = designated.countBy(self.topology.cores)
if n_cores < 2:
self.skipTest('This test requires at least 2 designated cores.')
n_cores //= 2
mask = yod.CpuSet(0)
for n in range(n_cores):
mask += self.topology.allcpus.selectNthBy(n + 1,
self.topology.cores)
for frac in ['.5', '0.5', '1/2']:
for alg in ['*', 'numa', 'simple']:
cmd = ['%RESOURCES%', frac]
if alg == '*':
mask = self.get_n_cores(n_cores, fromcpus=designated)
else:
cmd += ['--resource_algorithm', alg]
mask = self.get_n_cores(n_cores,
fromcpus=designated,
algorithm=alg)
cmd += [
'%AFFINITY_TEST%', '--affinity', mask,
'--lwkcpus_reserved', mask, '--lwkmem_reserved',
self.total_lwkmem // 2
]
self.expand_and_run(cmd, 0)
def setUpClass(cls):
super().setUpClass()
# We'll plug-in two memory groups a la SNC-4 mode on KNL
cls.var['I_LWKMEM_GROUPS'] = '0-3 4-7'
cls.distances = [
"10 21 21 21 31 41 41 41",
"21 10 21 21 41 31 41 41",
"21 21 10 21 41 41 31 41",
"21 21 21 10 41 41 41 31",
"31 41 41 41 10 41 41 41",
"41 31 41 41 41 10 41 41",
"41 41 31 41 41 41 10 41",
"41 41 41 31 41 41 41 10",
]
for i, dist in enumerate(cls.distances):
dmap_path = '/'.join([cls.var['FS_DIR'], 'distance%d' % i])
with open(dmap_path, 'w') as dmap:
dmap.write(dist)
logger.debug('KNL plugin established %s --> #-CPUS=%s -> %s',
cls.yod_plugin, cls.topology.allcpus.countCpus(),
cls.topology.allcpus)
# The (default) designated CPU set for KNL:
cls.designated = yod.CpuSet(0).fromList(
'2-17,20-67,70-85,88-135,138-153,156-203,206-221,224-271')
cls.test_env['YOD_MAX_CPUS'] = str(cls.topology.allcpus.countCpus())
cls.var['I_LWKCPUS'] = str(cls.designated)
def test_cmask(self):
# Test the options to specify compute CPUs via mask.
mask = 0xfedc
if not yod.CpuSet(mask).isSubsetOf(self.get_designated_lwkcpus()):
self.skipTest('This test requires that {} be LWK CPUs.'.format(
hex(mask)))
self.lwkcpus_request = yod.CpuSet(mask)
self.compute_lwkmem_request(fraction=1.0)
cmd = [
'%CPUS%',
hex(mask), '%MEM%', 'all', '%AFFINITY_TEST%', '--lwkcpus_reserved',
hex(mask)
]
self.expand_and_run(cmd, 0)
def test_multilaunch(self):
lwkcpus_mask = int(LWK_CPUS)
lwkcpus = yod.CpuSet(lwkcpus_mask)
n_lwk_cores = lwkcpus.countBy(self.topology.cores)
@contextlib.contextmanager
def process(i, n):
with _yod(self,
'-R',
'1/{}'.format(n),
'-u',
0,
'--resource_algorithm',
'simple',
'./affinity_test',
'wait',
i,
env=self.test_env,
bg=True,
pipe='io') as p:
# wait for it to get started
o = p.stdout.readline()
self.assertEqual(o.strip(), 'ready')
yield
# wait for it to terminate
p.stdin.write('done')
for n in range(1, n_lwk_cores + 1):
with contextlib.ExitStack() as stack:
stack.enter_context(self.subTest(concurrent=n))
# launch n processes waiting on stdin
expected = yod.CpuSet(0)
cores_per_proc = n_lwk_cores // n
for i in range(n):
stack.enter_context(process(i + 1, n))
# Each process consumes 1/nth of the LWK cores:
for j in range(1, cores_per_proc + 1):
expected += lwkcpus.selectNthBy(
i * cores_per_proc + j, self.topology.cores)
# compare expected and actual reserved CPUs
actual = cpulist(get_file('/sys/kernel/mOS/lwkcpus_reserved'))
self.assertEqual(int(actual), int(expected))
def test_list(self):
# Launch using "yod -c <list> foo".
# Construct a list using every third CPU.
ncpus = self.topology.allcpus.countCpus()
mask = yod.CpuSet(0)
for n in range(1, ncpus, 3):
mask += yod.CpuSet(1 << n)
# Use both simple and stride forms of the list:
masks = [str(mask), '1-' + str(ncpus - 1) + ':3']
for m in masks:
cmd = ['%CPUS%'] + [m] + [
'%MEM%', 'all', '%AFFINITY_TEST%', '--affinity',
str(mask), '--lwkcpus_reserved',
str(mask)
]
self.expand_and_run(cmd, 0)
def test_list(self):
# Launch using "yod -c <list> foo".
# Construct a list using every third CPU.
n_cpus = self.get_designated_lwkcpus().countCpus()
self.lwkcpus_request = yod.CpuSet(0)
for n in range(1, n_cpus, 3):
self.lwkcpus_request += yod.CpuSet(1 << n)
self.compute_lwkmem_request(fraction=1.0)
# Use both simple and stride forms of the list:
masks = [str(self.lwkcpus_request), '1-' + str(n_cpus - 1) + ':3']
for m in masks:
cmd = ['%CPUS%'] + [m] + [
'%MEM%', 'all', '%AFFINITY_TEST%', '--lwkcpus_reserved',
self.lwkcpus_request
]
self.expand_and_run(cmd, 0)
def test_numa_one_domain_cpus_resvd(self):
# For every CPU domain D, reserve a CPU in all other domains
# and launch a job. The NUMA alogorithm should place the job
# on domain D, reserving memory that is near D.
# Construct a list of designted CPUs by domain
nodes = []
for nid in range(len(self.topology.nodes)):
nodes.append(self.designated & self.topology.allcpus.selectNthBy(
nid + 1, self.topology.nodes))
# For every CPU domain ...
for nid, node in enumerate(self.topology.nodes):
self.lwkcpus_request = nodes[nid]
self.lwkmem_request = [0] * self.n_mem_nodes
lwkcpus_reserved = yod.CpuSet(0)
# Pre-reserve one CPU from every other CPU domain:
for other_nid in range(len(self.topology.nodes)):
if nid == other_nid:
continue
lwkcpus_reserved += nodes[other_nid].nthCpu(1)
self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved)
# Find the memory in each group that is nearest
# to the CPU domain:
self.domain_info = ''
prefix = ''
for g in range(self.n_mem_groups):
nearest_nid = self.nearest(nid, g)
self.lwkmem_request[nearest_nid] = self.lwkmem[nearest_nid]
self.domain_info += '{}{}={}'.format(prefix,
self.mem_group_names[g],
nearest_nid)
prefix = ' '
self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved)
cmd = [
'-v', 2, '%RESOURCES%', '.25', '%AFFINITY_TEST%',
'--lwkcpus_reserved',
str(self.lwkcpus_request + lwkcpus_reserved),
'--lwkmem_reserved',
strseq(self.lwkmem_request, ',')
]
self.expand_and_run(cmd,
0,
postrun=[self.check_lwkmem_domain_info])
def test_all_none_available(self):
# Test "yod -C all foo" when no cores are available.
# Grab one CPU from every core and pre-reserve it. Then launch all
# cores, which will, of course, fail.
rsvd = yod.CpuSet(0)
for c in self.topology.cores:
rsvd += c.nthCpu(1)
self.var['I_LWKCPUS_RESERVED'] = str(rsvd)
cmd = [
'%CORES%', 'all', '%MEM%', 'all', '%HELLO%', 'should not get here'
]
self.expand_and_run(cmd, EBUSY)
def test_cmask(self):
# Test the options to specify compute CPUs via mask.
mask = 0xfedc
if not yod.CpuSet(mask).isSubsetOf(self.topology.allcpus):
self.skipTest('This test requires that {} be LWK CPUs.'.format(
hex(mask)))
cmd = [
'%CPUS%',
hex(mask), '%MEM%', 'all', '%AFFINITY_TEST%', '--affinity',
hex(mask), '--lwkcpus_reserved',
hex(mask)
]
self.expand_and_run(cmd, 0)
def test_reserve_memory_from_cpus(self):
# Tests scenarios where CPUs (not cores) are selected from the
# same node. The reserved memory should match that node.
lwkmem_by_group = self.sumByGroup(self.lwkmem)
total_mem = sum(lwkmem_by_group)
# for every node and for every n in [1, ...,
# number-of-cpus-this-node], launch on n CPUs from that node.
# The reserved memory should correspond to the node.
for nid, node in enumerate(self.topology.nodes):
node = node & self.designated
for ncpus in range(1, node.countCpus() + 1):
self.lwkcpus_request = yod.CpuSet(0)
self.lwkmem_request = [0] * self.n_mem_nodes
for c in range(ncpus):
self.lwkcpus_request += node.nthCpu(c + 1)
self.domain_info = ''
prefix = ''
for g in range(self.n_mem_groups):
nearest_nid = self.nearest(nid, g)
ratio = lwkmem_by_group[g] / total_mem
self.lwkmem_request[nearest_nid] = int(20 * 1024 * 1024 *
ratio)
self.domain_info += '{}{}={}'.format(
prefix, self.mem_group_names[g], nearest_nid)
prefix = ' '
cmd = [
'--verbose', 2, '-c',
str(self.lwkcpus_request), '-M', '20M', '%AFFINITY_TEST%',
'--lwkcpus_reserved',
str(self.lwkcpus_request), '--lwkmem_reserved',
strseq(self.lwkmem_request, ',')
]
self.expand_and_run(cmd,
0,
postrun=[self.check_lwkmem_domain_info])
def pre_reserve(i, envelopes):
lwkcpus_reserved = yod.CpuSet(0)
for j, envelope in enumerate(envelopes):
if j == i:
continue
# --------------------------------------------------------------
# Reserve one core and memory from envelope j. To mix things
# up, we use the envelope's index (modulo its length) to target
# one of the NIDs in the envelope:
# --------------------------------------------------------------
nid = envelope[j % len(envelope)]
lwkcpus_reserved += (self.designated & self.topology.nodes[nid]).selectNthBy(1, self.topology.cores)
self.lwkmem_reserved[nid] = self.total_mem_designated // self.n_desig_cores
self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved)
return lwkcpus_reserved
def test_frac(self):
# Test "yod -C <frac> foo", i.e. the factional core specifier.
designated = self.get_designated_lwkcpus()
ncores = designated.countBy(self.topology.cores)
if ncores < 2:
self.skipTest('This test requires at least 2 designated cores.')
ncores //= 2
mask = yod.CpuSet(0)
for n in range(ncores):
mask += self.topology.allcpus.selectNthBy(n + 1,
self.topology.cores)
for frac in ['.5', '0.5', '1/2']:
for alg in ['*', 'numa', 'simple']:
for spec, extra in [('%CORES%', ['%MEM%', '.5']),
('%RESOURCES%', None)]:
cmd = [spec] + [frac]
if extra is not None:
cmd += extra
if alg == '*':
mask = self.get_n_cores(ncores, fromcpus=designated)
else:
cmd += ['--resource_algorithm', alg]
mask = self.get_n_cores(ncores,
fromcpus=designated,
algorithm=alg)
cmd += [
'%AFFINITY_TEST%', '--affinity', mask,
'--lwkcpus_reserved', mask
]
self.expand_and_run(cmd, 0)
def test_indirect_memory_ratios(self):
# --------------------------------------------------------------
# Select 1 CPU from node 1, 2 CPUs from node 2, and so on up
# to 5 nodes. Validate that the reserved memory is in
# proportion.
# --------------------------------------------------------------
self.lwkmem_request = [0] * self.n_nids
lwkcpu_request = yod.CpuSet(0)
for nid in range(1,6):
for i in range(1, nid + 1):
lwkcpu_request += (self.designated & self.topology.nodes[nid]).nthCpu(i)
self.lwkmem_request[nid] += 16 * 1024 * 1024
mem = sum(self.lwkmem_request)
cmd = ['-v', '2', '--cpu', str(lwkcpu_request), '--mem', str(mem),
'%AFFINITY_TEST%', '--lwkcpus_reserved', str(lwkcpu_request),
'--lwkmem_reserved', strseq(self.lwkmem_request)]
self.expand_and_run(cmd, 0)
def _test_order_n(self, N):
# --------------------------------------------------------------
# Test the NUMA algorithm for envelopes of depth N. This is
# done as follows:
# 1. Iterate through the envelopes for depth N.
# 2. Given the selected envelope, pre-reserve some small
# amount of resource from all of the other envelopes.
# 3. Launch a job that will fit exactly into an envelop for
# depth N and ensure that it lands on the selected
# envelope.
# --------------------------------------------------------------
#self.test_env['YOD_VERBOSE'] = '2'
def pre_reserve(i, envelopes):
lwkcpus_reserved = yod.CpuSet(0)
for j, envelope in enumerate(envelopes):
if j == i:
continue
# --------------------------------------------------------------
# Reserve one core and memory from envelope j. To mix things
# up, we use the envelope's index (modulo its length) to target
# one of the NIDs in the envelope:
# --------------------------------------------------------------
nid = envelope[j % len(envelope)]
lwkcpus_reserved += (self.designated & self.topology.nodes[nid]).selectNthBy(1, self.topology.cores)
self.lwkmem_reserved[nid] = self.total_mem_designated // self.n_desig_cores
self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved)
return lwkcpus_reserved
# --------------------------------------------------------------
# Construct the envelopes for order N
# --------------------------------------------------------------
envelopes = []
for nid in range(self.n_nids):
nearest = self.nearest(nid, N)
if not nearest in envelopes:
envelopes.append(nearest)
logger.debug('Discovered envelope: {}'.format(nearest))
# --------------------------------------------------------------
# Per the above above description, iterate, pre-reserve and
# test:
# --------------------------------------------------------------
for i, envelope in enumerate(envelopes):
self.lwkmem_request = [0] * self.n_nids
self.lwkmem_reserved = [0] * self.n_nids
self.lwkcpus_request = yod.CpuSet(0)
lwkcpus_reserved = pre_reserve(i, envelopes)
for nid in envelope:
cpus = self.designated & self.topology.nodes[nid]
self.lwkcpus_request += cpus
self.lwkmem_request[nid] = self.total_mem_designated // self.n_desig_cores * cpus.countBy(self.topology.cores)
lwkcpus_reserved_after = self.lwkcpus_request + lwkcpus_reserved
lwkmem_reserved_after = list(a + b for a,b in zip(self.lwkmem_reserved, self.lwkmem_request))
cmd = ['-v', '2', '-R', '{}/{}'.format(self.lwkcpus_request.countBy(self.topology.cores), self.n_desig_cores),
'%AFFINITY_TEST%', '--lwkcpus_reserved', str(lwkcpus_reserved_after),
'--lwkmem_reserved', strseq(lwkmem_reserved_after, ',')]
self.expand_and_run(cmd, 0)
def test_numa_half_node(self):
# Pre-reserve portions of two NUMA domains and launch a half-node
# job. Validate that the completely free domains are the ones
# selected.
# Construct a list of designted CPUs by domain
nodes = []
for nid in range(len(self.topology.nodes)):
nodes.append(self.designated & self.topology.allcpus.selectNthBy(
nid + 1, self.topology.nodes))
for c in range(len(self.topology.nodes)):
for m in range(len(self.topology.nodes)):
m = (c + 1 + m) % len(self.topology.nodes)
if m == c:
continue
self.lwkcpus_request = yod.CpuSet(0)
self.lwkmem_reserved = [0] * self.n_mem_nodes
self.lwkmem_request = [0] * self.n_mem_nodes
# Reserve a CPU from the "c" node
lwkcpus_reserved = nodes[c].nthCpu(1)
# Reserve memory from the "m" node's nearest memory
for g in range(self.n_mem_groups):
n = self.nearest(m, g)
self.lwkmem_reserved[n] = 2 * 1024 * 1024
self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved)
# Now gather up resources from the free domains:
dom_info = {'dram': [], 'hbm': []}
for n in range(len(self.topology.nodes)):
if n == c or n == m:
continue
self.lwkcpus_request += nodes[n]
for g in range(self.n_mem_groups):
self.lwkmem_request[self.nearest(
n, g)] = self.lwkmem[self.nearest(n, g)]
dom_info[self.mem_group_names[g]].append(
str(self.nearest(n, g)))
self.domain_info = 'dram={} hbm={}'.format(
','.join(dom_info['dram']), ','.join(dom_info['hbm']))
# The reserved memory (as seen by the launched process) is the total
# of the pre-reserved and requested memory:
lwkmem_reserved_after = list(
a + b
for a, b in zip(self.lwkmem_reserved, self.lwkmem_request))
cmd = [
'-v', 2, '%RESOURCES%', '.5', '%AFFINITY_TEST%',
'--lwkcpus_reserved',
str(lwkcpus_reserved + self.lwkcpus_request),
'--lwkmem_reserved',
strseq(lwkmem_reserved_after, ',')
]
self.expand_and_run(cmd,
0,
postrun=[self.check_lwkmem_domain_info])
