def test_numa_pack_domain(self): # Pre-reserve a portion of NUMA domain N and launch a job that # will logically fit into the remainder of domain N. Ensure that # it is, indeed, packed into the remaining space. # Construct a list of designated CPUs by domain nodes = [] for nid in range(len(self.topology.nodes)): nodes.append(self.designated & self.topology.allcpus.selectNthBy( nid + 1, self.topology.nodes)) # Pre-reserve the first half of domain 1, and select the second half # for this request. n_cores = nodes[1].countBy(self.topology.cores) n_half = n_cores // 2 lwkcpus_reserved = yod.CpuSet(0) self.lwkcpus_request = yod.CpuSet(0) for n in range(1, n_half + 1): lwkcpus_reserved += nodes[1].selectNthBy(n, self.topology.cores) self.lwkcpus_request += nodes[1].selectNthBy( n_half + n, self.topology.cores) lwkcpus_reserved += nodes[0] # Now pre-reserve the memory from domain 0 and half of the memory # from domain 1. Note that n_cores might be odd and therefore # 2 * n_half might not equal n_cores. self.domain_info = '' prefix = '' for g in range(self.n_mem_groups): n = self.nearest(0, g) self.lwkmem_reserved[n] = self.lwkmem[n] n = self.nearest(1, g) self.lwkmem_reserved[n] = self.lwkmem[n] // 2 self.lwkmem_request[n] = self.lwkmem[n] // 2 self.domain_info += '{}{}={}'.format(prefix, self.mem_group_names[g], n) prefix = ' ' self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved) # The reserved memory (as seen by the launched process) is the total # of the pre-reserved and requested memory: lwkmem_reserved_after = list( a + b for a, b in zip(self.lwkmem_reserved, self.lwkmem_request)) cmd = [ '-v', 2, '%RESOURCES%', '.125', '%AFFINITY_TEST%', '--lwkcpus_reserved', str(lwkcpus_reserved + self.lwkcpus_request), '--lwkmem_reserved', strseq(lwkmem_reserved_after, ',') ] self.expand_and_run(cmd, 0, postrun=[self.check_lwkmem_domain_info])
def test_numa_half_node_scattered(self): # Pre-reserve portions of all four NUMA domains and launch a # half-node job. # Construct a list of designted CPUs by domain nodes = [] lwkcpus_reserved = yod.CpuSet(0) self.lwkcpus_request = yod.CpuSet(0) cores_remaining = self.designated.countBy(self.topology.cores) // 2 dom_info = {'dram': [], 'hbm': []} for nid in range(len(self.topology.nodes)): nodes.append(self.designated & self.topology.allcpus.selectNthBy( nid + 1, self.topology.nodes)) remaining = nodes[nid] n_cores = remaining.countBy(self.topology.cores) n_half = n_cores // 2 for core in range(n_half): selected = remaining.selectNthBy(1, self.topology.cores) lwkcpus_reserved += selected remaining -= selected while not remaining.isEmpty() and cores_remaining > 0: selected = remaining.selectNthBy(1, self.topology.cores) self.lwkcpus_request += selected remaining -= selected cores_remaining -= 1 for group in range(self.n_mem_groups): nearest = self.nearest(nid, group) self.lwkmem_reserved[nearest] = self.lwkmem[nearest] // 2 self.lwkmem_request[nearest] = self.lwkmem[nearest] // 2 dom_info[self.mem_group_names[group]].append(str(nearest)) self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved) self.domain_info = 'dram={} hbm={}'.format(','.join(dom_info['dram']), ','.join(dom_info['hbm'])) # The lwkmem_reserved status (as seen by the launched process) is the total # of the pre-reserved and the requested memory: lwkmem_reserved_after = list( a + b for a, b in zip(self.lwkmem_reserved, self.lwkmem_request)) cmd = [ '-v', 2, '%RESOURCES%', '.5', '%AFFINITY_TEST%', '--lwkcpus_reserved', str(lwkcpus_reserved + self.lwkcpus_request), '--lwkmem_reserved', strseq(lwkmem_reserved_after, ',') ] self.expand_and_run(cmd, 0, postrun=[self.check_lwkmem_domain_info])
def test_numa_half_node_scattered(self): # Pre-reserve portions of all four NUMA domains and launch a # half-node job. # Construct a list of designted CPUs by domain nodes = [] lwkcpus_reserved = yod.CpuSet(0) lwkcpus_request = yod.CpuSet(0) lwkmem_reserved_before = [0] * self.n_mem_nodes lwkmem_reserved_after = [0] * self.n_mem_nodes cores_remaining = self.designated.countBy(self.topology.cores) // 2 dom_info = {'dram': [], 'hbm': []} for nid in range(len(self.topology.nodes)): nodes.append(self.designated & self.topology.allcpus.selectNthBy( nid + 1, self.topology.nodes)) remaining = nodes[nid] n_cores = remaining.countBy(self.topology.cores) n_half = n_cores // 2 for core in range(n_half): selected = remaining.selectNthBy(1, self.topology.cores) lwkcpus_reserved += selected remaining -= selected while not remaining.isEmpty() and cores_remaining > 0: selected = remaining.selectNthBy(1, self.topology.cores) lwkcpus_request += selected remaining -= selected cores_remaining -= 1 for group in range(self.n_mem_groups): nearest = self.nearest(nid, group) lwkmem_reserved_before[nearest] = self.lwkmem[nearest] // 2 lwkmem_reserved_after[nearest] = int(self.lwkmem[nearest]) dom_info[self.mem_group_names[group]].append(str(nearest)) self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved) self.var['I_LWKMEM_RESERVED'] = strseq(lwkmem_reserved_before) self.domain_info = 'dram={} hbm={}'.format(','.join(dom_info['dram']), ','.join(dom_info['hbm'])) cmd = [ '-v', 2, '%RESOURCES%', '.5', '--resource_algorithm', 'numa', '%AFFINITY_TEST%', '--lwkcpus_reserved', str(lwkcpus_reserved + lwkcpus_request), '--lwkmem_reserved', strseq(lwkmem_reserved_after, ',') ] self.expand_and_run(cmd, 0, postrun=[self.check_lwkmem_domain_info])
def test_cores(self): lwkcpus_mask = int(LWK_CPUS) lwkcpus = yod.CpuSet(lwkcpus_mask) n_lwk_cores = lwkcpus.countBy(self.topology.cores) ncores = len(self.topology.cores) + 1 mask = yod.CpuSet(0) for n in range(-1, ncores+1): with self.subTest(n=n): should_work = 1 <= n <= n_lwk_cores mask += lwkcpus.selectNthBy(n, self.topology.cores) cmd = ['-C', n, '-u', 0, '-M', 'all', '--resource_algorithm', 'simple', './affinity_test', '--affinity', str(mask)] out, rc = yod.launch(self, cmd, self.test_env) self.assertCommand(rc, should_work)
def test_fraction(self): designated = self.get_designated_lwkcpus() n_cores = designated.countBy(self.topology.cores) if n_cores < 2: self.skipTest('This test requires at least 2 designated cores.') n_cores //= 2 mask = yod.CpuSet(0) for n in range(n_cores): mask += self.topology.allcpus.selectNthBy(n + 1, self.topology.cores) for frac in ['.5', '0.5', '1/2']: for alg in ['*', 'numa', 'simple']: cmd = ['%RESOURCES%', frac] if alg == '*': mask = self.get_n_cores(n_cores, fromcpus=designated) else: cmd += ['--resource_algorithm', alg] mask = self.get_n_cores(n_cores, fromcpus=designated, algorithm=alg) cmd += [ '%AFFINITY_TEST%', '--affinity', mask, '--lwkcpus_reserved', mask, '--lwkmem_reserved', self.total_lwkmem // 2 ] self.expand_and_run(cmd, 0)
def setUpClass(cls): super().setUpClass() # We'll plug-in two memory groups a la SNC-4 mode on KNL cls.var['I_LWKMEM_GROUPS'] = '0-3 4-7' cls.distances = [ "10 21 21 21 31 41 41 41", "21 10 21 21 41 31 41 41", "21 21 10 21 41 41 31 41", "21 21 21 10 41 41 41 31", "31 41 41 41 10 41 41 41", "41 31 41 41 41 10 41 41", "41 41 31 41 41 41 10 41", "41 41 41 31 41 41 41 10", ] for i, dist in enumerate(cls.distances): dmap_path = '/'.join([cls.var['FS_DIR'], 'distance%d' % i]) with open(dmap_path, 'w') as dmap: dmap.write(dist) logger.debug('KNL plugin established %s --> #-CPUS=%s -> %s', cls.yod_plugin, cls.topology.allcpus.countCpus(), cls.topology.allcpus) # The (default) designated CPU set for KNL: cls.designated = yod.CpuSet(0).fromList( '2-17,20-67,70-85,88-135,138-153,156-203,206-221,224-271') cls.test_env['YOD_MAX_CPUS'] = str(cls.topology.allcpus.countCpus()) cls.var['I_LWKCPUS'] = str(cls.designated)
def test_cmask(self): # Test the options to specify compute CPUs via mask. mask = 0xfedc if not yod.CpuSet(mask).isSubsetOf(self.get_designated_lwkcpus()): self.skipTest('This test requires that {} be LWK CPUs.'.format( hex(mask))) self.lwkcpus_request = yod.CpuSet(mask) self.compute_lwkmem_request(fraction=1.0) cmd = [ '%CPUS%', hex(mask), '%MEM%', 'all', '%AFFINITY_TEST%', '--lwkcpus_reserved', hex(mask) ] self.expand_and_run(cmd, 0)
def test_multilaunch(self): lwkcpus_mask = int(LWK_CPUS) lwkcpus = yod.CpuSet(lwkcpus_mask) n_lwk_cores = lwkcpus.countBy(self.topology.cores) @contextlib.contextmanager def process(i, n): with _yod(self, '-R', '1/{}'.format(n), '-u', 0, '--resource_algorithm', 'simple', './affinity_test', 'wait', i, env=self.test_env, bg=True, pipe='io') as p: # wait for it to get started o = p.stdout.readline() self.assertEqual(o.strip(), 'ready') yield # wait for it to terminate p.stdin.write('done') for n in range(1, n_lwk_cores + 1): with contextlib.ExitStack() as stack: stack.enter_context(self.subTest(concurrent=n)) # launch n processes waiting on stdin expected = yod.CpuSet(0) cores_per_proc = n_lwk_cores // n for i in range(n): stack.enter_context(process(i + 1, n)) # Each process consumes 1/nth of the LWK cores: for j in range(1, cores_per_proc + 1): expected += lwkcpus.selectNthBy( i * cores_per_proc + j, self.topology.cores) # compare expected and actual reserved CPUs actual = cpulist(get_file('/sys/kernel/mOS/lwkcpus_reserved')) self.assertEqual(int(actual), int(expected))
def test_list(self): # Launch using "yod -c <list> foo". # Construct a list using every third CPU. ncpus = self.topology.allcpus.countCpus() mask = yod.CpuSet(0) for n in range(1, ncpus, 3): mask += yod.CpuSet(1 << n) # Use both simple and stride forms of the list: masks = [str(mask), '1-' + str(ncpus - 1) + ':3'] for m in masks: cmd = ['%CPUS%'] + [m] + [ '%MEM%', 'all', '%AFFINITY_TEST%', '--affinity', str(mask), '--lwkcpus_reserved', str(mask) ] self.expand_and_run(cmd, 0)
def test_list(self): # Launch using "yod -c <list> foo". # Construct a list using every third CPU. n_cpus = self.get_designated_lwkcpus().countCpus() self.lwkcpus_request = yod.CpuSet(0) for n in range(1, n_cpus, 3): self.lwkcpus_request += yod.CpuSet(1 << n) self.compute_lwkmem_request(fraction=1.0) # Use both simple and stride forms of the list: masks = [str(self.lwkcpus_request), '1-' + str(n_cpus - 1) + ':3'] for m in masks: cmd = ['%CPUS%'] + [m] + [ '%MEM%', 'all', '%AFFINITY_TEST%', '--lwkcpus_reserved', self.lwkcpus_request ] self.expand_and_run(cmd, 0)
def test_numa_one_domain_cpus_resvd(self): # For every CPU domain D, reserve a CPU in all other domains # and launch a job. The NUMA alogorithm should place the job # on domain D, reserving memory that is near D. # Construct a list of designted CPUs by domain nodes = [] for nid in range(len(self.topology.nodes)): nodes.append(self.designated & self.topology.allcpus.selectNthBy( nid + 1, self.topology.nodes)) # For every CPU domain ... for nid, node in enumerate(self.topology.nodes): self.lwkcpus_request = nodes[nid] self.lwkmem_request = [0] * self.n_mem_nodes lwkcpus_reserved = yod.CpuSet(0) # Pre-reserve one CPU from every other CPU domain: for other_nid in range(len(self.topology.nodes)): if nid == other_nid: continue lwkcpus_reserved += nodes[other_nid].nthCpu(1) self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved) # Find the memory in each group that is nearest # to the CPU domain: self.domain_info = '' prefix = '' for g in range(self.n_mem_groups): nearest_nid = self.nearest(nid, g) self.lwkmem_request[nearest_nid] = self.lwkmem[nearest_nid] self.domain_info += '{}{}={}'.format(prefix, self.mem_group_names[g], nearest_nid) prefix = ' ' self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved) cmd = [ '-v', 2, '%RESOURCES%', '.25', '%AFFINITY_TEST%', '--lwkcpus_reserved', str(self.lwkcpus_request + lwkcpus_reserved), '--lwkmem_reserved', strseq(self.lwkmem_request, ',') ] self.expand_and_run(cmd, 0, postrun=[self.check_lwkmem_domain_info])
def test_all_none_available(self): # Test "yod -C all foo" when no cores are available. # Grab one CPU from every core and pre-reserve it. Then launch all # cores, which will, of course, fail. rsvd = yod.CpuSet(0) for c in self.topology.cores: rsvd += c.nthCpu(1) self.var['I_LWKCPUS_RESERVED'] = str(rsvd) cmd = [ '%CORES%', 'all', '%MEM%', 'all', '%HELLO%', 'should not get here' ] self.expand_and_run(cmd, EBUSY)
def test_cmask(self): # Test the options to specify compute CPUs via mask. mask = 0xfedc if not yod.CpuSet(mask).isSubsetOf(self.topology.allcpus): self.skipTest('This test requires that {} be LWK CPUs.'.format( hex(mask))) cmd = [ '%CPUS%', hex(mask), '%MEM%', 'all', '%AFFINITY_TEST%', '--affinity', hex(mask), '--lwkcpus_reserved', hex(mask) ] self.expand_and_run(cmd, 0)
def test_reserve_memory_from_cpus(self): # Tests scenarios where CPUs (not cores) are selected from the # same node. The reserved memory should match that node. lwkmem_by_group = self.sumByGroup(self.lwkmem) total_mem = sum(lwkmem_by_group) # for every node and for every n in [1, ..., # number-of-cpus-this-node], launch on n CPUs from that node. # The reserved memory should correspond to the node. for nid, node in enumerate(self.topology.nodes): node = node & self.designated for ncpus in range(1, node.countCpus() + 1): self.lwkcpus_request = yod.CpuSet(0) self.lwkmem_request = [0] * self.n_mem_nodes for c in range(ncpus): self.lwkcpus_request += node.nthCpu(c + 1) self.domain_info = '' prefix = '' for g in range(self.n_mem_groups): nearest_nid = self.nearest(nid, g) ratio = lwkmem_by_group[g] / total_mem self.lwkmem_request[nearest_nid] = int(20 * 1024 * 1024 * ratio) self.domain_info += '{}{}={}'.format( prefix, self.mem_group_names[g], nearest_nid) prefix = ' ' cmd = [ '--verbose', 2, '-c', str(self.lwkcpus_request), '-M', '20M', '%AFFINITY_TEST%', '--lwkcpus_reserved', str(self.lwkcpus_request), '--lwkmem_reserved', strseq(self.lwkmem_request, ',') ] self.expand_and_run(cmd, 0, postrun=[self.check_lwkmem_domain_info])
def pre_reserve(i, envelopes): lwkcpus_reserved = yod.CpuSet(0) for j, envelope in enumerate(envelopes): if j == i: continue # -------------------------------------------------------------- # Reserve one core and memory from envelope j. To mix things # up, we use the envelope's index (modulo its length) to target # one of the NIDs in the envelope: # -------------------------------------------------------------- nid = envelope[j % len(envelope)] lwkcpus_reserved += (self.designated & self.topology.nodes[nid]).selectNthBy(1, self.topology.cores) self.lwkmem_reserved[nid] = self.total_mem_designated // self.n_desig_cores self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved) return lwkcpus_reserved
def test_frac(self): # Test "yod -C <frac> foo", i.e. the factional core specifier. designated = self.get_designated_lwkcpus() ncores = designated.countBy(self.topology.cores) if ncores < 2: self.skipTest('This test requires at least 2 designated cores.') ncores //= 2 mask = yod.CpuSet(0) for n in range(ncores): mask += self.topology.allcpus.selectNthBy(n + 1, self.topology.cores) for frac in ['.5', '0.5', '1/2']: for alg in ['*', 'numa', 'simple']: for spec, extra in [('%CORES%', ['%MEM%', '.5']), ('%RESOURCES%', None)]: cmd = [spec] + [frac] if extra is not None: cmd += extra if alg == '*': mask = self.get_n_cores(ncores, fromcpus=designated) else: cmd += ['--resource_algorithm', alg] mask = self.get_n_cores(ncores, fromcpus=designated, algorithm=alg) cmd += [ '%AFFINITY_TEST%', '--affinity', mask, '--lwkcpus_reserved', mask ] self.expand_and_run(cmd, 0)
def test_indirect_memory_ratios(self): # -------------------------------------------------------------- # Select 1 CPU from node 1, 2 CPUs from node 2, and so on up # to 5 nodes. Validate that the reserved memory is in # proportion. # -------------------------------------------------------------- self.lwkmem_request = [0] * self.n_nids lwkcpu_request = yod.CpuSet(0) for nid in range(1,6): for i in range(1, nid + 1): lwkcpu_request += (self.designated & self.topology.nodes[nid]).nthCpu(i) self.lwkmem_request[nid] += 16 * 1024 * 1024 mem = sum(self.lwkmem_request) cmd = ['-v', '2', '--cpu', str(lwkcpu_request), '--mem', str(mem), '%AFFINITY_TEST%', '--lwkcpus_reserved', str(lwkcpu_request), '--lwkmem_reserved', strseq(self.lwkmem_request)] self.expand_and_run(cmd, 0)
def _test_order_n(self, N): # -------------------------------------------------------------- # Test the NUMA algorithm for envelopes of depth N. This is # done as follows: # 1. Iterate through the envelopes for depth N. # 2. Given the selected envelope, pre-reserve some small # amount of resource from all of the other envelopes. # 3. Launch a job that will fit exactly into an envelop for # depth N and ensure that it lands on the selected # envelope. # -------------------------------------------------------------- #self.test_env['YOD_VERBOSE'] = '2' def pre_reserve(i, envelopes): lwkcpus_reserved = yod.CpuSet(0) for j, envelope in enumerate(envelopes): if j == i: continue # -------------------------------------------------------------- # Reserve one core and memory from envelope j. To mix things # up, we use the envelope's index (modulo its length) to target # one of the NIDs in the envelope: # -------------------------------------------------------------- nid = envelope[j % len(envelope)] lwkcpus_reserved += (self.designated & self.topology.nodes[nid]).selectNthBy(1, self.topology.cores) self.lwkmem_reserved[nid] = self.total_mem_designated // self.n_desig_cores self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved) return lwkcpus_reserved # -------------------------------------------------------------- # Construct the envelopes for order N # -------------------------------------------------------------- envelopes = [] for nid in range(self.n_nids): nearest = self.nearest(nid, N) if not nearest in envelopes: envelopes.append(nearest) logger.debug('Discovered envelope: {}'.format(nearest)) # -------------------------------------------------------------- # Per the above above description, iterate, pre-reserve and # test: # -------------------------------------------------------------- for i, envelope in enumerate(envelopes): self.lwkmem_request = [0] * self.n_nids self.lwkmem_reserved = [0] * self.n_nids self.lwkcpus_request = yod.CpuSet(0) lwkcpus_reserved = pre_reserve(i, envelopes) for nid in envelope: cpus = self.designated & self.topology.nodes[nid] self.lwkcpus_request += cpus self.lwkmem_request[nid] = self.total_mem_designated // self.n_desig_cores * cpus.countBy(self.topology.cores) lwkcpus_reserved_after = self.lwkcpus_request + lwkcpus_reserved lwkmem_reserved_after = list(a + b for a,b in zip(self.lwkmem_reserved, self.lwkmem_request)) cmd = ['-v', '2', '-R', '{}/{}'.format(self.lwkcpus_request.countBy(self.topology.cores), self.n_desig_cores), '%AFFINITY_TEST%', '--lwkcpus_reserved', str(lwkcpus_reserved_after), '--lwkmem_reserved', strseq(lwkmem_reserved_after, ',')] self.expand_and_run(cmd, 0)
def test_numa_half_node(self): # Pre-reserve portions of two NUMA domains and launch a half-node # job. Validate that the completely free domains are the ones # selected. # Construct a list of designted CPUs by domain nodes = [] for nid in range(len(self.topology.nodes)): nodes.append(self.designated & self.topology.allcpus.selectNthBy( nid + 1, self.topology.nodes)) for c in range(len(self.topology.nodes)): for m in range(len(self.topology.nodes)): m = (c + 1 + m) % len(self.topology.nodes) if m == c: continue self.lwkcpus_request = yod.CpuSet(0) self.lwkmem_reserved = [0] * self.n_mem_nodes self.lwkmem_request = [0] * self.n_mem_nodes # Reserve a CPU from the "c" node lwkcpus_reserved = nodes[c].nthCpu(1) # Reserve memory from the "m" node's nearest memory for g in range(self.n_mem_groups): n = self.nearest(m, g) self.lwkmem_reserved[n] = 2 * 1024 * 1024 self.var['I_LWKCPUS_RESERVED'] = str(lwkcpus_reserved) # Now gather up resources from the free domains: dom_info = {'dram': [], 'hbm': []} for n in range(len(self.topology.nodes)): if n == c or n == m: continue self.lwkcpus_request += nodes[n] for g in range(self.n_mem_groups): self.lwkmem_request[self.nearest( n, g)] = self.lwkmem[self.nearest(n, g)] dom_info[self.mem_group_names[g]].append( str(self.nearest(n, g))) self.domain_info = 'dram={} hbm={}'.format( ','.join(dom_info['dram']), ','.join(dom_info['hbm'])) # The reserved memory (as seen by the launched process) is the total # of the pre-reserved and requested memory: lwkmem_reserved_after = list( a + b for a, b in zip(self.lwkmem_reserved, self.lwkmem_request)) cmd = [ '-v', 2, '%RESOURCES%', '.5', '%AFFINITY_TEST%', '--lwkcpus_reserved', str(lwkcpus_reserved + self.lwkcpus_request), '--lwkmem_reserved', strseq(lwkmem_reserved_after, ',') ] self.expand_and_run(cmd, 0, postrun=[self.check_lwkmem_domain_info])