def setrankaffinity(self):
"""pins the rank to an available core on its node"""
ranknodes = self.comm.alltoall([self.name]*self.size)
ranksonnode = [i for i, j in enumerate(ranknodes) if j == self.name]
rankaffinity = sched_getaffinity()
self.log.debug("affinity pre-set: %s", rankaffinity)
cores = [i for i, j in enumerate(rankaffinity.cpus) if j == 1L]
topin = None
for index, iterrank in enumerate(ranksonnode):
if iterrank == self.rank:
topin = cores[index % len(cores)]
self.log.debug("setting affinity to core: %s", topin)
if topin is None:
topin = cores[0]
self.log.warning("could not determine core to pin the rank to. automatically set it to %s", topin)
rankaffinity.convert_hr_bits(str(topin))
rankaffinity.set_bits()
sched_setaffinity(rankaffinity)
rankaffinity = sched_getaffinity()
self.log.debug("affinity post-set: %s", rankaffinity)
return str(rankaffinity)
def setrankaffinity(self):
"""pins the rank to an available core on its node"""
ranknodes = self.comm.alltoall([self.name] * self.size)
ranksonnode = [i for i, j in enumerate(ranknodes) if j == self.name]
rankaffinity = sched_getaffinity()
self.log.debug("affinity pre-set: %s", rankaffinity)
cores = [i for i, j in enumerate(rankaffinity.cpus) if j == 1L]
topin = None
for index, iterrank in enumerate(ranksonnode):
if iterrank == self.rank:
topin = cores[index % len(cores)]
self.log.debug("setting affinity to core: %s", topin)
if topin is None:
topin = cores[0]
self.log.warning(
"could not determine core to pin the rank to. automatically set it to %s",
topin)
rankaffinity.convert_hr_bits(str(topin))
rankaffinity.set_bits()
sched_setaffinity(rankaffinity)
rankaffinity = sched_getaffinity()
self.log.debug("affinity post-set: %s", rankaffinity)
return str(rankaffinity)
def go(self):
"""A wrapper around some common functions"""
self.fqdn = socket.getfqdn()
self.network = sorted_network(get_networks())
self.pid = os.getpid()
self.usablecores = [
idx for idx, used in enumerate(sched_getaffinity().cpus) if used
]
self.cores = len(self.usablecores)
self.totalcores = os.sysconf('SC_NPROCESSORS_ONLN')
self.memory = get_memory()
self.num_nodes = int(os.getenv('PBS_NUM_NODES', 1))
descr = {
'fqdn': self.fqdn,
'network': self.network,
'pid': self.pid,
'cores': self.cores,
'usablecores': self.usablecores,
'totalcores': self.totalcores,
'num_nodes': self.num_nodes,
'memory': self.memory,
}
return descr
def go(self):
"""A wrapper around some common functions"""
self.fqdn = socket.getfqdn()
self.network = sorted_network(get_networks())
self.pid = os.getpid()
self.usablecores = [idx for idx, used in enumerate(sched_getaffinity().cpus) if used]
self.cores = len(self.usablecores)
self.totalcores = os.sysconf('SC_NPROCESSORS_ONLN')
self.memory = get_memory()
self.num_nodes = int(os.getenv('PBS_NUM_NODES', 1))
descr = {
'fqdn': self.fqdn,
'network': self.network,
'pid': self.pid,
'cores': self.cores,
'usablecores': self.usablecores,
'totalcores': self.totalcores,
'num_nodes': self.num_nodes,
'memory': self.memory,
}
return descr
def add_affinity(self):
self.update({
'affinity': [
idx for idx, x in enumerate(
affinity.sched_getaffinity().get_cpus()) if x == 1
],
})
def get_avail_core_count():
"""
Returns the number of available CPUs, according to cgroups and taskssets limits
"""
core_cnt = None
os_type = get_os_type()
if os_type == LINUX:
# simple use available sched_getaffinity() function (yields a long, so cast it down to int)
core_cnt = int(sum(sched_getaffinity().cpus))
else:
# BSD-type systems
out, _ = run_cmd('sysctl -n hw.ncpu',
force_in_dry_run=True,
trace=False)
try:
if int(out) > 0:
core_cnt = int(out)
except ValueError:
pass
if core_cnt is None:
raise SystemToolsException(
'Can not determine number of cores on this system')
else:
return core_cnt
def get_avail_core_count():
"""
Returns the number of available CPUs, according to cgroups and taskssets limits
"""
# tiny inner function to help figure out number of available cores in a cpuset
def count_bits(n):
"""Count the number of set bits for a given integer."""
bit_cnt = 0
while n > 0:
n &= n - 1
bit_cnt += 1
return bit_cnt
os_type = get_os_type()
if os_type == LINUX:
try:
# the preferred approach is via sched_getaffinity (yields a long, so cast it down to int)
num_cores = int(sum(sched_getaffinity().cpus))
return num_cores
except NameError:
pass
# in case sched_getaffinity isn't available, fall back to relying on /proc/cpuinfo
# determine total number of cores via /proc/cpuinfo
try:
txt = read_file('/proc/cpuinfo', log_error=False)
# sometimes this is uppercase
max_num_cores = txt.lower().count('processor\t:')
except IOError, err:
raise SystemToolsException(
"An error occured while determining total core count: %s" %
err)
# determine cpuset we're in (if any)
mypid = os.getpid()
try:
f = open("/proc/%s/status" % mypid, 'r')
txt = f.read()
f.close()
cpuset = re.search("^Cpus_allowed:\s*([0-9,a-f]+)", txt,
re.M | re.I)
except IOError:
cpuset = None
if cpuset is not None:
# use cpuset mask to determine actual number of available cores
mask_as_int = long(cpuset.group(1).replace(',', ''), 16)
num_cores_in_cpuset = count_bits((2**max_num_cores - 1)
& mask_as_int)
_log.info("In cpuset with %s CPUs" % num_cores_in_cpuset)
return num_cores_in_cpuset
else:
_log.debug("No list of allowed CPUs found, not in a cpuset.")
return max_num_cores
def get_avail_core_count():
"""
Returns the number of available CPUs, according to cgroups and taskssets limits
"""
# tiny inner function to help figure out number of available cores in a cpuset
def count_bits(n):
"""Count the number of set bits for a given integer."""
bit_cnt = 0
while n > 0:
n &= n - 1
bit_cnt += 1
return bit_cnt
os_type = get_os_type()
if os_type == LINUX:
try:
# the preferred approach is via sched_getaffinity (yields a long, so cast it down to int)
num_cores = int(sum(sched_getaffinity().cpus))
return num_cores
except NameError:
pass
# in case sched_getaffinity isn't available, fall back to relying on /proc/cpuinfo
# determine total number of cores via /proc/cpuinfo
try:
txt = read_file('/proc/cpuinfo', log_error=False)
# sometimes this is uppercase
max_num_cores = txt.lower().count('processor\t:')
except IOError, err:
raise SystemToolsException("An error occured while determining total core count: %s" % err)
# determine cpuset we're in (if any)
mypid = os.getpid()
try:
f = open("/proc/%s/status" % mypid, 'r')
txt = f.read()
f.close()
cpuset = re.search("^Cpus_allowed:\s*([0-9,a-f]+)", txt, re.M|re.I)
except IOError:
cpuset = None
if cpuset is not None:
# use cpuset mask to determine actual number of available cores
mask_as_int = long(cpuset.group(1).replace(',', ''), 16)
num_cores_in_cpuset = count_bits((2**max_num_cores - 1) & mask_as_int)
_log.info("In cpuset with %s CPUs" % num_cores_in_cpuset)
return num_cores_in_cpuset
else:
_log.debug("No list of allowed CPUs found, not in a cpuset.")
return max_num_cores
def set_cpus(self):
"""
Determine which cpus on the node can be used
are we running in a cpuset?
- and how big is it (nr of procs compared to local number of cores)
stores local core ids in array
"""
try:
proc_affinity = sched_getaffinity() # get affinity for current proc
self.cpus = [idx for idx, cpu in enumerate(proc_affinity.cpus) if cpu == 1]
self.log.debug("found cpus from affinity: %s", self.cpus)
except Exception:
self.cpus = range(self.cores_per_node)
self.log.debug("could not find cpus from affinity, simulating with range(cores_per_node): %s", self.cpus)
def go(self, ret=True):
"""A wrapper around some common functions"""
self.fqdn = socket.getfqdn()
self.network = get_networks()
self.order_network() # order the network
self.pid = os.getpid()
self.usablecores = [idx for idx, used in enumerate(sched_getaffinity().cpus) if used]
self.cores = len(self.usablecores)
self.memory = get_memory()
if ret:
descr = {
'fqdn': self.fqdn,
'network': self.network,
'pid': self.pid,
'cores': self.cores,
'usablecores': self.usablecores,
'topology': self.topology,
'memory': self.memory,
}
return descr
import struct
def calc_dotted_netmask(mask):
bits = 0
for i in xrange(32 - mask, 32):
bits |= (1 << i)
return "%d.%d.%d.%d" % ((bits & 0xff000000) >> 24, (bits & 0xff0000) >> 16, (bits & 0xff00) >> 8, (bits & 0xff))
endianness = '=L'
ipaddr = struct.unpack(endianness, socket.inet_aton(ip))[0]
netaddr, bits = net.split('/')
netmask = struct.unpack(
'=L', socket.inet_aton(calc_dotted_netmask(int(bits))))[0]
network = struct.unpack('=L', socket.inet_aton(netaddr))[0] & netmask
ans = (ipaddr & netmask) == (network & netmask)
self.log.debug("ip %s in net %s : %s" % (ip, net, ans))
return ans
def which_cpus(self):
"""
Determine which cpus can be used
are we running in a cpuset?
- and how big is it (nr of procs compared to local number of cores)
stores local core ids in array
# TODO fix remote cpusets
- what with remote ones?
"""
if self.foundppn is None:
self._cores_on_this_node()
try:
cs = sched_getaffinity() # get affinity for current proc
self.cpus = [idx for idx, cpu in enumerate(cs.cpus) if cpu == 1]
except:
self.cpus = range(self.foundppn)
self.log.debug("which_cpus: using cpus %s" % (self.cpus))
def which_cpus(self):
"""
Determine which cpus can be used
are we running in a cpuset?
- and how big is it (nr of procs compared to local number of cores)
stores local core ids in array
# TODO fix remote cpusets
- what with remote ones?
"""
if self.foundppn is None:
self._cores_on_this_node()
try:
cs = sched_getaffinity() # get affinity for current proc
self.cpus = [idx for idx, cpu in enumerate(cs.cpus) if cpu == 1]
except:
self.cpus = range(self.foundppn)
self.log.debug("which_cpus: using cpus %s" % (self.cpus))
def get_avail_core_count():
"""
Returns the number of available CPUs, according to cgroups and taskssets limits
"""
core_cnt = None
os_type = get_os_type()
if os_type == LINUX:
# simple use available sched_getaffinity() function (yields a long, so cast it down to int)
core_cnt = int(sum(sched_getaffinity().cpus))
else:
# BSD-type systems
out, _ = run_cmd('sysctl -n hw.ncpu', force_in_dry_run=True)
try:
if int(out) > 0:
core_cnt = int(out)
except ValueError:
pass
if core_cnt is None:
raise SystemToolsException('Can not determine number of cores on this system')
else:
return core_cnt
def which_cpus(self):
"""
Determine which cpus on the node can be used
are we running in a cpuset?
- and how big is it (nr of procs compared to local number of cores)
stores local core ids in array
"""
if self.foundppn is None:
self._cores_on_this_node()
try:
proc_affinity = sched_getaffinity(
) # get affinity for current proc
self.cpus = [
idx for idx, cpu in enumerate(proc_affinity.cpus) if cpu == 1
]
self.log.debug("found cpus from affinity: %s", self.cpus)
except Exception:
self.cpus = range(self.foundppn)
self.log.debug(
"could not find cpus from affinity, simulating with range(foundppn): %s",
self.cpus)
def add_affinity(self):
self.update({
'affinity':[idx for idx, x in enumerate(affinity.sched_getaffinity().get_cpus()) if x == 1],
})
def _get_affinity(self):
ctypes_cpuset_t = sched_getaffinity(pid=self.pid)
self.cpusett.set_bits(cpus=ctypes_cpuset_t.cpus)
def _get_affinity(self):
ctypes_cpuset_t = sched_getaffinity(pid=self.pid)
self.cpusett.set_bits(cpus=ctypes_cpuset_t.cpus)
