def findMyJob(self):
"""Call FindJobsFromUser, passing the output of whoami, and keeping only the FIRST job returned
Return a tuple (jobid,partition,user,nodeset), or ("","","")
WARNING - Jobs in any other state than RUNNING are ignored"""
user = runCmd('whoami').rstrip()
return self.findJobsFromUser(user)[0]
def nodesetToHosts(self,nodeset):
try:
nodes = runCmd('nodeset -e ' + nodeset).rstrip().split(' ')
except:
return []
return nodes
def nodesetToHosts(self, nodeset):
"""Expand the nodeset to a list of hosts"""
try:
nodes = runCmd('nodeset -e ' + nodeset).rstrip().split(' ')
except:
return []
return nodes
def findJobFromId(self,jobid):
cmd = 'squeue -t RUNNING -j ' + str(jobid) + ' --noheader -o %.16R@%.15u@%.7A'
try:
rvl = runCmd(cmd)
except:
return ("","","")
if rvl == "":
return ("","","")
# host[0-4]@ user@jobid @partition ==> (host[0-4],user,jobid,partition)
return tuple(map(str.strip,rvl.split('@')))
def __callNumactl(self):
"""Call numactl, detecting reserved sockets and physical cores
return the list of reserved sockets, and the list of physical cores """
cmd = "numactl --show"
output = runCmd(cmd).split('\n')
# l_sockets is generated from line nodebind of numactl
# nodebind: 4 5 6 => [4,5,6]
for l in output:
if l.startswith('nodebind:'):
l_sockets = list(map(int,l.rpartition(':')[2].strip().split(' ')))
elif l.startswith('physcpubind:'):
physcpubind = list(map(int,l.rpartition(':')[2].strip().split(' ')))
return [l_sockets,physcpubind]
def findJobsFromUser(self,user):
"""Call squeue and return a list of tuples corresponding to the jobs running for this user"""
cmd = 'squeue -t RUNNING -u ' + user + ' --noheader -o %.16R@%.15u@%.7A'
try:
rvl = runCmd(cmd)
except:
return [("","","")]
if rvl == "":
return [("","","")]
tuples = []
# host[0-4]@ user@jobid @partition ==> (host[0-4],user,jobid,partition)
for j in rvl.split('\n'):
tuples.append(tuple(map(str.strip,j.split('@'))))
return tuples
def __callNumactl(self):
"""Call numactl, detecting sockets and core addressing
return An array of int (concatenation of node X cpus: outputs of numactl --hardware)
"""
rvl = []
cmd = "numactl --hardware"
output = runCmd(cmd).split('\n')
# Looking for lines (in this order !)
# node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30
# node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31
sock_cnt = 0
for l in output:
if l.startswith('node ' + str(sock_cnt) + ' cpus:'):
cores = l.partition(':')[2]
rvl.append(list(map(int, cores.strip().split(' '))))
sock_cnt += 1
return flatten(rvl)
def __initDataStructures(self):
'''Init self.__pid2jobid, self.__core2jobid and self._job2tag
Explore the /sys/fs/cgroup/cpuset pseudo filesystem'''
# If the data do not exist, build them, else return
if self.__pid2jobid == None or self.__core2jobid == None:
pid2jobid = {}
core2jobid = {}
# Looking for /sys/fs/cgroup/cpuset/slurm/uid_xxx/job_yyyyyy/step_batch
top_dir = "/sys/fs/cgroup/cpuset/slurm/"
# This is a cache, avoiding more squeue than necessary
jobids = {}
for root, dirs, files in os.walk(top_dir, False):
leaf = os.path.basename(root)
if leaf.startswith('step_'):
job_path = os.path.split(root)[0]
# => .../slurm/uid_xxx/job_yyyyyy
job_dir = os.path.split(job_path)[1] # => job_yyyyyy
jobid = job_dir.replace('job_', '') # => yyyyyy
# If the job is not recognized by slurm, ignore it: this is an old trace
if jobid in jobids:
if not jobid in jobids:
continue
else:
# If we do not know, execute squeue
try:
# We do not need the output, only the exit status
runCmd(f'squeue -j {jobid} -o ""')
jobids[jobid] = True
except PlacementException as e:
jobids[jobid] = False
continue
# The pids are in the file cgroup.procs
pids = []
cgroup_procs = root + '/cgroup.procs'
with open(cgroup_procs, 'r') as infile:
for line in infile:
line = line.strip()
if line != '':
pid2jobid[line] = jobid
# The cores are in the file cpuset.cpus
cpuset_cpus = root + '/cpuset.cpus'
with open(cpuset_cpus, 'r') as infile:
for line in infile:
line = line.strip()
if line != '':
# Nearly same format for the cpusets as for the nodesets !
cores = self.nodesetToHosts('[' + line + ']')
for core in cores:
core2jobid[core] = jobid
# build the map self._job2tag
jobids = sorted(list(set(core2jobid.values())))
t = 0
m = {}
for j in jobids:
t += 1
m[j] = t
self.__pid2jobid = pid2jobid
self.__core2jobid = core2jobid
self._job2tag = m