def ejobs(args, bjobsargs):
"""Wrapper script with bjobs functionality."""
# handle arguments
if args.pending:
bjobsargs = ["-p"] + bjobsargs
args.groupby = "pend_reason"
for shortcutname, shortcutargs in ejobsshortcuts.items():
if getattr(args, shortcutname):
bjobsargs = shortcutargs + bjobsargs
for l in list("rsda"):
if args.__dict__[l]:
bjobsargs = ["-" + l] + bjobsargs
if args.u:
unames = map(lookupalias, args.u.split())
bjobsargs = ["-u", " ".join(unames)] + bjobsargs
if args.jid:
args.output = ["id"]
args.fast = True
args.noheader = True
if args.output:
args.output = sum([fields.split() for fields in args.output], [])
if len(args.output) == 1:
args.noheader = True
# read
jobs = readjobs(bjobsargs, fast=args.fast)
if not jobs:
return
# sort
jobs.sort(key=lambda j: j["submit_time"])
jobs.sort(key=lambda j: j["priority"], reverse=True) # can be None
jobs.sort(key=lambda j: -j["run_time"])
jobs.sort(key=lambda j: -statorder[j["stat"]])
if args.sort:
try:
jobs.sort(key=lambda j: j[args.sort])
except:
print("Unknown sorting key \"%s\"!" % args.sort, file=sys.stderr)
# no grouping
if not args.groupby or args.groupby not in jobs[0]:
if args.sum:
jobs = [sumjobs(jobs)]
printjobs(jobs, wide=args.wide, long=args.long, output=args.output,
header=not args.noheader)
return
# grouping
jobgroups = groupjobs(jobs, args.groupby)
if not args.pending:
if args.sum:
jobs = []
for title in sorted(jobgroups.keys()):
sumjob = sumjobs(jobgroups[title])
if args.groupby not in ("name", "jobname", "user"):
sumjob["title"] = title
jobs.append(sumjob)
printjobs(jobs, wide=args.wide, long=args.long, output=args.output,
header=not args.noheader)
else:
for title in sorted(jobgroups.keys()):
printjobs(jobgroups[title], wide=args.wide, long=args.long,
output=args.output, header=not args.noheader,
title=title)
return
# pending
for title in sorted(jobgroups.keys()):
jobs = jobgroups[title]
reasons = jobs[0]["pend_reason"]
resreq = jobs[0]["resreq"]
hostreq = jobs[0]["host_req"]
if not reasons or len(reasons) != 1:
title = None
else:
# use singular reason as title
reason = reasons[0]
title = reason[0]
if not isinstance(reason[1], bool):
title += ": %d" % reason[1]
if args.sum:
jobs = [sumjobs(jobs)]
printjobs(jobs, wide=args.wide, long=args.long, output=args.output,
header=not args.noheader, title=title)
if reasons and len(reasons) > 1:
# show pending reasons
for reason, count in reasons:
for pattern in pendingcolors:
if re.match(pattern, reason):
reason = color(reason, pendingcolors[pattern])
break
if count is True:
print(" " + reason)
else:
print(" %4d %s" % (count, reason))
# show potential hosts
if resreq and not args.fast:
resreq = re.sub(" && \(hostok\)", "", resreq)
resreq = re.sub(" && \(mem>\d+\)", "", resreq)
hosts = readhosts(["-R", resreq] + hostreq)
hostnames = [h["host_name"] for h in hosts]
jobs = readjobs(["-u", "all", "-r", "-m", " ".join(hostnames)])
hosts.sort(key=lambda h: h["host_name"])
printhosts(hosts, jobs, wide=args.wide,
header=not args.noheader)
if len(jobgroups) > 1:
print()
def ejobs(args, bjobsargs):
if args.pending:
bjobsargs = ["-p"] + bjobsargs
args.groupby = "pend_reason"
if args.sort:
args.sortby = "jobid"
if args.aices:
bjobsargs = ["-P", "aices", "-G", "p_aices"] + bjobsargs
if args.aices2:
bjobsargs = ["-P", "aices2", "-G", "p_aices"] + bjobsargs
if args.aices24:
bjobsargs = ["-P", "aices-24", "-G", "p_aices"] + bjobsargs
for l in list("rsda"):
if args.__dict__[l]:
bjobsargs = ["-" + l] + bjobsargs
if args.u:
unames = []
for alias in args.u.split():
unames += lookupalias(alias)
bjobsargs += ["-u", " ".join(unames)]
# read
jobs = readjobs(bjobsargs, fast=args.fast)
if not jobs:
return
# sort
statorder = {
"RUN": 4,
"PROV": 4,
"PSUSP": 3,
"USUSP": 3,
"SSUSP": 3,
"PEND": 2,
"WAIT": 2,
"UNKWN": 1,
"DONE": 0,
"ZOMBI": 0,
"EXIT": 0,
}
jobs.sort(key=lambda j: j["submit_time"])
jobs.sort(key=lambda j: j["priority"], reverse=True)
jobs.sort(key=lambda j: -j["run_time"])
jobs.sort(key=lambda j: -statorder[j["stat"]])
if args.sortby:
jobs.sort(key=lambda j: j[args.sortby])
# no grouping
if not args.groupby or args.groupby not in jobs[0]:
if args.sum:
printjobs([sumjobs(jobs)], wide=args.wide, long=args.long,
header=not args.noheader)
else:
printjobs(jobs, wide=args.wide, long=args.long,
header=not args.noheader)
return
# grouping
jobgroups = groupjobs(jobs, args.groupby)
if not args.pending:
if args.sum:
jobs = []
for title in sorted(jobgroups.keys()):
jobgroup = jobgroups[title]
sumjob = sumjobs(jobgroup)
sumjob["title"] = title
jobs.append(sumjob)
printjobs(jobs, wide=args.wide, long=args.long,
header=not args.noheader)
else:
for title in sorted(jobgroups.keys()):
jobs = jobgroups[title]
printjobs(jobs, wide=args.wide, long=args.long,
header=not args.noheader, title=title)
return
# pending
for title in sorted(jobgroups.keys()):
jobs = jobgroups[title]
reasons = jobs[0]["pend_reason"]
if not reasons or len(reasons) != 1:
title = None
else: # only use singular reason as title
reason = reasons[0]
title = reason[0]
if not isinstance(reason[1], bool):
title += ": %d" % reason[1]
if args.sum:
printjobs([sumjobs(jobs)], wide=args.wide, long=args.long,
header=not args.noheader, title=title)
else:
printjobs(jobs, wide=args.wide, long=args.long,
header=not args.noheader, title=title)
if reasons and len(reasons) > 1:
# show pending reasons
for reason, count in reasons:
if reason in pendingcolors:
reason = color(reason, pendingcolors[reason])
if count is True:
print(" " + reason)
else:
print(" %4d %s" % (count, reason))
# show potential hosts
if jobs[0]["resreq"] and not args.fast:
req = jobs[0]["resreq"]
req = re.sub(" && \(hostok\)", "", req)
req = re.sub(" && \(mem>\d+\)", "", req)
hosts = readhosts(["-R", req])
hostnames = [h["host_name"] for h in hosts]
jobs = readjobs(["-u", "all", "-r", "-m", " ".join(hostnames)])
hosts.sort(key=lambda h: h["host_name"])
printhosts(hosts, jobs, wide=args.wide,
header=not args.noheader)
if len(jobgroups) > 1:
print()
def ejobs(args, bjobsargs):
"""Wrapper script with bjobs functionality."""
# handle arguments
if args.pending:
bjobsargs = ["-p"] + bjobsargs
args.groupby = "pend_reason"
for shortcutname, shortcutargs in ejobsshortcuts.items():
if getattr(args, shortcutname):
bjobsargs = shortcutargs + bjobsargs
for l in list("rsda"):
if args.__dict__[l]:
bjobsargs = ["-" + l] + bjobsargs
if args.u:
unames = map(lookupalias, args.u.split())
bjobsargs = ["-u", " ".join(unames)] + bjobsargs
if args.jid:
args.output = ["id"]
args.fast = True
args.noheader = True
if args.output:
args.output = sum([fields.split() for fields in args.output], [])
if len(args.output) == 1:
args.noheader = True
# read
jobs = readjobs(bjobsargs, fast=args.fast)
if not jobs:
return
# sort
jobs.sort(key=lambda j: j["submit_time"])
jobs.sort(key=lambda j: j["priority"], reverse=True) # can be None
jobs.sort(key=lambda j: -j["run_time"])
jobs.sort(key=lambda j: -statorder[j["stat"]])
if args.sort:
try:
jobs.sort(key=lambda j: j[args.sort])
except:
print("Unknown sorting key \"%s\"!" % args.sort, file=sys.stderr)
# no grouping
if not args.groupby or args.groupby not in jobs[0]:
if args.sum:
jobs = [sumjobs(jobs)]
printjobs(jobs,
wide=args.wide,
long=args.long,
output=args.output,
header=not args.noheader)
return
# grouping
jobgroups = groupjobs(jobs, args.groupby)
if not args.pending:
if args.sum:
jobs = []
for title in sorted(jobgroups.keys()):
sumjob = sumjobs(jobgroups[title])
if args.groupby not in ("name", "jobname", "user"):
sumjob["title"] = title
jobs.append(sumjob)
printjobs(jobs,
wide=args.wide,
long=args.long,
output=args.output,
header=not args.noheader)
else:
for title in sorted(jobgroups.keys()):
printjobs(jobgroups[title],
wide=args.wide,
long=args.long,
output=args.output,
header=not args.noheader,
title=title)
return
# pending
for title in sorted(jobgroups.keys()):
jobs = jobgroups[title]
reasons = jobs[0]["pend_reason"]
resreq = jobs[0]["resreq"]
hostreq = jobs[0]["host_req"]
if not reasons or len(reasons) != 1:
title = None
else:
# use singular reason as title
reason = reasons[0]
title = reason[0]
if not isinstance(reason[1], bool):
title += ": %d" % reason[1]
if args.sum:
jobs = [sumjobs(jobs)]
printjobs(jobs,
wide=args.wide,
long=args.long,
output=args.output,
header=not args.noheader,
title=title)
if reasons and len(reasons) > 1:
# show pending reasons
for reason, count in reasons:
for pattern in pendingcolors:
if re.match(pattern, reason):
reason = color(reason, pendingcolors[pattern])
break
if count is True:
print(" " + reason)
else:
print(" %4d %s" % (count, reason))
# show potential hosts
if resreq and not args.fast:
resreq = re.sub(" && \(hostok\)", "", resreq)
resreq = re.sub(" && \(mem>\d+\)", "", resreq)
hosts = readhosts(["-R", resreq] + hostreq)
hostnames = [h["host_name"] for h in hosts]
jobs = readjobs(["-u", "all", "-r", "-m", " ".join(hostnames)])
hosts.sort(key=lambda h: h["host_name"])
printhosts(hosts,
jobs,
wide=args.wide,
header=not args.noheader)
if len(jobgroups) > 1:
print()
def printhosts(hosts, jobs=[], wide=False, header=True, file=sys.stdout):
"""list the hosts."""
if len(hosts) == 0:
return
sumhosts = not isinstance(hosts[0]["status"], str)
jobsbyhost = groupjobs(jobs, "exec_host")
# begin output
screencols = int(check_output(["tput", "cols"]))
whoami = os.getenv("USER")
namelen = max(map(len, (host["host_name"] for host in hosts)))
lens = {
"host_name": min(20, max(6, namelen + 1)),
"status": 8,
"title": 15,
"cpus": 10
}
if wide:
lens["title"] = 20
lens["host_name"] = max(6, namelen + 1)
lens["model"] = 14
if sumhosts:
lens["status"] = 12
lens["cpus"] = 14
if header:
h = ""
if sumhosts and "title" in hosts[0]:
h += "group".ljust(lens["title"])
h += "".join(n.ljust(lens[n]) for n in ("host_name", "status", "cpus"))
h += "mem (free/total)"
if wide:
h += " " + "model".ljust(lens["model"])
h = h.upper()
print(h, file=file)
for host in hosts:
l = ""
if sumhosts and "title" in host:
# title
title = host["title"]
if not wide:
if len(title) >= lens["title"]:
title = title[:lens["title"] - 2] + "*"
l += color(title.ljust(lens["title"]), "b")
# host_name
l += host["host_name"].ljust(lens["host_name"])
# status
if sumhosts:
l += color("%3d " % host["status"]["ok"], "g")
closed = sum(n for stat, n in host["status"].iteritems() if
stat.startswith("closed_"))
l += color("%3d " % closed, "r")
other = len(host["host_names"]) - host["status"]["ok"] - closed
if other:
l += color("%3d " % other, "y")
else:
l += " "
else:
if host["status"] == "ok":
l += color("ok".ljust(lens["status"]), "g")
elif "closed_" in host["status"]:
l += color(host["status"][7:].ljust(lens["status"]), "r")
else:
l += color(host["status"].ljust(lens["status"]), "y")
# cpus
total = host["max"]
used = host["njobs"]
free = total - used
c = fractioncolor(free / total)
if sumhosts:
l += color("%4d" % free, c) + "/%4d" % total
else:
l += color("%2d" % free, c) + "/%2d" % total
# mem
if "mem" in host["load"]:
free, used = host["load"]["mem"]
total = free + used
if "maxmem" in host and host["maxmem"]:
total = host["maxmem"]
c = fractioncolor(free / total)
l += " " + format_mem(free, c) + "/" + format_mem(total)
if wide:
if sumhosts:
if len(host["model"]) == 1:
l += host["model"][0].ljust(lens["model"])
else:
nmodel = len(host["model"])
l += color((" %d" % nmodel).ljust(lens["model"]), "b")
else:
hoststr = host["model"]
# Xeon Phi(s)
phis = 0
if "mic0" in host["load"]:
phis += int(bool(host["load"]["mic0"][0]))
if "mic1" in host["load"]:
phis += int(bool(host["load"]["mic1"][0]))
if phis > 0:
hoststr += "+%dPhi" % phis
# GPU
if "gpu" in host["resources"]:
hoststr += "+GPU"
l += " " + hoststr.ljust(14)
l += " "
if host["rsv"] > 0:
l += " %3d*" % host["rsv"] + color("reserved", "y")
if sumhosts:
hostnames = host["host_names"]
else:
hostnames = [host["host_name"]]
jobs = []
for hostname in hostnames:
if hostname in jobsbyhost:
for job in jobsbyhost[hostname]:
if job not in jobs:
jobs.append(job)
if sumhosts:
jobgroups = groupjobs(jobs, "user")
jobs = []
for user in sorted(jobgroups.keys()):
jobs.append(sumjobs(jobgroups[user]))
if jobs:
for job in jobs:
exclusive = job["exclusive"]
if sumhosts:
exclusive = len(exclusive) == 1 and True in exclusive
times = color("x", "r") if exclusive else "*"
nslots = sum(job["exec_host"][hn] for hn in hostnames
if hn in job["exec_host"])
c = "r" if nslots >= 100 else "y" if nslots >= 20 else 0
l += color(" %3d" % nslots, c)
user = job["user"]
if sumhosts:
user = user.keys()[0]
c = "g" if user == whoami else 0
l += times + color(getuseralias(user).ljust(8), c)
if wide and not sumhosts:
if job["mem"]:
l += format_mem(job["mem"])
else:
l += " "
if job["%complete"] and job["runlimit"]:
ptime = job["%complete"]
c = fractioncolor(1 - ptime / 100)
l += color("%3d" % ptime, c) + "% "
l += format_duration(job["runlimit"])
if host["comment"]:
if sumhosts:
for key, val in host["comment"].iteritems():
if key:
l += " %3dx" % val + color(key, "b")
else:
l += " " + color(host["comment"], "b")
print(l, file=file)
file.flush()
def ejobs(args, bjobsargs):
if args.pending:
bjobsargs = ["-p"] + bjobsargs
args.groupby = "pend_reason"
if args.sort:
args.sortby = "jobid"
if args.aices:
bjobsargs = ["-P", "aices", "-G", "p_aices"] + bjobsargs
if args.aices2:
bjobsargs = ["-P", "aices2", "-G", "p_aices"] + bjobsargs
if args.aices24:
bjobsargs = ["-P", "aices-24", "-G", "p_aices"] + bjobsargs
for l in list("rsda"):
if args.__dict__[l]:
bjobsargs = ["-" + l] + bjobsargs
if args.u:
unames = []
for alias in args.u.split():
unames += lookupalias(alias)
bjobsargs += ["-u", " ".join(unames)]
# read
jobs = readjobs(bjobsargs, fast=args.fast)
if not jobs:
return
# sort
statorder = {
"RUN": 4,
"PROV": 4,
"PSUSP": 3,
"USUSP": 3,
"SSUSP": 3,
"PEND": 2,
"WAIT": 2,
"UNKWN": 1,
"DONE": 0,
"ZOMBI": 0,
"EXIT": 0,
}
jobs.sort(key=lambda j: j["submit_time"])
jobs.sort(key=lambda j: j["priority"], reverse=True)
jobs.sort(key=lambda j: -j["run_time"])
jobs.sort(key=lambda j: -statorder[j["stat"]])
if args.sortby:
jobs.sort(key=lambda j: j[args.sortby])
# no grouping
if not args.groupby or args.groupby not in jobs[0]:
if args.sum:
printjobs([sumjobs(jobs)],
wide=args.wide,
long=args.long,
header=not args.noheader)
else:
printjobs(jobs,
wide=args.wide,
long=args.long,
header=not args.noheader)
return
# grouping
jobgroups = groupjobs(jobs, args.groupby)
if not args.pending:
if args.sum:
jobs = []
for title in sorted(jobgroups.keys()):
jobgroup = jobgroups[title]
sumjob = sumjobs(jobgroup)
sumjob["title"] = title
jobs.append(sumjob)
printjobs(jobs,
wide=args.wide,
long=args.long,
header=not args.noheader)
else:
for title in sorted(jobgroups.keys()):
jobs = jobgroups[title]
printjobs(jobs,
wide=args.wide,
long=args.long,
header=not args.noheader,
title=title)
return
# pending
for title in sorted(jobgroups.keys()):
jobs = jobgroups[title]
reasons = jobs[0]["pend_reason"]
if not reasons or len(reasons) != 1:
title = None
else: # only use singular reason as title
reason = reasons[0]
title = reason[0]
if not isinstance(reason[1], bool):
title += ": %d" % reason[1]
if args.sum:
printjobs([sumjobs(jobs)],
wide=args.wide,
long=args.long,
header=not args.noheader,
title=title)
else:
printjobs(jobs,
wide=args.wide,
long=args.long,
header=not args.noheader,
title=title)
if reasons and len(reasons) > 1:
# show pending reasons
for reason, count in reasons:
if reason in pendingcolors:
reason = color(reason, pendingcolors[reason])
if count is True:
print(" " + reason)
else:
print(" %4d %s" % (count, reason))
# show potential hosts
if jobs[0]["resreq"] and not args.fast:
req = jobs[0]["resreq"]
req = re.sub(" && \(hostok\)", "", req)
req = re.sub(" && \(mem>\d+\)", "", req)
hosts = readhosts(["-R", req])
hostnames = [h["host_name"] for h in hosts]
jobs = readjobs(["-u", "all", "-r", "-m", " ".join(hostnames)])
hosts.sort(key=lambda h: h["host_name"])
printhosts(hosts,
jobs,
wide=args.wide,
header=not args.noheader)
if len(jobgroups) > 1:
print()
def printhosts(hosts, jobs=[], wide=False, header=True, file=sys.stdout):
"""Print a list of hosts."""
if len(hosts) == 0:
return
sumhosts = not isinstance(hosts[0]["status"], str)
jobsbyhost = groupjobs(jobs, "exec_host")
# begin output
screencols = int(check_output(["tput", "cols"]))
whoami = os.getenv("USER")
namelen = max(map(len, (host["host_name"] for host in hosts)))
lens = {
"host_name": min(20, max(6, namelen + 1)),
"status": 8,
"title": 15,
"cpus": 10
}
if wide:
lens["title"] = 20
lens["host_name"] = max(6, namelen + 1)
lens["model"] = 14
if sumhosts:
lens["status"] = 12
lens["cpus"] = 14
if header:
h = ""
if sumhosts and "title" in hosts[0]:
h += "group".ljust(lens["title"])
h += "".join(n.ljust(lens[n]) for n in ("host_name", "status", "cpus"))
h += "mem (free/total)"
if wide:
h += " " + "model".ljust(lens["model"])
h = h.upper()
print(h, file=file)
for host in hosts:
l = ""
if sumhosts and "title" in host:
# title
title = host["title"]
if not wide:
if len(title) >= lens["title"]:
title = title[:lens["title"] - 2] + "*"
l += color(title.ljust(lens["title"]), "b")
# host_name
l += host["host_name"].ljust(lens["host_name"])
# status
if sumhosts:
l += color("%3d " % host["status"]["ok"], "g")
closed = sum(n for stat, n in host["status"].iteritems()
if stat.startswith("closed_"))
l += color("%3d " % closed, "r")
other = len(host["host_names"]) - host["status"]["ok"] - closed
if other:
l += color("%3d " % other, "y")
else:
l += " "
else:
if host["status"] == "ok":
l += color("ok".ljust(lens["status"]), "g")
elif "closed_" in host["status"]:
l += color(host["status"][7:].ljust(lens["status"]), "r")
else:
l += color(host["status"].ljust(lens["status"]), "y")
# cpus
total = host["max"]
used = host["njobs"]
free = total - used
c = fractioncolor(free, total)
if sumhosts:
l += color("%4d" % free, c) + "/%4d" % total
else:
l += color("%2d" % free, c) + "/%2d" % total
# mem
if "mem" in host["load"]:
free, used = host["load"]["mem"]
total = free
if used: # used can be None
total += used
if "maxmem" in host and host["maxmem"]:
total = host["maxmem"]
c = fractioncolor(free, total)
l += " " + format_mem(free, c) + "/" + format_mem(total)
if wide:
if sumhosts:
if len(host["model"]) == 1:
l += host["model"][0].ljust(lens["model"])
else:
nmodel = len(host["model"])
l += color((" %d" % nmodel).ljust(lens["model"]), "b")
else:
hoststr = host["model"]
# Xeon Phi(s)
phis = 0
if "mic0" in host["load"]:
phis += int(bool(host["load"]["mic0"][0]))
phis += int(bool(host["load"]["mic0"][1]))
if "mic1" in host["load"]:
phis += int(bool(host["load"]["mic1"][0]))
phis += int(bool(host["load"]["mic1"][1]))
if phis > 0:
hoststr += "+%dPhi" % phis
# GPU
if "gpu" in host["resources"]:
hoststr += "+GPU"
l += " " + hoststr.ljust(14)
l += " "
if host["rsv"] > 0:
l += " %3d*" % host["rsv"] + color("reserved", "y")
if sumhosts:
hostnames = host["host_names"]
else:
hostnames = [host["host_name"]]
jobs = []
for hostname in hostnames:
if hostname in jobsbyhost:
for job in jobsbyhost[hostname]:
if job not in jobs:
jobs.append(job)
if sumhosts:
jobgroups = groupjobs(jobs, "user")
jobs = []
for user in sorted(jobgroups.keys()):
jobs.append(sumjobs(jobgroups[user]))
if jobs:
for job in jobs:
exclusive = job["exclusive"]
if sumhosts:
exclusive = len(exclusive) == 1 and True in exclusive
times = color("x", "r") if exclusive else "*"
nslots = sum(job["exec_host"][hn] for hn in hostnames
if hn in job["exec_host"])
c = "r" if nslots >= 100 else "y" if nslots >= 20 else 0
l += color(" %3d" % nslots, c)
user = job["user"]
if sumhosts:
user = user.keys()[0]
c = "g" if user == whoami else 0
l += times + color(getuseralias(user).ljust(8), c)
if wide and not sumhosts:
if job["mem"]:
l += format_mem(job["mem"])
else:
l += " "
if job["%complete"] and job["runlimit"]:
ptime = job["%complete"]
c = fractioncolor(1 - ptime / 100)
l += color("%3d" % ptime, c) + "% "
l += format_duration(job["runlimit"])
if host["comment"]:
if sumhosts:
for key, val in host["comment"].iteritems():
if key:
l += " %3dx" % val + color(key, "b")
else:
l += " " + color(host["comment"], "b")
print(l, file=file)
file.flush()