def check_total_pmem(self, job):
'''check total memory requirements of job as aggregated of pmem'''
partition = job.resource_spec('partition')
mem_sizes = self._mem_sizes(partition)
for nodes_spec in job.resource_spec('nodes'):
orig_nodes = nodes_spec['nodes']
nodes = orig_nodes
if 'ppn' in nodes_spec:
ppn = nodes_spec['ppn']
else:
ppn = None
pmem = job.resource_spec('pmem')
if ppn and pmem:
node_mem = ppn*pmem
else:
continue
for mem in sorted(mem_sizes.keys()):
if node_mem < mem:
if nodes <= mem_sizes[mem]:
mem_sizes[mem] -= nodes
nodes = 0
break
else:
nodes -= mem_sizes[mem]
if nodes > 0:
self.reg_event('insufficient_nodes_mem',
{'mem': bytes2size(node_mem, 'gb'),
'nodes': orig_nodes})
def check_mem(self, job):
"""check total memory requirements of job as mem"""
partition = job.resource_spec("partition")
mem_sizes = self._mem_sizes(partition)
mem_spec = job.resource_spec("mem")
if mem_spec:
for nodes_spec in job.resource_spec("nodes"):
nodes = nodes_spec["nodes"]
for mem in sorted(mem_sizes.keys()):
mem_spec -= mem * nodes
if mem_spec <= 0:
break
if mem_spec > 0:
self.reg_event("insufficient_mem", {"mem": bytes2size(mem_spec, "gb")})
def check_mem(self, job):
'''check total memory requirements of job as mem'''
partition = job.resource_spec('partition')
mem_sizes = self._mem_sizes(partition)
mem_spec = job.resource_spec('mem')
if mem_spec:
for nodes_spec in job.resource_spec('nodes'):
nodes = nodes_spec['nodes']
for mem in sorted(mem_sizes.keys()):
mem_spec -= mem*nodes
if mem_spec <= 0:
break
if mem_spec > 0:
self.reg_event('insufficient_mem',
{'mem': bytes2size(mem_spec, 'gb')})
def _job_to_tuple(job):
'''returns a tuple with the job information that will be inserted
into a pandas data frame'''
if job.start:
start_time = job.start.strftime(PbsLogAnalysis.time_fmt)
else:
start_time = '1970-01-01 00:00:00'
if job.end:
end_time = job.end.strftime(PbsLogAnalysis.time_fmt)
else:
end_time = datetime.now().strftime(PbsLogAnalysis.time_fmt)
nodes = job.resource_spec('nodes')[0]['nodes']
if 'ppn' in job.resource_spec('nodes')[0]:
ppn = job.resource_spec('nodes')[0]['ppn']
else:
ppn = None
if job.resource_used('mem'):
mem_used = float(
bytes2size(job.resource_used('mem'),
'gb',
no_unit=True,
fraction=True))
else:
mem_used = None
if job.resource_used('walltime'):
walltime = seconds2walltime(job.resource_used('walltime'))
else:
walltime = None
spec_walltime = seconds2walltime(job.resource_spec('walltime'))
if job.exec_host:
hosts = ' '.join(job.exec_host.keys())
else:
hosts = None
return PbsLogAnalysis.JobTuple(start=start_time,
end=end_time,
job_id=job.job_id,
user=job.user,
state=job.state,
partition=job.partition,
used_mem=mem_used,
used_walltime=walltime,
spec_walltime=spec_walltime,
nodes=nodes,
ppn=ppn,
hosts=hosts,
exit_status=job.exit_status)
def _job_to_tuple(job):
'''returns a tuple with the job information that will be inserted
into a pandas data frame'''
if job.start:
start_time = job.start.strftime(PbsLogAnalysis.time_fmt)
else:
start_time = '1970-01-01 00:00:00'
if job.end:
end_time = job.end.strftime(PbsLogAnalysis.time_fmt)
else:
end_time = datetime.now().strftime(PbsLogAnalysis.time_fmt)
nodes = job.resource_spec('nodes')[0]['nodes']
if 'ppn' in job.resource_spec('nodes')[0]:
ppn = job.resource_spec('nodes')[0]['ppn']
else:
ppn = None
if job.resource_used('mem'):
mem_used = float(bytes2size(job.resource_used('mem'), 'gb',
no_unit=True, fraction=True))
else:
mem_used = None
if job.resource_used('walltime'):
walltime = seconds2walltime(job.resource_used('walltime'))
else:
walltime = None
spec_walltime = seconds2walltime(job.resource_spec('walltime'))
if job.exec_host:
hosts = ' '.join(job.exec_host.keys())
else:
hosts = None
return PbsLogAnalysis.JobTuple(
start=start_time,
end=end_time,
job_id=job.job_id,
user=job.user,
state=job.state,
partition=job.partition,
used_mem=mem_used,
used_walltime=walltime,
spec_walltime=spec_walltime,
nodes=nodes,
ppn=ppn,
hosts=hosts,
exit_status=job.exit_status
)
def check_pmem(self, job):
"""Check whether the requested memory per node is available"""
partition = job.resource_spec("partition")
mem_sizes = self._mem_sizes(partition).keys()
for nodes_spec in job.resource_spec("nodes"):
satisfied = False
ppn = nodes_spec["ppn"]
pmem = job.resource_spec("pmem")
if ppn and pmem:
node_mem = ppn * pmem
else:
continue
for mem in mem_sizes:
if node_mem < mem:
satisfied = True
break
if not satisfied:
self.reg_event("insufficient_node_pmem", {"mem": bytes2size(node_mem, "gb")})
def check_pmem(self, job):
'''Check whether the requested memory per node is available'''
partition = job.resource_spec('partition')
mem_sizes = self._mem_sizes(partition).keys()
for nodes_spec in job.resource_spec('nodes'):
satisfied = False
ppn = nodes_spec['ppn']
pmem = job.resource_spec('pmem')
if ppn and pmem:
node_mem = ppn*pmem
else:
continue
for mem in mem_sizes:
if node_mem < mem:
satisfied = True
break
if not satisfied:
self.reg_event('insufficient_node_pmem',
{'mem': bytes2size(node_mem, 'gb')})
def check_total_pmem(self, job):
"""check total memory requirements of job as aggregated of pmem"""
partition = job.resource_spec("partition")
mem_sizes = self._mem_sizes(partition)
for nodes_spec in job.resource_spec("nodes"):
satisfied = False
orig_nodes = nodes_spec["nodes"]
nodes = orig_nodes
ppn = nodes_spec["ppn"]
pmem = job.resource_spec("pmem")
if ppn and pmem:
node_mem = ppn * pmem
else:
continue
for mem in sorted(mem_sizes.keys()):
if node_mem < mem:
if nodes <= mem_sizes[mem]:
mem_sizes[mem] -= nodes
nodes = 0
break
else:
nodes -= mem_sizes[mem]
if nodes > 0:
self.reg_event("insufficient_nodes_mem", {"mem": bytes2size(node_mem, "gb"), "nodes": orig_nodes})
if loadaves:
print 'nodes = {0}, cores = {1}'.format(len(loadaves), core_count)
if cput:
print 'cpu time: {0}'.format(seconds2walltime(cput))
if walltime:
print 'walltime: {0}'.format(seconds2walltime(walltime))
if cput and walltime and core_count:
efficiency = 100.0 * int(cput) / (int(walltime) * core_count)
print 'parallel efficiency: {0:.1f} %'.format(efficiency)
print 'loadave:'
stats = compute_stats(loadaves)
fmt_str = ' min: {0:.1f}, median: {1:.1f}, max: {2:.1f}'
print(fmt_str.format(*stats))
print 'mem:'
stats = map(
lambda x: float(bytes2size(x, 'gb', no_unit=True, fraction=True)),
compute_stats(mems))
fmt_str = ' min: {0:.1f} GB, median: {1:.1f} GB, max: {2:.1f} GB'
print(fmt_str.format(*stats))
print 'netload:'
stats = map(
lambda x: float(bytes2size(x, 'gb', no_unit=True, fraction=True)),
compute_stats(netloads))
fmt_str = ' min: {0:.1f} GB, median: {1:.1f} GB, max: {2:.1f} GB'
print(fmt_str.format(*stats))
if options.nodes:
print('nodes:')
print(' ' + ','.join(node_names))
else:
print 'job {0} is not running'.format(options.job_id)