def efficiency(wdir, details, verbose):
stats = JobsReportStats.from_workdir(wdir, verbose=verbose)
if not stats.has_realtime_stats():
real = False
print(
f'warning: real time log files not found - the generated chart might be not accurate'
)
report = stats.efficiency(details)
print('\t{:>40}: {}'.format('used cores', report.get('total_cores', 0)))
print('\t{:>40}: {:.1f}%'.format('average core utilization (%)',
report.get('avg_core_utilization', 0)))
if details:
if report.get('nodes'):
sorted_node_list = sorted(report.get('nodes', {}).keys())
for node_name in sorted_node_list:
cores = report['nodes'][node_name]
print(f'\t{node_name}')
sorted_core_list = sorted(cores.keys(),
key=lambda c: int(c.split('&')[0])
if '&' in str(c) else int(c))
for core_id in sorted_core_list:
core_name = str(core_id)
core_spec = cores[core_name]
print(
f'\t\t{core_name:4}: {core_spec.get("utilization", 0):.1f}%, unused {core_spec.get("unused", 0):.4f} s'
)
def launch_stats(wdir, details, verbose):
stats = JobsReportStats.from_workdir(wdir, verbose)
report = stats.job_start_finish_launch_overheads(details=details)
print('\t{:>40}: {:.4f}'.format('total start overhead',
report.get('start', 0)))
print('\t{:>40}: {:.4f}'.format('total finish overhead',
report.get('finish', 0)))
print('\t{:>40}: {:.4f}'.format('total start and finish overhead',
report.get('total', 0)))
print('\t{:>40}: {:.4f}'.format('average job start overhead',
report.get('job_start_avg', 0)))
print('\t{:>40}: {:.4f}'.format('average job finish overhead',
report.get('job_finish_avg', 0)))
print('\t{:>40}: {:.4f}'.format('average job total overhead',
report.get('job_avg', 0)))
print('\t{:>40}: {:.4f}'.format('average real job run time',
report.get('job_real_rt_avg', 0)))
print('\t{:>40}: {:.4f}'.format('average qcg job run time',
report.get('job_qcg_rt_avg', 0)))
print('\t{:>40}: {:.2f}'.format('average job overhead per runtime (%)',
report.get('job_avg_per_rt', 0)))
print('\t{:>40}: {}'.format('generated for total jobs',
report.get('analyzed_jobs', 0)))
if details and 'jobs' in report:
print('\t{:>40}'.format('individual job statistics'))
for job_name, job_overheads in report.get('jobs', {}).items():
print(
f'\t\t{job_name}: total {job_overheads.get("start", 0) + job_overheads.get("finish", 0):.4f}, '
f'start {job_overheads.get("start"):.4f}, stop {job_overheads.get("finish"):.4f}'
)
def allocation(wdir, node, core, verbose):
stats = JobsReportStats.from_workdir(wdir, verbose)
jobs = stats.allocation_jobs(node, core)
print(f'jobs launched on {node}:{core}')
for job in jobs:
print(
f'\t{job.get("name")} for {job.get("r_time")}, between {job.get("real_start")} - {job.get("real_finish")}'
)
print(f'\t\ton nodes {job.get("nodes")}')
def efficiency_core(wdir, node, core, details, verbose):
stats = JobsReportStats.from_workdir(wdir, verbose=verbose)
if not stats.has_realtime_stats():
real = False
print(
f'warning: real time log files not found - the generated chart might be not accurate'
)
stats.efficiency_core(node, core, details)
def rusage(wdir, details, wo_init, until_last_job, verbose):
stats = JobsReportStats.from_workdir(wdir, verbose=verbose)
if not stats.has_realtime_stats():
sys.stderr.write(
f'error: real time log files not found - cannot generate statistics'
)
sys.exit(1)
report = stats.resource_usage(from_first_job=wo_init,
until_last_job=until_last_job,
details=details)
if report.get('method') != 'from_service_start':
print(
'WARNING: due to not sufficient data, resource usage is measured from first job start, NOT from service start'
)
if until_last_job:
print(
'WARNING: resource usage is counted until last job on given core finished, NOT when allocation finished'
)
print('\t{:>40}: {}'.format('used cores', report.get('total_cores', 0)))
if report.get('not_used_cores', None):
print(
f'WARNING: {report.get("not_used_cores")} has not been used by any job'
)
print('\t{:>40}: {:.1f}%'.format('average USED core utilization (%)',
report.get('avg_core_utilization',
0)))
print('\t{:>40}: {:.1f}%'.format(
'average ALL cores utilization (%)',
report.get('avg_all_cores_utilization', 0)))
else:
print('\t{:>40}: {:.1f}%'.format('average core utilization (%)',
report.get('avg_core_utilization',
0)))
if details:
if report.get('nodes'):
sorted_node_list = sorted(report.get('nodes', {}).keys())
for node_name in sorted_node_list:
cores = report['nodes'][node_name]
print(f'\t{node_name}')
sorted_core_list = sorted(cores.keys(),
key=lambda c: int(c.split('&')[0])
if '&' in str(c) else int(c))
for core_id in sorted_core_list:
core_name = str(core_id)
core_spec = cores[core_name]
print(
f'\t\t{core_name:4}: {core_spec.get("utilization", 0):.1f}%, unused {core_spec.get("unused", 0):.4f} s'
)
def gantt_gaps(wdir, output, real, verbose):
stats = JobsReportStats.from_workdir(wdir, verbose=verbose)
if not stats.has_realtime_stats():
real = False
print(
f'warning: real time log files not found - the generated chart might be not accurate'
)
try:
stats.gantt_gaps(output, real)
except ImportError as exc:
sys.stderr.write(str(exc) + '\n')
sys.exit(1)
def jobs(wdir, state, node, core, sort, details, verbose):
stats = JobsReportStats.from_workdir(wdir, verbose)
def job_filter(job_data):
ok = True
if ok and state and job_data.get('state', '').lower() != state.lower():
# print(f'searched state {state} does not match job state {job_data.get("state")}')
ok = False
if ok and (node or core):
job_nodes = job_data.get('nodes', {})
job_node = None
if job_nodes is None:
ok = False
else:
if node:
job_node = job_nodes.get(node, [])
if not job_node:
ok = False
if ok and core:
if job_node:
if core not in job_node:
ok = False
else:
ok = any(core in job_node
for _, job_node in job_nodes.items())
if ok and job_data.get('r_time') is None:
ok = False
return ok
jobs = list(stats.filter_jobs(job_filter))
if sort:
if sort == 'start':
jobs.sort(key=lambda job: job['s_time'])
elif sort == 'finish':
jobs.sort(key=lambda job: job['f_time'])
elif sort == 'runtime':
jobs.sort(key=lambda job: job['r_time'])
elif sort == 'name':
jobs.sort(key=lambda job: job['name'])
elif sort == 'state':
jobs.sort(key=lambda job: job['state'])
for job in jobs:
if details:
print(job_details_desc(job))
else:
print(job_short_desc(job))
def read_logs(wdir, verbose):
jobs_report_paths = find_report_files(wdir)
proc_traces_paths = find_proc_traces_files(wdir)
if not proc_traces_paths:
sys.stderr.write(
f'error: process traces log files not found in "{wdir}"')
sys.exit(2)
if verbose:
print(f'found {len(proc_traces_paths)} process traces log files')
stats = JobsReportStats(jobs_report_paths).job_stats()
if verbose:
print(f'job report file "{jobs_report_paths}" read')
procs = ProcTraces(proc_traces_paths)
return stats, procs
def job(wdir, jobids, verbose):
stats = JobsReportStats.from_workdir(wdir, verbose)
info = stats.job_info(*jobids)
for job_name, job_data in info.items():
print(job_details_desc(job_data))
def text(wdir, verbose):
stats = JobsReportStats.from_workdir(wdir, verbose)
print_text(stats)
def stats(wdir, details, verbose):
stats = JobsReportStats.from_workdir(wdir, verbose)
print_stats(stats, details)