def set_tool_sanity(self):
self.sanity_patterns = sn.all([
# check the job output:
sn.assert_found(r'Total time for iteration\(0\)', self.stdout),
sn.assert_true(sphsscorep.scorep_info_papi_support(self)),
sn.assert_true(sphsscorep.scorep_info_perf_support(self)),
sn.assert_true(sphsscorep.scorep_info_unwinding_support(self)),
# check the report:
sn.assert_eq(
sphsscorep.program_begin_count(self),
self.num_tasks,
msg='sphsscorep.program_begin_count failed: {0} != {1}'),
sn.assert_eq(
sphsscorep.program_end_count(self),
self.num_tasks,
msg='sphsscorep.program_end_count failed: {0} != {1}'),
# TODO: create derived metric (ipc) in cube
])
def set_tool_sanity(self):
self.sanity_patterns = sn.all([
# check the job output:
sn.assert_found(r'Total time for iteration\(0\)', self.stdout),
# check the tool's version and configuration:
# sn.assert_true(sphsscorep.scorep_assert_version(self)),
sn.assert_true(sphsscorep.scorep_info_papi_support(self)),
sn.assert_true(sphsscorep.scorep_info_perf_support(self)),
sn.assert_true(sphsscorep.scorep_info_unwinding_support(self)),
# check the tool report:
sn.assert_found(r'^S=C=A=N: \S+ complete\.', self.stderr),
sn.assert_eq(
sphsscorep.program_begin_count(self),
self.num_tasks,
msg='sphsscorep.program_begin_count failed: {0} != {1}'),
sn.assert_eq(
sphsscorep.program_end_count(self),
self.num_tasks,
msg='sphsscorep.program_end_count failed: {0} != {1}'),
])
def __init__(self, mpi_task):
# {{{ pe
self.descr = 'Tool validation'
self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-intel', 'PrgEnv-pgi',
'PrgEnv-cray']
# self.valid_systems = ['daint:gpu', 'dom:gpu']
self.valid_systems = ['*']
self.maintainers = ['JG']
self.tags = {'sph', 'hpctools', 'cpu'}
# }}}
# {{{ compile
self.testname = 'sqpatch'
self.tool = 'scalasca'
self.prebuild_cmds = ['module rm xalt', 'module list -t']
tool_ver = '2.5'
tc_ver = '20.08'
self.tool_modules = {
'PrgEnv-gnu': [f'Scalasca/{tool_ver}-CrayGNU-{tc_ver}'],
'PrgEnv-intel': [f'Scalasca/{tool_ver}-CrayIntel-{tc_ver}'],
'PrgEnv-cray': [f'Scalasca/{tool_ver}-CrayCCE-{tc_ver}'],
'PrgEnv-pgi': [f'Scalasca/{tool_ver}-CrayPGI-{tc_ver}'],
}
self.prgenv_flags = {
'PrgEnv-gnu': ['-I.', '-I./include', '-std=c++14', '-g', '-O3',
'-DUSE_MPI', '-DNDEBUG'],
'PrgEnv-intel': ['-I.', '-I./include', '-std=c++14', '-g', '-O3',
'-DUSE_MPI', '-DNDEBUG'],
'PrgEnv-cray': ['-I.', '-I./include', '-std=c++17', '-g', '-Ofast',
'-DUSE_MPI', '-DNDEBUG'],
'PrgEnv-pgi': ['-I.', '-I./include', '-std=c++14', '-g', '-O3',
'-DUSE_MPI', '-DNDEBUG'],
}
self.build_system = 'SingleSource'
self.build_system.cxx = 'scorep --mpp=mpi --nocompiler CC'
self.sourcepath = f'{self.testname}.cpp'
self.executable = f'./{self.testname}.exe'
# }}}
# {{{ run
ompthread = 1
self.num_tasks = mpi_task
self.cubeside = cubeside_dict[mpi_task]
self.steps = steps_dict[mpi_task]
cycles = cycles_dict[mpi_task]
self.name = \
'sphexa_scalascaS+T_{}_{:03d}mpi_{:03d}omp_{}n_{}steps_{}cycles'. \
format(self.testname, mpi_task, ompthread, self.cubeside,
self.steps, cycles)
self.num_tasks_per_node = 24
# {{{ ht:
# self.num_tasks_per_node = mpitask if mpitask < 36 else 36 # noht
# self.use_multithreading = False # noht
# self.num_tasks_per_core = 1 # noht
# self.num_tasks_per_node = mpitask if mpitask < 72 else 72
# self.use_multithreading = True # ht
# self.num_tasks_per_core = 2 # ht
# }}}
self.num_cpus_per_task = ompthread
self.num_tasks_per_core = 2
self.use_multithreading = True
self.exclusive = True
self.time_limit = '10m'
self.variables = {
'CRAYPE_LINK_TYPE': 'dynamic',
'OMP_NUM_THREADS': str(self.num_cpus_per_task),
'SCOREP_ENABLE_PROFILING': 'false',
'SCOREP_ENABLE_TRACING': 'true',
'SCOREP_ENABLE_UNWINDING': 'true',
'SCOREP_SAMPLING_EVENTS': 'perf_cycles@%s' % cycles,
# 'SCOREP_SAMPLING_EVENTS': 'PAPI_TOT_CYC@1000000',
# 'SCOREP_SAMPLING_EVENTS': 'PAPI_TOT_CYC@%s' % cycles,
# export SCOREP_SAMPLING_EVENTS=PAPI_TOT_CYC@1000000
# empty SCOREP_SAMPLING_EVENTS will profile mpi calls only:
# ok: 'SCOREP_SAMPLING_EVENTS': '',
# 'SCOREP_METRIC_PAPI': 'PAPI_TOT_INS,PAPI_TOT_CYC',
# 'SCOREP_METRIC_RUSAGE': 'ru_maxrss',
# 'SCOREP_TIMER': 'clock_gettime',
# 'SCOREP_PROFILING_MAX_CALLPATH_DEPTH': '1',
# 'SCOREP_VERBOSE': 'true',
# To avoid "No free memory page available":
'SCOREP_TOTAL_MEMORY': '1G',
# Advanced performance metrics:
'SCOREP_METRIC_RUSAGE': 'ru_maxrss',
'SCOREP_METRIC_PAPI': 'PAPI_TOT_INS,PAPI_TOT_CYC',
}
self.version_rpt = 'version.rpt'
self.which_rpt = 'which.rpt'
self.info_rpt = 'info.rpt'
self.rpt = 'rpt'
# must use scorep.score:
self.score_rpt = '%s.postproc' % self.rpt
self.stat_rpt = 'scorep_%s_%s_trace/trace.stat' % \
(self.testname, self.num_tasks)
# self.rpt_inclusive = '%s.inclusive' % self.rpt
# self.rpt_exclusive = '%s.exclusive' % self.rpt
# self.cubetool = 'cube_calltree'
self.executable_opts = [
f'-n {self.cubeside}', f'-s {self.steps}', '2>&1']
self.prerun_cmds = [
'module rm xalt',
f'{self.tool} -V &> {self.version_rpt}',
f'scorep --version >> {self.version_rpt}',
f'which {self.tool} &> {self.which_rpt}',
f'which scorep >> {self.which_rpt}',
# f'which {self.cubetool} >> {self.which_rpt}',
f'scorep-info config-summary &> {self.info_rpt}',
]
cubetree = 'cube_calltree -m time -p -t 1'
# -m metricname -- print out values for the metric <metricname>
# -i -- calculate inclusive values instead of exclusive
# -t treshold -- print out only call path with a value larger
# than <treshold>%
# -p -- diplay percent value
self.postrun_cmds = [
# can't test directly from vampir gui, dumping tracefile content:
'otf2-print scorep_*_trace/traces.otf2 > %s' % self.rpt
# 'otf2-print scorep-*/traces.otf2 > %s' % self.rpt
]
# }}}
# {{{ sanity
self.sanity_patterns = sn.all([
# check the job output:
sn.assert_found(r'Total time for iteration\(0\)', self.stdout),
# check the tool's version and configuration:
sn.assert_true(sphsscorep.scorep_version(self)),
sn.assert_true(sphsscorep.scorep_info_papi_support(self)),
sn.assert_true(sphsscorep.scorep_info_perf_support(self)),
sn.assert_true(sphsscorep.scorep_info_unwinding_support(self)),
# check the report:
sn.assert_eq(sphsscorep.program_begin_count(self), self.num_tasks),
sn.assert_eq(sphsscorep.program_end_count(self), self.num_tasks),
# check the summary report:
# sn.assert_found(r'^S=C=A=N: \S+ complete\.', self.stderr)
])
# }}}
# {{{ performance
# {{{ internal timers
self.prerun_cmds += ['echo starttime=`date +%s`']
self.postrun_cmds += ['echo stoptime=`date +%s`']
# }}}
# {{{ perf_patterns:
basic_perf_patterns = sn.evaluate(sphs.basic_perf_patterns(self))
# tool: scalasca
tool_perf_patterns = sn.evaluate(sphssca.rpt_trace_stats_d(self))
self.perf_patterns = {**basic_perf_patterns, **tool_perf_patterns}
# tool: scorep
self.perf_patterns.update({
'max_ipc_rk0': sphsscorep.ipc_rk0(self),
'max_rumaxrss_rk0': sphsscorep.ru_maxrss_rk0(self),
})
# }}}
# {{{ reference:
self.reference = sn.evaluate(sphs.basic_reference_scoped_d(self))
# tool
myzero_n = (0, None, None, 'count')
myzero_ipc = (0, None, None, 'ins/cyc')
myzero_kb = (0, None, None, 'kilobytes')
# tool
self.reference['*:mpi_latesender'] = myzero_n
self.reference['*:mpi_latesender_wo'] = myzero_n
self.reference['*:mpi_latereceiver'] = myzero_n
self.reference['*:mpi_wait_nxn'] = myzero_n
self.reference['*:max_ipc_rk0'] = myzero_ipc
self.reference['*:max_rumaxrss_rk0'] = myzero_kb
def __init__(self, mpitask, steps, cycles, rumetric):
# {{{ pe
self.descr = 'Tool validation'
self.valid_prog_environs = ['PrgEnv-pgi']
# self.valid_systems = ['daint:gpu', 'dom:gpu']
self.valid_systems = ['*']
self.maintainers = ['JG']
self.tags = {'sph', 'hpctools', 'gpu', 'openacc'}
# }}}
# {{{ compile
self.testname = 'sqpatch'
self.prebuild_cmds = ['module rm xalt']
self.prgenv_flags = {
'PrgEnv-pgi': [
'-I.', '-I./include', '-std=c++14', '-g', '-O3', '-DNDEBUG',
'-DUSE_MPI', '-DUSE_ACC', '-DUSE_STD_MATH_IN_KERNELS', '-acc',
'-ta=tesla:managed,cc60'
], # -mp
}
# ---------------------------------------------------------------- tool
self.modules = ['craype-accel-nvidia60']
tc_ver = '19.10'
tool_ver = '6.0'
postproc_tool_ver = '4ef9d3f'
postproc_tool_serial = 'otf-profiler'
self.postproc_tool = 'otf-profiler-mpi'
self.tool_modules = {
'PrgEnv-pgi': ['Score-P/%s-CrayPGI-%s' % (tool_ver, tc_ver)]
}
# ---------------------------------------------------------------- tool
self.build_system = 'SingleSource'
self.build_system.cxx = 'scorep-CC'
self.sourcepath = '%s.cpp' % self.testname
self.executable = '%s.exe' % self.testname
# {{{ openmp:
# 'PrgEnv-intel': ['-qopenmp'],
# 'PrgEnv-gnu': ['-fopenmp'],
# 'PrgEnv-pgi': ['-mp'],
# 'PrgEnv-cray_classic': ['-homp'],
# 'PrgEnv-cray': ['-fopenmp'],
# # '-homp' if lang == 'F90' else '-fopenmp',
# }}}
# }}}
# {{{ run
ompthread = 1
# weak scaling = 10^6 p/cn:
size_dict = {
1: 100,
2: 126,
4: 159,
8: 200,
16: 252,
32: 318,
64: 400,
128: 504,
256: 635
}
cubesize = size_dict[mpitask]
self.name = \
'openacc_scorepT_{}_{:03d}mpi_{:03d}omp_{}n_{}steps_{}cycles_{}'. \
format(self.testname, mpitask, ompthread, cubesize, steps, cycles,
rumetric)
self.num_tasks = mpitask
self.num_tasks_per_node = 1
# {{{ ht:
# self.num_tasks_per_node = mpitask if mpitask < 36 else 36 # noht
# self.use_multithreading = False # noht
# self.num_tasks_per_core = 1 # noht
# self.num_tasks_per_node = mpitask if mpitask < 72 else 72
# self.use_multithreading = True # ht
# self.num_tasks_per_core = 2 # ht
# }}}
self.num_cpus_per_task = ompthread
self.num_tasks_per_core = 1
self.use_multithreading = False
self.exclusive = True
self.time_limit = '10m'
self.variables = {
'CRAYPE_LINK_TYPE': 'dynamic',
'SCOREP_OPENACC_ENABLE': 'yes',
'OMP_NUM_THREADS': str(self.num_cpus_per_task),
'SCOREP_WRAPPER_INSTRUMENTER_FLAGS': '"--mpp=mpi --openacc"',
'SCOREP_ENABLE_PROFILING': 'false',
'SCOREP_ENABLE_TRACING': 'true',
'SCOREP_FILTERING_FILE': 'myfilt',
'SCOREP_VERBOSE': 'true',
# Needed to avoid "No free memory page available"
'SCOREP_TOTAL_MEMORY': '1G',
# Adding some performance metrics:
# http://scorepci.pages.jsc.fz-juelich.de/scorep-pipelines/docs/
# => scorep-6.0/html/measurement.html#rusage_counters
# => https://vampir.eu/public/files/pdf/spcheatsheet_letter.pdf
# 'SCOREP_METRIC_RUSAGE': 'ru_maxrss',
# 'SCOREP_METRIC_RUSAGE': 'ru_maxrss,ru_utime',
# 'SCOREP_METRIC_RUSAGE': 'ru_maxrss',
# 'SCOREP_METRIC_RUSAGE': '',
'SCOREP_METRIC_RUSAGE': rumetric,
'SCOREP_METRIC_PAPI': 'PAPI_TOT_INS,PAPI_TOT_CYC',
}
self.rusage_name = sn.evaluate(sphsscacc.otf2cli_metric_name(self))
if cycles > 0:
self.variables['SCOREP_SAMPLING_EVENTS'] \
= 'perf_cycles@%s' % cycles
self.version_rpt = 'version.rpt'
self.which_rpt = 'which.rpt'
self.info_rpt = 'scorep-info.rpt'
self.rpt = 'rpt'
self.rpt_jsn = 'result.json'
self.rpt_inclusive = '%s.inclusive' % self.rpt
self.rpt_exclusive = '%s.exclusive' % self.rpt
self.tool = 'scorep'
self.executable_opts = ['-n %s' % cubesize, '-s %s' % steps]
self.prerun_cmds = [
'module rm xalt',
'%s --version &> %s' % (self.tool, self.version_rpt),
'which %s &> %s' % (self.tool, self.which_rpt),
'scorep-info config-summary &> %s' % self.info_rpt,
]
self.postrun_cmds = [
# otf-profiler is needed for postprocessing but i managed to
# compile only gnu version => removing CubeLib to avoid conflict
# with CrayPGI:
'module rm CubeLib',
'module load otf2_cli_profile/%s-CrayGNU-%s' %
(postproc_tool_ver, tc_ver),
# report post-processing tools version
'%s --version' % postproc_tool_serial,
# OTF-Profiler version 2.0.0
'which %s %s' % (postproc_tool_serial, self.postproc_tool),
# create result.json performance report from tracefile
# see otf_profiler method (@run_after)
]
# }}}
# {{{ sanity
# sanity
self.sanity_patterns = sn.all([
# check the job output:
sn.assert_found(r'Total time for iteration\(0\)', self.stdout),
# check the tool's version and configuration:
sn.assert_true(sphsscorep.scorep_version(self)),
# Needed when using papi counters:
# sn.assert_true(sphsscorep.scorep_info_papi_support(self)),
sn.assert_true(sphsscorep.scorep_info_perf_support(self)),
sn.assert_true(sphsscorep.scorep_info_unwinding_support(self)),
])
# }}}
# {{{ performance
# {{{ internal timers
# use linux date as timer:
self.prerun_cmds += ['echo starttime=`date +%s`']
# }}}
# {{{ perf_patterns:
basic_perf_patterns = sn.evaluate(sphs.basic_perf_patterns(self))
tool_perf_patterns = sn.evaluate(sphsscacc.otf2cli_perf_patterns(self))
self.perf_patterns = {**basic_perf_patterns, **tool_perf_patterns}
# }}}
# {{{ reference:
self.reference = sn.evaluate(sphs.basic_reference_scoped_d(self))
self.reference = sn.evaluate(sphsscacc.otf2cli_tool_reference(self))
def __init__(self, mpi_task):
# {{{ pe
self.descr = 'Tool validation'
self.valid_prog_environs = ['PrgEnv-gnu']
# self.valid_systems = ['daint:gpu', 'dom:gpu']
self.valid_systems = ['*']
self.maintainers = ['JG']
self.tags = {'sph', 'hpctools', 'gpu'}
# }}}
# {{{ compile
self.testname = 'sqpatch'
self.tool = 'scorep'
self.prebuild_cmds = ['module rm xalt', 'module list -t']
tool_ver = '6.0'
tc_ver = '20.08'
self.tool_modules = {
'PrgEnv-gnu': [f'Score-P/{tool_ver}-CrayGNU-{tc_ver}-cuda'],
}
self.build_system = 'Make'
self.build_system.makefile = 'Makefile'
self.build_system.nvcc = 'nvcc'
self.build_system.cxx = 'CC'
self.build_system.max_concurrency = 2
self.sourcepath = f'{self.testname}.cpp'
self.executable = f'./{self.testname}.exe'
self.target_executable = 'mpi+omp+cuda'
self.build_system.cxx = 'scorep --mpp=mpi --cuda --nocompiler CC'
self.build_system.nvcc = 'scorep --cuda --nocompiler nvcc'
self.build_system.options = [
self.target_executable,
f'MPICXX="{self.build_system.cxx}"',
'SRCDIR=.',
'BUILDDIR=.',
'BINDIR=.',
'CXXFLAGS=-std=c++14',
'CUDA_PATH=$CUDATOOLKIT_HOME',
# The makefile adds -DUSE_MPI
]
self.postbuild_cmds = [
f'mv {self.target_executable}.app '
f'{self.executable}'
]
# }}}
# {{{ run
ompthread = 1
self.num_tasks = mpi_task
self.cubeside = cubeside_dict[mpi_task]
self.steps = steps_dict[mpi_task]
# cycles = cycles_dict[mpi_task]
self.name = \
'sphexa_scorep+cuda_{}_{:03d}mpi_{:03d}omp_{}n_{}steps'. \
format(self.testname, mpi_task, ompthread, self.cubeside,
self.steps)
self.num_tasks_per_node = 1
self.num_cpus_per_task = ompthread
self.num_tasks_per_core = 1
self.use_multithreading = False
self.exclusive = True
self.time_limit = '10m'
self.variables = {
'CRAYPE_LINK_TYPE': 'dynamic',
'OMP_NUM_THREADS': str(self.num_cpus_per_task),
'SCOREP_ENABLE_PROFILING': 'false',
'SCOREP_ENABLE_TRACING': 'true',
'SCOREP_CUDA_ENABLE': 'yes',
'SCOREP_ENABLE_UNWINDING': 'true',
# 'SCOREP_SAMPLING_EVENTS': 'perf_cycles@%s' % cycles,
# 'SCOREP_SAMPLING_EVENTS': 'PAPI_TOT_CYC@1000000',
# 'SCOREP_SAMPLING_EVENTS': 'PAPI_TOT_CYC@%s' % cycles,
# export SCOREP_SAMPLING_EVENTS=PAPI_TOT_CYC@1000000
# empty SCOREP_SAMPLING_EVENTS will profile mpi calls only:
# ok: 'SCOREP_SAMPLING_EVENTS': '',
# 'SCOREP_METRIC_PAPI': 'PAPI_TOT_INS,PAPI_TOT_CYC',
# 'SCOREP_METRIC_RUSAGE': 'ru_maxrss',
# 'SCOREP_TIMER': 'clock_gettime',
# 'SCOREP_PROFILING_MAX_CALLPATH_DEPTH': '1',
# 'SCOREP_VERBOSE': 'true',
# 'SCOREP_TOTAL_MEMORY': '1G',
}
self.version_rpt = 'version.rpt'
self.which_rpt = 'which.rpt'
self.info_rpt = 'info.rpt'
self.executable_opts = [
f'-n {self.cubeside}', f'-s {self.steps}', '2>&1'
]
self.prerun_cmds = [
'module rm xalt',
f'{self.tool} --version &> {self.version_rpt}',
f'which {self.tool} &> {self.which_rpt}',
'scorep-info config-summary &> %s' % self.info_rpt,
]
# }}}
# {{{ sanity
self.sanity_patterns = sn.all([
# check the job output:
sn.assert_found(r'Total time for iteration\(0\)', self.stdout),
# check the tool's version and configuration:
sn.assert_true(sphsscorep.scorep_version(self)),
sn.assert_true(sphsscorep.scorep_info_papi_support(self)),
sn.assert_true(sphsscorep.scorep_info_perf_support(self)),
sn.assert_true(sphsscorep.scorep_info_unwinding_support(self)),
sn.assert_true(sphsscorep.scorep_info_cuda_support(self)),
])
# }}}
# {{{ performance
# {{{ internal timers
self.prerun_cmds += ['echo starttime=`date +%s`']
self.postrun_cmds += ['echo stoptime=`date +%s`']
# }}}
# {{{ perf_patterns:
basic_perf_patterns = sn.evaluate(sphs.basic_perf_patterns(self))
self.perf_patterns = {**basic_perf_patterns}
# }}}
# {{{ reference:
self.reference = sn.evaluate(sphs.basic_reference_scoped_d(self))
def __init__(self):
# {{{ pe
self.descr = 'Tool validation'
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
self.modules = ['Scalasca']
self.maintainers = ['JG']
self.tags = {'sph', 'hpctools', 'cpu'}
# }}}
# {{{ compile
self.testname = 'sedov'
self.tool = 'scalasca'
self.version_rpt = 'version.rpt'
self.which_rpt = 'which.rpt'
self.info_rpt = 'info.rpt'
self.cubetool = 'cube_calltree'
self.prebuild_cmds = [
f'{self.tool} -V &> {self.version_rpt}',
f'which {self.tool} &> {self.which_rpt}',
f'scorep --version >> {self.version_rpt}',
f'which scorep >> {self.which_rpt}',
# f'which vampir >> {self.which_rpt}',
f'which {self.cubetool} >> {self.which_rpt}',
f'scorep-info config-summary &> {self.info_rpt}',
f'# step1: prepare executable with: scalasca -instrument (skin)',
f'# step2: run executable with: scalasca -analyze (scan)',
f'# step3: explore report with: scalasca -examine (square)',
f'# step4: get calltree with: cube_calltree'
]
# }}}
# {{{ run
self.variables = {
'SCOREP_ENABLE_UNWINDING': 'true',
'SCOREP_SAMPLING_EVENTS': f'perf_cycles@{self.cycles}',
'SCOREP_PROFILING_MAX_CALLPATH_DEPTH': '10',
'SCOREP_TOTAL_MEMORY': '1G',
}
self.rpt = 'rpt'
self.rpt_score = 'scorep-score.rpt'
self.rpt_exclusive = 'cube_calltree_exclusive.rpt'
self.rpt_inclusive = 'cube_calltree_inclusive.rpt'
#
cubetree = 'cube_calltree -m time -p -t 1'
# -m metricname -- print out values for the metric <metricname>
# -i -- calculate inclusive values instead of exclusive
# -t treshold -- print out only call path with a value larger
# than <treshold>%
# -p -- diplay percent value
self.postrun_cmds += [
f'# -------------------------------------------------------------',
# generate summary.cubex from profile.cubex with: scalasca -examine
# (it will report scoring too)
f'{self.tool} -examine -s scorep_*sum/profile.cubex &> {self.rpt}',
# rpt will always be written to scorep.score, not into self.rpt
f'rm -f core*',
# this file is used for sanity checks:
f'cp scorep_*_sum/scorep.score {self.rpt_score}',
# exclusive time:
f'({cubetree} scorep_*_sum/summary.cubex ;rm -f core*) &>'
f' {self.rpt_exclusive}',
# inclusive time:
f'({cubetree} -i scorep_*_sum/summary.cubex ;rm -f core*) &>'
f' {self.rpt_inclusive}',
f'# -------------------------------------------------------------',
]
# }}}
# {{{ sanity
self.sanity_patterns = sn.all([
# check the job output:
sn.assert_found(r'Total time for iteration\(0\)', self.stdout),
# check the tool's version and configuration:
# sn.assert_true(sphsscorep.scorep_assert_version(self)),
sn.assert_true(sphsscorep.scorep_info_papi_support(self)),
sn.assert_true(sphsscorep.scorep_info_perf_support(self)),
sn.assert_true(sphsscorep.scorep_info_unwinding_support(self)),
# check the tool report:
sn.assert_found(r'Estimated aggregate size of event trace',
self.rpt_score),
sn.assert_found(r'^S=C=A=N: \S+ complete\.', self.stderr)
])
def __init__(self):
# {{{ pe
self.descr = 'Tool validation'
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
self.modules = ['Scalasca']
self.maintainers = ['JG']
self.tags = {'sph', 'hpctools', 'cpu'}
# }}}
# {{{ compile
self.testname = 'sedov'
self.executable = 'mpi+omp'
self.tool = 'scalasca'
self.version_rpt = 'version.rpt'
self.which_rpt = 'which.rpt'
self.info_rpt = 'info.rpt'
self.cubetool = 'cube_calltree'
self.prebuild_cmds = [
f'{self.tool} -V &> {self.version_rpt}',
f'which {self.tool} &> {self.which_rpt}',
f'scorep --version >> {self.version_rpt}',
f'which scorep >> {self.which_rpt}',
f'which cube_remap2 >> {self.which_rpt}',
f'which cube_dump >> {self.which_rpt}',
f'which {self.cubetool} >> {self.which_rpt}',
# f'which vampir >> {self.which_rpt}',
f'scorep-info config-summary &> {self.info_rpt}',
f'# step1: prepare executable with: scalasca -instrument (skin)',
f'# step2: run executable with: scalasca -analyze (scan)',
f'# step3: explore report with: scalasca -examine (square)',
f'# step4: get calltree with: cube_calltree'
]
# }}}
# {{{ run
self.variables = {
# 'SCOREP_ENABLE_UNWINDING': 'true',
# 'SCOREP_SAMPLING_EVENTS': f'perf_cycles@{self.cycles}',
# 'SCOREP_PROFILING_MAX_CALLPATH_DEPTH': '10',
'SCOREP_TOTAL_MEMORY': '1G',
# 'SCOREP_TIMER': 'gettimeofday',
'SCAN_ANALYZE_OPTS': '--time-correct',
}
self.rpt = 'rpt'
self.rpt_score = 'scorep-score.rpt'
self.rpt_exclusive = 'cube_calltree_exclusive.rpt'
self.rpt_inclusive = 'cube_calltree_inclusive.rpt'
#
cubetree = 'cube_calltree -m time -p -t 1'
self.postrun_cmds += [
'# {{{ --- Postprocessing steps: ---',
f'# -------------------------------------------------------------',
'# profile.cubex - scalasca -examine -s = square -> scorep.score:',
f'# ({self.rpt_score} is used by sanity checks)',
f'scalasca -examine -s scorep_*sum/profile.cubex &> {self.rpt}',
f'cp scorep_*_sum/scorep.score {self.rpt_score}',
'# --------------------------------------------------------------',
'# transform metric tree into metric hierarchy with remap2',
'# profile.cubex - cube_remap2 (slow) -> summary.cubex: ',
f'# time -p cube_remap2 -d -o summary.cubex */profile.cubex',
f'# scorep-score summary.cubex &> {self.rpt_score}',
'# --------------------------------------------------------------',
'# exclusive time: summary.cubex - cubetree -> rpt_exclusive:',
f'# ({cubetree} scorep_*_sum/summary.cubex ;rm -f core*) &>'
f' {self.rpt_exclusive}',
'# --------------------------------------------------------------',
'# inclusive time: summary.cubex - cubetree -i -> rpt_inclusive:',
f'# ({cubetree} -i scorep_*_sum/summary.cubex ;rm -f core*) &>'
f' {self.rpt_inclusive}',
'# -m metricname -- print out values for the metric <metricname>',
'# -i -- calculate inclusive values instead of',
'# exclusive',
'# -t treshold -- print out only call path with a value larger',
'# than <treshold>%',
'# -p -- diplay percent value',
'# --------------------------------------------------------- }}}',
]
# }}}
# {{{ sanity
self.sanity_patterns = sn.all([
# check the job output:
sn.assert_found(r'Total time for iteration\(0\)', self.stdout),
# check the tool's version and configuration:
# sn.assert_true(sphsscorep.scorep_assert_version(self)),
sn.assert_true(sphsscorep.scorep_info_papi_support(self)),
sn.assert_true(sphsscorep.scorep_info_perf_support(self)),
sn.assert_true(sphsscorep.scorep_info_unwinding_support(self)),
# check the tool report:
sn.assert_found(r'Estimated aggregate size of event trace',
self.rpt_score),
sn.assert_found(r'^S=C=A=N: \S+ complete\.', self.stderr)
])
def __init__(self):
# {{{ pe
self.descr = 'Tool validation'
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
self.modules = ['Score-P']
self.maintainers = ['JG']
self.tags = {'sph', 'hpctools', 'cpu'}
# }}}
# {{{ compile
self.testname = 'sedov'
self.tool = 'scorep'
self.version_rpt = 'version.rpt'
self.which_rpt = 'which.rpt'
self.info_rpt = 'info.rpt'
self.prebuild_cmds = [
f'{self.tool} --version &> {self.version_rpt}',
f'which {self.tool} &> {self.which_rpt}',
f'which vampir >> {self.which_rpt}',
f'scorep-info config-summary &> {self.info_rpt}',
]
# }}}
# {{{ run
self.variables = {
# 'CRAYPE_LINK_TYPE': 'dynamic',
# 'OMP_NUM_THREADS': str(self.num_cpus_per_task),
'SCOREP_ENABLE_PROFILING': 'true',
'SCOREP_ENABLE_TRACING': 'false',
'SCOREP_ENABLE_UNWINDING': 'true',
'SCOREP_SAMPLING_EVENTS': f'perf_cycles@{self.cycles}',
# 'SCOREP_SAMPLING_EVENTS': 'PAPI_TOT_CYC@1000000',
# 'SCOREP_SAMPLING_EVENTS': 'PAPI_TOT_CYC@%s' % cycles,
# export SCOREP_SAMPLING_EVENTS=PAPI_TOT_CYC@1000000
# empty SCOREP_SAMPLING_EVENTS will profile mpi calls only:
# ok: 'SCOREP_SAMPLING_EVENTS': '',
# 'SCOREP_METRIC_PAPI': 'PAPI_TOT_INS,PAPI_TOT_CYC',
# 'SCOREP_METRIC_RUSAGE': 'ru_maxrss',
# 'SCOREP_TIMER': 'clock_gettime',
# 'SCOREP_VERBOSE': 'true',
'SCOREP_PROFILING_MAX_CALLPATH_DEPTH': '10',
'SCOREP_TOTAL_MEMORY': '1G',
# 'PATH': f'{tool_path}:{cube_path}:$PATH',
}
self.rpt = 'rpt'
self.rpt_score = 'scorep-score.rpt'
self.rpt_inclusive = 'cube_calltree_inclusive.rpt'
self.rpt_exclusive = 'cube_calltree_exclusive.rpt'
self.rpt_otf2 = 'otf2-print.rpt'
#
cubetree = 'cube_calltree -m time -p -t 1'
# -m metricname -- print out values for the metric <metricname>
# -i -- calculate inclusive values instead of exclusive
# -t treshold -- print out only call path with a value larger
# than <treshold>%
# -p -- diplay percent value
self.postrun_cmds += [
f'# -------------------------------------------------------------',
# working around memory crash in scorep-score:
f'(scorep-score -r scorep-*/profile.cubex ;rm -f core*) >'
f'{self.rpt_score}',
# exclusive time (+ workaround memory crash):
f'({cubetree} scorep-*/profile.cubex ;rm -f core*) &>'
f' {self.rpt_exclusive}',
# inclusive time (+ workaround memory crash):
f'({cubetree} -i scorep-*/profile.cubex ;rm -f core*) &>'
f' {self.rpt_inclusive}',
f'# -------------------------------------------------------------',
]
# }}}
# {{{ sanity
self.sanity_patterns = sn.all([
# check the job output:
sn.assert_found(r'Total time for iteration\(0\)', self.stdout),
# check the tool's version and configuration:
# sn.assert_true(sphsscorep.scorep_assert_version(self)),
sn.assert_true(sphsscorep.scorep_info_papi_support(self)),
sn.assert_true(sphsscorep.scorep_info_perf_support(self)),
sn.assert_true(sphsscorep.scorep_info_unwinding_support(self)),
# check the summary report:
sn.assert_found(r'Estimated aggregate size of event trace',
self.rpt_score)
])
def __init__(self, mpi_task):
# {{{ pe
self.descr = 'Tool validation'
self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-intel', 'PrgEnv-pgi',
'PrgEnv-cray']
# self.valid_systems = ['daint:gpu', 'dom:gpu']
self.valid_systems = ['*']
self.maintainers = ['JG']
self.tags = {'sph', 'hpctools', 'cpu'}
# }}}
# {{{ compile
self.testname = 'sqpatch'
self.tool = 'scorep'
self.prebuild_cmds = ['module rm xalt', 'module list -t']
tool_ver = '6.0'
tc_ver = '20.08'
self.tool_modules = {
'PrgEnv-gnu': [f'Score-P/{tool_ver}-CrayGNU-{tc_ver}'],
'PrgEnv-intel': [f'Score-P/{tool_ver}-CrayIntel-{tc_ver}'],
'PrgEnv-cray': [f'Score-P/{tool_ver}-CrayCCE-{tc_ver}'],
'PrgEnv-pgi': [f'Score-P/{tool_ver}-CrayPGI-{tc_ver}'],
}
self.prgenv_flags = {
'PrgEnv-gnu': ['-I.', '-I./include', '-std=c++14', '-g', '-O3',
'-DUSE_MPI', '-DNDEBUG'],
'PrgEnv-intel': ['-I.', '-I./include', '-std=c++14', '-g', '-O3',
'-DUSE_MPI', '-DNDEBUG'],
'PrgEnv-cray': ['-I.', '-I./include', '-std=c++17', '-g', '-Ofast',
'-DUSE_MPI', '-DNDEBUG'],
'PrgEnv-pgi': ['-I.', '-I./include', '-std=c++14', '-g', '-O3',
'-DUSE_MPI', '-DNDEBUG'],
}
self.build_system = 'SingleSource'
self.build_system.cxx = 'scorep --mpp=mpi --nocompiler CC'
self.sourcepath = f'{self.testname}.cpp'
self.executable = f'./{self.testname}.exe'
# }}}
# {{{ run
ompthread = 1
self.num_tasks = mpi_task
self.cubeside = cubeside_dict[mpi_task]
self.steps = steps_dict[mpi_task]
cycles = cycles_dict[mpi_task]
self.name = \
'sphexa_scorepS+P_{}_{:03d}mpi_{:03d}omp_{}n_{}steps_{}cycles'. \
format(self.testname, mpi_task, ompthread, self.cubeside,
self.steps, cycles)
self.num_tasks_per_node = 24
# {{{ ht:
# self.num_tasks_per_node = mpitask if mpitask < 36 else 36 # noht
# self.use_multithreading = False # noht
# self.num_tasks_per_core = 1 # noht
# self.num_tasks_per_node = mpitask if mpitask < 72 else 72
# self.use_multithreading = True # ht
# self.num_tasks_per_core = 2 # ht
# }}}
self.num_cpus_per_task = ompthread
self.num_tasks_per_core = 2
self.use_multithreading = True
self.exclusive = True
self.time_limit = '10m'
self.variables = {
'CRAYPE_LINK_TYPE': 'dynamic',
'OMP_NUM_THREADS': str(self.num_cpus_per_task),
'SCOREP_ENABLE_PROFILING': 'true',
'SCOREP_ENABLE_TRACING': 'false',
'SCOREP_ENABLE_UNWINDING': 'true',
'SCOREP_SAMPLING_EVENTS': 'perf_cycles@%s' % cycles,
# 'SCOREP_SAMPLING_EVENTS': 'PAPI_TOT_CYC@1000000',
# 'SCOREP_SAMPLING_EVENTS': 'PAPI_TOT_CYC@%s' % cycles,
# export SCOREP_SAMPLING_EVENTS=PAPI_TOT_CYC@1000000
# empty SCOREP_SAMPLING_EVENTS will profile mpi calls only:
# ok: 'SCOREP_SAMPLING_EVENTS': '',
# 'SCOREP_METRIC_PAPI': 'PAPI_TOT_INS,PAPI_TOT_CYC',
# 'SCOREP_METRIC_RUSAGE': 'ru_maxrss',
# 'SCOREP_TIMER': 'clock_gettime',
# 'SCOREP_PROFILING_MAX_CALLPATH_DEPTH': '1',
# 'SCOREP_VERBOSE': 'true',
# 'SCOREP_TOTAL_MEMORY': '1G',
}
self.version_rpt = 'version.rpt'
self.which_rpt = 'which.rpt'
self.info_rpt = 'info.rpt'
self.rpt = 'rpt'
self.rpt_inclusive = '%s.inclusive' % self.rpt
self.rpt_exclusive = '%s.exclusive' % self.rpt
self.executable_opts = [
f'-n {self.cubeside}', f'-s {self.steps}', '2>&1']
self.prerun_cmds = [
'module rm xalt',
f'{self.tool} --version &> {self.version_rpt}',
f'which {self.tool} &> {self.which_rpt}',
'scorep-info config-summary &> %s' % self.info_rpt,
]
cubetree = 'cube_calltree -m time -p -t 1'
# -m metricname -- print out values for the metric <metricname>
# -i -- calculate inclusive values instead of exclusive
# -t treshold -- print out only call path with a value larger
# than <treshold>%
# -p -- diplay percent value
self.postrun_cmds = [
# working around memory crash in scorep-score:
'(scorep-score -r scorep-*/profile.cubex ;rm -f core*) > %s' \
% self.rpt,
'(%s scorep-*/profile.cubex ;rm -f core*) >> %s' \
% (cubetree, self.rpt_exclusive),
'(%s -i scorep-*/profile.cubex ;rm -f core*) >> %s' \
% (cubetree, self.rpt_inclusive),
]
# }}}
# {{{ sanity
self.sanity_patterns = sn.all([
# check the job output:
sn.assert_found(r'Total time for iteration\(0\)', self.stdout),
# check the tool's version and configuration:
sn.assert_true(sphsscorep.scorep_version(self)),
sn.assert_true(sphsscorep.scorep_info_papi_support(self)),
sn.assert_true(sphsscorep.scorep_info_perf_support(self)),
sn.assert_true(sphsscorep.scorep_info_unwinding_support(self)),
# check the summary report:
sn.assert_found(r'Estimated aggregate size of event trace',
self.rpt)
])
# }}}
# {{{ performance
# {{{ internal timers
self.prerun_cmds += ['echo starttime=`date +%s`']
self.postrun_cmds += ['echo stoptime=`date +%s`']
# }}}
# {{{ perf_patterns:
self.perf_patterns = sn.evaluate(sphs.basic_perf_patterns(self))
# tool
self.perf_patterns.update({
'scorep_elapsed': sphsscorep.scorep_elapsed(self),
'%scorep_USR': sphsscorep.scorep_usr_pct(self),
'%scorep_MPI': sphsscorep.scorep_mpi_pct(self),
'scorep_top1': sphsscorep.scorep_top1_pct(self),
'%scorep_Energy_exclusive':
sphsscorep.scorep_exclusivepct_energy(self),
'%scorep_Energy_inclusive':
sphsscorep.scorep_inclusivepct_energy(self),
})
# }}}
# {{{ reference:
self.reference = sn.evaluate(sphs.basic_reference_scoped_d(self))
# tool:
self.reference['*:scorep_elapsed'] = (0, None, None, 's')
self.reference['*:%scorep_USR'] = (0, None, None, '%')
self.reference['*:%scorep_MPI'] = (0, None, None, '%')
top1_name = sphsscorep.scorep_top1_name(self)
# TODO: self.reference['*:scorep_top1'] = (0, None, None, top1_name)
self.reference['*:scorep_top1'] = (0, None, None, '')
self.reference['*:%scorep_Energy_exclusive'] = (0, None, None, '%')
self.reference['*:%scorep_Energy_inclusive'] = (0, None, None, '%')
def __init__(self, mpi_task):
# {{{ pe
self.descr = 'Tool validation'
self.valid_prog_environs = [
'PrgEnv-gnu', 'PrgEnv-intel', 'PrgEnv-pgi', 'PrgEnv-cray'
]
# self.valid_systems = ['daint:gpu', 'dom:gpu']
self.valid_systems = ['*']
self.maintainers = ['JG']
self.tags = {'sph', 'hpctools', 'cpu'}
# }}}
# {{{ compile
self.testname = 'sqpatch'
self.tool = 'scorep'
self.prebuild_cmds = ['module rm xalt', 'module list -t']
tool_ver = '6.0'
tc_ver = '20.08'
self.tool_modules = {
'PrgEnv-gnu': [f'Score-P/{tool_ver}-CrayGNU-{tc_ver}'],
'PrgEnv-intel': [f'Score-P/{tool_ver}-CrayIntel-{tc_ver}'],
'PrgEnv-cray': [f'Score-P/{tool_ver}-CrayCCE-{tc_ver}'],
'PrgEnv-pgi': [f'Score-P/{tool_ver}-CrayPGI-{tc_ver}'],
}
self.prgenv_flags = {
'PrgEnv-gnu': [
'-I.', '-I./include', '-std=c++14', '-g', '-O3', '-DUSE_MPI',
'-DNDEBUG'
],
'PrgEnv-intel': [
'-I.', '-I./include', '-std=c++14', '-g', '-O3', '-DUSE_MPI',
'-DNDEBUG'
],
'PrgEnv-cray': [
'-I.', '-I./include', '-std=c++17', '-g', '-Ofast',
'-DUSE_MPI', '-DNDEBUG'
],
'PrgEnv-pgi': [
'-I.', '-I./include', '-std=c++14', '-g', '-O3', '-DUSE_MPI',
'-DNDEBUG'
],
}
self.build_system = 'SingleSource'
self.build_system.cxx = 'scorep --mpp=mpi --nocompiler CC'
self.sourcepath = f'{self.testname}.cpp'
self.executable = f'./{self.testname}.exe'
# }}}
# {{{ run
ompthread = 1
self.num_tasks = mpi_task
self.cubeside = cubeside_dict[mpi_task]
self.steps = steps_dict[mpi_task]
cycles = cycles_dict[mpi_task]
self.name = \
'sphexa_scorepS+T_{}_{:03d}mpi_{:03d}omp_{}n_{}steps_{}cycles'. \
format(self.testname, mpi_task, ompthread, self.cubeside,
self.steps, cycles)
self.num_tasks_per_node = 24
# {{{ ht:
# self.num_tasks_per_node = mpitask if mpitask < 36 else 36 # noht
# self.use_multithreading = False # noht
# self.num_tasks_per_core = 1 # noht
# self.num_tasks_per_node = mpitask if mpitask < 72 else 72
# self.use_multithreading = True # ht
# self.num_tasks_per_core = 2 # ht
# }}}
self.num_cpus_per_task = ompthread
self.num_tasks_per_core = 2
self.use_multithreading = True
self.exclusive = True
self.time_limit = '10m'
self.variables = {
'CRAYPE_LINK_TYPE': 'dynamic',
'OMP_NUM_THREADS': str(self.num_cpus_per_task),
'SCOREP_ENABLE_PROFILING': 'false',
'SCOREP_ENABLE_TRACING': 'true',
'SCOREP_ENABLE_UNWINDING': 'true',
'SCOREP_SAMPLING_EVENTS': 'perf_cycles@%s' % cycles,
# 'SCOREP_SAMPLING_EVENTS': 'PAPI_TOT_CYC@1000000',
# 'SCOREP_SAMPLING_EVENTS': 'PAPI_TOT_CYC@%s' % cycles,
# export SCOREP_SAMPLING_EVENTS=PAPI_TOT_CYC@1000000
# empty SCOREP_SAMPLING_EVENTS will profile mpi calls only:
# ok: 'SCOREP_SAMPLING_EVENTS': '',
# 'SCOREP_METRIC_PAPI': 'PAPI_TOT_INS,PAPI_TOT_CYC',
# 'SCOREP_METRIC_RUSAGE': 'ru_maxrss',
# 'SCOREP_TIMER': 'clock_gettime',
# 'SCOREP_PROFILING_MAX_CALLPATH_DEPTH': '1',
# 'SCOREP_VERBOSE': 'true',
# ---
# Needed to avoid "No free memory page available"
# (SCOREP_TOTAL_MEMORY=16384000 bytes)
'SCOREP_TOTAL_MEMORY': '1G',
# ---
# adding some metrics to test my sanity_functions:
'SCOREP_METRIC_RUSAGE': 'ru_maxrss',
'SCOREP_METRIC_PAPI': 'PAPI_TOT_INS,PAPI_TOT_CYC',
}
self.version_rpt = 'version.rpt'
self.which_rpt = 'which.rpt'
self.info_rpt = 'scorep-info.rpt'
self.rpt = 'rpt'
self.rpt_inclusive = '%s.inclusive' % self.rpt
self.rpt_exclusive = '%s.exclusive' % self.rpt
self.executable_opts = [
f'-n {self.cubeside}', f'-s {self.steps}', '2>&1'
]
self.prerun_cmds = [
'module rm xalt',
f'{self.tool} --version &> {self.version_rpt}',
f'which {self.tool} &> {self.which_rpt}',
'scorep-info config-summary &> %s' % self.info_rpt,
]
self.postrun_cmds = [
# can't test directly from vampir gui:
'otf2-print scorep-*/traces.otf2 > %s' % self.rpt
]
# }}}
# {{{ sanity
self.sanity_patterns = sn.all([
# check the job output:
sn.assert_found(r'Total time for iteration\(0\)', self.stdout),
# check the tool's version and configuration:
sn.assert_true(sphsscorep.scorep_version(self)),
sn.assert_true(sphsscorep.scorep_info_papi_support(self)),
sn.assert_true(sphsscorep.scorep_info_perf_support(self)),
sn.assert_true(sphsscorep.scorep_info_unwinding_support(self)),
# check the report:
sn.assert_eq(sphsscorep.program_begin_count(self), self.num_tasks),
sn.assert_eq(sphsscorep.program_end_count(self), self.num_tasks),
# TODO: create derived metric (ipc) in cube
])
# }}}
# {{{ performance
# {{{ internal timers
self.prerun_cmds += ['echo starttime=`date +%s`']
self.postrun_cmds += ['echo stoptime=`date +%s`']
# }}}
# {{{ perf_patterns:
self.perf_patterns = sn.evaluate(sphs.basic_perf_patterns(self))
# tool
self.perf_patterns.update({
'max_ipc_rk0':
sphsscorep.ipc_rk0(self),
'max_rumaxrss_rk0':
sphsscorep.ru_maxrss_rk0(self),
})
# }}}
# {{{ reference:
self.reference = sn.evaluate(sphs.basic_reference_scoped_d(self))
# tool:
self.reference['*:max_ipc_rk0'] = (0, None, None, 'ins/cyc')
self.reference['*:max_rumaxrss_rk0'] = (0, None, None, 'kilobytes')