def postprocess(self):
"""Submit a postprocessing script after collation"""
assert self.postscript
envmod.setup()
envmod.module('load', 'pbs')
cmd = 'qsub {script}'.format(script=self.postscript)
cmd = shlex.split(cmd)
rc = sp.call(cmd)
assert rc == 0, 'Postprocessing script submission failed.'
def postprocess(self):
"""Submit a postprocessing script after collation"""
assert self.postscript
envmod.setup()
envmod.module('load', 'pbs')
cmd = 'qsub {script}'.format(script=self.postscript)
cmd = shlex.split(cmd)
rc = sp.call(cmd)
assert rc == 0, 'Postprocessing script submission failed.'
def profile(self):
# TODO: Replace with call to "profile" drivers
if self.expt.config.get('hpctoolkit', False) and self.exec_name:
envmod.module('load', 'hpctoolkit')
# Create the code structure file
hpcstruct_fname = '{0}.hpcstruct'.format(self.exec_name)
hpcstruct_path = os.path.join(self.expt.lab.bin_path,
hpcstruct_fname)
# TODO: Validate struct file
if not os.path.isfile(hpcstruct_path):
cmd = 'hpcstruct -o {0} {1}'.format(hpcstruct_path,
self.exec_path)
sp.check_call(shlex.split(cmd))
# Parse the profile output
hpctk_header = 'hpctoolkit-{0}-measurements'.format(self.exec_name)
hpctk_measure_dir = [os.path.join(self.output_path, f)
for f in os.listdir(self.output_path)
if f.startswith(hpctk_header)][0]
hpctk_db_dir = hpctk_measure_dir.replace('measurements',
'database')
# TODO: This needs to be model-specifc
src_path = os.path.join(self.codebase_path, 'src')
cmd = 'hpcprof-mpi -S {0} -I {1} -o {2} {3}'.format(
hpcstruct_path, src_path, hpctk_db_dir, hpctk_measure_dir)
sp.check_call(shlex.split(cmd))
if self.expt.config.get('scalasca', False):
envmod.module('use', '/home/900/mpc900/my_modules')
envmod.module('load', 'scalasca')
scorep_path = [os.path.join(self.output_path, f)
for f in os.listdir(self.output_path)
if f.startswith('scorep')][0]
cmd = 'scalasca -examine -s {0}'.format(scorep_path)
sp.check_call(shlex.split(cmd))
if self.expt.config.get('scorep', False):
envmod.module('load', 'scorep')
scorep_path = [os.path.join(self.output_path, f)
for f in os.listdir(self.output_path)
if f.startswith('scorep')][0]
cube_path = [os.path.join(scorep_path, f)
for f in os.listdir(scorep_path)
if f.endswith('.cubex')][0]
cmd = 'scorep-score {0}'.format(cube_path)
sp.check_call(shlex.split(cmd))
def profile(self):
# TODO: Replace with call to "profile" drivers
if self.expt.config.get('hpctoolkit', False) and self.exec_name:
envmod.module('load', 'hpctoolkit')
# Create the code structure file
hpcstruct_fname = '{}.hpcstruct'.format(self.exec_name)
hpcstruct_path = os.path.join(self.expt.lab.bin_path,
hpcstruct_fname)
# TODO: Validate struct file
if not os.path.isfile(hpcstruct_path):
cmd = 'hpcstruct -o {} {}'.format(hpcstruct_path,
self.exec_path)
sp.check_call(shlex.split(cmd))
# Parse the profile output
hpctk_header = 'hpctoolkit-{}-measurements'.format(self.exec_name)
hpctk_measure_dir = [os.path.join(self.output_path, f)
for f in os.listdir(self.output_path)
if f.startswith(hpctk_header)][0]
hpctk_db_dir = hpctk_measure_dir.replace('measurements',
'database')
# TODO: This needs to be model-specifc
src_path = os.path.join(self.codebase_path, 'src')
cmd = 'hpcprof-mpi -S {} -I {} -o {} {}'.format(
hpcstruct_path, src_path, hpctk_db_dir, hpctk_measure_dir)
sp.check_call(shlex.split(cmd))
if self.expt.config.get('scalasca', False):
envmod.module('use', '/home/900/mpc900/my_modules')
envmod.module('load', 'scalasca')
scorep_path = [os.path.join(self.output_path, f)
for f in os.listdir(self.output_path)
if f.startswith('scorep')][0]
cmd = 'scalasca -examine -s {}'.format(scorep_path)
sp.check_call(shlex.split(cmd))
if self.expt.config.get('scorep', False):
envmod.module('load', 'scorep')
scorep_path = [os.path.join(self.output_path, f)
for f in os.listdir(self.output_path)
if f.startswith('scorep')][0]
cube_path = [os.path.join(scorep_path, f)
for f in os.listdir(scorep_path)
if f.endswith('.cubex')][0]
cmd = 'scorep-score {}'.format(cube_path)
sp.check_call(shlex.split(cmd))
def repython(version, script_path):
"""Update the Python environment modules to the specified ``version`` and
replace the current process with an updated Python execution running the
script specified by ``script_path``.
"""
# Establish the environment modules
envmod.setup()
if not os.environ['MODULESHOME']:
print('payu: warning: Environment modules unavailable; aborting '
'reversion.')
return
# Ensure that payu is loaded
try:
envmod.module('use', os.environ['PAYU_MODULEPATH'])
envmod.module('load', os.environ['PAYU_MODULENAME'])
except KeyError:
pass
# NOTE: Older versions (<2.7) require the version as a tuple
version_tuple = tuple(int(i) for i in version.split('.'))
module_name = os.path.join('python', version)
python_modules = [
m for m in os.environ['LOADEDMODULES'].split(':')
if m.startswith('python')
]
if sys.version_info < version_tuple or module_name not in python_modules:
# First unload all python (and supporting) modules
python_modules = [
m for m in os.environ['LOADEDMODULES'].split(':')
if m.startswith('python')
]
for mod in python_modules:
envmod.module('unload', mod)
# Replace with specified version
envmod.module('load', module_name)
# Replace the current python process with the updated version
os.execl(script_path, *sys.argv)
def repython(version, script_path):
"""Update the Python environment modules to the specified ``version`` and
replace the current process with an updated Python execution running the
script specified by ``script_path``.
"""
# Establish the environment modules
envmod.setup()
if not os.environ['MODULESHOME']:
print('payu: warning: Environment modules unavailable; aborting '
'reversion.')
return
# Ensure that payu is loaded
try:
envmod.module('use', os.environ['PAYU_MODULEPATH'])
envmod.module('load', os.environ['PAYU_MODULENAME'])
except KeyError:
pass
# NOTE: Older versions (<2.7) require the version as a tuple
version_tuple = tuple(int(i) for i in version.split('.'))
module_name = os.path.join('python', version)
python_modules = [m for m in os.environ['LOADEDMODULES'].split(':')
if m.startswith('python')]
if sys.version_info < version_tuple or module_name not in python_modules:
# First unload all python (and supporting) modules
python_modules = [m for m in os.environ['LOADEDMODULES'].split(':')
if m.startswith('python')]
for mod in python_modules:
envmod.module('unload', mod)
# Replace with specified version
envmod.module('load', module_name)
# Replace the current python process with the updated version
os.execl(script_path, *sys.argv)
def load_modules(self):
# Scheduler
sched_modname = self.config.get('scheduler', 'pbs')
self.modules.add(sched_modname)
# MPI library
mpi_config = self.config.get('mpi', {})
mpi_modname = mpi_config.get('module', 'openmpi')
self.modules.add(mpi_modname)
# Unload non-essential modules
loaded_mods = os.environ.get('LOADEDMODULES', '').split(':')
for mod in loaded_mods:
mod_base = mod.split('/')[0]
if mod_base not in core_modules:
envmod.module('unload', mod)
# Now load model-dependent modules
for mod in self.modules:
envmod.module('load', mod)
# User-defined modules
user_modules = self.config.get('modules', [])
for mod in user_modules:
envmod.module('load', mod)
envmod.module('list')
for prof in self.profilers:
prof.load_modules()
# TODO: Consolidate this profiling stuff
c_ipm = self.config.get('ipm', False)
if c_ipm:
if isinstance(c_ipm, str):
ipm_mod = os.path.join('ipm', c_ipm)
else:
ipm_mod = 'ipm/2.0.2'
envmod.module('load', ipm_mod)
os.environ['IPM_LOGDIR'] = self.work_path
if self.config.get('mpiP', False):
envmod.module('load', 'mpiP')
if self.config.get('hpctoolkit', False):
envmod.module('load', 'hpctoolkit')
if self.debug:
envmod.module('load', 'totalview')
def load_modules(self):
# NOTE: This function is increasingly irrelevant, and may be removable.
# Scheduler
sched_modname = self.config.get('scheduler', 'pbs')
self.modules.add(sched_modname)
# MPI library
mpi_config = self.config.get('mpi', {})
# Assign MPI module paths
mpi_modpath = mpi_config.get('modulepath', None)
if mpi_modpath:
envmod.module('use', mpi_modpath)
mpi_modname = mpi_config.get('module', 'openmpi')
self.modules.add(mpi_modname)
# Unload non-essential modules
loaded_mods = os.environ.get('LOADEDMODULES', '').split(':')
for mod in loaded_mods:
mod_base = mod.split('/')[0]
if mod_base not in core_modules:
envmod.module('unload', mod)
# Now load model-dependent modules
for mod in self.modules:
envmod.module('load', mod)
# User-defined modules
user_modules = self.config.get('modules', [])
for mod in user_modules:
envmod.module('load', mod)
envmod.module('list')
for prof in self.profilers:
prof.load_modules()
# TODO: Consolidate this profiling stuff
c_ipm = self.config.get('ipm', False)
if c_ipm:
if isinstance(c_ipm, str):
ipm_mod = os.path.join('ipm', c_ipm)
else:
ipm_mod = 'ipm/2.0.2'
envmod.module('load', ipm_mod)
os.environ['IPM_LOGDIR'] = self.work_path
if self.config.get('mpiP', False):
envmod.module('load', 'mpiP')
if self.config.get('hpctoolkit', False):
envmod.module('load', 'hpctoolkit')
if self.debug:
envmod.module('load', 'totalview')
def submit_job(pbs_script, pbs_config, pbs_vars=None):
"""Submit a userscript the scheduler."""
# Initialisation
if pbs_vars is None:
pbs_vars = {}
pbs_flags = []
pbs_queue = pbs_config.get('queue', 'normal')
pbs_flags.append('-q {queue}'.format(queue=pbs_queue))
pbs_project = pbs_config.get('project', os.environ['PROJECT'])
pbs_flags.append('-P {project}'.format(project=pbs_project))
pbs_resources = ['walltime', 'ncpus', 'mem', 'jobfs']
for res_key in pbs_resources:
res_flags = []
res_val = pbs_config.get(res_key)
if res_val:
res_flags.append('{key}={val}'.format(key=res_key, val=res_val))
if res_flags:
pbs_flags.append('-l {res}'.format(res=','.join(res_flags)))
# TODO: Need to pass lab.config_path somehow...
pbs_jobname = pbs_config.get('jobname', os.path.basename(os.getcwd()))
if pbs_jobname:
# PBSPro has a 15-character jobname limit
pbs_flags.append('-N {name}'.format(name=pbs_jobname[:15]))
pbs_priority = pbs_config.get('priority')
if pbs_priority:
pbs_flags.append('-p {priority}'.format(priority=pbs_priority))
pbs_flags.append('-l wd')
pbs_join = pbs_config.get('join', 'n')
if pbs_join not in ('oe', 'eo', 'n'):
print('payu: error: unknown qsub IO stream join setting.')
sys.exit(-1)
else:
pbs_flags.append('-j {join}'.format(join=pbs_join))
# Append environment variables to qsub command
# TODO: Support full export of environment variables: `qsub -V`
pbs_vstring = ','.join('{0}={1}'.format(k, v) for k, v in pbs_vars.items())
pbs_flags.append('-v ' + pbs_vstring)
# Append any additional qsub flags here
pbs_flags_extend = pbs_config.get('qsub_flags')
if pbs_flags_extend:
pbs_flags.append(pbs_flags_extend)
if not os.path.isabs(pbs_script):
# NOTE: PAYU_PATH is always set if `set_env_vars` was always called.
# This is currently always true, but is not explicitly enforced.
# So this conditional check is a bit redundant.
payu_bin = pbs_vars.get('PAYU_PATH', os.path.dirname(sys.argv[0]))
pbs_script = os.path.join(payu_bin, pbs_script)
assert os.path.isfile(pbs_script)
# Set up environment modules here for PBS.
envmod.setup()
envmod.module('load', 'pbs')
# Construct job submission command
cmd = 'qsub {flags} -- {python} {script}'.format(flags=' '.join(pbs_flags),
python=sys.executable,
script=pbs_script)
print(cmd)
subprocess.check_call(shlex.split(cmd))
def submit_job(pbs_script, pbs_config, pbs_vars=None):
"""Submit a userscript the scheduler."""
pbs_flags = []
pbs_queue = pbs_config.get('queue', 'normal')
pbs_flags.append('-q {}'.format(pbs_queue))
pbs_project = pbs_config.get('project', os.environ['PROJECT'])
pbs_flags.append('-P {}'.format(pbs_project))
pbs_resources = ['walltime', 'ncpus', 'mem', 'jobfs']
for res_key in pbs_resources:
res_flags = []
res_val = pbs_config.get(res_key)
if res_val:
res_flags.append('{}={}'.format(res_key, res_val))
if res_flags:
pbs_flags.append('-l {}'.format(','.join(res_flags)))
pbs_jobname = pbs_config.get('jobname')
if pbs_jobname:
# PBSPro has a 15-character jobname limit
pbs_flags.append('-N {}'.format(pbs_jobname[:15]))
pbs_priority = pbs_config.get('priority')
if pbs_priority:
pbs_flags.append('-p {}'.format(pbs_priority))
pbs_flags.append('-l wd')
pbs_join = pbs_config.get('join', 'oe')
if pbs_join not in ('oe', 'eo', 'n'):
print('payu: error: unknown qsub IO stream join setting.')
sys.exit(-1)
else:
pbs_flags.append('-j {}'.format(pbs_join))
if pbs_vars:
pbs_vstring = ','.join('{}={}'.format(k, v)
for k, v in pbs_vars.iteritems())
pbs_flags.append('-v ' + pbs_vstring)
# Append any additional qsub flags here
pbs_flags_extend = pbs_config.get('qsub_flags')
if pbs_flags_extend:
pbs_flags.append(pbs_flags_extend)
# Enable PBS, in case it's not available
envmod.setup()
envmod.module('load', 'pbs')
# If script path does not exist, then check the PATH directories
if not os.path.isabs(pbs_script):
for path in os.environ['PATH'].split(':'):
if os.path.isdir(path) and pbs_script in os.listdir(path):
pbs_script = os.path.join(path, pbs_script)
break
# Construct full command
cmd = 'qsub {} {}'.format(' '.join(pbs_flags), pbs_script)
print(cmd)
subprocess.check_call(shlex.split(cmd))
def load_modules(self):
envmod.module('load', 'openspeedshop')
def submit_job(pbs_script, pbs_config, pbs_vars=None):
"""Submit a userscript the scheduler."""
# Initialisation
if pbs_vars is None:
pbs_vars = {}
pbs_flags = []
pbs_queue = pbs_config.get('queue', 'normal')
pbs_flags.append('-q {queue}'.format(queue=pbs_queue))
pbs_project = pbs_config.get('project', os.environ['PROJECT'])
pbs_flags.append('-P {project}'.format(project=pbs_project))
pbs_resources = ['walltime', 'ncpus', 'mem', 'jobfs']
for res_key in pbs_resources:
res_flags = []
res_val = pbs_config.get(res_key)
if res_val:
res_flags.append('{key}={val}'.format(key=res_key, val=res_val))
if res_flags:
pbs_flags.append('-l {res}'.format(res=','.join(res_flags)))
# TODO: Need to pass lab.config_path somehow...
pbs_jobname = pbs_config.get('jobname', os.path.basename(os.getcwd()))
if pbs_jobname:
# PBSPro has a 15-character jobname limit
pbs_flags.append('-N {name}'.format(name=pbs_jobname[:15]))
pbs_priority = pbs_config.get('priority')
if pbs_priority:
pbs_flags.append('-p {priority}'.format(priority=pbs_priority))
pbs_flags.append('-l wd')
pbs_join = pbs_config.get('join', 'n')
if pbs_join not in ('oe', 'eo', 'n'):
print('payu: error: unknown qsub IO stream join setting.')
sys.exit(-1)
else:
pbs_flags.append('-j {join}'.format(join=pbs_join))
# Append environment variables to qsub command
# TODO: Support full export of environment variables: `qsub -V`
pbs_vstring = ','.join('{0}={1}'.format(k, v)
for k, v in pbs_vars.items())
pbs_flags.append('-v ' + pbs_vstring)
# Append any additional qsub flags here
pbs_flags_extend = pbs_config.get('qsub_flags')
if pbs_flags_extend:
pbs_flags.append(pbs_flags_extend)
if not os.path.isabs(pbs_script):
# NOTE: PAYU_PATH is always set if `set_env_vars` was always called.
# This is currently always true, but is not explicitly enforced.
# So this conditional check is a bit redundant.
payu_bin = pbs_vars.get('PAYU_PATH', os.path.dirname(sys.argv[0]))
pbs_script = os.path.join(payu_bin, pbs_script)
assert os.path.isfile(pbs_script)
# Set up environment modules here for PBS.
envmod.setup()
envmod.module('load', 'pbs')
# Construct job submission command
cmd = 'qsub {flags} -- {python} {script}'.format(
flags=' '.join(pbs_flags),
python=sys.executable,
script=pbs_script
)
print(cmd)
subprocess.check_call(shlex.split(cmd))
def generate_command(pbs_script, pbs_config, pbs_vars=None, python_exe=None):
"""Prepare a correct PBS command string"""
pbs_env_init()
# Initialisation
if pbs_vars is None:
pbs_vars = {}
# Necessary for testing
if python_exe is None:
python_exe = sys.executable
pbs_flags = []
pbs_queue = pbs_config.get('queue', 'normal')
pbs_flags.append('-q {queue}'.format(queue=pbs_queue))
pbs_project = pbs_config.get('project', os.environ['PROJECT'])
pbs_flags.append('-P {project}'.format(project=pbs_project))
pbs_resources = ['walltime', 'ncpus', 'mem', 'jobfs']
for res_key in pbs_resources:
res_flags = []
res_val = pbs_config.get(res_key)
if res_val:
res_flags.append('{key}={val}'.format(key=res_key, val=res_val))
if res_flags:
pbs_flags.append('-l {res}'.format(res=','.join(res_flags)))
# TODO: Need to pass lab.config_path somehow...
pbs_jobname = pbs_config.get('jobname', os.path.basename(os.getcwd()))
if pbs_jobname:
# PBSPro has a 15-character jobname limit
pbs_flags.append('-N {name}'.format(name=pbs_jobname[:15]))
pbs_priority = pbs_config.get('priority')
if pbs_priority:
pbs_flags.append('-p {priority}'.format(priority=pbs_priority))
pbs_flags.append('-l wd')
pbs_join = pbs_config.get('join', 'n')
if pbs_join not in ('oe', 'eo', 'n'):
print('payu: error: unknown qsub IO stream join setting.')
sys.exit(-1)
else:
pbs_flags.append('-j {join}'.format(join=pbs_join))
# Append environment variables to qsub command
# TODO: Support full export of environment variables: `qsub -V`
pbs_vstring = ','.join('{0}={1}'.format(k, v)
for k, v in pbs_vars.items())
pbs_flags.append('-v ' + pbs_vstring)
storages = set()
storage_config = pbs_config.get('storage', {})
mounts = set(['/scratch', '/g/data'])
for mount in storage_config:
mounts.add(mount)
for project in storage_config[mount]:
storages.add(make_mount_string(encode_mount(mount), project))
# Append any additional qsub flags here
pbs_flags_extend = pbs_config.get('qsub_flags')
if pbs_flags_extend:
pbs_flags.append(pbs_flags_extend)
payu_path = pbs_vars.get('PAYU_PATH', os.path.dirname(sys.argv[0]))
pbs_script = check_exe_path(payu_path, pbs_script)
# Check for storage paths that might need to be mounted in the
# python and script paths
extra_search_paths = [python_exe, payu_path, pbs_script]
laboratory_path = pbs_config.get('laboratory', None)
if laboratory_path is not None:
extra_search_paths.append(laboratory_path)
short_path = pbs_config.get('shortpath', None)
if short_path is not None:
extra_search_paths.append(short_path)
storages.update(find_mounts(extra_search_paths, mounts))
storages.update(find_mounts(get_manifest_paths(), mounts))
# Add storage flags. Note that these are sorted to get predictable
# behaviour for testing
pbs_flags_extend = '+'.join(sorted(storages))
if pbs_flags_extend:
pbs_flags.append("-l storage={}".format(pbs_flags_extend))
# Set up environment modules here for PBS.
envmod.setup()
envmod.module('load', 'pbs')
# Construct job submission command
cmd = 'qsub {flags} -- {python} {script}'.format(
flags=' '.join(pbs_flags),
python=python_exe,
script=pbs_script
)
return cmd
def submit_job(pbs_script, pbs_config, pbs_vars=None):
"""Submit a userscript the scheduler."""
pbs_flags = []
pbs_queue = pbs_config.get('queue', 'normal')
pbs_flags.append('-q {}'.format(pbs_queue))
pbs_project = pbs_config.get('project', os.environ['PROJECT'])
pbs_flags.append('-P {}'.format(pbs_project))
pbs_resources = ['walltime', 'ncpus', 'mem', 'jobfs']
for res_key in pbs_resources:
res_flags = []
res_val = pbs_config.get(res_key)
if res_val:
res_flags.append('{}={}'.format(res_key, res_val))
if res_flags:
pbs_flags.append('-l {}'.format(','.join(res_flags)))
# TODO: Need to pass lab.config_path somehow...
pbs_jobname = pbs_config.get('jobname', os.path.basename(os.getcwd()))
if pbs_jobname:
# PBSPro has a 15-character jobname limit
pbs_flags.append('-N {}'.format(pbs_jobname[:15]))
pbs_priority = pbs_config.get('priority')
if pbs_priority:
pbs_flags.append('-p {}'.format(pbs_priority))
pbs_flags.append('-l wd')
pbs_join = pbs_config.get('join', 'n')
if pbs_join not in ('oe', 'eo', 'n'):
print('payu: error: unknown qsub IO stream join setting.')
sys.exit(-1)
else:
pbs_flags.append('-j {}'.format(pbs_join))
if pbs_vars:
pbs_vstring = ','.join('{}={}'.format(k, v)
for k, v in pbs_vars.iteritems())
pbs_flags.append('-v ' + pbs_vstring)
# Append any additional qsub flags here
pbs_flags_extend = pbs_config.get('qsub_flags')
if pbs_flags_extend:
pbs_flags.append(pbs_flags_extend)
# Enable PBS, in case it's not available
envmod.setup()
envmod.module('load', 'pbs')
# If script path does not exist, then check the PATH directories
if not os.path.isabs(pbs_script):
for path in os.environ['PATH'].split(':'):
if os.path.isdir(path) and pbs_script in os.listdir(path):
pbs_script = os.path.join(path, pbs_script)
break
# Construct full command
cmd = 'qsub {} {}'.format(' '.join(pbs_flags), pbs_script)
print(cmd)
subprocess.check_call(shlex.split(cmd))
def load_modules(self):
# Scheduler
sched_modname = self.config.get('scheduler', 'pbs')
self.modules.add(sched_modname)
# MPI library
mpi_config = self.config.get('mpi', {})
mpi_modname = mpi_config.get('module', 'mpprun')
self.modules.add(mpi_modname)
# Unload non-essential modules
loaded_mods = os.environ.get('LOADEDMODULES', '').split(':')
for mod in loaded_mods:
mod_base = mod.split('/')[0]
if mod_base not in core_modules:
envmod.module('unload', mod)
# Now load model-dependent modules
for mod in self.modules:
envmod.module('load', mod)
# TODO: Consolidate this profiling stuff
c_ipm = self.config.get('ipm', False)
if c_ipm:
if isinstance(c_ipm, str):
ipm_mod = os.path.join('ipm', c_ipm)
else:
ipm_mod = 'ipm'
envmod.module('load', ipm_mod)
os.environ['IPM_LOGDIR'] = self.work_path
if self.config.get('mpiP', False):
envmod.module('load', 'mpiP')
if self.config.get('hpctoolkit', False):
envmod.module('load', 'hpctoolkit')
if self.config.get('scalasca', False):
envmod.module('use', '/home/900/mpc900/my_modules')
envmod.module('load', 'scalasca')
if self.config.get('scorep', False):
envmod.module('load', 'scorep')
if self.debug:
envmod.module('load', 'totalview')
