def __init__(self, model_type=None, config_path=None, lab_path=None):
"""Create the Payu laboratory interface."""
config = read_config(config_path)
# Set the file permission mask
perms = config.get('umask', 0o0027)
os.umask(perms)
# Set model type
if not model_type:
model_type = config.get('model')
if not model_type:
raise ValueError('Cannot determine model type.')
self.model_type = model_type
# Set top-level lab path if provided
if 'PAYU_LAB_PATH' in os.environ:
self.basepath = os.environ.get('PAYU_LAB_PATH')
else:
self.basepath = lab_path
# If no lab path is set, generate a default path
if not self.basepath:
self.basepath = self.get_default_lab_path(config)
# Set subdirectory paths
self.archive_path = os.path.join(self.basepath, 'archive')
self.bin_path = os.path.join(self.basepath, 'bin')
self.codebase_path = os.path.join(self.basepath, 'codebase')
self.input_basepath = os.path.join(self.basepath, 'input')
self.work_path = os.path.join(self.basepath, 'work')
def __init__(self, lab):
self.lab = lab
# TODO: replace with dict, check versions via key-value pairs
self.modules = set()
# TODO: __init__ should not be a config dumping ground!
self.config = read_config()
# Payu experiment type
self.debug = self.config.get('debug', False)
self.postscript = self.config.get('postscript')
self.repeat_run = self.config.get('repeat', False)
# Configuration
self.expand_shell_vars = True # TODO: configurable
# Model run time
self.runtime = None
if ('calendar' in self.config and
'runtime' in self.config['calendar']):
self.runtime = self.config['calendar']['runtime']
# Stacksize
# NOTE: Possible PBS issue in setting non-unlimited stacksizes
stacksize = self.config.get('stacksize', 'unlimited')
self.set_stacksize(stacksize)
# Initialize the submodels
self.init_models()
# TODO: Move to run/collate/sweep?
self.set_expt_pathnames()
self.set_counters()
for model in self.models:
model.set_input_paths()
self.set_output_paths()
# Miscellaneous configurations
# TODO: Move this stuff somewhere else
self.userscripts = self.config.get('userscripts', {})
self.profilers = []
init_script = self.userscripts.get('init')
if init_script:
self.run_userscript(init_script)
self.runlog = Runlog(self)
# XXX: Temporary spot for the payu path
# This is horrible; payu/cli.py does this much more safely!
# But also does not even store it in os.environ!
default_payu_bin = os.path.dirname(sys.argv[0])
payu_bin = os.environ.get('PAYU_PATH', default_payu_bin)
self.payu_path = os.path.join(payu_bin, 'payu')
def __init__(self, lab):
self.lab = lab
# TODO: replace with dict, check versions via key-value pairs
self.modules = set()
# TODO: __init__ should not be a config dumping ground!
self.config = read_config()
# Payu experiment type
self.debug = self.config.get('debug', False)
self.postscript = self.config.get('postscript')
self.repeat_run = self.config.get('repeat', False)
# Configuration
self.expand_shell_vars = True # TODO: configurable
# Model run time
self.runtime = None
if ('calendar' in self.config
and 'runtime' in self.config['calendar']):
self.runtime = self.config['calendar']['runtime']
# Stacksize
# NOTE: Possible PBS issue in setting non-unlimited stacksizes
stacksize = self.config.get('stacksize', 'unlimited')
self.set_stacksize(stacksize)
# Initialize the submodels
self.init_models()
# TODO: Move to run/collate/sweep?
self.set_expt_pathnames()
self.set_counters()
for model in self.models:
model.set_input_paths()
self.set_output_paths()
# Miscellaneous configurations
# TODO: Move this stuff somewhere else
self.userscripts = self.config.get('userscripts', {})
self.profilers = []
init_script = self.userscripts.get('init')
if init_script:
self.run_userscript(init_script)
# Logging
if self.config.get('runlog', True):
self.runlog = Runlog(self)
else:
self.runlog = None
def __init__(self, lab):
self.lab = lab
# TODO: replace with dict, check versions via key-value pairs
self.modules = set()
# TODO: __init__ should not be a config dumping ground!
self.config = read_config()
# Payu experiment type
self.debug = self.config.get('debug', False)
self.postscript = self.config.get('postscript')
self.repeat_run = self.config.get('repeat', False)
# Model run time
self.runtime = None
if ('calendar' in self.config and
'runtime' in self.config['calendar']):
self.runtime = self.config['calendar']['runtime']
# Stacksize
# NOTE: Possible PBS issue in setting non-unlimited stacksizes
stacksize = self.config.get('stacksize', 'unlimited')
self.set_stacksize(stacksize)
# Initialize the submodels
self.init_models()
# TODO: Move to run/collate/sweep?
self.set_expt_pathnames()
self.set_counters()
for model in self.models:
model.set_input_paths()
self.set_output_paths()
# Miscellaneous configurations
# TODO: Move this stuff somewhere else
self.userscripts = self.config.get('userscripts', {})
self.profilers = []
init_script = self.userscripts.get('init')
if init_script:
self.run_userscript(init_script)
# Logging
if self.config.get('runlog', True):
self.runlog = Runlog(self)
else:
self.runlog = None
def __init__(self, model_type=None, config_path=None, lab_path=None):
"""Create the Payu laboratory interface."""
config = read_config(config_path)
# Set the file permission mask
perms = config.get('umask', 0o0027)
os.umask(perms)
# Set model type
if not model_type:
model_type = config.get('model')
if not model_type:
raise ValueError('Cannot determine model type.')
self.model_type = model_type
# Set top-level lab path if provided
if 'PAYU_LAB_PATH' in os.environ:
self.basepath = os.environ.get('PAYU_LAB_PATH')
else:
self.basepath = lab_path
# Support multiple bases for default short/scratch
# locations. Fall back to control directory if others
# don't exist
for path in ['/short', '/scratch', '.']:
if os.path.exists(path):
self.base = path
break
# If no lab path is set, generate a default path
if not self.basepath:
self.basepath = self.get_default_lab_path(config)
# Set subdirectory paths
self.archive_path = os.path.join(self.basepath, 'archive')
self.bin_path = os.path.join(self.basepath, 'bin')
self.codebase_path = os.path.join(self.basepath, 'codebase')
self.input_basepath = os.path.join(self.basepath, 'input')
self.work_path = os.path.join(self.basepath, 'work')
print("laboratory path: ", self.basepath)
print("binary path: ", self.bin_path)
print("input path: ", self.input_basepath)
print("work path: ", self.work_path)
print("archive path: ", self.archive_path)
def runcmd(model_type, config_path, init_run, n_runs, lab_path):
pbs_config = fsops.read_config(config_path)
pbs_vars = cli.set_env_vars(init_run=init_run,
n_runs=n_runs,
lab_path=lab_path)
pbs_config['queue'] = pbs_config.get('profile_queue', 'normal')
# Collation jobs are (currently) serial
pbs_config['ncpus'] = 1
# Modify jobname
pbs_config['jobname'] = pbs_config['jobname'][:13] + '_p'
# Replace (or remove) walltime
profile_walltime = pbs_config.get('profile_walltime')
if profile_walltime:
pbs_config['walltime'] = profile_walltime
else:
# Remove the model walltime if set
try:
pbs_config.pop('walltime')
except KeyError:
pass
# Replace (or remove) memory request
profile_mem = pbs_config.get('profile_mem')
if profile_mem:
pbs_config['mem'] = profile_mem
else:
# Remove the model memory request if set
try:
pbs_config.pop('mem')
except KeyError:
pass
# Disable hyperthreading
qsub_flags = []
for flag in pbs_config.get('qsub_flags', '').split():
if 'hyperthread' not in flag:
qsub_flags.append(flag)
pbs_config['qsub_flags'] = ' '.join(qsub_flags)
cli.submit_job('payu-profile', pbs_config, pbs_vars)
def __init__(self, lab, reproduce=False, force=False):
self.lab = lab
if not force:
# check environment for force flag under PBS
self.force = os.environ.get('PAYU_FORCE', False)
else:
self.force = force
self.start_time = datetime.datetime.now()
# TODO: replace with dict, check versions via key-value pairs
self.modules = set()
# TODO: __init__ should not be a config dumping ground!
self.config = read_config()
# Payu experiment type
self.debug = self.config.get('debug', False)
self.postscript = self.config.get('postscript')
self.repeat_run = self.config.get('repeat', False)
# Configuration
self.expand_shell_vars = True # TODO: configurable
# Model run time
self.runtime = None
if ('calendar' in self.config and
'runtime' in self.config['calendar']):
self.runtime = self.config['calendar']['runtime']
# Stacksize
# NOTE: Possible PBS issue in setting non-unlimited stacksizes
stacksize = self.config.get('stacksize', 'unlimited')
self.set_stacksize(stacksize)
# Initialize the submodels
self.init_models()
# TODO: Move to run/collate/sweep?
self.set_expt_pathnames()
self.set_counters()
for model in self.models:
model.set_input_paths()
self.set_output_paths()
if not reproduce:
# check environment for reproduce flag under PBS
reproduce = os.environ.get('PAYU_REPRODUCE', False)
# Initialize manifest
self.manifest = Manifest(self.config.get('manifest', {}),
reproduce=reproduce)
# Miscellaneous configurations
# TODO: Move this stuff somewhere else
self.userscripts = self.config.get('userscripts', {})
self.profilers = []
init_script = self.userscripts.get('init')
if init_script:
self.run_userscript(init_script)
self.runlog = Runlog(self)
# XXX: Temporary spot for the payu path
# This is horrible; payu/cli.py does this much more safely!
# But also does not even store it in os.environ!
default_payu_bin = os.path.dirname(sys.argv[0])
payu_bin = os.environ.get('PAYU_PATH', default_payu_bin)
self.payu_path = os.path.join(payu_bin, 'payu')
self.run_id = None
def runcmd(model_type, config_path, init_run, n_runs, lab_path, reproduce):
# Get job submission configuration
pbs_config = fsops.read_config(config_path)
pbs_vars = cli.set_env_vars(init_run, n_runs, lab_path)
# Set the queue
# NOTE: Maybe force all jobs on the normal queue
if 'queue' not in pbs_config:
pbs_config['queue'] = 'normal'
# TODO: Create drivers for servers
platform = pbs_config.get('platform', {})
max_cpus_per_node = platform.get('nodesize', 16)
max_ram_per_node = platform.get('nodemem', 32)
# Adjust the CPUs for any model-specific settings
# TODO: Incorporate this into the Model driver
mask_table = pbs_config.get('mask_table', False)
if mask_table:
# Check if a mask table exists
# TODO: Is control_path defined at this stage?
mask_table_fname = None
for fname in os.listdir(os.curdir):
if f.startswith('mask_table'):
mask_table_fname = fname
# TODO TODO
if 'ncpureq' in pbs_config:
# Hard override of CPU request
n_cpus_request = pbs_config.get('ncpureq')
elif 'submodels' in pbs_config and 'ncpus' not in pbs_config:
# Increase the cpu request to match a complete node
n_cpus_request = 0
submodel_configs = pbs_config['submodels']
for model_config in submodel_configs:
n_cpus_request += model_config.get('ncpus', 0)
else:
n_cpus_request = pbs_config.get('ncpus', 1)
n_cpus = n_cpus_request
n_cpus_per_node = pbs_config.get('npernode', max_cpus_per_node)
assert n_cpus_per_node <= max_cpus_per_node
node_misalignment = n_cpus % max_cpus_per_node != 0
node_increase = n_cpus_per_node < max_cpus_per_node
# Increase the CPUs to accomodate the cpu-per-node request
if n_cpus > max_cpus_per_node and (node_increase or node_misalignment):
# Number of requested nodes
n_nodes = 1 + (n_cpus - 1) // n_cpus_per_node
n_cpu_request = max_cpus_per_node * n_nodes
n_inert_cpus = n_cpu_request - n_cpus
print('payu: warning: Job request includes {n} unused CPUs.'
''.format(n=n_inert_cpus))
# Increase CPU request to match the effective node request
n_cpus = max_cpus_per_node * n_nodes
# Update the ncpus field in the config
if n_cpus != n_cpus_request:
print('payu: warning: CPU request increased from {n_req} to {n}'
''.format(n_req=n_cpus_request, n=n_cpus))
# Update the (possibly unchanged) value of ncpus
pbs_config['ncpus'] = n_cpus
# Set memory to use the complete node if unspecified
pbs_mem = pbs_config.get('mem')
if not pbs_mem:
if n_cpus > max_cpus_per_node:
pbs_mem = (n_cpus // max_cpus_per_node) * max_ram_per_node
else:
pbs_mem = n_cpus * (max_ram_per_node // max_cpus_per_node)
pbs_config['mem'] = '{0}GB'.format(pbs_mem)
cli.submit_job('payu-run', pbs_config, pbs_vars)
def runcmd(model_type, config_path, init_run, lab_path, dir_path):
pbs_config = fsops.read_config(config_path)
pbs_vars = cli.set_env_vars(init_run=init_run,
lab_path=lab_path,
dir_path=dir_path)
collate_config = pbs_config.get('collate', {})
# The mpi flag implies using mppnccombine-fast
mpi = collate_config.get('mpi', False)
default_ncpus = 1
default_queue = 'copyq'
if mpi:
default_ncpus = 2
default_queue = 'express'
collate_queue = collate_config.get('queue', default_queue)
pbs_config['queue'] = collate_queue
n_cpus_request = collate_config.get('ncpus', default_ncpus)
pbs_config['ncpus'] = n_cpus_request
collate_jobname = collate_config.get('jobname')
if not collate_jobname:
pbs_jobname = pbs_config.get('jobname')
if not pbs_jobname:
if dir_path and os.path.isdir(dir_path):
pbs_jobname = os.path.basename(dir_path)
else:
pbs_jobname = os.path.basename(os.getcwd())
collate_jobname = pbs_jobname[:13] + '_c'
# NOTE: Better to construct `collate_config` to pass to `submit_job`
pbs_config['jobname'] = collate_jobname[:15]
# Replace (or remove) walltime
collate_walltime = collate_config.get('walltime')
if collate_walltime:
pbs_config['walltime'] = collate_walltime
else:
# Remove the model walltime if set
try:
pbs_config.pop('walltime')
except KeyError:
pass
# TODO: calcualte default memory request based on ncpus and platform
pbs_config['mem'] = collate_config.get('mem', '2GB')
# Disable hyperthreading
qsub_flags = []
iflags = iter(pbs_config.get('qsub_flags', '').split())
for flag in iflags:
if flag == '-l':
try:
flag += ' ' + next(iflags)
except StopIteration:
break
# TODO: Test the sequence, not just existence of characters in string
if 'hyperthread' not in flag:
qsub_flags.append(flag)
pbs_config['qsub_flags'] = ' '.join(qsub_flags)
cli.submit_job('payu-collate', pbs_config, pbs_vars)
def runcmd(model_type, config_path, init_run, n_runs, lab_path, dir_path):
pbs_config = fsops.read_config(config_path)
pbs_vars = cli.set_env_vars(init_run, n_runs, lab_path, dir_path)
collate_config = pbs_config.get('collate', {})
# The mpi flag implies using mppnccombine-fast
mpi = collate_config.get('mpi', False)
default_ncpus = 1
default_queue = 'copyq'
if mpi:
default_ncpus = 2
default_queue = 'express'
collate_queue = collate_config.get('queue', default_queue)
pbs_config['queue'] = collate_queue
n_cpus_request = collate_config.get('ncpus', default_ncpus)
pbs_config['ncpus'] = n_cpus_request
collate_jobname = collate_config.get('jobname')
if not collate_jobname:
pbs_jobname = pbs_config.get('jobname')
if not pbs_jobname:
if dir_path and os.path.isdir(dir_path):
pbs_jobname = os.path.basename(dir_path)
else:
pbs_jobname = os.path.basename(os.getcwd())
collate_jobname = pbs_jobname[:13] + '_c'
# NOTE: Better to construct `collate_config` to pass to `submit_job`
pbs_config['jobname'] = collate_jobname[:15]
# Replace (or remove) walltime
collate_walltime = collate_config.get('walltime')
if collate_walltime:
pbs_config['walltime'] = collate_walltime
else:
# Remove the model walltime if set
try:
pbs_config.pop('walltime')
except KeyError:
pass
# TODO: calcualte default memory request based on ncpus and platform
pbs_config['mem'] = collate_config.get('mem', '2GB')
# Disable hyperthreading
qsub_flags = []
iflags = iter(pbs_config.get('qsub_flags', '').split())
for flag in iflags:
if flag == '-l':
try:
flag += ' ' + next(iflags)
except StopIteration:
break
# TODO: Test the sequence, not just existence of characters in string
if 'hyperthread' not in flag:
qsub_flags.append(flag)
pbs_config['qsub_flags'] = ' '.join(qsub_flags)
cli.submit_job('payu-collate', pbs_config, pbs_vars)
