def get_manifest_paths():
"""
Return a list of paths from manifest files to use to check for
storage paths
"""
tmpmanifest = Manifest(config={}, reproduce=False)
tmpmanifest.load()
return tmpmanifest.get_all_fullpaths()
def runscript():
parser = argparse.ArgumentParser()
for arg in arguments:
parser.add_argument(*arg['flags'], **arg['parameters'])
run_args = parser.parse_args()
lab = Laboratory(run_args.model_type, run_args.config_path,
run_args.lab_path)
expt = Experiment(lab, reproduce=run_args.reproduce, force=run_args.force)
n_runs_per_submit = expt.config.get('runspersub', 1)
subrun = 1
while True:
print('nruns: {0} nruns_per_submit: {1} subrun: {2}'
''.format(expt.n_runs, n_runs_per_submit, subrun))
expt.setup()
expt.run()
expt.archive()
# Finished runs
if expt.n_runs == 0:
break
# Need to manually increment the run counter if still looping
if n_runs_per_submit > 1 and subrun < n_runs_per_submit:
expt.counter += 1
# Re-initialize manifest: important to clear out restart manifest
# note no attempt to preserve reproduce flag, it makes no sense
# to on subsequent runs
expt.manifest = Manifest(expt.config.get('manifest', {}),
reproduce=False)
expt.set_output_paths()
# Does not make sense to reproduce a multiple run.
# Take care of this with argument processing?
expt.reproduce = False
else:
break
subrun += 1
if expt.n_runs > 0:
expt.resubmit()
def __init__(self, lab, reproduce=False, force=False):
self.lab = lab
if not force:
# check environment for force flag under PBS
self.force = os.environ.get('PAYU_FORCE', False)
else:
self.force = force
self.start_time = datetime.datetime.now()
# TODO: replace with dict, check versions via key-value pairs
self.modules = set()
# TODO: __init__ should not be a config dumping ground!
self.config = read_config()
# Payu experiment type
self.debug = self.config.get('debug', False)
self.postscript = self.config.get('postscript')
self.repeat_run = self.config.get('repeat', False)
# Configuration
self.expand_shell_vars = True # TODO: configurable
# Model run time
self.runtime = None
if ('calendar' in self.config and
'runtime' in self.config['calendar']):
self.runtime = self.config['calendar']['runtime']
# Stacksize
# NOTE: Possible PBS issue in setting non-unlimited stacksizes
stacksize = self.config.get('stacksize', 'unlimited')
self.set_stacksize(stacksize)
# Initialize the submodels
self.init_models()
# TODO: Move to run/collate/sweep?
self.set_expt_pathnames()
self.set_counters()
for model in self.models:
model.set_input_paths()
self.set_output_paths()
if not reproduce:
# check environment for reproduce flag under PBS
reproduce = os.environ.get('PAYU_REPRODUCE', False)
# Initialize manifest
self.manifest = Manifest(self.config.get('manifest', {}),
reproduce=reproduce)
# Miscellaneous configurations
# TODO: Move this stuff somewhere else
self.userscripts = self.config.get('userscripts', {})
self.profilers = []
init_script = self.userscripts.get('init')
if init_script:
self.run_userscript(init_script)
self.runlog = Runlog(self)
# XXX: Temporary spot for the payu path
# This is horrible; payu/cli.py does this much more safely!
# But also does not even store it in os.environ!
default_payu_bin = os.path.dirname(sys.argv[0])
payu_bin = os.environ.get('PAYU_PATH', default_payu_bin)
self.payu_path = os.path.join(payu_bin, 'payu')
self.run_id = None