def get_manifest_paths(): """ Return a list of paths from manifest files to use to check for storage paths """ tmpmanifest = Manifest(config={}, reproduce=False) tmpmanifest.load() return tmpmanifest.get_all_fullpaths()
def runscript(): parser = argparse.ArgumentParser() for arg in arguments: parser.add_argument(*arg['flags'], **arg['parameters']) run_args = parser.parse_args() lab = Laboratory(run_args.model_type, run_args.config_path, run_args.lab_path) expt = Experiment(lab, reproduce=run_args.reproduce, force=run_args.force) n_runs_per_submit = expt.config.get('runspersub', 1) subrun = 1 while True: print('nruns: {0} nruns_per_submit: {1} subrun: {2}' ''.format(expt.n_runs, n_runs_per_submit, subrun)) expt.setup() expt.run() expt.archive() # Finished runs if expt.n_runs == 0: break # Need to manually increment the run counter if still looping if n_runs_per_submit > 1 and subrun < n_runs_per_submit: expt.counter += 1 # Re-initialize manifest: important to clear out restart manifest # note no attempt to preserve reproduce flag, it makes no sense # to on subsequent runs expt.manifest = Manifest(expt.config.get('manifest', {}), reproduce=False) expt.set_output_paths() # Does not make sense to reproduce a multiple run. # Take care of this with argument processing? expt.reproduce = False else: break subrun += 1 if expt.n_runs > 0: expt.resubmit()
def __init__(self, lab, reproduce=False, force=False): self.lab = lab if not force: # check environment for force flag under PBS self.force = os.environ.get('PAYU_FORCE', False) else: self.force = force self.start_time = datetime.datetime.now() # TODO: replace with dict, check versions via key-value pairs self.modules = set() # TODO: __init__ should not be a config dumping ground! self.config = read_config() # Payu experiment type self.debug = self.config.get('debug', False) self.postscript = self.config.get('postscript') self.repeat_run = self.config.get('repeat', False) # Configuration self.expand_shell_vars = True # TODO: configurable # Model run time self.runtime = None if ('calendar' in self.config and 'runtime' in self.config['calendar']): self.runtime = self.config['calendar']['runtime'] # Stacksize # NOTE: Possible PBS issue in setting non-unlimited stacksizes stacksize = self.config.get('stacksize', 'unlimited') self.set_stacksize(stacksize) # Initialize the submodels self.init_models() # TODO: Move to run/collate/sweep? self.set_expt_pathnames() self.set_counters() for model in self.models: model.set_input_paths() self.set_output_paths() if not reproduce: # check environment for reproduce flag under PBS reproduce = os.environ.get('PAYU_REPRODUCE', False) # Initialize manifest self.manifest = Manifest(self.config.get('manifest', {}), reproduce=reproduce) # Miscellaneous configurations # TODO: Move this stuff somewhere else self.userscripts = self.config.get('userscripts', {}) self.profilers = [] init_script = self.userscripts.get('init') if init_script: self.run_userscript(init_script) self.runlog = Runlog(self) # XXX: Temporary spot for the payu path # This is horrible; payu/cli.py does this much more safely! # But also does not even store it in os.environ! default_payu_bin = os.path.dirname(sys.argv[0]) payu_bin = os.environ.get('PAYU_PATH', default_payu_bin) self.payu_path = os.path.join(payu_bin, 'payu') self.run_id = None