def test_pgd(self): exp_name = "pgd" root = os.getcwd() wd = "/tmp/host0/hm_wd/" + exp_name self._create_exp(wd, root) exp = experiment.ExpFromFiles(exp_name, wd, debug=True) experiment.init_run(exp, self.stream) wd = "/tmp/host1/scratch/sfx_home/" + exp_name + "/lib" exp = experiment.ExpFromFiles(exp_name, wd, host=self.host, stream=self.stream, progress=self.progress) ecf_name = "/" + exp_name + "/task" task = scheduler.EcflowTask(ecf_name, self.ecf_tryno, self.ecf_pass, self.ecf_rid, self.submission_id) experiment.Pgd(task, exp, host=self.host, mbr=self.mbr, stream=self.stream, args=self.args).run(wrapper=self.wrapper) os.chdir(root)
def test_quality_control_t2m(self): exp_name = "quality_control_t2m_task" root = os.getcwd() wd = "/tmp/host0/hm_wd/" + exp_name self._create_exp(wd, root) exp = experiment.ExpFromFiles(exp_name, wd) experiment.init_run(exp, self.stream) wd = "/tmp/host1/scratch/sfx_home/" + exp_name + "/lib" exp = experiment.ExpFromFiles(exp_name, wd, host=self.host, stream=self.stream, progress=self.progress) ecf_name = "/" + exp_name + "/task" task = scheduler.EcflowTask(ecf_name, self.ecf_tryno, self.ecf_pass, self.ecf_rid, self.submission_id) task.family1 = "t2m" experiment.QualityControl(task, exp, host=self.host, mbr=self.mbr, stream=self.stream, args=self.args).run(wrapper=self.wrapper) os.chdir(root)
def status_cmd_exp(**kwargs): debug = False if "debug" in kwargs: debug = kwargs["debug"] exp = kwargs["exp"] del kwargs["exp"] lib = kwargs["lib"] del kwargs["lib"] libs = ["surfex", "scheduler", "experiment"] for ll in libs: if os.path.exists(lib + "/" + ll): sys.path.insert(0, lib + "/" + ll) if debug: print("Set first in system path: ", lib + "/" + ll) exp = experiment.ExpFromFiles(exp, lib) kwargs.update({"env_submit": exp.env_submit}) hosts = exp.system.hosts joboutdir = {} for host in range(0, len(hosts)): joboutdir.update( {str(host): exp.system.get_var("JOBOUTDIR", str(host))}) kwargs.update({"joboutdirs": joboutdir}) scheduler.status_cmd(**kwargs)
def test_submission(self): exp = "test_submission" hm_wd = "/tmp/host0/hm_wd/" + exp self._create_exp(hm_wd) # Set progress dtg = datetime(2020, 6, 9, 9) progress = {"DTG": dtg.strftime("%Y%m%d%H")} progress_pp = {"DTGPP": dtg.strftime("%Y%m%d%H")} progress = experiment.Progress(progress, progress_pp) sfx_exp = experiment.ExpFromFiles(exp, hm_wd, progress=progress) ecf_name = "/" + exp + "/Forecasting/Forecast" ecf_tryno = 1 ecf_pass = "******" task = scheduler.EcflowTask(ecf_name, ecf_tryno, ecf_pass, ecf_rid=int(os.getpid()), submission_id=None) os.makedirs("/tmp/host1/job/" + exp + "/Forecasting/", exist_ok=True) fh = open("/tmp/host1/job/" + exp + "/Forecasting/Forecast.job1", "w") fh.write("My job generated by ecflow") fh.close() exceptions = { "complete": { "task": { "Forecast": "is_coldstart" }, "family": { "Forecasting": "is_coldstart" } } } env_file = "/tmp/host1/job/" + exp + "/Forecasting/Env" fh = open(env_file, "w") fh.write("print(\"Oh my environment\n\")") fh.close() hosts = sfx_exp.system.hosts joboutdir = {} for host in range(0, len(hosts)): joboutdir.update( {str(host): sfx_exp.system.get_var("JOBOUTDIR", str(host))}) submit = scheduler.EcflowSubmitTask(task, sfx_exp.env_submit, sfx_exp.server, joboutdir, submit_exceptions=exceptions, env_file=env_file) submit.write_job()
def submit_cmd_exp(**kwargs): debug = False if "debug" in kwargs: debug = kwargs["debug"] exp = kwargs["exp"] lib = kwargs["lib"] dtg = kwargs["dtg"] if isinstance(dtg, str): dtg = datetime.strptime(dtg, "%Y%m%d%H") dtgbeg = dtg if "dtgbeg" in kwargs: dtgbeg = kwargs["dtgbeg"] if isinstance(dtgbeg, str): dtgbeg = datetime.strptime(dtgbeg, "%Y%m%d%H") libs = ["surfex", "scheduler", "experiment"] for ll in libs: if os.path.exists(lib + "/" + ll): sys.path.insert(0, lib + "/" + ll) if debug: print("Set first in system path: ", lib + "/" + ll) progress = {"DTG": dtg, "DTGBEG": dtgbeg, "DTGEND": dtg} progress_pp = {"DTGPP": dtg} progress = experiment.Progress(progress, progress_pp) stream = None if "stream" in kwargs: stream = kwargs["stream"] exp = experiment.ExpFromFiles(exp, lib, stream=stream, progress=progress) wd = exp.wd submit_exceptions = wd + "/config//submit/submission.json" submit_exceptions = json.load(open(submit_exceptions, "r")) kwargs.update({"submit_exceptions": submit_exceptions}) kwargs.update( {"env_file": exp.get_file_name(exp.wd, "env", full_path=True)}) kwargs.update({"env_submit": exp.env_submit}) kwargs.update({"env_server": exp.server}) hosts = exp.system.hosts joboutdir = {} for host in range(0, len(hosts)): joboutdir.update( {str(host): exp.system.get_var("JOBOUTDIR", str(host))}) kwargs.update({"joboutdir": joboutdir}) scheduler.submit_cmd(**kwargs)
dtgbeg = "%DTGBEG%" if stream == "": stream = None if mbr == "" or int(mbr) < 0: mbr = None libs = ["surfex", "scheduler", "experiment"] for ll in libs: if os.path.exists(lib + "/" + ll): sys.path.insert(0, lib + "/" + ll) progress = {"DTG": dtg, "DTGBEG": dtgbeg} progress_pp = {"DTGPP": dtg} progress = experiment.Progress(progress, progress_pp) exp = experiment.ExpFromFiles(exp, lib, host=host, progress=progress) server = exp.server ecf_name = "%ECF_NAME%" ecf_pass = "******" ecf_tryno = "%ECF_TRYNO%" ecf_rid = "%ECF_RID%" submission_id = "%SUBMISSION_ID%" task_name = "%TASK%" args = "%ARGS%" if args == "": args = None task = scheduler.EcflowTask(ecf_name, ecf_tryno, ecf_pass, ecf_rid, submission_id) # Dummy commands to try out your self
def surfex_script(**kwargs): debug = False if "debug" in kwargs: debug = kwargs["debug"] action = kwargs["action"] exp = None if "exp" in kwargs: exp = kwargs["exp"] # Setup wd = None if "wd" in kwargs: wd = kwargs["wd"] rev = None if "rev" in kwargs: rev = kwargs["rev"] pysurfex = None if "pysurfex" in kwargs: pysurfex = kwargs["pysurfex"] pysurfex_experiment = None if "pysurfex_experiment" in kwargs: pysurfex_experiment = kwargs["pysurfex_experiment"] pysurfex_scheduler = None if "pysurfex_scheduler" in kwargs: pysurfex_scheduler = kwargs["pysurfex_scheduler"] host = kwargs["host"] config = None if "config" in kwargs: config = kwargs["config"] config_file = None if "config_file" in kwargs: config_file = kwargs["config_file"] domain_name = None if "domain_name" in kwargs: domain_name = kwargs["domain_name"] domain_file = None if "domain_file" in kwargs: domain_file = kwargs["domain_file"] # Co file = kwargs["file"] # Others dtg = kwargs["dtg"] dtgend = kwargs["dtgend"] suite = kwargs["suite"] stream = kwargs["stream"] begin = True if "begin" in kwargs: begin = kwargs["begin"] # Find experiment if wd is None: wd = os.getcwd() print("Setting current working directory as WD: " + wd) if exp is None: print("Setting EXP from WD:" + wd) exp = wd.split("/")[-1] print("EXP = " + exp) libs = ["surfex", "scheduler", "experiment"] for lib in libs: if os.path.exists(wd + "/" + lib): sys.path.insert(0, wd + "/" + lib) if debug: print("Set first in system path: ", wd + "/" + lib) if action == "setup": # Copy files to WD from REV if rev is None: raise Exception("You must set REV") if host is None: raise Exception("You must set host") if pysurfex is None: print("Using " + rev + " as pysurfex") pysurfex = rev if pysurfex_scheduler is None: print("Using " + rev + " as pysurfex_scheduler") pysurfex_scheduler = rev if pysurfex_experiment is None: print("Using " + rev + " as pysurfex_experiment") pysurfex_experiment = rev experiment_is_locked = False sfx_exp = experiment.Exp(exp, wd, rev, pysurfex, pysurfex_scheduler, pysurfex_experiment, experiment_is_locked, setup=True, host=host, configuration=config, configuration_file=config_file, debug=debug) exp_domain_file = sfx_exp.get_file_name(wd, "domain", full_path=True) # print(domain_file, exp_domain_file) if domain_file is None: if domain_name is not None: domain_json = surfex.set_domain(json.load( open(wd + "/config/domains/Harmonie_domains.json", "r")), domain_name, hm_mode=True) json.dump(domain_json, open(exp_domain_file, "w"), indent=2) else: if not os.path.exists(exp_domain_file): raise Exception("You must provide a domain!") else: domain_json = json.load(open(domain_file)) if os.path.abspath(domain_file) != os.path.abspath( exp_domain_file): json.dump(domain_json, open(exp_domain_file, "w"), indent=2) elif "action" == "mon": # TODO raise NotImplementedError elif action == "co": if file is None: raise Exception("Checkout requires a file (--file)") experiment.ExpFromFiles(exp, wd).checkout(file) else: # Some kind of start if action == "start" and dtg is None: raise Exception("You must provide -dtg to start a simulation") elif action == "climate": if dtg is None: dtg = "2008061600" if suite is not None and suite != "climate": raise Exception( "Action was climate but you also specified a suite not being climate: " + suite) suite = "climate" elif action == "testbed": if dtg is None: dtg = "2008061600" if suite is not None and suite != "testbed": raise Exception( "Action was climate but you also specified a suite not being testbed: " + suite) suite = "testbed" elif action == "install": if dtg is None: dtg = "2008061600" suite = "Makeup" updated_progress = {} updated_progress_pp = {} if dtg is not None: updated_progress.update({"DTG": dtg}) if dtgend is not None: updated_progress.update({"DTGEND": dtgend}) if action == "start": if dtg is None: raise Exception("No DTG was provided!") updated_progress.update({"DTG": dtg}) updated_progress.update({"DTGBEG": dtg}) if dtgend is not None: updated_progress.update({"DTGEND": dtgend}) else: updated_progress.update({"DTGEND": dtg}) updated_progress_pp.update({"DTGPP": dtg}) progress_file = experiment.Exp.get_file_name(wd, "progress", stream=stream, full_path=True) progress_pp_file = experiment.Exp.get_file_name(wd, "progressPP", stream=stream, full_path=True) if action.lower() == "prod" or action.lower() == "continue": progress = experiment.ProgressFromFile(progress_file, progress_pp_file) if dtgend is not None: progress.dtgend = datetime.strptime(dtgend, "%Y%m%d%H") else: progress = experiment.Progress(updated_progress, updated_progress_pp) # Update progress progress.save(progress_file, progress_pp_file) # Set experiment from files. Should be existing now after setup sfx_exp = experiment.ExpFromFiles(exp, wd) system = sfx_exp.system # Merge config # all_merged_settings = surfex.merge_toml_env_from_config_dicts(sfx_exp.config_files) # merged_config, member_merged_config = surfex.process_merged_settings(all_merged_settings) # Create configuration # config = surfex.Configuration(merged_config, member_merged_config) data0 = system.get_var("SFX_EXP_DATA", "0") lib0 = system.get_var("SFX_EXP_LIB", "0") logfile = data0 + "/ECF.log" # Create LIB0 and copy init run if WD != lib0 if wd.rstrip("/") != lib0.rstrip("/"): ecf_init_run = lib0 + "/ecf/InitRun.py" dirname = os.path.dirname(ecf_init_run) # print(dirname) dirs = dirname.split("/") # print(dirs) if len(dirs) > 1: p = "/" for d in dirs[1:]: p = p + d # print(p) os.makedirs(p, exist_ok=True) p = p + "/" shutil.copy2(wd + "/ecf/InitRun.py", ecf_init_run) # Create the scheduler env_server = sfx_exp.wd + "/Env_server" my_scheduler = scheduler.EcflowServerFromFile(env_server, logfile) print(scheduler.__file__) # Create and start the suite def_file = data0 + "/" + suite + ".def" defs = experiment.get_defs(sfx_exp, suite, def_file) defs.save_as_defs() my_scheduler.start_suite(defs, begin=begin)
import scheduler import experiment import os import sys lib = "%LIB%" exp = "%EXP%" stream = "%STREAM%" if stream == "": stream = None libs = ["surfex", "scheduler", "experiment"] for ll in libs: if os.path.exists(lib + "/" + ll): sys.path.insert(0, lib + "/" + ll) # InitRun always runs from HOST0 exp = experiment.ExpFromFiles(exp, lib) server = exp.server ecf_name = "%ECF_NAME%" ecf_pass = "******" ecf_tryno = "%ECF_TRYNO%" ecf_rid = "%ECF_RID%" submission_id = "%SUBMISSION_ID%" task = scheduler.EcflowTask(ecf_name, ecf_tryno, ecf_pass, ecf_rid, submission_id) # This will also handle call to sys.exit(), i.e. Client.__exit__ will still be called. with scheduler.EcflowClient(server, task) as ci: experiment.init_run(exp, stream=stream)
def test_start_and_run(self): exp = "test_start_and_run" wd = "/tmp/host0/hm_wd/" + exp self._create_exp(wd) # Dry submit argv = [ "-exp", exp, "-lib", wd, "-ecf_name", exp + "/dry_submit", "-ecf_pass", "12345", "-ecf_tryno", "1" ] print(argv) kwargs = experiment.parse_submit_cmd_exp(argv) kwargs.update({"dry_run": True}) experiment.submit_cmd_exp(**kwargs) argv = [ "start", "--wd", wd, "-dtg", "2020111303", "-dtgend", "2020111306", "--suite", "unittest" ] kwargs = experiment.parse_surfex_script(argv) experiment.surfex_script(**kwargs) # Test if init run has synced test_file = "/tmp/host1/scratch/sfx_home/" + exp + "/unittest_ok" found = False for t in range(0, 15): print(t) if os.path.exists(test_file): found = True break time.sleep(1) if not found: raise FileNotFoundError(test_file + " not found!") test_file = "/tmp/host1/scratch/sfx_home/" + exp + "/SleepingBeauty" found = False for t in range(0, 15): print(t) if os.path.exists(test_file): found = True break time.sleep(1) if not found: raise FileNotFoundError(test_file + " not found!") ecf_name = None ecf_pass = None ecf_tryno = None ecf_rid = None submission_id = None # job_file = "/tmp/host1/job/" + exp + "/SleepingBeauty.job1" # for line in open(job_file): # for match in re.finditer("^ecf_name = ", line): # print(match) # print(line) # ecf_name = line.split()[2].replace('"', '') # for match in re.finditer("^ecf_pass = "******"', '') # for match in re.finditer("^ecf_tryno = ", line): # ecf_tryno = line.split()[2].replace('"', '') # for match in re.finditer("^ecf_rid = ", line): # ecf_rid = line.split()[2].replace('"', '') # for match in re.finditer("^submission_id = ", line): # submission_id = line.split()[2].replace('"', '') # Find sleeping beauty information from job file sfx_exp = experiment.ExpFromFiles(exp, wd) sfx_exp.server.ecf_client.sync_local() defs = sfx_exp.server.ecf_client.get_defs() for item in defs.suites: for node in item: n = node.get_abs_node_path() print(n) if n == "/" + exp + "/SleepingBeauty": print("There she is") ecf_name = node.find_gen_variable("ECF_NAME").value() ecf_tryno = node.find_gen_variable("ECF_TRYNO").value() ecf_pass = node.find_gen_variable("ECF_PASS").value() ecf_rid = node.find_gen_variable("ECF_RID").value() submission_id = node.find_variable("SUBMISSION_ID").value() print(ecf_name) print(submission_id) argv = [ "-exp", exp, "-lib", "/tmp/host1/scratch/sfx_home/" + exp + "/lib", "-ecf_name", ecf_name, "-ecf_pass", ecf_pass, "-ecf_tryno", ecf_tryno, "-ecf_rid", ecf_rid, "-submission_id", submission_id ] print(argv) kwargs = experiment.parse_status_cmd_exp(argv) experiment.status_cmd_exp(**kwargs) print("kill", argv) kwargs = experiment.parse_kill_cmd_exp(argv) experiment.kill_cmd_exp(**kwargs) test_file = "/tmp/host1/scratch/sfx_home/" + exp + "/SleepingBeauty2" found = False for t in range(0, 15): print(t) if os.path.exists(test_file): found = True break time.sleep(1) if not found: raise FileNotFoundError(test_file + " not found!")