def test_pgd(self):
exp_name = "pgd"
root = os.getcwd()
wd = "/tmp/host0/hm_wd/" + exp_name
self._create_exp(wd, root)
exp = experiment.ExpFromFiles(exp_name, wd, debug=True)
experiment.init_run(exp, self.stream)
wd = "/tmp/host1/scratch/sfx_home/" + exp_name + "/lib"
exp = experiment.ExpFromFiles(exp_name,
wd,
host=self.host,
stream=self.stream,
progress=self.progress)
ecf_name = "/" + exp_name + "/task"
task = scheduler.EcflowTask(ecf_name, self.ecf_tryno, self.ecf_pass,
self.ecf_rid, self.submission_id)
experiment.Pgd(task,
exp,
host=self.host,
mbr=self.mbr,
stream=self.stream,
args=self.args).run(wrapper=self.wrapper)
os.chdir(root)
def test_quality_control_t2m(self):
exp_name = "quality_control_t2m_task"
root = os.getcwd()
wd = "/tmp/host0/hm_wd/" + exp_name
self._create_exp(wd, root)
exp = experiment.ExpFromFiles(exp_name, wd)
experiment.init_run(exp, self.stream)
wd = "/tmp/host1/scratch/sfx_home/" + exp_name + "/lib"
exp = experiment.ExpFromFiles(exp_name,
wd,
host=self.host,
stream=self.stream,
progress=self.progress)
ecf_name = "/" + exp_name + "/task"
task = scheduler.EcflowTask(ecf_name, self.ecf_tryno, self.ecf_pass,
self.ecf_rid, self.submission_id)
task.family1 = "t2m"
experiment.QualityControl(task,
exp,
host=self.host,
mbr=self.mbr,
stream=self.stream,
args=self.args).run(wrapper=self.wrapper)
os.chdir(root)
def status_cmd_exp(**kwargs):
debug = False
if "debug" in kwargs:
debug = kwargs["debug"]
exp = kwargs["exp"]
del kwargs["exp"]
lib = kwargs["lib"]
del kwargs["lib"]
libs = ["surfex", "scheduler", "experiment"]
for ll in libs:
if os.path.exists(lib + "/" + ll):
sys.path.insert(0, lib + "/" + ll)
if debug:
print("Set first in system path: ", lib + "/" + ll)
exp = experiment.ExpFromFiles(exp, lib)
kwargs.update({"env_submit": exp.env_submit})
hosts = exp.system.hosts
joboutdir = {}
for host in range(0, len(hosts)):
joboutdir.update(
{str(host): exp.system.get_var("JOBOUTDIR", str(host))})
kwargs.update({"joboutdirs": joboutdir})
scheduler.status_cmd(**kwargs)
def test_submission(self):
exp = "test_submission"
hm_wd = "/tmp/host0/hm_wd/" + exp
self._create_exp(hm_wd)
# Set progress
dtg = datetime(2020, 6, 9, 9)
progress = {"DTG": dtg.strftime("%Y%m%d%H")}
progress_pp = {"DTGPP": dtg.strftime("%Y%m%d%H")}
progress = experiment.Progress(progress, progress_pp)
sfx_exp = experiment.ExpFromFiles(exp, hm_wd, progress=progress)
ecf_name = "/" + exp + "/Forecasting/Forecast"
ecf_tryno = 1
ecf_pass = "******"
task = scheduler.EcflowTask(ecf_name,
ecf_tryno,
ecf_pass,
ecf_rid=int(os.getpid()),
submission_id=None)
os.makedirs("/tmp/host1/job/" + exp + "/Forecasting/", exist_ok=True)
fh = open("/tmp/host1/job/" + exp + "/Forecasting/Forecast.job1", "w")
fh.write("My job generated by ecflow")
fh.close()
exceptions = {
"complete": {
"task": {
"Forecast": "is_coldstart"
},
"family": {
"Forecasting": "is_coldstart"
}
}
}
env_file = "/tmp/host1/job/" + exp + "/Forecasting/Env"
fh = open(env_file, "w")
fh.write("print(\"Oh my environment\n\")")
fh.close()
hosts = sfx_exp.system.hosts
joboutdir = {}
for host in range(0, len(hosts)):
joboutdir.update(
{str(host): sfx_exp.system.get_var("JOBOUTDIR", str(host))})
submit = scheduler.EcflowSubmitTask(task,
sfx_exp.env_submit,
sfx_exp.server,
joboutdir,
submit_exceptions=exceptions,
env_file=env_file)
submit.write_job()
def submit_cmd_exp(**kwargs):
debug = False
if "debug" in kwargs:
debug = kwargs["debug"]
exp = kwargs["exp"]
lib = kwargs["lib"]
dtg = kwargs["dtg"]
if isinstance(dtg, str):
dtg = datetime.strptime(dtg, "%Y%m%d%H")
dtgbeg = dtg
if "dtgbeg" in kwargs:
dtgbeg = kwargs["dtgbeg"]
if isinstance(dtgbeg, str):
dtgbeg = datetime.strptime(dtgbeg, "%Y%m%d%H")
libs = ["surfex", "scheduler", "experiment"]
for ll in libs:
if os.path.exists(lib + "/" + ll):
sys.path.insert(0, lib + "/" + ll)
if debug:
print("Set first in system path: ", lib + "/" + ll)
progress = {"DTG": dtg, "DTGBEG": dtgbeg, "DTGEND": dtg}
progress_pp = {"DTGPP": dtg}
progress = experiment.Progress(progress, progress_pp)
stream = None
if "stream" in kwargs:
stream = kwargs["stream"]
exp = experiment.ExpFromFiles(exp, lib, stream=stream, progress=progress)
wd = exp.wd
submit_exceptions = wd + "/config//submit/submission.json"
submit_exceptions = json.load(open(submit_exceptions, "r"))
kwargs.update({"submit_exceptions": submit_exceptions})
kwargs.update(
{"env_file": exp.get_file_name(exp.wd, "env", full_path=True)})
kwargs.update({"env_submit": exp.env_submit})
kwargs.update({"env_server": exp.server})
hosts = exp.system.hosts
joboutdir = {}
for host in range(0, len(hosts)):
joboutdir.update(
{str(host): exp.system.get_var("JOBOUTDIR", str(host))})
kwargs.update({"joboutdir": joboutdir})
scheduler.submit_cmd(**kwargs)
dtgbeg = "%DTGBEG%"
if stream == "":
stream = None
if mbr == "" or int(mbr) < 0:
mbr = None
libs = ["surfex", "scheduler", "experiment"]
for ll in libs:
if os.path.exists(lib + "/" + ll):
sys.path.insert(0, lib + "/" + ll)
progress = {"DTG": dtg, "DTGBEG": dtgbeg}
progress_pp = {"DTGPP": dtg}
progress = experiment.Progress(progress, progress_pp)
exp = experiment.ExpFromFiles(exp, lib, host=host, progress=progress)
server = exp.server
ecf_name = "%ECF_NAME%"
ecf_pass = "******"
ecf_tryno = "%ECF_TRYNO%"
ecf_rid = "%ECF_RID%"
submission_id = "%SUBMISSION_ID%"
task_name = "%TASK%"
args = "%ARGS%"
if args == "":
args = None
task = scheduler.EcflowTask(ecf_name, ecf_tryno, ecf_pass, ecf_rid, submission_id)
# Dummy commands to try out your self
def surfex_script(**kwargs):
debug = False
if "debug" in kwargs:
debug = kwargs["debug"]
action = kwargs["action"]
exp = None
if "exp" in kwargs:
exp = kwargs["exp"]
# Setup
wd = None
if "wd" in kwargs:
wd = kwargs["wd"]
rev = None
if "rev" in kwargs:
rev = kwargs["rev"]
pysurfex = None
if "pysurfex" in kwargs:
pysurfex = kwargs["pysurfex"]
pysurfex_experiment = None
if "pysurfex_experiment" in kwargs:
pysurfex_experiment = kwargs["pysurfex_experiment"]
pysurfex_scheduler = None
if "pysurfex_scheduler" in kwargs:
pysurfex_scheduler = kwargs["pysurfex_scheduler"]
host = kwargs["host"]
config = None
if "config" in kwargs:
config = kwargs["config"]
config_file = None
if "config_file" in kwargs:
config_file = kwargs["config_file"]
domain_name = None
if "domain_name" in kwargs:
domain_name = kwargs["domain_name"]
domain_file = None
if "domain_file" in kwargs:
domain_file = kwargs["domain_file"]
# Co
file = kwargs["file"]
# Others
dtg = kwargs["dtg"]
dtgend = kwargs["dtgend"]
suite = kwargs["suite"]
stream = kwargs["stream"]
begin = True
if "begin" in kwargs:
begin = kwargs["begin"]
# Find experiment
if wd is None:
wd = os.getcwd()
print("Setting current working directory as WD: " + wd)
if exp is None:
print("Setting EXP from WD:" + wd)
exp = wd.split("/")[-1]
print("EXP = " + exp)
libs = ["surfex", "scheduler", "experiment"]
for lib in libs:
if os.path.exists(wd + "/" + lib):
sys.path.insert(0, wd + "/" + lib)
if debug:
print("Set first in system path: ", wd + "/" + lib)
if action == "setup":
# Copy files to WD from REV
if rev is None:
raise Exception("You must set REV")
if host is None:
raise Exception("You must set host")
if pysurfex is None:
print("Using " + rev + " as pysurfex")
pysurfex = rev
if pysurfex_scheduler is None:
print("Using " + rev + " as pysurfex_scheduler")
pysurfex_scheduler = rev
if pysurfex_experiment is None:
print("Using " + rev + " as pysurfex_experiment")
pysurfex_experiment = rev
experiment_is_locked = False
sfx_exp = experiment.Exp(exp,
wd,
rev,
pysurfex,
pysurfex_scheduler,
pysurfex_experiment,
experiment_is_locked,
setup=True,
host=host,
configuration=config,
configuration_file=config_file,
debug=debug)
exp_domain_file = sfx_exp.get_file_name(wd, "domain", full_path=True)
# print(domain_file, exp_domain_file)
if domain_file is None:
if domain_name is not None:
domain_json = surfex.set_domain(json.load(
open(wd + "/config/domains/Harmonie_domains.json", "r")),
domain_name,
hm_mode=True)
json.dump(domain_json, open(exp_domain_file, "w"), indent=2)
else:
if not os.path.exists(exp_domain_file):
raise Exception("You must provide a domain!")
else:
domain_json = json.load(open(domain_file))
if os.path.abspath(domain_file) != os.path.abspath(
exp_domain_file):
json.dump(domain_json, open(exp_domain_file, "w"), indent=2)
elif "action" == "mon":
# TODO
raise NotImplementedError
elif action == "co":
if file is None:
raise Exception("Checkout requires a file (--file)")
experiment.ExpFromFiles(exp, wd).checkout(file)
else:
# Some kind of start
if action == "start" and dtg is None:
raise Exception("You must provide -dtg to start a simulation")
elif action == "climate":
if dtg is None:
dtg = "2008061600"
if suite is not None and suite != "climate":
raise Exception(
"Action was climate but you also specified a suite not being climate: "
+ suite)
suite = "climate"
elif action == "testbed":
if dtg is None:
dtg = "2008061600"
if suite is not None and suite != "testbed":
raise Exception(
"Action was climate but you also specified a suite not being testbed: "
+ suite)
suite = "testbed"
elif action == "install":
if dtg is None:
dtg = "2008061600"
suite = "Makeup"
updated_progress = {}
updated_progress_pp = {}
if dtg is not None:
updated_progress.update({"DTG": dtg})
if dtgend is not None:
updated_progress.update({"DTGEND": dtgend})
if action == "start":
if dtg is None:
raise Exception("No DTG was provided!")
updated_progress.update({"DTG": dtg})
updated_progress.update({"DTGBEG": dtg})
if dtgend is not None:
updated_progress.update({"DTGEND": dtgend})
else:
updated_progress.update({"DTGEND": dtg})
updated_progress_pp.update({"DTGPP": dtg})
progress_file = experiment.Exp.get_file_name(wd,
"progress",
stream=stream,
full_path=True)
progress_pp_file = experiment.Exp.get_file_name(wd,
"progressPP",
stream=stream,
full_path=True)
if action.lower() == "prod" or action.lower() == "continue":
progress = experiment.ProgressFromFile(progress_file,
progress_pp_file)
if dtgend is not None:
progress.dtgend = datetime.strptime(dtgend, "%Y%m%d%H")
else:
progress = experiment.Progress(updated_progress,
updated_progress_pp)
# Update progress
progress.save(progress_file, progress_pp_file)
# Set experiment from files. Should be existing now after setup
sfx_exp = experiment.ExpFromFiles(exp, wd)
system = sfx_exp.system
# Merge config
# all_merged_settings = surfex.merge_toml_env_from_config_dicts(sfx_exp.config_files)
# merged_config, member_merged_config = surfex.process_merged_settings(all_merged_settings)
# Create configuration
# config = surfex.Configuration(merged_config, member_merged_config)
data0 = system.get_var("SFX_EXP_DATA", "0")
lib0 = system.get_var("SFX_EXP_LIB", "0")
logfile = data0 + "/ECF.log"
# Create LIB0 and copy init run if WD != lib0
if wd.rstrip("/") != lib0.rstrip("/"):
ecf_init_run = lib0 + "/ecf/InitRun.py"
dirname = os.path.dirname(ecf_init_run)
# print(dirname)
dirs = dirname.split("/")
# print(dirs)
if len(dirs) > 1:
p = "/"
for d in dirs[1:]:
p = p + d
# print(p)
os.makedirs(p, exist_ok=True)
p = p + "/"
shutil.copy2(wd + "/ecf/InitRun.py", ecf_init_run)
# Create the scheduler
env_server = sfx_exp.wd + "/Env_server"
my_scheduler = scheduler.EcflowServerFromFile(env_server, logfile)
print(scheduler.__file__)
# Create and start the suite
def_file = data0 + "/" + suite + ".def"
defs = experiment.get_defs(sfx_exp, suite, def_file)
defs.save_as_defs()
my_scheduler.start_suite(defs, begin=begin)
import scheduler
import experiment
import os
import sys
lib = "%LIB%"
exp = "%EXP%"
stream = "%STREAM%"
if stream == "":
stream = None
libs = ["surfex", "scheduler", "experiment"]
for ll in libs:
if os.path.exists(lib + "/" + ll):
sys.path.insert(0, lib + "/" + ll)
# InitRun always runs from HOST0
exp = experiment.ExpFromFiles(exp, lib)
server = exp.server
ecf_name = "%ECF_NAME%"
ecf_pass = "******"
ecf_tryno = "%ECF_TRYNO%"
ecf_rid = "%ECF_RID%"
submission_id = "%SUBMISSION_ID%"
task = scheduler.EcflowTask(ecf_name, ecf_tryno, ecf_pass, ecf_rid, submission_id)
# This will also handle call to sys.exit(), i.e. Client.__exit__ will still be called.
with scheduler.EcflowClient(server, task) as ci:
experiment.init_run(exp, stream=stream)
def test_start_and_run(self):
exp = "test_start_and_run"
wd = "/tmp/host0/hm_wd/" + exp
self._create_exp(wd)
# Dry submit
argv = [
"-exp", exp, "-lib", wd, "-ecf_name", exp + "/dry_submit",
"-ecf_pass", "12345", "-ecf_tryno", "1"
]
print(argv)
kwargs = experiment.parse_submit_cmd_exp(argv)
kwargs.update({"dry_run": True})
experiment.submit_cmd_exp(**kwargs)
argv = [
"start", "--wd", wd, "-dtg", "2020111303", "-dtgend", "2020111306",
"--suite", "unittest"
]
kwargs = experiment.parse_surfex_script(argv)
experiment.surfex_script(**kwargs)
# Test if init run has synced
test_file = "/tmp/host1/scratch/sfx_home/" + exp + "/unittest_ok"
found = False
for t in range(0, 15):
print(t)
if os.path.exists(test_file):
found = True
break
time.sleep(1)
if not found:
raise FileNotFoundError(test_file + " not found!")
test_file = "/tmp/host1/scratch/sfx_home/" + exp + "/SleepingBeauty"
found = False
for t in range(0, 15):
print(t)
if os.path.exists(test_file):
found = True
break
time.sleep(1)
if not found:
raise FileNotFoundError(test_file + " not found!")
ecf_name = None
ecf_pass = None
ecf_tryno = None
ecf_rid = None
submission_id = None
# job_file = "/tmp/host1/job/" + exp + "/SleepingBeauty.job1"
# for line in open(job_file):
# for match in re.finditer("^ecf_name = ", line):
# print(match)
# print(line)
# ecf_name = line.split()[2].replace('"', '')
# for match in re.finditer("^ecf_pass = "******"', '')
# for match in re.finditer("^ecf_tryno = ", line):
# ecf_tryno = line.split()[2].replace('"', '')
# for match in re.finditer("^ecf_rid = ", line):
# ecf_rid = line.split()[2].replace('"', '')
# for match in re.finditer("^submission_id = ", line):
# submission_id = line.split()[2].replace('"', '')
# Find sleeping beauty information from job file
sfx_exp = experiment.ExpFromFiles(exp, wd)
sfx_exp.server.ecf_client.sync_local()
defs = sfx_exp.server.ecf_client.get_defs()
for item in defs.suites:
for node in item:
n = node.get_abs_node_path()
print(n)
if n == "/" + exp + "/SleepingBeauty":
print("There she is")
ecf_name = node.find_gen_variable("ECF_NAME").value()
ecf_tryno = node.find_gen_variable("ECF_TRYNO").value()
ecf_pass = node.find_gen_variable("ECF_PASS").value()
ecf_rid = node.find_gen_variable("ECF_RID").value()
submission_id = node.find_variable("SUBMISSION_ID").value()
print(ecf_name)
print(submission_id)
argv = [
"-exp", exp, "-lib", "/tmp/host1/scratch/sfx_home/" + exp + "/lib",
"-ecf_name", ecf_name, "-ecf_pass", ecf_pass, "-ecf_tryno",
ecf_tryno, "-ecf_rid", ecf_rid, "-submission_id", submission_id
]
print(argv)
kwargs = experiment.parse_status_cmd_exp(argv)
experiment.status_cmd_exp(**kwargs)
print("kill", argv)
kwargs = experiment.parse_kill_cmd_exp(argv)
experiment.kill_cmd_exp(**kwargs)
test_file = "/tmp/host1/scratch/sfx_home/" + exp + "/SleepingBeauty2"
found = False
for t in range(0, 15):
print(t)
if os.path.exists(test_file):
found = True
break
time.sleep(1)
if not found:
raise FileNotFoundError(test_file + " not found!")
