def test_pgd(self):
exp_name = "pgd"
root = os.getcwd()
wd = "/tmp/host0/hm_wd/" + exp_name
self._create_exp(wd, root)
exp = experiment.ExpFromFiles(exp_name, wd, debug=True)
experiment.init_run(exp, self.stream)
wd = "/tmp/host1/scratch/sfx_home/" + exp_name + "/lib"
exp = experiment.ExpFromFiles(exp_name,
wd,
host=self.host,
stream=self.stream,
progress=self.progress)
ecf_name = "/" + exp_name + "/task"
task = scheduler.EcflowTask(ecf_name, self.ecf_tryno, self.ecf_pass,
self.ecf_rid, self.submission_id)
experiment.Pgd(task,
exp,
host=self.host,
mbr=self.mbr,
stream=self.stream,
args=self.args).run(wrapper=self.wrapper)
os.chdir(root)
def test_quality_control_t2m(self):
exp_name = "quality_control_t2m_task"
root = os.getcwd()
wd = "/tmp/host0/hm_wd/" + exp_name
self._create_exp(wd, root)
exp = experiment.ExpFromFiles(exp_name, wd)
experiment.init_run(exp, self.stream)
wd = "/tmp/host1/scratch/sfx_home/" + exp_name + "/lib"
exp = experiment.ExpFromFiles(exp_name,
wd,
host=self.host,
stream=self.stream,
progress=self.progress)
ecf_name = "/" + exp_name + "/task"
task = scheduler.EcflowTask(ecf_name, self.ecf_tryno, self.ecf_pass,
self.ecf_rid, self.submission_id)
task.family1 = "t2m"
experiment.QualityControl(task,
exp,
host=self.host,
mbr=self.mbr,
stream=self.stream,
args=self.args).run(wrapper=self.wrapper)
os.chdir(root)
def test_submission(self):
exp = "test_submission"
hm_wd = "/tmp/host0/hm_wd/" + exp
self._create_exp(hm_wd)
# Set progress
dtg = datetime(2020, 6, 9, 9)
progress = {"DTG": dtg.strftime("%Y%m%d%H")}
progress_pp = {"DTGPP": dtg.strftime("%Y%m%d%H")}
progress = experiment.Progress(progress, progress_pp)
sfx_exp = experiment.ExpFromFiles(exp, hm_wd, progress=progress)
ecf_name = "/" + exp + "/Forecasting/Forecast"
ecf_tryno = 1
ecf_pass = "******"
task = scheduler.EcflowTask(ecf_name,
ecf_tryno,
ecf_pass,
ecf_rid=int(os.getpid()),
submission_id=None)
os.makedirs("/tmp/host1/job/" + exp + "/Forecasting/", exist_ok=True)
fh = open("/tmp/host1/job/" + exp + "/Forecasting/Forecast.job1", "w")
fh.write("My job generated by ecflow")
fh.close()
exceptions = {
"complete": {
"task": {
"Forecast": "is_coldstart"
},
"family": {
"Forecasting": "is_coldstart"
}
}
}
env_file = "/tmp/host1/job/" + exp + "/Forecasting/Env"
fh = open(env_file, "w")
fh.write("print(\"Oh my environment\n\")")
fh.close()
hosts = sfx_exp.system.hosts
joboutdir = {}
for host in range(0, len(hosts)):
joboutdir.update(
{str(host): sfx_exp.system.get_var("JOBOUTDIR", str(host))})
submit = scheduler.EcflowSubmitTask(task,
sfx_exp.env_submit,
sfx_exp.server,
joboutdir,
submit_exceptions=exceptions,
env_file=env_file)
submit.write_job()
def status_cmd(argv):
kwargs = parse_status_cmd(argv)
exp = kwargs["exp"]
lib = kwargs["lib"]
ecf_name = kwargs["ecf_name"]
ecf_tryno = kwargs["ecf_tryno"]
ecf_pass = kwargs["ecf_pass"]
ecf_rid = kwargs["ecf_rid"]
submission_id = kwargs["submission_id"]
exp = scheduler.ExpFromFiles(exp, lib)
task = scheduler.EcflowTask(ecf_name, ecf_tryno, ecf_pass, ecf_rid, submission_id)
task_settings = scheduler.TaskSettings(task, exp.env_submit, exp.system)
sub = scheduler.get_submission_object(task, task_settings)
sub.status()
def submit_cmd(argv):
kwargs = parse_submit_cmd(argv)
exp = kwargs["exp"]
lib = kwargs["lib"]
ecf_name = kwargs["ecf_name"]
ensmbr = kwargs["ensmbr"]
dtg = kwargs["dtg"]
if dtg is not None:
dtg = datetime.strptime(dtg, "%Y%m%d%H")
ecf_tryno = kwargs["ecf_tryno"]
ecf_pass = kwargs["ecf_pass"]
ecf_rid = kwargs["ecf_rid"]
submission_id = None
stream = kwargs["stream"]
db = kwargs["db"]
try:
if stream is not None:
if stream == "":
stream = None
if ecf_rid is not None:
if ecf_rid == "":
ecf_rid = os.getpid()
else:
ecf_rid = os.getpid()
exp = scheduler.ExpFromFiles(exp, lib)
wd = exp.wd
# coldstart = config.is_coldstart(dtg)
submit_exceptions = wd + "/config//submit/submission.json"
submit_exceptions = json.load(open(submit_exceptions, "r"))
task = scheduler.EcflowTask(ecf_name, ecf_tryno, ecf_pass, ecf_rid, submission_id)
sub = scheduler.EcflowSubmitTask(exp, task, submit_exceptions=submit_exceptions,
ensmbr=ensmbr, dbfile=db)
sub.submit()
except Exception as e:
raise e
def kill_cmd(argv):
kwargs = parse_status_cmd(argv)
exp = kwargs["exp"]
lib = kwargs["lib"]
ecf_name = kwargs["ecf_name"]
ecf_tryno = kwargs["ecf_tryno"]
ecf_pass = kwargs["ecf_pass"]
ecf_rid = kwargs["ecf_rid"]
submission_id = ""
if "submission_id" in kwargs:
submission_id = kwargs["submission_id"]
if submission_id == "":
submission_id = None
exp = scheduler.ExpFromFiles(exp, lib)
task = scheduler.EcflowTask(ecf_name, ecf_tryno, ecf_pass, ecf_rid, submission_id)
task_settings = scheduler.TaskSettings(task, exp.env_submit, exp.system)
print(task.submission_id)
sub = scheduler.get_submission_object(task, task_settings)
print(sub)
sub.kill()
exp.server.force_aborted(task)
#!/usr/bin/env python3
import scheduler
lib = "%LIB%"
exp = "%EXP%"
stream = "%STREAM%"
if stream == "":
stream = None
# InitRun always runs from HOST0
exp = scheduler.ExpFromFiles(exp, lib)
server = exp.server
ecf_name = "%ECF_NAME%"
ecf_pass = "******"
ecf_tryno = "%ECF_TRYNO%"
ecf_rid = "%ECF_RID%"
submission_id = "%SUBMISSION_ID%"
task = scheduler.EcflowTask(ecf_name, ecf_tryno, ecf_pass, ecf_rid,
submission_id)
# This will also handle call to sys.exit(), i.e. Client.__exit__ will still be called.
with scheduler.EcflowClient(server, task) as ci:
scheduler.init_run(exp, stream=stream)