def get_config(wdir, conf):
# Check existence of needed config files
local_config = wdir + "/config/config.toml"
rev_config = conf + "/scheduler/config/config.toml"
if os.path.exists(local_config):
c_files = surfex.toml_load(local_config)
elif os.path.exists(rev_config):
c_files = surfex.toml_load(rev_config)
else:
raise Exception("No config found in " + wdir + " or " + conf)
return c_files
def __init__(self, progress, progress_pp):
self.progress_file = progress
self.progress_pp_file = progress_pp
if os.path.exists(self.progress_file):
progress = surfex.toml_load(self.progress_file)
else:
progress = {"DTGBEG": None, "DTG": None, "DTGEND": None}
if os.path.exists(self.progress_pp_file):
progress_pp = surfex.toml_load(self.progress_pp_file)
else:
progress_pp = {"DTGPP": None}
Progress.__init__(self, progress, progress_pp)
def __init__(self, env_system_file, exp_name):
# system = System.get_file_name(wdir, full_path=True)
print(env_system_file)
if os.path.exists(env_system_file):
host_system = surfex.toml_load(env_system_file)
else:
raise FileNotFoundError(env_system_file)
# print(host_system)
System.__init__(self, host_system, exp_name)
def __init__(self, ecflow_server_file, logfile):
if os.path.exists(ecflow_server_file):
self.settings = surfex.toml_load(ecflow_server_file)
else:
raise FileNotFoundError("Could not find " + ecflow_server_file)
ecf_host = self.get_var("ECF_HOST")
ecf_port_offset = int(self.get_var("ECF_PORT_OFFSET", default=1500))
ecf_port = int(self.get_var("ECF_PORT", default=int(os.getuid())))
ecf_port = ecf_port + ecf_port_offset
# logfile = self.get_var("SERVER_LOG")
EcflowServer.__init__(self, ecf_host, ecf_port, logfile)
def merge_testbed_configurations(self, testbed_confs):
merged_conf = {}
for testbed_configuration in testbed_confs:
print("Merging testbed configuration: " + testbed_configuration)
testbed_configuration = "/config/testbed/" + testbed_configuration
if os.path.exists(self.wd + testbed_configuration):
testbed_configuration = self.wd + testbed_configuration
else:
if os.path.exists(self.rev + testbed_configuration):
testbed_configuration = self.rev + testbed_configuration
else:
raise Exception("Testbed configuration not existing: " + testbed_configuration)
conf = surfex.toml_load(testbed_configuration)
merged_conf = surfex.merge_toml_env(merged_conf, conf)
return merged_conf
def get_config_files(self):
# Check existence of needed config files
config_file = self.wd + "/config/config.toml"
config = toml.load(open(config_file, "r"))
c_files = config["config_files"]
config_files = {}
for f in c_files:
lfile = self.wd + "/config/" + f
if os.path.exists(lfile):
# print("lfile", lfile)
toml_dict = surfex.toml_load(lfile)
else:
raise Exception("No config file found for " + lfile)
config_files.update({
f: {
"toml": toml_dict,
"blocks": config[f]["blocks"]
}
})
return config_files
def setup_files(self, host):
rev_file = Exp.get_file_name(self.wd, "rev")
conf_file = Exp.get_file_name(self.wd, "conf")
open(rev_file, "w").write(self.rev + "\n")
open(conf_file, "w").write(self.conf + "\n")
env_system = Exp.get_file_name(self.wd, "system", full_path=False)
env = Exp.get_file_name(self.wd, "env", full_path=False)
env_submit = Exp.get_file_name(self.wd, "submit", full_path=False)
env_server = Exp.get_file_name(self.wd, "server", full_path=False)
input_paths = Exp.get_file_name(self.wd,
"input_paths",
full_path=False)
# Create needed system files
system_files = {
env_system: "",
env: "",
env_submit: "",
env_server: "",
input_paths: "",
}
system_files.update({
env_system:
"config/system/" + host + ".toml",
env:
"config/env/" + host + ".py",
env_submit:
"config/submit/" + host + ".json",
env_server:
"config/server/" + host + ".toml",
input_paths:
"config/input_paths/" + host + ".json",
})
for key in system_files:
target = self.wd + "/" + key
lfile = self.wd + "/" + system_files[key]
rfile = self.conf + "/scheduler/" + system_files[key]
dirname = os.path.dirname(lfile)
os.makedirs(dirname, exist_ok=True)
if os.path.exists(lfile):
print("System file " + lfile +
" already exists, is not fetched again")
else:
shutil.copy2(rfile, lfile)
if os.path.exists(target):
print("System target file " + lfile +
" already exists, is not fetched again")
else:
os.symlink(system_files[key], target)
self.env_submit = json.load(open(self.wd + "/Env_submit", "r"))
plib = self.wd + "/pysurfex"
config_dirs = ["surfex", "scheduler", "bin"]
for cdir in config_dirs:
if not os.path.exists(plib + "/" + cdir):
print("Copy " + cdir + " from " + self.conf)
shutil.copytree(self.conf + "/" + cdir,
plib + "/" + cdir,
ignore=shutil.ignore_patterns(
"config", "ecf", "nam", "toml"))
else:
print(cdir + " already exists in " + self.wd + "/pysurfex")
# Check existence of needed config files
local_config = self.wd + "/config/config.toml"
rev_config = self.conf + "/scheduler/config/config.toml"
config = surfex.toml_load(rev_config)
c_files = config["config_files"]
if os.path.exists(local_config):
config = surfex.toml_load(local_config)
c_files = config["config_files"]
config_files = []
for f in c_files:
lfile = self.wd + "/config/" + f
config_files.append(lfile)
os.makedirs(self.wd + "/config", exist_ok=True)
rfile = self.conf + "/scheduler/config/" + f
if not os.path.exists(lfile):
# print(rfile, lfile)
shutil.copy2(rfile, lfile)
else:
print("Config file " + lfile +
" already exists, is not fetched again")
dirs = ["config/domains"]
# Copy dirs
for dir_path in dirs:
os.makedirs(self.wd + "/" + dir_path, exist_ok=True)
files = [
f for f in os.listdir(self.conf + "/scheduler/" + dir_path)
if os.path.isfile(
os.path.join(self.conf + "/scheduler/" + dir_path, f))
]
for f in files:
print("f", f)
fname = self.wd + "/" + dir_path + "/" + f
rfname = self.conf + "/scheduler/" + dir_path + "/" + f
if not os.path.exists(fname):
print("Copy " + rfname + " -> " + fname)
shutil.copy2(rfname, fname)
# ECF_submit exceptions
f = "config/submit/submission.json"
fname = self.wd + "/" + f
rfname = self.conf + "/scheduler/" + f
if not os.path.exists(fname):
print("Copy " + rfname + " -> " + fname)
shutil.copy2(rfname, fname)
# Init run
files = ["ecf/InitRun.py", "ecf/default.py"]
os.makedirs(self.wd + "/ecf", exist_ok=True)
for f in files:
fname = self.wd + "/" + f
rfname = self.conf + "/scheduler/" + f
if not os.path.exists(fname):
print("Copy " + rfname + " -> " + fname)
shutil.copy2(rfname, fname)
exp_dirs = ["nam", "toml"]
for exp_dir in exp_dirs:
rdir = self.conf + "/scheduler/" + exp_dir
ldir = self.wd + "/" + exp_dir
print("Copy " + rdir + " -> " + ldir)
shutil.copytree(rdir, ldir)
def __init__(self,
name,
wdir,
rev,
conf,
experiment_is_locked,
system_file_paths=None,
system=None,
server=None,
configuration=None,
geo=None,
env_submit=None,
write_config_files=False,
progress=None):
self.name = name
self.wd = wdir
self.rev = rev
self.conf = conf
self.experiment_is_locked = experiment_is_locked
self.system = system
self.server = server
self.env_submit = env_submit
self.system_file_paths = system_file_paths
self.progress = progress
# Check existence of needed config files
config = Exp.get_config(self.wd, self.conf)
c_files = config["config_files"]
config_files = {}
for f in c_files:
lfile = self.wd + "/config/" + f
rfile = self.conf + "/scheduler/config/" + f
if os.path.exists(lfile):
# print("lfile", lfile)
toml_dict = surfex.toml_load(lfile)
else:
if os.path.exists(rfile):
# print("rfile", rfile)
toml_dict = surfex.toml_load(rfile)
else:
raise Exception("No config file found for " + f)
config_files.update(
{f: {
"toml": toml_dict,
"blocks": config[f]["blocks"]
}})
self.config_files = config_files
do_merge = False
if configuration is not None:
do_merge = True
conf = self.wd + "/config/configurations/" + configuration.lower(
) + ".toml"
if not os.path.exists(conf):
conf = self.conf + "/scheduler/config/configurations/" + configuration.lower(
) + ".toml"
if not os.path.exists(conf):
raise Exception("Can not find configuration " +
configuration + " in: " + conf)
configuration = surfex.toml_load(conf)
if do_merge:
self.merge_to_toml_config_files(
configuration=configuration,
write_config_files=write_config_files)
# Merge config
all_merged_settings = surfex.merge_toml_env_from_config_dicts(
self.config_files)
merged_config, member_merged_config = surfex.process_merged_settings(
all_merged_settings)
# Create configuration
self.config = surfex.Configuration(merged_config,
member_merged_config,
geo=geo)
def setup_files(self, host, configuration=None, configuration_file=None):
rev_file = Exp.get_file_name(self.wd, "rev", full_path=True)
pysurfex_file = Exp.get_file_name(self.wd, "pysurfex", full_path=True)
pysurfex_scheduler_file = Exp.get_file_name(self.wd, "pysurfex_scheduler", full_path=True)
pysurfex_experiment_file = Exp.get_file_name(self.wd, "pysurfex_experiment", full_path=True)
open(rev_file, "w").write(self.rev + "\n")
open(pysurfex_file, "w").write(self.pysurfex + "\n")
open(pysurfex_scheduler_file, "w").write(self.pysurfex_scheduler + "\n")
open(pysurfex_experiment_file, "w").write(self.pysurfex_experiment + "\n")
print("rev: " + self.rev + " stored in:" + rev_file)
print("pysurfex: " + self.pysurfex + " stored in:" + pysurfex_file)
print("pysurfex_scheduler: " + self.pysurfex_scheduler + " stored in:" + pysurfex_scheduler_file)
print("pysurfex_experiment: " + self.pysurfex_experiment + " stored in:" + pysurfex_experiment_file)
env_system = Exp.get_file_name(self.wd, "system", full_path=False)
env = Exp.get_file_name(self.wd, "env", full_path=False)
env_submit = Exp.get_file_name(self.wd, "submit", full_path=False)
env_server = Exp.get_file_name(self.wd, "server", full_path=False)
input_paths = Exp.get_file_name(self.wd, "input_paths", full_path=False)
# Create needed system files
system_files = {
env_system: "",
env: "",
env_submit: "",
env_server: "",
input_paths: "",
}
system_files.update({
env_system: "config/system/" + host + ".toml",
env: "config/env/" + host + ".py",
env_submit: "config/submit/" + host + ".json",
env_server: "config/server/" + host + ".toml",
input_paths: "config/input_paths/" + host + ".json",
})
fetched = []
revs = [self.rev, self.pysurfex_experiment]
for key in system_files:
for rev in revs:
lfile = self.wd + "/" + system_files[key]
rfile = rev + "/" + system_files[key]
dirname = os.path.dirname(lfile)
print(rfile, lfile)
print(dirname)
os.makedirs(dirname, exist_ok=True)
if os.path.exists(lfile):
if lfile not in fetched:
print("System file " + lfile + " already exists, is not fetched again from " + rev)
else:
if os.path.exists(rfile) and lfile != rfile:
print("Copy " + rfile + " to " + lfile)
shutil.copy2(rfile, lfile)
fetched.append(lfile)
target = self.wd + "/" + key
lfile = self.wd + "/" + system_files[key]
if os.path.islink(target):
if target not in fetched:
print("System target file " + target + " already exists")
else:
os.symlink(lfile, target)
fetched.append(target)
self.env_submit = json.load(open(self.wd + "/Env_submit", "r"))
# Set up pysurfex_experiment
print("Set up experiment from ", revs)
config_dirs = ["experiment", "bin"]
for rev in revs:
for cdir in config_dirs:
os.makedirs(self.wd + "/" + cdir, exist_ok=True)
files = [f for f in os.listdir(rev + "/" + cdir)
if os.path.isfile(os.path.join(rev + "/" + cdir, f))]
for f in files:
lfile = self.wd + "/" + cdir + "/" + f
rfile = rev + "/" + cdir + "/" + f
dirname = os.path.dirname(lfile)
os.makedirs(dirname, exist_ok=True)
if not os.path.exists(lfile):
if os.path.exists(rfile) and lfile != rfile:
# print("Copy " + rfile + " to " + lfile)
shutil.copy2(rfile, lfile)
fetched.append(lfile)
else:
if lfile not in fetched:
print(lfile + " already exists, is not fetched again from " + rev)
# Check existence of needed config files
config_file = self.wd + "/config/config.toml"
os.makedirs(self.wd + "/config/", exist_ok=True)
for rev in revs:
lfile = config_file
rfile = rev + "/config/config.toml"
if not os.path.exists(lfile):
if os.path.exists(rfile) and lfile != rfile:
print("Copy " + rfile + " to " + lfile)
shutil.copy2(rfile, lfile)
fetched.append(lfile)
else:
if lfile not in fetched:
print(lfile + " already exists, is not fetched again from " + rev)
c_files = toml.load(open(config_file, "r"))["config_files"]
for rev in [self.pysurfex] + revs:
cfg = "/config/"
if rev == self.pysurfex:
cfg = "/cfg/"
config_files = []
os.makedirs(self.wd + "/config/", exist_ok=True)
for f in c_files:
lfile = self.wd + "/config/" + f
config_files.append(lfile)
os.makedirs(self.wd + "/config", exist_ok=True)
rfile = rev + cfg + f
if not os.path.exists(lfile):
if self.debug:
print(rfile, lfile)
if os.path.exists(rfile) and lfile != rfile:
print("Copy " + rfile + " -> " + lfile)
shutil.copy2(rfile, lfile)
fetched.append(lfile)
else:
if lfile not in fetched:
print("Config file " + lfile + " already exists, is not fetched again from " + rev)
for f in c_files:
lfile = self.wd + "/config/" + f
if not os.path.exists(lfile):
raise Exception("Config file not found ", lfile)
conf = None
if configuration is not None:
print("Using configuration ", configuration)
conf = self.wd + "/config/configurations/" + configuration.lower() + ".toml"
if not os.path.exists(conf):
conf = self.pysurfex_experiment + "/config/configurations/" + configuration.lower() + ".toml"
print("Configuration file ", configuration_file)
elif configuration_file is not None:
print("Using configuration from file ", configuration_file)
conf = configuration_file
self.config_files = self.get_config_files()
if conf is not None:
if not os.path.exists(conf):
raise Exception("Can not find configuration " + configuration + " in: " + conf)
configuration = surfex.toml_load(conf)
self.merge_to_toml_config_files(configuration=configuration, write_config_files=True)
print("Set up domains from ", revs)
dirs = ["config/domains"]
# Copy dirs
for dir_path in dirs:
os.makedirs(self.wd + "/" + dir_path, exist_ok=True)
for rev in revs:
files = [f for f in os.listdir(rev + "/" + dir_path)
if os.path.isfile(os.path.join(rev + "/" + dir_path, f))]
for f in files:
lfile = self.wd + "/" + dir_path + "/" + f
rfile = rev + "/" + dir_path + "/" + f
if not os.path.exists(lfile):
if self.debug:
print(rfile, lfile)
if os.path.exists(rfile) and lfile != rfile:
if self.debug:
print("Copy " + rfile + " -> " + lfile)
shutil.copy2(rfile, lfile)
fetched.append(lfile)
else:
if lfile not in fetched:
print(lfile + " already exists, is not fetched again from " + rev)
print("Set up submission exceptions from ", revs)
for rev in revs:
# ECF_submit exceptions
f = "config/submit/submission.json"
lfile = self.wd + "/" + f
rfile = rev + "/" + f
if not os.path.exists(lfile):
if os.path.exists(rfile) and lfile != rfile:
if self.debug:
print("Copy " + rfile + " -> " + lfile)
shutil.copy2(rfile, lfile)
fetched.append(lfile)
else:
if lfile not in fetched:
print(lfile + " already exists, is not fetched again from " + rev)
print("Set up ecflow default containers from ", revs)
# Init run
files = ["ecf/InitRun.py", "ecf/default.py"]
os.makedirs(self.wd + "/ecf", exist_ok=True)
for rev in revs:
for f in files:
lfile = self.wd + "/" + f
rfile = rev + "/" + f
if not os.path.exists(lfile):
if self.debug:
print(rfile, lfile)
if os.path.exists(rfile) and lfile != rfile:
if self.debug:
print("Copy " + rfile + " -> " + lfile)
shutil.copy2(rfile, lfile)
fetched.append(lfile)
else:
if lfile not in fetched:
print(lfile + " already exists, is not fetched again from " + rev)
print("Copy namelists from ", revs)
exp_dirs = ["nam"]
for rev in revs:
for exp_dir in exp_dirs:
os.makedirs(self.wd + "/" + exp_dir, exist_ok=True)
rdir = rev + "/" + exp_dir
ldir = self.wd + "/" + exp_dir
files = [f for f in os.listdir(rev + "/" + exp_dir)
if os.path.isfile(os.path.join(rev + "/" + exp_dir, f))]
for f in files:
lfile = ldir + "/" + f
rfile = rdir + "/" + f
dirname = os.path.dirname(lfile)
os.makedirs(dirname, exist_ok=True)
if not os.path.exists(lfile):
if os.path.exists(rfile) and lfile != rfile:
if self.debug:
print("Copy " + rfile + " -> " + lfile)
# shutil.copytree(rdir, ldir)
shutil.copy2(rfile, lfile)
fetched.append(lfile)
else:
if lfile not in fetched:
print(lfile + " already exists, is not fetched again from " + rev)