def _read_date_file(self, config, date_file=None):
if not date_file:
date_file = (
config["general"]["base_dir"]
+ "/"
+ config["general"]["expid"]
+ "/scripts/"
+ config["general"]["expid"]
+ "_"
+ config["general"]["setup_name"]
+ ".date"
)
if os.path.isfile(date_file):
logging.info("Date file read from %s", date_file)
with open(date_file) as date_file:
date, self.run_number = date_file.readline().strip().split()
write_file = False
else:
logging.info("No date file found %s", date_file)
logging.info("Initializing run_number=1 and date=18500101")
date = config["general"].get("initial_date", "18500101")
self.run_number = 1
write_file = True
config["general"]["run_number"] = self.run_number
self.current_date = Date(date)
# needs to happen AFTER a run!
# if write_file:
# self._write_date_file()
logging.info("current_date = %s", self.current_date)
logging.info("run_number = %s", self.run_number)
def _initialize_calendar(self, config):
nyear, nmonth, nday, nhour, nminute, nsecond = 0, 0, 0, 0, 0, 0
nyear = int(config["general"].get("nyear", nyear))
if not nyear:
nmonth = int(config["general"].get("nmonth", nmonth))
if not nyear and not nmonth:
nday = int(config["general"].get("nday", nday))
if not nyear and not nmonth and not nday:
nhour = int(config["general"].get("nhour", nhour))
if not nyear and not nmonth and not nday and not nhour:
nminute = int(config["general"].get("nminute", nminute))
if not nyear and not nmonth and not nday and not nhour and not nminute:
nsecond = int(config["general"].get("nsecond", nsecond))
if (
not nyear
and not nmonth
and not nday
and not nhour
and not nminute
and not nsecond
):
nyear = 1
self.delta_date = (nyear, nmonth, nday, nhour, nminute, nsecond)
config["general"]["current_date"] = self.current_date
config["general"]["start_date"] = self.current_date
config["general"]["initial_date"] = Date(config["general"]["initial_date"])
config["general"]["final_date"] = Date(config["general"]["final_date"])
config["general"]["prev_date"] = self.current_date.sub((0, 0, 1, 0, 0, 0))
config["general"]["next_date"] = self.current_date.add(self.delta_date)
config["general"]["end_date"] = config["general"]["next_date"].sub(
(0, 0, 1, 0, 0, 0)
)
config["general"]["runtime"] = (
config["general"]["next_date"] - config["general"]["current_date"]
)
config["general"]["total_runtime"] = (
config["general"]["next_date"] - config["general"]["initial_date"]
)
self.run_datestamp = (
config["general"]["current_date"].format(
form=9, givenph=False, givenpm=False, givenps=False
)
+ "-"
+ config["general"]["end_date"].format(
form=9, givenph=False, givenpm=False, givenps=False
)
)
def set_most_dates(config):
from esm_calendar import Calendar, Date
calendar = config["general"]["calendar"]
if isinstance(config["general"]["current_date"], Date):
current_date = config["general"]["current_date"]
else:
current_date = Date(config["general"]["current_date"], calendar)
delta_date = (
config["general"]["nyear"],
config["general"]["nmonth"],
config["general"]["nday"],
config["general"]["nhour"],
config["general"]["nminute"],
config["general"]["nsecond"],
)
config["general"]["delta_date"] = delta_date
config["general"]["current_date"] = current_date
config["general"]["start_date"] = current_date
config["general"]["initial_date"] = Date(config["general"]["initial_date"],
calendar)
config["general"]["final_date"] = Date(config["general"]["final_date"],
calendar)
config["general"]["prev_date"] = current_date - (0, 0, 1, 0, 0, 0)
config["general"]["next_date"] = current_date.add(delta_date)
config["general"]["last_start_date"] = current_date - delta_date
config["general"]["end_date"] = config["general"]["next_date"] - (0, 0, 1,
0, 0, 0)
config["general"]["runtime"] = (config["general"]["next_date"] -
config["general"]["current_date"])
config["general"]["total_runtime"] = (config["general"]["next_date"] -
config["general"]["initial_date"])
config["general"]["run_datestamp"] = (
config["general"]["current_date"].format(
form=9, givenph=False, givenpm=False, givenps=False) + "-" +
config["general"]["end_date"].format(
form=9, givenph=False, givenpm=False, givenps=False))
config["general"]["last_run_datestamp"] = (
config["general"]["last_start_date"].format(
form=9, givenph=False, givenpm=False, givenps=False) + "-" +
config["general"]["prev_date"].format(
form=9, givenph=False, givenpm=False, givenps=False))
return config
def test_date_math_add(self):
new_entry = esm_parser.do_math_in_entry(
["date_math_add", None],
self.math_test_dict["date_math_add"],
self.math_test_dict,
)
self.assertEqual(str(Date.from_list([1851, 1, 1, 0, 0, 0])), new_entry)
def setUp(self):
self.date_test_dict = {
"skip_var": "${something.something}",
"skip_date": Date.fromlist([1850, 1, 1, 0, 0, 0]),
"create_date": "18500101" + esm_parser.DATE_MARKER,
"just_year_date": "18500101!year" + esm_parser.DATE_MARKER,
"yearmonth_date": "18500101!syear!smonth" + esm_parser.DATE_MARKER,
"return_num": 1,
"return_bool": True,
}
def find_last_prepared_run(config):
from esm_calendar import Date, Calendar
import os
import sys
calendar = config["general"]["calendar"]
current_date = Date(config["general"]["current_date"], calendar)
initial_date = Date(config["general"]["initial_date"], calendar)
delta_date = (
config["general"]["nyear"],
config["general"]["nmonth"],
config["general"]["nday"],
config["general"]["nhour"],
config["general"]["nminute"],
config["general"]["nsecond"],
)
while True:
if current_date < initial_date:
break
next_date = current_date.add(delta_date)
end_date = next_date - (0, 0, 1, 0, 0, 0)
datestamp = (current_date.format(
form=9, givenph=False, givenpm=False, givenps=False) + "-" +
end_date.format(
form=9, givenph=False, givenpm=False, givenps=False))
if os.path.isdir(config["general"]["base_dir"] +
config["general"]["expid"] + "/run_" + datestamp):
config["general"]["current_date"] = current_date
return config
current_date = current_date - delta_date
print("Could not find a prepared run.")
sys.exit(42)
class SimulationSetup(object):
def __init__(self, name, user_config):
self.check = False
if not "expid" in user_config["general"]:
user_config["general"]["expid"] = "test"
if user_config["general"]["setup_name"] in user_config:
user_config["general"].update(
user_config[user_config["general"]["setup_name"]]
)
del user_config[user_config["general"]["setup_name"]]
self._read_date_file(user_config)
self._initialize_calendar(user_config)
self.config = esm_parser.ConfigSetup(name, user_config)
del user_config
if "check" in self.config["general"]:
self.check = self.config["general"]["check"]
self._add_all_folders()
self.config.calendar()
self.config.finalize()
self._initialize_components()
self._finalize_components()
self._finalize_attributes()
self._write_finalized_config()
self._copy_preliminary_files_from_experiment_to_thisrun()
self._show_simulation_info()
self.write_batch_system_hostfile()
self.prepare()
self.write_simple_runscript()
if self.check:
sys.exit()
self.submit()
def submit(self):
six.print_("\n", 40 * "+ ")
print ("Submitting sad jobscript to batch system...")
for command in self.submit_command:
print (command)
six.print_("\n", 40 * "+ ")
for command in self.submit_command:
os.system(command)
sys.exit()
def write_batch_system_hostfile(self):
import esm_batch_system
self.batch = esm_batch_system.esm_batch_system(self.config, self.config["computer"]["batch_system"])
def get_sad_filename(self):
folder = self.config["general"]["thisrun_scripts_dir"]
expid = self.config["general"]["expid"]
startdate = self.config["general"]["current_date"]
enddate = self.config["general"]["end_date"]
return folder + "/" + expid+"_"+self.run_datestamp+".sad"
def get_environment(self):
environment = []
import esm_environment
env = esm_environment.environment_infos()
if "module_actions" in env.config:
for action in env.config["module_actions"]:
environment.append("module " + action)
environment.append("")
if "export_vars" in env.config:
for var in env.config["export_vars"].keys():
environment.append(
"export " + var + "=" + env.config["export_vars"][var]
)
return environment
def calculate_requirements(self):
tasks = 0
for model in self.config["general"]["models"]:
if "nproc" in self.config[model]:
tasks += self.config[model]["nproc"]
elif "nproca" in self.config[model] and "nprocb" in self.config[model]:
tasks += self.config[model]["nproca"] * self.config[model]["nprocb"]
return tasks
def get_batch_header(self):
header = []
batch_system = self.config["computer"]
if "sh_interpreter" in batch_system:
header.append("#!"+batch_system["sh_interpreter"])
tasks = self.calculate_requirements()
replacement_tags = [("@tasks@", tasks)]
all_flags = ["partition_flag",
"time_flag",
"tasks_flag",
"output_flags",
"name_flag",
]
conditional_flags = ["exclusive_flag",
"accounting_flag",
"notification_flag",
"hyperthreading_flag",
"additional_flags"
]
for flag in conditional_flags:
if flag in batch_system and not batch_system[flag].strip() == "":
all_flags.append(flag)
for flag in all_flags:
for (tag, repl) in replacement_tags:
batch_system[flag] = batch_system[flag].replace(tag, str(repl))
header.append(batch_system["header_start"] + " " + batch_system[flag])
return header
def get_run_commands(self):
commands = []
batch_system = self.config["computer"]
if "execution_command" in batch_system:
commands.append(batch_system["execution_command"])
return commands
def get_submit_command(self, sadfilename):
commands = []
batch_system = self.config["computer"]
if "submit" in batch_system:
commands.append("cd " + self.config["general"]["thisrun_scripts_dir"] + "; " + batch_system["submit"] + " " +sadfilename)
return commands
def write_simple_runscript(self):
sadfilename = self.get_sad_filename()
header = self.get_batch_header()
environment = self.get_environment()
commands = self.get_run_commands()
with open(sadfilename, "w") as sadfile:
for line in header:
sadfile.write(line + "\n")
sadfile.write("\n")
for line in environment:
sadfile.write(line + "\n")
sadfile.write("\n")
sadfile.write("cd "+ self.config["general"]["thisrun_work_dir"] + "\n")
for line in commands:
sadfile.write(line + "\n")
self.submit_command = self.get_submit_command(sadfilename)
six.print_("\n", 40 * "+ ")
six.print_("Contents of ",sadfilename, ":")
with open(sadfilename, "r") as fin:
print (fin.read())
six.print_("\n", 40 * "+ ")
six.print_("Contents of ",self.batch.bs.filename, ":")
with open(self.batch.bs.path, "r") as fin:
print (fin.read())
def _add_all_folders(self):
self.all_filetypes = ["analysis", "config", "log", "mon", "scripts"]
for filetype in self.all_filetypes:
setattr(
self,
"experiment_" + filetype + "_dir",
self.config["general"]["base_dir"]
+ "/"
+ self.config["general"]["expid"]
+ "/"
+ filetype
+ "/",
)
self.config["general"]["experiment_" + filetype + "_dir"] = getattr(
self, "experiment_" + filetype + "_dir"
)
self.all_filetypes.append("work")
for filetype in self.all_filetypes:
setattr(
self,
"thisrun_" + filetype + "_dir",
self.config["general"]["base_dir"]
+ "/"
+ self.config["general"]["expid"]
+ "/run_"
+ self.run_datestamp
+ "/"
+ filetype
+ "/",
)
self.config["general"]["thisrun_" + filetype + "_dir"] = getattr(
self, "thisrun_" + filetype + "_dir"
)
self.all_model_filetypes = [
"analysis",
"bin",
"config",
"couple",
"forcing",
"input",
"log",
"mon",
"outdata",
"restart",
"viz",
]
for model in self.config["general"]["valid_model_names"]:
for filetype in self.all_model_filetypes:
setattr(
self,
"experiment_" + model + "_" + filetype + "_dir",
self.config["general"]["base_dir"]
+ "/"
+ self.config["general"]["expid"]
+ "/"
+ filetype
+ "/"
+ model
+ "/",
)
setattr(
self,
"thisrun_" + model + "_" + filetype + "_dir",
self.config["general"]["base_dir"]
+ "/"
+ self.config["general"]["expid"]
+ "/run_"
+ self.run_datestamp
+ "/"
+ filetype
+ "/"
+ model
+ "/",
)
self.config[model][
"experiment_" + model + "_" + filetype + "_dir"
] = getattr(self, "experiment_" + model + "_" + filetype + "_dir")
self.config[model][
"thisrun_" + model + "_" + filetype + "_dir"
] = getattr(self, "thisrun_" + model + "_" + filetype + "_dir")
def _initialize_components(self):
components = []
for component in self.config["general"]["valid_model_names"]:
components.append(
SimulationComponent(self.config["general"], self.config[component])
)
self.components = components
def _create_folders(self):
for filetype in self.all_filetypes:
if not os.path.exists(getattr(self, "experiment_" + filetype + "_dir")):
os.makedirs(getattr(self, "experiment_" + filetype + "_dir"))
def _dump_final_yaml(self):
with open(
self.experiment_config_dir
+ "/"
+ self.config["general"]["expid"]
+ "_preconfig.yaml",
"w",
) as config_file:
yaml.dump(self.config, config_file)
def _show_simulation_info(self):
six.print_(80 * "=")
six.print_("STARTING SIMULATION JOB!")
six.print_("Experiment ID = %s" % self.config["general"]["expid"])
six.print_("Setup = %s" % self.config["general"]["setup_name"])
six.print_("This setup consists of:")
for model in self.config["general"]["valid_model_names"]:
six.print_("- %s" % model)
six.print_("You are using the Python version.")
def _finalize_attributes(self):
for filetype in self.all_filetypes:
setattr(
self,
"thisrun_" + filetype + "_dir",
self.config["general"]["base_dir"]
+ "/"
+ self.config["general"]["expid"]
+ "/run_"
+ self.config["general"]["current_date"].format(
form=9, givenph=False, givenpm=False, givenps=False
)
+ "-"
+ self.config["general"]["end_date"].format(
form=9, givenph=False, givenpm=False, givenps=False
)
+ "/"
+ filetype
+ "/",
)
if not os.path.exists(getattr(self, "thisrun_" + filetype + "_dir")):
os.makedirs(getattr(self, "thisrun_" + filetype + "_dir"))
self.run_datestamp = (
self.config["general"]["current_date"].format(
form=9, givenph=False, givenpm=False, givenps=False
)
+ "-"
+ self.config["general"]["end_date"].format(
form=9, givenph=False, givenpm=False, givenps=False
)
)
def _write_finalized_config(self):
pp.pprint(self.config)
with open(
self.thisrun_config_dir
+ "/"
+ self.config["general"]["expid"]
+ "_finished_config.yaml",
"w",
) as config_file:
yaml.dump(self.config, config_file)
def _finalize_components(self):
for component in self.components:
component.general_config = self.config["general"]
component.finalize_attributes()
def _copy_preliminary_files_from_experiment_to_thisrun(self):
filelist = [
(
"scripts",
self.config["general"]["expid"]
+ "_"
+ self.config["general"]["setup_name"]
+ ".date",
"copy",
)
]
for filetype, filename, copy_or_link in filelist:
source = getattr(self, "experiment_" + filetype + "_dir")
dest = getattr(self, "thisrun_" + filetype + "_dir")
if copy_or_link == "copy":
method = shutil.copy2
elif copy_or_link == "link":
method = os.symlink
if os.path.isfile(source + "/" + filename):
method(source + "/" + filename, dest + "/" + filename)
def _read_date_file(self, config, date_file=None):
if not date_file:
date_file = (
config["general"]["base_dir"]
+ "/"
+ config["general"]["expid"]
+ "/scripts/"
+ config["general"]["expid"]
+ "_"
+ config["general"]["setup_name"]
+ ".date"
)
if os.path.isfile(date_file):
logging.info("Date file read from %s", date_file)
with open(date_file) as date_file:
date, self.run_number = date_file.readline().strip().split()
write_file = False
else:
logging.info("No date file found %s", date_file)
logging.info("Initializing run_number=1 and date=18500101")
date = config["general"].get("initial_date", "18500101")
self.run_number = 1
write_file = True
config["general"]["run_number"] = self.run_number
self.current_date = Date(date)
# needs to happen AFTER a run!
# if write_file:
# self._write_date_file()
logging.info("current_date = %s", self.current_date)
logging.info("run_number = %s", self.run_number)
def _initialize_calendar(self, config):
nyear, nmonth, nday, nhour, nminute, nsecond = 0, 0, 0, 0, 0, 0
nyear = int(config["general"].get("nyear", nyear))
if not nyear:
nmonth = int(config["general"].get("nmonth", nmonth))
if not nyear and not nmonth:
nday = int(config["general"].get("nday", nday))
if not nyear and not nmonth and not nday:
nhour = int(config["general"].get("nhour", nhour))
if not nyear and not nmonth and not nday and not nhour:
nminute = int(config["general"].get("nminute", nminute))
if not nyear and not nmonth and not nday and not nhour and not nminute:
nsecond = int(config["general"].get("nsecond", nsecond))
if (
not nyear
and not nmonth
and not nday
and not nhour
and not nminute
and not nsecond
):
nyear = 1
self.delta_date = (nyear, nmonth, nday, nhour, nminute, nsecond)
config["general"]["current_date"] = self.current_date
config["general"]["start_date"] = self.current_date
config["general"]["initial_date"] = Date(config["general"]["initial_date"])
config["general"]["final_date"] = Date(config["general"]["final_date"])
config["general"]["prev_date"] = self.current_date.sub((0, 0, 1, 0, 0, 0))
config["general"]["next_date"] = self.current_date.add(self.delta_date)
config["general"]["end_date"] = config["general"]["next_date"].sub(
(0, 0, 1, 0, 0, 0)
)
config["general"]["runtime"] = (
config["general"]["next_date"] - config["general"]["current_date"]
)
config["general"]["total_runtime"] = (
config["general"]["next_date"] - config["general"]["initial_date"]
)
self.run_datestamp = (
config["general"]["current_date"].format(
form=9, givenph=False, givenpm=False, givenps=False
)
+ "-"
+ config["general"]["end_date"].format(
form=9, givenph=False, givenpm=False, givenps=False
)
)
def _increment_date_and_run_number(self):
self.run_number += 1
self.current_date += self.delta_date
def _write_date_file(self, date_file=None):
if not date_file:
date_file = (
self.experiment_scripts_dir
+ "/"
+ self.config["general"]["expid"]
+ "_"
+ self.config["general"]["setup_name"]
+ ".date"
)
with open(date_file, "w") as date_file:
date_file.write(self.current_date.output() + " " + str(self.run_number))
def prepare(self):
six.print_("=" * 80, "\n")
six.print_("PREPARING EXPERIMENT")
# Copy files:
all_files_to_copy = []
six.print_("\n" "- Generating file lists for this run...")
for component in self.components:
six.print_("-" * 80)
six.print_("* %s" % component.config["model"], "\n")
all_component_files, filetype_specific_dict = (
component.filesystem_to_experiment()
)
with open(
component.thisrun_config_dir
+ "/"
+ self.config["general"]["expid"]
+ "_filelist_"
+ self.run_datestamp,
"w",
) as flist:
flist.write(
"These files are used for \nexperiment %s\ncomponent %s\ndate %s"
% (
self.config["general"]["expid"],
component.config["model"],
self.run_datestamp,
)
)
flist.write("\n")
flist.write(80 * "-")
for filetype in filetype_specific_dict:
flist.write("\n" + filetype.upper() + ":\n")
for source, exp_tree_name, work_dir_name in filetype_specific_dict[
filetype
]:
flist.write("\nSource: " + source)
flist.write("\nExp Tree: " + exp_tree_name)
flist.write("\nWork Dir: " + work_dir_name)
flist.write("\n")
flist.write("\n")
flist.write(80 * "-")
esm_parser.pprint_config(filetype_specific_dict)
all_files_to_copy += all_component_files
six.print_("\n" "- File lists populated, proceeding with copy...")
six.print_("- Note that you can see your file lists in the config folder")
six.print_("- You will be informed about missing files")
self._prepare_copy_files(all_files_to_copy)
# Load and modify namelists:
six.print_("\n" "- Setting up namelists for this run...")
all_namelists = {}
for component in self.components:
six.print_("-" * 80)
six.print_("* %s" % component.config["model"], "\n")
component.nmls_load()
component.nmls_remove()
component.nmls_modify()
component.nmls_finalize(all_namelists)
six.print_(
"\n" "- Namelists modified according to experiment specifications..."
)
for nml_name, nml in all_namelists.items():
six.print_("Contents of ", nml_name, ":")
nml.write(sys.stdout)
six.print_("\n", 40 * "+ ")
self._prepare_modify_files()
for model in list(self.config):
if model in esm_coupler.known_couplers:
coupler_config_dir = (
self.config["general"]["base_dir"]
+ "/"
+ self.config["general"]["expid"]
+ "/run_"
+ self.run_datestamp
+ "/config/"
+ model
+ "/"
)
self.coupler = esm_coupler.esm_coupler(self.config, model)
self.coupler.prepare(self.config, coupler_config_dir)
coupler_filename="namcouple" # needs to be set by function above
all_files_to_copy.append(
(
"",
"",
coupler_config_dir + "/" + coupler_filename,
)
)
#print (coupler_config_dir + "/" + coupler_filename)
all_files_to_copy.append(
(
"",
"",
self.batch.bs.path,
)
)
self._prepare_copy_files_to_work(all_files_to_copy)
def _prepare_copy_files(self, flist):
successful_files = []
missing_files = []
# TODO: Check if we are on login node or elsewhere for the progress
# bar, it doesn't make sense on the compute nodes:
for ftuple in tqdm.tqdm(
flist,
bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]",
):
logging.debug(ftuple)
(file_source, file_intermediate, file_target) = ftuple
try:
shutil.copy2(file_source, file_intermediate)
if not os.path.isfile(file_target):
os.symlink(file_intermediate, file_target)
successful_files.append(file_target)
except IOError:
missing_files.append(file_target)
if missing_files:
six.print_("--- WARNING: These files were missing:")
for missing_file in missing_files:
six.print_("- %s" % missing_file)
def _prepare_copy_files_to_work(self, flist):
successful_files = []
missing_files = []
# TODO: Check if we are on login node or elsewhere for the progress
# bar, it doesn't make sense on the compute nodes:
for ftuple in tqdm.tqdm(
flist,
bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]",
):
logging.debug(ftuple)
(file_source, file_intermediate, file_target) = ftuple
file_in_work = self.thisrun_work_dir + "/" + file_target.split("/", -1)[-1]
try:
shutil.copy2(file_target, file_in_work)
successful_files.append(file_target)
except IOError:
missing_files.append(file_target)
if missing_files:
six.print_("--- WARNING: These files were missing:")
for missing_file in missing_files:
six.print_("- %s" % missing_file)
#sys.exit()
def _prepare_modify_files(self):
for model in self.config['general']['valid_model_names']:
for filetype in self.all_model_filetypes:
#print(self.config[model].get(filetype+"_modifications"))
if filetype == "restart":
nothing = "nothing"