def write_to_log(config, message, message_sep=None):
"""
Puts a message into the experiment log file
Parameters
----------
message : list
A list of the message elements; which is joined by either (highest
to lowest): 1) the message_sep argument passed to the method, 2)
The user's chosen seperator, as written in
``config["general"]["experiment_log_file_message_sep"]``, 3)
An empty space ``" "``.
message_sep : None
The hard-coded message seperator to use; which ignores user choices.
Note
----
The user can control two things regarding the logfile format:
1) The datestamp formatting, whjich is taken from the config
section ``general.experiment_log_file_dateformat``.
2) The message seperators; taken from
``general.experiment_log_file_message_sep``. Note that if the
programmer passes a ``message_sep`` argument; this one wins over
the user choice.
"""
try:
with open(config["general"]["experiment_log_file"], "a+") as logfile:
line = assemble_log_message(config, message, message_sep)
logfile.write(line + "\n")
except KeyError:
import esm_parser
print(
"Sorry; couldn't find 'experiment_log_file' in config['general']..."
)
esm_parser.pprint_config(config["general"])
raise
def model_env_into_computer(config):
'''
This function allows to store in the ``computer`` dictionary, variables that were
defined inside ``environment_changes`` or ``compile/runtime_environment_changes``
in the components.
It excludes ``module_actions`` and ``export_vars`` dictionaries as those are
resolved later.
This function is necessary for controlling ``choose_`` blocks in the computer file
from the component configuration file (i.e. add ``useMPI`` case to the component to
control which ``useMPI`` case is selected in the computer file).
This function works both for compilation time and run time, and the result is that
all components work under the same environment. The only exception is for
the compilation of components, where ``add_export_vars`` and ``add_module_actions``
are excluded from the merging into ``computer``, and are included individually in
respective compilation scripts.
Later on, it might be desirable to always split the environments both for compiling
(done by Paul Gierz, but this function would need to be adapted) and running (not
done yet).
If this script gives you problems contact Miguel Andres-Martinez
([email protected]).
Parameters
----------
config : dict
Dictionary containing the simulation/compilation information
Raises
------
User Note/Error
If the same variable is found in two or more different component environments.
Asks the user how to proceed.
'''
import copy
import logging
from esm_parser import basic_choose_blocks, dict_merge, user_note, user_error, pprint_config
# Get which type of changes are to be applied to the environment
run_or_compile = config.get("general", {}).get("run_or_compile", "runtime")
thesechanges = run_or_compile + "_environment_changes"
# List the component names
models = config.get("general", {}).get("models", [])
# ``env_vars`` stores information about the environment changes to inform
# about conflicts
env_vars = {}
# Loop through the models
for model in models:
# Update the models ``environment_changes`` with the ``compiletime/runtime_
# environment_changes
modelconfig = copy.deepcopy(config[model])
modelconfig["environment_changes"] = modelconfig.get(
"environment_changes", {})
if thesechanges in modelconfig:
if "environment_changes" in modelconfig:
modelconfig["environment_changes"].update(
modelconfig[thesechanges])
else:
modelconfig["environment_changes"] = modelconfig[thesechanges]
# Resolve ``choose_`` blocks, ``add_`` and ``remove_`` inside ``environment_
# changes``
basic_choose_blocks(modelconfig["environment_changes"], config)
# Set to true when specified by the user in ``env_overwrite`` or when this
# method has been already called once in this run
overwrite = config[model].get("env_overwrite", False)
# Loop through every variable in ``environment_changes`` except ``export_vars``
# and ``module_actions`` as those are solved by ``esm_environment`` later and
# need of the solving of later ``choose_`` blocks.
for key, value in modelconfig["environment_changes"].items():
if (key not in [
"export_vars", "module_actions", "add_export_vars",
"add_module_actions"
] and "computer" in config and not overwrite
#and run_or_compile=="runtime"
):
# If the key is already included in ``env_vars``, the key variable has
# been already modified by a previous model and a warning needs to be
# raised. Do this check only on the first step (user interaction step).
if key in env_vars and config["general"]["run_number"] == 1:
# Previous model with the same key
model0 = env_vars[key][1]
while True:
# Warn the user about the overwriting of the variable
user_note("Environment conflict", f"In '{model0}':")
pprint_config({key: env_vars[key][0]})
logging.info("\nIn '" + model + "':")
pprint_config({key: value})
# Ask the user how to proceed if it is not a ``tidy_and_
# resubmit`` job
if not config["general"][
"jobtype"] == "tidy_and_resubmit":
user_answer = input(
f"Environment variable '{key}' defined in '{model0}' is "
+
"going to be overwritten by the one defined in "
+
f"'{model}'. Are you okay with that? (Y/n): ")
# If it is a ``tidy_and_resubmit`` job, the user has already
# interacted and accepted the overwriting.
else:
user_answer = "Y"
logging.info(
f"Environment variable '{key}' defined in '{model0}' is "
+
f"overwritten by the one defined in '{model}'."
)
# If the user selects ``Y``, overwrite the environment variable
if user_answer == "Y":
config[model]["env_overwrite"] = True
break
# If the user selects ``n`` raise a user error with recommendations
elif user_answer == "n":
config[model]["env_overwrite"] = False
user_error(
"Environment conflict",
"You were not happy with the environment variable "
+
f"'{key}' in '{model0}' being overwritten by the same "
+
f"variable in '{model}'. If you are running a "
+
"coupled setup, we recommend that you resolve this "
+
"conflict inside the coupled setup file, by " +
"specifying unconflicting environments for each "
+ "model, or contact the setup developers.",
)
else:
logging.info("Wrong answer, please choose Y/n.")
# Merge variable into the ``computer`` dictionary so that it becomes
# part of the general environment.
dict_merge(config["computer"], {key: value})
# Add the variable to ``env_vars`` so it can be checked for conflicts
# with other models.
env_vars[key] = [value, model]
def output():
esm_parser.pprint_config(self.config)
def output(self):
print()
esm_parser.pprint_config(self.config)
print("Known repos: " + str(self.known_repos))
print("Known vcs-commands: " + str(self.known_todos))
def inspect_config(config):
if config["general"]["inspect"] == "config":
pprint_config(config)
sys.exit(0)
return config
#!/usr/bin/env python import esm_parser import esm_environment env = esm_environment.environment_infos() esm_parser.pprint_config(env)
def prepare(self):
six.print_("=" * 80, "\n")
six.print_("PREPARING EXPERIMENT")
# Copy files:
all_files_to_copy = []
six.print_("\n" "- Generating file lists for this run...")
for component in self.components:
six.print_("-" * 80)
six.print_("* %s" % component.config["model"], "\n")
all_component_files, filetype_specific_dict = (
component.filesystem_to_experiment()
)
with open(
component.thisrun_config_dir
+ "/"
+ self.config["general"]["expid"]
+ "_filelist_"
+ self.run_datestamp,
"w",
) as flist:
flist.write(
"These files are used for \nexperiment %s\ncomponent %s\ndate %s"
% (
self.config["general"]["expid"],
component.config["model"],
self.run_datestamp,
)
)
flist.write("\n")
flist.write(80 * "-")
for filetype in filetype_specific_dict:
flist.write("\n" + filetype.upper() + ":\n")
for source, exp_tree_name, work_dir_name in filetype_specific_dict[
filetype
]:
flist.write("\nSource: " + source)
flist.write("\nExp Tree: " + exp_tree_name)
flist.write("\nWork Dir: " + work_dir_name)
flist.write("\n")
flist.write("\n")
flist.write(80 * "-")
esm_parser.pprint_config(filetype_specific_dict)
all_files_to_copy += all_component_files
six.print_("\n" "- File lists populated, proceeding with copy...")
six.print_("- Note that you can see your file lists in the config folder")
six.print_("- You will be informed about missing files")
self._prepare_copy_files(all_files_to_copy)
# Load and modify namelists:
six.print_("\n" "- Setting up namelists for this run...")
all_namelists = {}
for component in self.components:
six.print_("-" * 80)
six.print_("* %s" % component.config["model"], "\n")
component.nmls_load()
component.nmls_remove()
component.nmls_modify()
component.nmls_finalize(all_namelists)
six.print_(
"\n" "- Namelists modified according to experiment specifications..."
)
for nml_name, nml in all_namelists.items():
six.print_("Contents of ", nml_name, ":")
nml.write(sys.stdout)
six.print_("\n", 40 * "+ ")
self._prepare_modify_files()
for model in list(self.config):
if model in esm_coupler.known_couplers:
coupler_config_dir = (
self.config["general"]["base_dir"]
+ "/"
+ self.config["general"]["expid"]
+ "/run_"
+ self.run_datestamp
+ "/config/"
+ model
+ "/"
)
self.coupler = esm_coupler.esm_coupler(self.config, model)
self.coupler.prepare(self.config, coupler_config_dir)
coupler_filename="namcouple" # needs to be set by function above
all_files_to_copy.append(
(
"",
"",
coupler_config_dir + "/" + coupler_filename,
)
)
#print (coupler_config_dir + "/" + coupler_filename)
all_files_to_copy.append(
(
"",
"",
self.batch.bs.path,
)
)
self._prepare_copy_files_to_work(all_files_to_copy)