def set_entry_point(self):
"""Locate the top-level driver script for the POD.
Raises: :class:`~util.PodRuntimeError` if driver script can't be found.
"""
if not self.driver:
self.log.warning("No valid driver script found for %s.",
self.full_name)
# try to find one anyway
script_names = [self.name, "driver"]
file_names = [f"{script}{ext}" for script in script_names \
for ext in self._interpreters.keys()]
for f in file_names:
path_ = os.path.join(self.POD_CODE_DIR, f)
if os.path.exists(path_):
self.log.debug("Setting driver script for %s to '%s'.",
self.full_name, f)
self.driver = path_
break # go with the first one found
if not self.driver:
raise util.PodRuntimeError(
(f"No driver script found in "
f"{self.POD_CODE_DIR}. Specify 'driver' in settings.jsonc."),
self)
if not os.path.isabs(self.driver): # expand relative path
self.driver = os.path.join(self.POD_CODE_DIR, self.driver)
if not os.path.exists(self.driver):
raise util.PodRuntimeError(
f"Unable to locate driver script '{self.driver}'.", self)
def check_pod_driver(self):
"""Private method called by :meth:`~diagnostic.Diagnostic.setup`.
Raises: :exc:`~diagnostic.PodRuntimeError` if driver script
can't be found.
"""
programs = util.get_available_programs()
if not self.driver:
_log.warning("No valid driver entry found for %s", self.full_name)
#try to find one anyway
try_filenames = [self.name + ".", "driver."]
file_combos = [ file_root + ext for file_root \
in try_filenames for ext in programs]
_log.debug("Checking for possible driver names in {} {}".format(
self.POD_CODE_DIR, file_combos))
for try_file in file_combos:
try_path = os.path.join(self.POD_CODE_DIR, try_file)
_log.debug(" looking for driver file " + try_path)
if os.path.exists(try_path):
self.driver = try_path
_log.debug("Found driver script for {}: {}".format(
self.name, self.driver))
break #go with the first one found
else:
_log.debug("\t " + try_path + " not found...")
if self.driver == '':
raise util.PodRuntimeError(
(f"No driver script found in "
f"{self.POD_CODE_DIR}. Specify 'driver' in settings.jsonc."),
self)
if not os.path.isabs(self.driver): # expand relative path
self.driver = os.path.join(self.POD_CODE_DIR, self.driver)
if not os.path.exists(self.driver):
raise util.PodRuntimeError(
f"Unable to locate driver script {self.driver}.", self)
if self.program == '':
# Find ending of filename to determine the program that should be used
driver_ext = self.driver.split('.')[-1]
# Possible error: Driver file type unrecognized
if driver_ext not in programs:
raise util.PodRuntimeError(
(f"Don't know how to call a .{driver_ext} "
f"file.\nSupported programs: {programs}"), self)
self.program = programs[driver_ext]
_log.debug("Found program " + programs[driver_ext])
def pre_run_setup(self):
"""Perform filesystem operations and checks prior to running the POD.
In order, this 1) sets environment variables specific to the POD, 2)
creates POD-specific working directories, and 3) checks for the existence
of the POD's driver script.
Note:
The existence of data files is checked with
:meth:`~data_manager.DataManager.fetchData`
and the runtime environment is validated separately as a function of
:meth:`~environment_manager.EnvironmentManager.run`. This is because
each POD is run in a subprocess (due to the necessity of supporting
multiple languages) so the validation must take place in that
subprocess.
Raises:
:exc:`~diagnostic.PodRuntimeError` if requirements aren't met. This
is re-raised from the :meth:`diagnostic.Diagnostic.set_entry_point`
and :meth:`diagnostic.Diagnostic._check_for_varlist_files`
subroutines.
"""
try:
self.set_pod_env_vars()
self.set_entry_point()
except Exception as exc:
raise util.PodRuntimeError("Caught exception during pre_run_setup",
self) from exc
def set_interpreter(self):
"""Determine what executable should be used to run the driver script.
.. note::
Existence of the program on the environment's ``$PATH`` isn't checked
until before the POD runs (see :mod:`src.environment_manager`.)
"""
if not self.program:
# Find ending of filename to determine the program that should be used
_, driver_ext = os.path.splitext(self.driver)
# Possible error: Driver file type unrecognized
if driver_ext not in self._interpreters:
raise util.PodRuntimeError((
f"Don't know how to call a '{driver_ext}' "
f"file.\nSupported programs: {list(self._interpreters.values())}"
), self)
self.program = self._interpreters[driver_ext]
self.log.debug("Set program for %s to '%s'.", self.full_name,
self.program)
def __init__(self, log=_log):
super(CondaEnvironmentManager, self).__init__(log=log)
paths = core.PathManager()
self.code_root = paths.CODE_ROOT
self.conda_dir = os.path.join(self.code_root, 'src', 'conda')
self.env_list = []
for file_ in os.listdir(self.conda_dir):
if file_.endswith('.yml'):
name, _ = os.path.splitext(file_)
self.env_list.append(name.split('env_')[-1])
# find conda executable
# conda_init for bash defines conda as a shell function; will get error
# if we try to call the conda executable directly
try:
conda_info = util.run_shell_command(
f"{self.conda_dir}/conda_init.sh {paths.get('conda_root','')}",
log=self.log)
for line in conda_info:
key, val = line.split('=')
if key == '_CONDA_EXE':
self.conda_exe = val
assert os.path.exists(self.conda_exe)
elif key == '_CONDA_ROOT':
self.conda_root = val
except Exception as exc:
raise util.PodRuntimeError("Can't find conda.") from exc
# find where environments are installed
if 'conda_env_root' in paths and paths.conda_env_root:
self.conda_env_root = paths.conda_env_root
if not os.path.isdir(self.conda_env_root):
self.log.warning(
"Conda env directory '%s' not found; creating.",
self.conda_env_root)
os.makedirs(self.conda_env_root) # recursive mkdir if needed
else:
# only true in default anaconda install, may need to fix
self.conda_env_root = os.path.join(self.conda_root, 'envs')
def pre_run_setup(self):
"""Extra code only applicable in frepp cooperative mode. If this code is
called, all the POD's model data has been generated. Write a placeholder
directory to POD_OUT_DIR, so if frepp invokes the MDTF package again
while we're running, only our results will be written to the overall
output.
"""
super(GfdlDiagnostic, self).pre_run_setup()
config = core.ConfigManager()
frepp_mode = config.get('frepp', False)
if frepp_mode and not os.path.exists(self.POD_OUT_DIR):
try:
gfdl_util.make_remote_dir(self.POD_OUT_DIR)
self._has_placeholder = True
except Exception as exc:
try:
raise util.PodRuntimeError((f"Caught exception making output "
f"directory at {self.POD_OUT_DIR}; deactivating {self.name}."),
self) from exc
except Exception as chained_exc:
_log.error(chained_exc)
self.exceptions.log(chained_exc)