def _create_py_venv(self, env_name, py_pkgs):
env_path = os.path.join(self.venv_root, env_name)
if not os.path.isdir(env_path):
os.makedirs(env_path) # recursive mkdir if needed
for cmd in [
f"python -m virtualenv {env_path}",
f"source {env_path}/bin/activate",
'pip install {}'.format(' '.join(py_pkgs)), 'deactivate'
]:
util.run_shell_command(cmd, log=self.log)
def _create_py_venv(self, env_name):
py_pkgs = set()
for pod in self.pods:
if pod.env == env_name:
py_pkgs.update(set(pod.required_python_modules))
env_path = os.path.join(self.venv_root, env_name)
if not os.path.isdir(env_path):
os.makedirs(env_path) # recursive mkdir if needed
for cmd in [
'python -m virtualenv {}'.format(env_path),
'source {}/bin/activate'.format(env_path),
'pip install {}'.format(' '.join(py_pkgs)), 'deactivate'
]:
util.run_shell_command(cmd)
def _create_r_venv(self, env_name, r_pkgs):
r_pkg_str = ', '.join(['"' + x + '"' for x in r_pkgs])
if self.r_lib_root != '':
env_path = os.path.join(self.r_lib_root, env_name)
if not os.path.isdir(env_path):
os.makedirs(env_path) # recursive mkdir if needed
cmds = [
f'export R_LIBS_USER="******"',
f'Rscript -e \'install.packages(c({r_pkg_str}), ' \
+ 'lib=Sys.getenv("R_LIBS_USER"))\''
]
else:
cmds = [f'Rscript -e \'install.packages(c({r_pkg_str}))\'']
for cmd in cmds:
util.run_shell_command(cmd, log=self.log)
def nc_dump_axis(cls, ax_name, in_file=None, cwd=None, dry_run=False):
# OK for 4.7.6, works on 4.5.4 if "--trd" flag removed
ax_vals = util.run_shell_command("ncks --trd -H -V -v '{}' {}".format(
ax_name, in_file),
cwd=cwd,
dry_run=dry_run)
return [float(val) for val in ax_vals if val]
def create_all_environments(self):
try:
_ = util.run_shell_command(
(f'{self.conda_dir}/conda_env_setup.sh -c "{self.conda_exe}" '
f'-d "{self.conda_env_root}" --all'))
except Exception:
raise
def create_all_environments(self):
try:
_ = util.run_shell_command(
'{}/conda_env_setup.sh -c "{}" -d "{}" --all'.format(
self.conda_dir, self.conda_exe, self.conda_env_root))
except Exception:
raise
def make_tar_file(self, case):
"""Make tar file of web/bitmap output.
"""
out_path = self._tarball_file_path
if not self.file_overwrite:
out_path, _ = util.bump_version(out_path)
_log.info("%s: Creating %s.", case.name, out_path)
elif os.path.exists(out_path):
_log.info("%s: Overwriting %s.", case.name, out_path)
tar_flags = [
f"--exclude=.{s}" for s in ('netCDF', 'nc', 'ps', 'PS', 'eps')
]
tar_flags = ' '.join(tar_flags)
util.run_shell_command(
f'tar {tar_flags} -czf {out_path} -C {self.WK_DIR} .',
dry_run=self.dry_run)
return out_path
def _make_tar_file(self, tar_dest_dir):
"""Make tar file of web/bitmap output.
"""
out_file = os.path.join(tar_dest_dir, self.MODEL_WK_DIR + '.tar')
if not self.file_overwrite:
out_file, _ = util_mdtf.bump_version(out_file)
print("Creating {}.".format(out_file))
elif os.path.exists(out_file):
print('Overwriting {}.'.format(out_file))
tar_flags = [
"--exclude=.{}".format(s)
for s in ['netCDF', 'nc', 'ps', 'PS', 'eps']
]
tar_flags = ' '.join(tar_flags)
util.run_shell_command('tar {} -czf {} -C {} .'.format(
tar_flags, out_file, self.MODEL_WK_DIR),
dry_run=self.dry_run)
return out_file
def create_environment(self, env_name):
# check to see if conda env exists, and if not, try to create it
conda_prefix = os.path.join(self.conda_env_root, env_name)
try:
_ = util.run_shell_command('{} env list | grep -qF "{}"'.format(
self.conda_exe, conda_prefix))
except Exception:
print('Conda env {} not found (grepped for {})'.format(
env_name, conda_prefix))
def convert_pod_figures(self, src_subdir, dest_subdir):
"""Convert all vector graphics in `POD_WK_DIR/subdir` to .png files using
ghostscript.
All vector graphics files (identified by extension) in any subdirectory
of `POD_WK_DIR/src_subdir` are converted to .png files by running
`ghostscript <https://www.ghostscript.com/>`__ in a subprocess.
Ghostscript is included in the _MDTF_base conda environment. Afterwards,
any bitmap files (identified by extension) in any subdirectory of
`POD_WK_DIR/src_subdir` are moved to `POD_WK_DIR/dest_subdir`, preserving
and subdirectories (see doc for :func:`~util.recursive_copy`.)
Args:
src_subdir: Subdirectory tree of `POD_WK_DIR` to search for vector
graphics files.
dest_subdir: Subdirectory tree of `POD_WK_DIR` to move converted
bitmap files to.
"""
config = util_mdtf.ConfigManager()
abs_src_subdir = os.path.join(self.POD_WK_DIR, src_subdir)
abs_dest_subdir = os.path.join(self.POD_WK_DIR, dest_subdir)
files = util.find_files(
abs_src_subdir,
['*.ps', '*.PS', '*.eps', '*.EPS', '*.pdf', '*.PDF'])
for f in files:
f_stem, _ = os.path.splitext(f)
_ = util.run_shell_command(
'gs {flags} -sOutputFile="{f_out}" {f_in}'.format(
flags=config.config.get('convert_flags', ''),
f_in=f,
f_out=f_stem + '_MDTF_TEMP_%d.png'))
# syntax for f_out above appends "_MDTF_TEMP" + page number to
# output files. If input .ps/.pdf file had multiple pages, this will
# generate 1 png per page. Page numbering starts at 1. Now check
# how many files gs created:
out_files = glob.glob(f_stem + '_MDTF_TEMP_?.png')
if not out_files:
raise OSError("Error: no png generated from {}".format(f))
elif len(out_files) == 1:
# got one .png, so remove suffix.
os.rename(out_files[0], f_stem + '.png')
else:
# Multiple .pngs. Drop the MDTF_TEMP suffix and renumber starting
# from zero (forget which POD requires this.)
for n in list(range(len(out_files))):
os.rename(f_stem + '_MDTF_TEMP_{}.png'.format(n + 1),
f_stem + '-{}.png'.format(n))
# move converted figures and any figures that were saved directly as bitmaps
files = util.find_files(abs_src_subdir,
['*.png', '*.gif', '*.jpg', '*.jpeg'])
util.recursive_copy(files,
abs_src_subdir,
abs_dest_subdir,
copy_function=shutil.move,
overwrite=False)
def _create_r_venv(self, env_name):
r_pkgs = set()
for pod in self.pods:
if pod.env == env_name:
r_pkgs.update(set(pod.required_r_packages))
r_pkg_str = ', '.join(['"' + x + '"' for x in r_pkgs])
if self.r_lib_root != '':
env_path = os.path.join(self.r_lib_root, env_name)
if not os.path.isdir(env_path):
os.makedirs(env_path) # recursive mkdir if needed
cmds = [
'export R_LIBS_USER="******"'.format(env_path),
'Rscript -e \'install.packages(c({}), '.format(r_pkg_str) \
+ 'lib=Sys.getenv("R_LIBS_USER"))\''
]
else:
cmds = ['Rscript -e \'install.packages(c({}))\''.format(r_pkg_str)]
for cmd in cmds:
util.run_shell_command(cmd)
def shell_command_wrapper(cmd, **kwargs):
print('SHELL RUN:')
print(' ', cmd)
try:
stdout = util.run_shell_command(cmd, **kwargs)
except Exception:
raise
if stdout:
print('SHELL STDOUT:')
for line in stdout:
print(' ', line)
else:
print('SHELL STDOUT: (no output returned)')
return stdout
def _call_conda_create(self, env_name):
if env_name.startswith(self.env_name_prefix):
short_name = env_name[(len(self.env_name_prefix)+1):]
else:
short_name = env_name
path = f"{self.conda_dir}/env_{short_name}.yml"
if not os.path.exists(path):
self.log.error("Can't find %s", path)
else:
conda_prefix = os.path.join(self.conda_env_root, env_name)
self.log.info("Creating conda env '%s' in '%s'.", env_name, conda_prefix)
command = (
f'source {self.conda_dir}/conda_init.sh {self.conda_root} && '
f'{self.conda_exe} env create --force -q -p "{conda_prefix}" -f "{path}"'
)
try:
_ = util.run_shell_command(command, log=self.log)
except Exception:
raise
def _call_conda_create(self, env_name):
if env_name.startswith(self.env_name_prefix):
short_name = env_name[(len(self.env_name_prefix) + 1):]
else:
short_name = env_name
path = '{}/env_{}.yml'.format(self.conda_dir, short_name)
if not os.path.exists(path):
print("Can't find {}".format(path))
else:
conda_prefix = os.path.join(self.conda_env_root, env_name)
print('Creating conda env {} in {}'.format(env_name, conda_prefix))
command = \
'source {}/conda_init.sh {} && '.format(
self.conda_dir, self.conda_root
) + '{} env create --force -q -p "{}" -f "{}"'.format(
self.conda_exe, conda_prefix, path
)
try:
_ = util.run_shell_command(command)
except Exception:
raise
def __init__(self, log=_log):
super(CondaEnvironmentManager, self).__init__(log=log)
paths = core.PathManager()
self.code_root = paths.CODE_ROOT
self.conda_dir = os.path.join(self.code_root, 'src', 'conda')
self.env_list = []
for file_ in os.listdir(self.conda_dir):
if file_.endswith('.yml'):
name, _ = os.path.splitext(file_)
self.env_list.append(name.split('env_')[-1])
# find conda executable
# conda_init for bash defines conda as a shell function; will get error
# if we try to call the conda executable directly
try:
conda_info = util.run_shell_command(
f"{self.conda_dir}/conda_init.sh {paths.get('conda_root','')}",
log=self.log)
for line in conda_info:
key, val = line.split('=')
if key == '_CONDA_EXE':
self.conda_exe = val
assert os.path.exists(self.conda_exe)
elif key == '_CONDA_ROOT':
self.conda_root = val
except Exception as exc:
raise util.PodRuntimeError("Can't find conda.") from exc
# find where environments are installed
if 'conda_env_root' in paths and paths.conda_env_root:
self.conda_env_root = paths.conda_env_root
if not os.path.isdir(self.conda_env_root):
self.log.warning(
"Conda env directory '%s' not found; creating.",
self.conda_env_root)
os.makedirs(self.conda_env_root) # recursive mkdir if needed
else:
# only true in default anaconda install, may need to fix
self.conda_env_root = os.path.join(self.conda_root, 'envs')
def __init__(self, verbose=0):
super(CondaEnvironmentManager, self).__init__(verbose)
config = util_mdtf.ConfigManager()
self.code_root = config.paths.CODE_ROOT
self.conda_dir = os.path.join(self.code_root, 'src', 'conda')
self.env_list = []
for file_ in os.listdir(self.conda_dir):
if file_.endswith('.yml'):
name, _ = os.path.splitext(file_)
self.env_list.append(name.split('env_')[-1])
# find conda executable
# conda_init for bash defines conda as a shell function; will get error
# if we try to call the conda executable directly
try:
conda_info = util.run_shell_command('{}/conda_init.sh {}'.format(
self.conda_dir, config.paths.get('conda_root', '')))
for line in conda_info:
key, val = line.split('=')
if key == '_CONDA_EXE':
self.conda_exe = val
assert os.path.exists(self.conda_exe)
elif key == '_CONDA_ROOT':
self.conda_root = val
except Exception:
print("Error: can't find conda.")
raise
# find where environments are installed
if 'conda_env_root' in config.paths and config.paths.conda_env_root:
self.conda_env_root = config.paths.conda_env_root
if not os.path.isdir(self.conda_env_root):
os.makedirs(self.conda_env_root) # recursive mkdir if needed
else:
# only true in default anaconda install, may need to fix
self.conda_env_root = os.path.join(self.conda_root, 'envs')
def test_run_shell_commands_stdout1(self):
input = 'echo "foo"'
out = util.run_shell_command(input)
self.assertEqual(len(out), 1)
self.assertEqual(out[0], 'foo')
def test_run_shell_commands_exitcode(self):
input = 'echo "foo"; false'
with self.assertRaises(Exception):
# I couldn't get this to catch CalledProcessError specifically,
# maybe because it takes args?
util.run_shell_command(input)
def test_run_shell_commands_envvars(self):
input = 'echo $FOO; export FOO="baz"; echo $FOO'
out = util.run_shell_command(input, env={'FOO': 'bar'})
self.assertEqual(len(out), 2)
self.assertEqual(out[0], 'bar')
self.assertEqual(out[1], 'baz')
def convert_pod_figures(self, pod, src_subdir, dest_subdir):
"""Convert all vector graphics in `POD_WK_DIR/subdir` to .png files using
ghostscript.
All vector graphics files (identified by extension) in any subdirectory
of `POD_WK_DIR/src_subdir` are converted to .png files by running
`ghostscript <https://www.ghostscript.com/>`__ in a subprocess.
Ghostscript is included in the _MDTF_base conda environment. Afterwards,
any bitmap files (identified by extension) in any subdirectory of
`POD_WK_DIR/src_subdir` are moved to `POD_WK_DIR/dest_subdir`, preserving
and subdirectories (see doc for :func:`~util.recursive_copy`.)
Args:
src_subdir: Subdirectory tree of `POD_WK_DIR` to search for vector
graphics files.
dest_subdir: Subdirectory tree of `POD_WK_DIR` to move converted
bitmap files to.
"""
# Flags to pass to ghostscript for PS -> PNG conversion (in particular
# bitmap resolution.)
eps_convert_flags = (
"-dSAFER -dBATCH -dNOPAUSE -dEPSCrop -r150 "
"-sDEVICE=png16m -dTextAlphaBits=4 -dGraphicsAlphaBits=4")
abs_src_subdir = os.path.join(pod.POD_WK_DIR, src_subdir)
abs_dest_subdir = os.path.join(pod.POD_WK_DIR, dest_subdir)
files = util.find_files(
abs_src_subdir,
['*.ps', '*.PS', '*.eps', '*.EPS', '*.pdf', '*.PDF'])
for f in files:
f_stem, _ = os.path.splitext(f)
# Append "_MDTF_TEMP" + page number to output files ("%d" = ghostscript's
# template for multi-page output). If input .ps/.pdf file has multiple
# pages, this will generate 1 png per page, counting from 1.
f_out = f_stem + '_MDTF_TEMP_%d.png'
try:
util.run_shell_command(
f'gs {eps_convert_flags} -sOutputFile="{f_out}" {f}')
except Exception as exc:
_log.error("%s produced malformed plot: %s", pod.name,
f[len(abs_src_subdir):])
if isinstance(exc, util.MDTFCalledProcessError):
_log.debug(
"gs error encountered when converting %s for %s:\n%s",
pod.name, f[len(abs_src_subdir):],
getattr(exc, "output", ""))
continue
# gs ran successfully; check how many files it created:
out_files = glob.glob(f_stem + '_MDTF_TEMP_?.png')
if not out_files:
raise util.MDTFFileNotFoundError(
f"No .png generated from {f}.")
elif len(out_files) == 1:
# got one .png, so remove suffix.
os.rename(out_files[0], f_stem + '.png')
else:
# Multiple .pngs. Drop the MDTF_TEMP suffix and renumber starting
# from zero (forget which POD requires this.)
for n in range(len(out_files)):
os.rename(f_stem + f'_MDTF_TEMP_{n+1}.png',
f_stem + f'-{n}.png')
# move converted figures and any figures that were saved directly as bitmaps
files = util.find_files(abs_src_subdir,
['*.png', '*.gif', '*.jpg', '*.jpeg'])
util.recursive_copy(files,
abs_src_subdir,
abs_dest_subdir,
copy_function=shutil.move,
overwrite=False)
def test_run_shell_commands_stdout2(self):
input = 'echo "foo" && echo "bar"'
out = util.run_shell_command(input)
self.assertEqual(len(out), 2)
self.assertEqual(out[0], 'foo')
self.assertEqual(out[1], 'bar')
