def _create_py_venv(self, env_name, py_pkgs): env_path = os.path.join(self.venv_root, env_name) if not os.path.isdir(env_path): os.makedirs(env_path) # recursive mkdir if needed for cmd in [ f"python -m virtualenv {env_path}", f"source {env_path}/bin/activate", 'pip install {}'.format(' '.join(py_pkgs)), 'deactivate' ]: util.run_shell_command(cmd, log=self.log)
def _create_py_venv(self, env_name): py_pkgs = set() for pod in self.pods: if pod.env == env_name: py_pkgs.update(set(pod.required_python_modules)) env_path = os.path.join(self.venv_root, env_name) if not os.path.isdir(env_path): os.makedirs(env_path) # recursive mkdir if needed for cmd in [ 'python -m virtualenv {}'.format(env_path), 'source {}/bin/activate'.format(env_path), 'pip install {}'.format(' '.join(py_pkgs)), 'deactivate' ]: util.run_shell_command(cmd)
def _create_r_venv(self, env_name, r_pkgs): r_pkg_str = ', '.join(['"' + x + '"' for x in r_pkgs]) if self.r_lib_root != '': env_path = os.path.join(self.r_lib_root, env_name) if not os.path.isdir(env_path): os.makedirs(env_path) # recursive mkdir if needed cmds = [ f'export R_LIBS_USER="******"', f'Rscript -e \'install.packages(c({r_pkg_str}), ' \ + 'lib=Sys.getenv("R_LIBS_USER"))\'' ] else: cmds = [f'Rscript -e \'install.packages(c({r_pkg_str}))\''] for cmd in cmds: util.run_shell_command(cmd, log=self.log)
def nc_dump_axis(cls, ax_name, in_file=None, cwd=None, dry_run=False): # OK for 4.7.6, works on 4.5.4 if "--trd" flag removed ax_vals = util.run_shell_command("ncks --trd -H -V -v '{}' {}".format( ax_name, in_file), cwd=cwd, dry_run=dry_run) return [float(val) for val in ax_vals if val]
def create_all_environments(self): try: _ = util.run_shell_command( (f'{self.conda_dir}/conda_env_setup.sh -c "{self.conda_exe}" ' f'-d "{self.conda_env_root}" --all')) except Exception: raise
def create_all_environments(self): try: _ = util.run_shell_command( '{}/conda_env_setup.sh -c "{}" -d "{}" --all'.format( self.conda_dir, self.conda_exe, self.conda_env_root)) except Exception: raise
def make_tar_file(self, case): """Make tar file of web/bitmap output. """ out_path = self._tarball_file_path if not self.file_overwrite: out_path, _ = util.bump_version(out_path) _log.info("%s: Creating %s.", case.name, out_path) elif os.path.exists(out_path): _log.info("%s: Overwriting %s.", case.name, out_path) tar_flags = [ f"--exclude=.{s}" for s in ('netCDF', 'nc', 'ps', 'PS', 'eps') ] tar_flags = ' '.join(tar_flags) util.run_shell_command( f'tar {tar_flags} -czf {out_path} -C {self.WK_DIR} .', dry_run=self.dry_run) return out_path
def _make_tar_file(self, tar_dest_dir): """Make tar file of web/bitmap output. """ out_file = os.path.join(tar_dest_dir, self.MODEL_WK_DIR + '.tar') if not self.file_overwrite: out_file, _ = util_mdtf.bump_version(out_file) print("Creating {}.".format(out_file)) elif os.path.exists(out_file): print('Overwriting {}.'.format(out_file)) tar_flags = [ "--exclude=.{}".format(s) for s in ['netCDF', 'nc', 'ps', 'PS', 'eps'] ] tar_flags = ' '.join(tar_flags) util.run_shell_command('tar {} -czf {} -C {} .'.format( tar_flags, out_file, self.MODEL_WK_DIR), dry_run=self.dry_run) return out_file
def create_environment(self, env_name): # check to see if conda env exists, and if not, try to create it conda_prefix = os.path.join(self.conda_env_root, env_name) try: _ = util.run_shell_command('{} env list | grep -qF "{}"'.format( self.conda_exe, conda_prefix)) except Exception: print('Conda env {} not found (grepped for {})'.format( env_name, conda_prefix))
def convert_pod_figures(self, src_subdir, dest_subdir): """Convert all vector graphics in `POD_WK_DIR/subdir` to .png files using ghostscript. All vector graphics files (identified by extension) in any subdirectory of `POD_WK_DIR/src_subdir` are converted to .png files by running `ghostscript <https://www.ghostscript.com/>`__ in a subprocess. Ghostscript is included in the _MDTF_base conda environment. Afterwards, any bitmap files (identified by extension) in any subdirectory of `POD_WK_DIR/src_subdir` are moved to `POD_WK_DIR/dest_subdir`, preserving and subdirectories (see doc for :func:`~util.recursive_copy`.) Args: src_subdir: Subdirectory tree of `POD_WK_DIR` to search for vector graphics files. dest_subdir: Subdirectory tree of `POD_WK_DIR` to move converted bitmap files to. """ config = util_mdtf.ConfigManager() abs_src_subdir = os.path.join(self.POD_WK_DIR, src_subdir) abs_dest_subdir = os.path.join(self.POD_WK_DIR, dest_subdir) files = util.find_files( abs_src_subdir, ['*.ps', '*.PS', '*.eps', '*.EPS', '*.pdf', '*.PDF']) for f in files: f_stem, _ = os.path.splitext(f) _ = util.run_shell_command( 'gs {flags} -sOutputFile="{f_out}" {f_in}'.format( flags=config.config.get('convert_flags', ''), f_in=f, f_out=f_stem + '_MDTF_TEMP_%d.png')) # syntax for f_out above appends "_MDTF_TEMP" + page number to # output files. If input .ps/.pdf file had multiple pages, this will # generate 1 png per page. Page numbering starts at 1. Now check # how many files gs created: out_files = glob.glob(f_stem + '_MDTF_TEMP_?.png') if not out_files: raise OSError("Error: no png generated from {}".format(f)) elif len(out_files) == 1: # got one .png, so remove suffix. os.rename(out_files[0], f_stem + '.png') else: # Multiple .pngs. Drop the MDTF_TEMP suffix and renumber starting # from zero (forget which POD requires this.) for n in list(range(len(out_files))): os.rename(f_stem + '_MDTF_TEMP_{}.png'.format(n + 1), f_stem + '-{}.png'.format(n)) # move converted figures and any figures that were saved directly as bitmaps files = util.find_files(abs_src_subdir, ['*.png', '*.gif', '*.jpg', '*.jpeg']) util.recursive_copy(files, abs_src_subdir, abs_dest_subdir, copy_function=shutil.move, overwrite=False)
def _create_r_venv(self, env_name): r_pkgs = set() for pod in self.pods: if pod.env == env_name: r_pkgs.update(set(pod.required_r_packages)) r_pkg_str = ', '.join(['"' + x + '"' for x in r_pkgs]) if self.r_lib_root != '': env_path = os.path.join(self.r_lib_root, env_name) if not os.path.isdir(env_path): os.makedirs(env_path) # recursive mkdir if needed cmds = [ 'export R_LIBS_USER="******"'.format(env_path), 'Rscript -e \'install.packages(c({}), '.format(r_pkg_str) \ + 'lib=Sys.getenv("R_LIBS_USER"))\'' ] else: cmds = ['Rscript -e \'install.packages(c({}))\''.format(r_pkg_str)] for cmd in cmds: util.run_shell_command(cmd)
def shell_command_wrapper(cmd, **kwargs): print('SHELL RUN:') print(' ', cmd) try: stdout = util.run_shell_command(cmd, **kwargs) except Exception: raise if stdout: print('SHELL STDOUT:') for line in stdout: print(' ', line) else: print('SHELL STDOUT: (no output returned)') return stdout
def _call_conda_create(self, env_name): if env_name.startswith(self.env_name_prefix): short_name = env_name[(len(self.env_name_prefix)+1):] else: short_name = env_name path = f"{self.conda_dir}/env_{short_name}.yml" if not os.path.exists(path): self.log.error("Can't find %s", path) else: conda_prefix = os.path.join(self.conda_env_root, env_name) self.log.info("Creating conda env '%s' in '%s'.", env_name, conda_prefix) command = ( f'source {self.conda_dir}/conda_init.sh {self.conda_root} && ' f'{self.conda_exe} env create --force -q -p "{conda_prefix}" -f "{path}"' ) try: _ = util.run_shell_command(command, log=self.log) except Exception: raise
def _call_conda_create(self, env_name): if env_name.startswith(self.env_name_prefix): short_name = env_name[(len(self.env_name_prefix) + 1):] else: short_name = env_name path = '{}/env_{}.yml'.format(self.conda_dir, short_name) if not os.path.exists(path): print("Can't find {}".format(path)) else: conda_prefix = os.path.join(self.conda_env_root, env_name) print('Creating conda env {} in {}'.format(env_name, conda_prefix)) command = \ 'source {}/conda_init.sh {} && '.format( self.conda_dir, self.conda_root ) + '{} env create --force -q -p "{}" -f "{}"'.format( self.conda_exe, conda_prefix, path ) try: _ = util.run_shell_command(command) except Exception: raise
def __init__(self, log=_log): super(CondaEnvironmentManager, self).__init__(log=log) paths = core.PathManager() self.code_root = paths.CODE_ROOT self.conda_dir = os.path.join(self.code_root, 'src', 'conda') self.env_list = [] for file_ in os.listdir(self.conda_dir): if file_.endswith('.yml'): name, _ = os.path.splitext(file_) self.env_list.append(name.split('env_')[-1]) # find conda executable # conda_init for bash defines conda as a shell function; will get error # if we try to call the conda executable directly try: conda_info = util.run_shell_command( f"{self.conda_dir}/conda_init.sh {paths.get('conda_root','')}", log=self.log) for line in conda_info: key, val = line.split('=') if key == '_CONDA_EXE': self.conda_exe = val assert os.path.exists(self.conda_exe) elif key == '_CONDA_ROOT': self.conda_root = val except Exception as exc: raise util.PodRuntimeError("Can't find conda.") from exc # find where environments are installed if 'conda_env_root' in paths and paths.conda_env_root: self.conda_env_root = paths.conda_env_root if not os.path.isdir(self.conda_env_root): self.log.warning( "Conda env directory '%s' not found; creating.", self.conda_env_root) os.makedirs(self.conda_env_root) # recursive mkdir if needed else: # only true in default anaconda install, may need to fix self.conda_env_root = os.path.join(self.conda_root, 'envs')
def __init__(self, verbose=0): super(CondaEnvironmentManager, self).__init__(verbose) config = util_mdtf.ConfigManager() self.code_root = config.paths.CODE_ROOT self.conda_dir = os.path.join(self.code_root, 'src', 'conda') self.env_list = [] for file_ in os.listdir(self.conda_dir): if file_.endswith('.yml'): name, _ = os.path.splitext(file_) self.env_list.append(name.split('env_')[-1]) # find conda executable # conda_init for bash defines conda as a shell function; will get error # if we try to call the conda executable directly try: conda_info = util.run_shell_command('{}/conda_init.sh {}'.format( self.conda_dir, config.paths.get('conda_root', ''))) for line in conda_info: key, val = line.split('=') if key == '_CONDA_EXE': self.conda_exe = val assert os.path.exists(self.conda_exe) elif key == '_CONDA_ROOT': self.conda_root = val except Exception: print("Error: can't find conda.") raise # find where environments are installed if 'conda_env_root' in config.paths and config.paths.conda_env_root: self.conda_env_root = config.paths.conda_env_root if not os.path.isdir(self.conda_env_root): os.makedirs(self.conda_env_root) # recursive mkdir if needed else: # only true in default anaconda install, may need to fix self.conda_env_root = os.path.join(self.conda_root, 'envs')
def test_run_shell_commands_stdout1(self): input = 'echo "foo"' out = util.run_shell_command(input) self.assertEqual(len(out), 1) self.assertEqual(out[0], 'foo')
def test_run_shell_commands_exitcode(self): input = 'echo "foo"; false' with self.assertRaises(Exception): # I couldn't get this to catch CalledProcessError specifically, # maybe because it takes args? util.run_shell_command(input)
def test_run_shell_commands_envvars(self): input = 'echo $FOO; export FOO="baz"; echo $FOO' out = util.run_shell_command(input, env={'FOO': 'bar'}) self.assertEqual(len(out), 2) self.assertEqual(out[0], 'bar') self.assertEqual(out[1], 'baz')
def convert_pod_figures(self, pod, src_subdir, dest_subdir): """Convert all vector graphics in `POD_WK_DIR/subdir` to .png files using ghostscript. All vector graphics files (identified by extension) in any subdirectory of `POD_WK_DIR/src_subdir` are converted to .png files by running `ghostscript <https://www.ghostscript.com/>`__ in a subprocess. Ghostscript is included in the _MDTF_base conda environment. Afterwards, any bitmap files (identified by extension) in any subdirectory of `POD_WK_DIR/src_subdir` are moved to `POD_WK_DIR/dest_subdir`, preserving and subdirectories (see doc for :func:`~util.recursive_copy`.) Args: src_subdir: Subdirectory tree of `POD_WK_DIR` to search for vector graphics files. dest_subdir: Subdirectory tree of `POD_WK_DIR` to move converted bitmap files to. """ # Flags to pass to ghostscript for PS -> PNG conversion (in particular # bitmap resolution.) eps_convert_flags = ( "-dSAFER -dBATCH -dNOPAUSE -dEPSCrop -r150 " "-sDEVICE=png16m -dTextAlphaBits=4 -dGraphicsAlphaBits=4") abs_src_subdir = os.path.join(pod.POD_WK_DIR, src_subdir) abs_dest_subdir = os.path.join(pod.POD_WK_DIR, dest_subdir) files = util.find_files( abs_src_subdir, ['*.ps', '*.PS', '*.eps', '*.EPS', '*.pdf', '*.PDF']) for f in files: f_stem, _ = os.path.splitext(f) # Append "_MDTF_TEMP" + page number to output files ("%d" = ghostscript's # template for multi-page output). If input .ps/.pdf file has multiple # pages, this will generate 1 png per page, counting from 1. f_out = f_stem + '_MDTF_TEMP_%d.png' try: util.run_shell_command( f'gs {eps_convert_flags} -sOutputFile="{f_out}" {f}') except Exception as exc: _log.error("%s produced malformed plot: %s", pod.name, f[len(abs_src_subdir):]) if isinstance(exc, util.MDTFCalledProcessError): _log.debug( "gs error encountered when converting %s for %s:\n%s", pod.name, f[len(abs_src_subdir):], getattr(exc, "output", "")) continue # gs ran successfully; check how many files it created: out_files = glob.glob(f_stem + '_MDTF_TEMP_?.png') if not out_files: raise util.MDTFFileNotFoundError( f"No .png generated from {f}.") elif len(out_files) == 1: # got one .png, so remove suffix. os.rename(out_files[0], f_stem + '.png') else: # Multiple .pngs. Drop the MDTF_TEMP suffix and renumber starting # from zero (forget which POD requires this.) for n in range(len(out_files)): os.rename(f_stem + f'_MDTF_TEMP_{n+1}.png', f_stem + f'-{n}.png') # move converted figures and any figures that were saved directly as bitmaps files = util.find_files(abs_src_subdir, ['*.png', '*.gif', '*.jpg', '*.jpeg']) util.recursive_copy(files, abs_src_subdir, abs_dest_subdir, copy_function=shutil.move, overwrite=False)
def test_run_shell_commands_stdout2(self): input = 'echo "foo" && echo "bar"' out = util.run_shell_command(input) self.assertEqual(len(out), 2) self.assertEqual(out[0], 'foo') self.assertEqual(out[1], 'bar')