def cleanup_pod_files(self, pod):
"""Copy and remove remaining files to `POD_WK_DIR`.
In order, this 1) copies any bitmap figures in any subdirectory of
`POD_OBS_DATA` to `POD_WK_DIR/obs` (needed for legacy PODs without
digested observational data), 2) removes vector graphics if requested,
3) removes netCDF scratch files in `POD_WK_DIR` if requested.
Settings are set at runtime, when :class:`~core.ConfigManager` is
initialized.
"""
# copy premade figures (if any) to output
files = util.find_files(pod.POD_OBS_DATA,
['*.gif', '*.png', '*.jpg', '*.jpeg'])
for f in files:
shutil.copy2(f, os.path.join(pod.POD_WK_DIR, 'obs'))
# remove .eps files if requested (actually, contents of any 'PS' subdirs)
if not self.save_ps:
for d in util.find_files(pod.POD_WK_DIR, 'PS' + os.sep):
shutil.rmtree(d)
# delete netCDF files, keep everything else
if self.save_non_nc:
for f in util.find_files(pod.POD_WK_DIR, '*.nc'):
os.remove(f)
# delete all generated data
# actually deletes contents of any 'netCDF' subdirs
elif not self.save_nc:
for d in util.find_files(pod.POD_WK_DIR, 'netCDF' + os.sep):
shutil.rmtree(d)
for f in util.find_files(pod.POD_WK_DIR, '*.nc'):
os.remove(f)
def convert_pod_figures(self, src_subdir, dest_subdir):
"""Convert all vector graphics in `POD_WK_DIR/subdir` to .png files using
ghostscript.
All vector graphics files (identified by extension) in any subdirectory
of `POD_WK_DIR/src_subdir` are converted to .png files by running
`ghostscript <https://www.ghostscript.com/>`__ in a subprocess.
Ghostscript is included in the _MDTF_base conda environment. Afterwards,
any bitmap files (identified by extension) in any subdirectory of
`POD_WK_DIR/src_subdir` are moved to `POD_WK_DIR/dest_subdir`, preserving
and subdirectories (see doc for :func:`~util.recursive_copy`.)
Args:
src_subdir: Subdirectory tree of `POD_WK_DIR` to search for vector
graphics files.
dest_subdir: Subdirectory tree of `POD_WK_DIR` to move converted
bitmap files to.
"""
config = util_mdtf.ConfigManager()
abs_src_subdir = os.path.join(self.POD_WK_DIR, src_subdir)
abs_dest_subdir = os.path.join(self.POD_WK_DIR, dest_subdir)
files = util.find_files(
abs_src_subdir,
['*.ps', '*.PS', '*.eps', '*.EPS', '*.pdf', '*.PDF'])
for f in files:
f_stem, _ = os.path.splitext(f)
_ = util.run_shell_command(
'gs {flags} -sOutputFile="{f_out}" {f_in}'.format(
flags=config.config.get('convert_flags', ''),
f_in=f,
f_out=f_stem + '_MDTF_TEMP_%d.png'))
# syntax for f_out above appends "_MDTF_TEMP" + page number to
# output files. If input .ps/.pdf file had multiple pages, this will
# generate 1 png per page. Page numbering starts at 1. Now check
# how many files gs created:
out_files = glob.glob(f_stem + '_MDTF_TEMP_?.png')
if not out_files:
raise OSError("Error: no png generated from {}".format(f))
elif len(out_files) == 1:
# got one .png, so remove suffix.
os.rename(out_files[0], f_stem + '.png')
else:
# Multiple .pngs. Drop the MDTF_TEMP suffix and renumber starting
# from zero (forget which POD requires this.)
for n in list(range(len(out_files))):
os.rename(f_stem + '_MDTF_TEMP_{}.png'.format(n + 1),
f_stem + '-{}.png'.format(n))
# move converted figures and any figures that were saved directly as bitmaps
files = util.find_files(abs_src_subdir,
['*.png', '*.gif', '*.jpg', '*.jpeg'])
util.recursive_copy(files,
abs_src_subdir,
abs_dest_subdir,
copy_function=shutil.move,
overwrite=False)
def make_pod_html(self):
"""Perform templating on POD's html results page(s).
A wrapper for :func:`~util_mdtf.append_html_template`. Looks for all
html files in POD_CODE_DIR, templates them, and copies them to
POD_WK_DIR, respecting subdirectory structure (see doc for
:func:`~util.recursive_copy`).
"""
config = util_mdtf.ConfigManager()
template = config.global_envvars.copy()
template.update(self.pod_env_vars)
source_files = util.find_files(self.POD_CODE_DIR, '*.html')
util.recursive_copy(
source_files,
self.POD_CODE_DIR,
self.POD_WK_DIR,
copy_function=lambda src, dest: util_mdtf.append_html_template(
src, dest, template_dict=template, append=False),
overwrite=True)
def make_pod_html(self, pod):
"""Perform templating on POD's html results page(s).
A wrapper for :func:`~util.append_html_template`. Looks for all
html files in POD_CODE_DIR, templates them, and copies them to
POD_WK_DIR, respecting subdirectory structure (see doc for
:func:`~util.recursive_copy`).
"""
test_path = os.path.join(pod.POD_CODE_DIR,
self.pod_html_template_file_name(pod))
if not os.path.isfile(test_path):
# POD's top-level HTML template needs to exist
raise util.MDTFFileNotFoundError(test_path)
template_d = html_templating_dict(pod)
# copy and template all .html files, since PODs can make sub-pages
source_files = util.find_files(pod.POD_CODE_DIR, '*.html')
util.recursive_copy(
source_files,
pod.POD_CODE_DIR,
pod.POD_WK_DIR,
copy_function=(lambda src, dest: util.append_html_template(
src, dest, template_dict=template_d, append=False)),
overwrite=True)
def convert_pod_figures(self, pod, src_subdir, dest_subdir):
"""Convert all vector graphics in `POD_WK_DIR/subdir` to .png files using
ghostscript.
All vector graphics files (identified by extension) in any subdirectory
of `POD_WK_DIR/src_subdir` are converted to .png files by running
`ghostscript <https://www.ghostscript.com/>`__ in a subprocess.
Ghostscript is included in the _MDTF_base conda environment. Afterwards,
any bitmap files (identified by extension) in any subdirectory of
`POD_WK_DIR/src_subdir` are moved to `POD_WK_DIR/dest_subdir`, preserving
and subdirectories (see doc for :func:`~util.recursive_copy`.)
Args:
src_subdir: Subdirectory tree of `POD_WK_DIR` to search for vector
graphics files.
dest_subdir: Subdirectory tree of `POD_WK_DIR` to move converted
bitmap files to.
"""
# Flags to pass to ghostscript for PS -> PNG conversion (in particular
# bitmap resolution.)
eps_convert_flags = (
"-dSAFER -dBATCH -dNOPAUSE -dEPSCrop -r150 "
"-sDEVICE=png16m -dTextAlphaBits=4 -dGraphicsAlphaBits=4")
abs_src_subdir = os.path.join(pod.POD_WK_DIR, src_subdir)
abs_dest_subdir = os.path.join(pod.POD_WK_DIR, dest_subdir)
files = util.find_files(
abs_src_subdir,
['*.ps', '*.PS', '*.eps', '*.EPS', '*.pdf', '*.PDF'])
for f in files:
f_stem, _ = os.path.splitext(f)
# Append "_MDTF_TEMP" + page number to output files ("%d" = ghostscript's
# template for multi-page output). If input .ps/.pdf file has multiple
# pages, this will generate 1 png per page, counting from 1.
f_out = f_stem + '_MDTF_TEMP_%d.png'
try:
util.run_shell_command(
f'gs {eps_convert_flags} -sOutputFile="{f_out}" {f}')
except Exception as exc:
_log.error("%s produced malformed plot: %s", pod.name,
f[len(abs_src_subdir):])
if isinstance(exc, util.MDTFCalledProcessError):
_log.debug(
"gs error encountered when converting %s for %s:\n%s",
pod.name, f[len(abs_src_subdir):],
getattr(exc, "output", ""))
continue
# gs ran successfully; check how many files it created:
out_files = glob.glob(f_stem + '_MDTF_TEMP_?.png')
if not out_files:
raise util.MDTFFileNotFoundError(
f"No .png generated from {f}.")
elif len(out_files) == 1:
# got one .png, so remove suffix.
os.rename(out_files[0], f_stem + '.png')
else:
# Multiple .pngs. Drop the MDTF_TEMP suffix and renumber starting
# from zero (forget which POD requires this.)
for n in range(len(out_files)):
os.rename(f_stem + f'_MDTF_TEMP_{n+1}.png',
f_stem + f'-{n}.png')
# move converted figures and any figures that were saved directly as bitmaps
files = util.find_files(abs_src_subdir,
['*.png', '*.gif', '*.jpg', '*.jpeg'])
util.recursive_copy(files,
abs_src_subdir,
abs_dest_subdir,
copy_function=shutil.move,
overwrite=False)
