def cleanup_pod_files(self, pod): """Copy and remove remaining files to `POD_WK_DIR`. In order, this 1) copies any bitmap figures in any subdirectory of `POD_OBS_DATA` to `POD_WK_DIR/obs` (needed for legacy PODs without digested observational data), 2) removes vector graphics if requested, 3) removes netCDF scratch files in `POD_WK_DIR` if requested. Settings are set at runtime, when :class:`~core.ConfigManager` is initialized. """ # copy premade figures (if any) to output files = util.find_files(pod.POD_OBS_DATA, ['*.gif', '*.png', '*.jpg', '*.jpeg']) for f in files: shutil.copy2(f, os.path.join(pod.POD_WK_DIR, 'obs')) # remove .eps files if requested (actually, contents of any 'PS' subdirs) if not self.save_ps: for d in util.find_files(pod.POD_WK_DIR, 'PS' + os.sep): shutil.rmtree(d) # delete netCDF files, keep everything else if self.save_non_nc: for f in util.find_files(pod.POD_WK_DIR, '*.nc'): os.remove(f) # delete all generated data # actually deletes contents of any 'netCDF' subdirs elif not self.save_nc: for d in util.find_files(pod.POD_WK_DIR, 'netCDF' + os.sep): shutil.rmtree(d) for f in util.find_files(pod.POD_WK_DIR, '*.nc'): os.remove(f)
def convert_pod_figures(self, src_subdir, dest_subdir): """Convert all vector graphics in `POD_WK_DIR/subdir` to .png files using ghostscript. All vector graphics files (identified by extension) in any subdirectory of `POD_WK_DIR/src_subdir` are converted to .png files by running `ghostscript <https://www.ghostscript.com/>`__ in a subprocess. Ghostscript is included in the _MDTF_base conda environment. Afterwards, any bitmap files (identified by extension) in any subdirectory of `POD_WK_DIR/src_subdir` are moved to `POD_WK_DIR/dest_subdir`, preserving and subdirectories (see doc for :func:`~util.recursive_copy`.) Args: src_subdir: Subdirectory tree of `POD_WK_DIR` to search for vector graphics files. dest_subdir: Subdirectory tree of `POD_WK_DIR` to move converted bitmap files to. """ config = util_mdtf.ConfigManager() abs_src_subdir = os.path.join(self.POD_WK_DIR, src_subdir) abs_dest_subdir = os.path.join(self.POD_WK_DIR, dest_subdir) files = util.find_files( abs_src_subdir, ['*.ps', '*.PS', '*.eps', '*.EPS', '*.pdf', '*.PDF']) for f in files: f_stem, _ = os.path.splitext(f) _ = util.run_shell_command( 'gs {flags} -sOutputFile="{f_out}" {f_in}'.format( flags=config.config.get('convert_flags', ''), f_in=f, f_out=f_stem + '_MDTF_TEMP_%d.png')) # syntax for f_out above appends "_MDTF_TEMP" + page number to # output files. If input .ps/.pdf file had multiple pages, this will # generate 1 png per page. Page numbering starts at 1. Now check # how many files gs created: out_files = glob.glob(f_stem + '_MDTF_TEMP_?.png') if not out_files: raise OSError("Error: no png generated from {}".format(f)) elif len(out_files) == 1: # got one .png, so remove suffix. os.rename(out_files[0], f_stem + '.png') else: # Multiple .pngs. Drop the MDTF_TEMP suffix and renumber starting # from zero (forget which POD requires this.) for n in list(range(len(out_files))): os.rename(f_stem + '_MDTF_TEMP_{}.png'.format(n + 1), f_stem + '-{}.png'.format(n)) # move converted figures and any figures that were saved directly as bitmaps files = util.find_files(abs_src_subdir, ['*.png', '*.gif', '*.jpg', '*.jpeg']) util.recursive_copy(files, abs_src_subdir, abs_dest_subdir, copy_function=shutil.move, overwrite=False)
def make_pod_html(self): """Perform templating on POD's html results page(s). A wrapper for :func:`~util_mdtf.append_html_template`. Looks for all html files in POD_CODE_DIR, templates them, and copies them to POD_WK_DIR, respecting subdirectory structure (see doc for :func:`~util.recursive_copy`). """ config = util_mdtf.ConfigManager() template = config.global_envvars.copy() template.update(self.pod_env_vars) source_files = util.find_files(self.POD_CODE_DIR, '*.html') util.recursive_copy( source_files, self.POD_CODE_DIR, self.POD_WK_DIR, copy_function=lambda src, dest: util_mdtf.append_html_template( src, dest, template_dict=template, append=False), overwrite=True)
def make_pod_html(self, pod): """Perform templating on POD's html results page(s). A wrapper for :func:`~util.append_html_template`. Looks for all html files in POD_CODE_DIR, templates them, and copies them to POD_WK_DIR, respecting subdirectory structure (see doc for :func:`~util.recursive_copy`). """ test_path = os.path.join(pod.POD_CODE_DIR, self.pod_html_template_file_name(pod)) if not os.path.isfile(test_path): # POD's top-level HTML template needs to exist raise util.MDTFFileNotFoundError(test_path) template_d = html_templating_dict(pod) # copy and template all .html files, since PODs can make sub-pages source_files = util.find_files(pod.POD_CODE_DIR, '*.html') util.recursive_copy( source_files, pod.POD_CODE_DIR, pod.POD_WK_DIR, copy_function=(lambda src, dest: util.append_html_template( src, dest, template_dict=template_d, append=False)), overwrite=True)
def convert_pod_figures(self, pod, src_subdir, dest_subdir): """Convert all vector graphics in `POD_WK_DIR/subdir` to .png files using ghostscript. All vector graphics files (identified by extension) in any subdirectory of `POD_WK_DIR/src_subdir` are converted to .png files by running `ghostscript <https://www.ghostscript.com/>`__ in a subprocess. Ghostscript is included in the _MDTF_base conda environment. Afterwards, any bitmap files (identified by extension) in any subdirectory of `POD_WK_DIR/src_subdir` are moved to `POD_WK_DIR/dest_subdir`, preserving and subdirectories (see doc for :func:`~util.recursive_copy`.) Args: src_subdir: Subdirectory tree of `POD_WK_DIR` to search for vector graphics files. dest_subdir: Subdirectory tree of `POD_WK_DIR` to move converted bitmap files to. """ # Flags to pass to ghostscript for PS -> PNG conversion (in particular # bitmap resolution.) eps_convert_flags = ( "-dSAFER -dBATCH -dNOPAUSE -dEPSCrop -r150 " "-sDEVICE=png16m -dTextAlphaBits=4 -dGraphicsAlphaBits=4") abs_src_subdir = os.path.join(pod.POD_WK_DIR, src_subdir) abs_dest_subdir = os.path.join(pod.POD_WK_DIR, dest_subdir) files = util.find_files( abs_src_subdir, ['*.ps', '*.PS', '*.eps', '*.EPS', '*.pdf', '*.PDF']) for f in files: f_stem, _ = os.path.splitext(f) # Append "_MDTF_TEMP" + page number to output files ("%d" = ghostscript's # template for multi-page output). If input .ps/.pdf file has multiple # pages, this will generate 1 png per page, counting from 1. f_out = f_stem + '_MDTF_TEMP_%d.png' try: util.run_shell_command( f'gs {eps_convert_flags} -sOutputFile="{f_out}" {f}') except Exception as exc: _log.error("%s produced malformed plot: %s", pod.name, f[len(abs_src_subdir):]) if isinstance(exc, util.MDTFCalledProcessError): _log.debug( "gs error encountered when converting %s for %s:\n%s", pod.name, f[len(abs_src_subdir):], getattr(exc, "output", "")) continue # gs ran successfully; check how many files it created: out_files = glob.glob(f_stem + '_MDTF_TEMP_?.png') if not out_files: raise util.MDTFFileNotFoundError( f"No .png generated from {f}.") elif len(out_files) == 1: # got one .png, so remove suffix. os.rename(out_files[0], f_stem + '.png') else: # Multiple .pngs. Drop the MDTF_TEMP suffix and renumber starting # from zero (forget which POD requires this.) for n in range(len(out_files)): os.rename(f_stem + f'_MDTF_TEMP_{n+1}.png', f_stem + f'-{n}.png') # move converted figures and any figures that were saved directly as bitmaps files = util.find_files(abs_src_subdir, ['*.png', '*.gif', '*.jpg', '*.jpeg']) util.recursive_copy(files, abs_src_subdir, abs_dest_subdir, copy_function=shutil.move, overwrite=False)