def time_2d_animation(self, output_path=None, dataset_selector=None, axes_selector=None, time_selector=None, dpi=200, fps=1, cmap=None, norm=None, rasterized=True, z_min=None, z_max=None, latex_label=True, interval=200): """ Generate a plot of 2d data as a color map which animated in time. If an output path with a suitable extension is supplied, the method will export it. Available formats are mp4 and gif. The returned objects allow for minimal customization and representation. For example in Jupyter you might use `IPython.display.HTML(animation.to_html5_video())`, where `animation` is the returned `FuncAnimation` instance. Note: Exporting a high resolution animated gif with many frames might eat your RAM. Args: output_path (str): The place where the plot is saved. If "" or None, the plot is shown in matplotlib. dataset_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method. axes_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method. time_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method. interval (float): Delay between frames in ms. If exporting to mp4, the fps is used instead to generate the file, although the returned objects do use this value. dpi (int): The resolution of the frames in dots per inch (only if exporting). fps (int): The frames per seconds (only if exporting to mp4). latex_label (bool): Whether for use LaTeX code for the plot. cmap (str or `matplotlib.colors.Colormap`): The Colormap to use in the plot. norm (str or `matplotlib.colors.Normalize`): How to scale the colormap. For advanced manipulation, use some Normalize subclass, e.g., colors.SymLogNorm(0.01). Automatic scales can be selected with the following strings: * "lin": Linear scale from minimum to maximum. * "log": Logarithmic scale from minimum to maximum up to vmax/vmin>1E9, otherwise increasing vmin. rasterized (bool): Whether the map is rasterized. This does not apply to axes, title... Note non-rasterized images with large amount of data exported to PDF might challenging to handle. Returns: (`matplotlib.figure.Figure`, `matplotlib.axes.Axes`, `matplotlib.animation.FuncAnimation`): Objects representing the generated plot and its animation. Raises: FileNotFoundError: If tried to export to mp4 but ffmpeg is not found in the system. """ if output_path: ensure_dir_exists(os.path.dirname(output_path)) axes = self.get_axes(dataset_selector=dataset_selector, axes_selector=axes_selector) if len(axes) != 2: raise ValueError("Expected 2 axes plot, but %d were provided" % len(axes)) gen = self.get_generator(dataset_selector=dataset_selector, axes_selector=axes_selector, time_selector=time_selector) # Set plot labels fig, ax = plt.subplots() fig.set_tight_layout(True) x_name = axes[0]["LONG_NAME"] x_units = axes[0]["UNITS"] y_name = axes[1]["LONG_NAME"] y_units = axes[1]["UNITS"] title_name = self.data_name title_units = self.units ax.set_xlabel(_create_label(x_name, x_units, latex_label)) ax.set_ylabel(_create_label(y_name, y_units, latex_label)) # Gather the points x_min, x_max = axes[0]["MIN"], axes[0]["MAX"] y_min, y_max = axes[1]["MIN"], axes[1]["MAX"] z = np.transpose(np.asarray(next(gen))) time_list = self.get_time_list(time_selector) if len(time_list) < 2: raise ValueError("At least two time snapshots are needed to make an animation") norm = _autonorm(norm, z) plot_function = ax.pcolormesh if rasterized: # Rasterizing in contourf is a bit tricky # Cf. http://stackoverflow.com/questions/33250005/size-of-matplotlib-contourf-image-files plot = plot_function(axes[0]["LIST"], axes[1]["LIST"], z, norm=norm, cmap=cmap, zorder=-9) ax.set_rasterization_zorder(-1) else: plot = plot_function(axes[0]["LIST"], axes[1]["LIST"], z, norm=norm, cmap=cmap) ax.set_xlim(x_min, x_max) ax.set_ylim(y_min, y_max) ax.set_title(_create_label(title_name, title_units, latex_label)) _fix_colorbar(fig.colorbar(plot)) # Prepare a function for the updates def update(i): """Update the plot, returning the artists which must be redrawn.""" try: new_dataset = np.transpose(np.asarray(next(gen))) except StopIteration: logger.warning("Tried to add a frame to the animation, but all data was used.") return label = 't = {0}'.format(time_list[i]) # BEWARE: The set_array syntax is rather problematic. Depending on the shading used in pcolormesh, the # following might not work. plot.set_array(new_dataset[:-1, :-1].ravel()) # For more details, check lumbric's answer to # https://stackoverflow.com/questions/18797175/animation-with-pcolormesh-routine-in-matplotlib-how-do-i-initialize-the-data ax.set_title(label) return plot, ax anim = FuncAnimation(fig, update, frames=range(1, len(time_list) - 2), interval=interval) if not output_path: # "" or None pass else: filename = os.path.basename(output_path) if "." in filename: extension = output_path.split(".")[-1].lower() else: extension = None if extension == "gif": anim.save(output_path, dpi=dpi, writer='imagemagick') elif extension == "mp4": metadata = dict(title=os.path.split(self.data_path)[-1], artist='duat', comment=self.data_path) writer = FFMpegWriter(fps=fps, metadata=metadata) with writer.saving(fig, output_path, dpi): # Iterate over frames for i in range(1, len(time_list) - 1): update(i) writer.grab_frame() # Keep showing the last frame for the fixed time writer.grab_frame() else: logger.warning("Unknown extension in path %s. No output produced." % output_path) plt.close() return fig, ax, anim
def axes_2d_colormap(self, output_path=None, dataset_selector=None, axes_selector=None, time_selector=None, dpi=200, latex_label=True, cmap=None, norm=None, show=True, rasterized=True, contour_plot=False): """ Generate a colormap in two axes. A single time snapshot must be selected with the time_selector parameter. For an animated version in time see the :func:`~duat.osiris.plot.Diagnostic.time_2d_animation` method. Note: For simple manipulation like labels or title you can make use of the returned tuple or a `matplotlib.pyplot.style.context`. More advanced manipulation can be done extracting the data with the :func:`~duat.osiris.plot.Diagnostic.get_generator` method instead. Args: output_path (str): The place where the plot is saved. If "" or None, the figure is not saved. dataset_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method. axes_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method. time_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method. dpi (int): The resolution of the file in dots per inch. latex_label (bool): Whether for use LaTeX code for the plot. cmap (str or `matplotlib.colors.Colormap`): The Colormap to use in the plot. norm (str or `matplotlib.colors.Normalize`): How to scale the colormap. For advanced manipulation, use some Normalize subclass, e.g., colors.SymLogNorm(0.01). Automatic scales can be selected with the following strings: * "lin": Linear scale from minimum to maximum. * "log": Logarithmic scale from minimum to maximum up to vmax/vmin>1E9, otherwise increasing vmin. show (bool): Whether to show the plot. This is blocking if matplotlib is in non-interactive mode. rasterized (bool): Whether the map is rasterized. This does not apply to axes, title... Note non-rasterized images with large amount of data exported to PDF might challenging to handle. contour_plot (bool): Whether contour lines are plot instead of the density map. Returns: (`matplotlib.figure.Figure`, `matplotlib.axes.Axes`): Objects representing the generated plot. """ if output_path: ensure_dir_exists(os.path.dirname(output_path)) axes = self.get_axes(dataset_selector=dataset_selector, axes_selector=axes_selector) if len(axes) != 2: raise ValueError("Expected 2 axes plot, but %d were provided" % len(axes)) gen = self.get_generator(dataset_selector=dataset_selector, axes_selector=axes_selector, time_selector=time_selector) # Set plot labels fig, ax = plt.subplots() x_name = axes[0]["LONG_NAME"] x_units = axes[0]["UNITS"] y_name = axes[1]["LONG_NAME"] y_units = axes[1]["UNITS"] title_name = self.data_name title_units = self.units ax.set_xlabel(_create_label(x_name, x_units, latex_label)) ax.set_ylabel(_create_label(y_name, y_units, latex_label)) time_list = self.get_time_list(time_selector) if len(time_list) == 0: raise ValueError("No time snapshot selected") if len(time_list) != 1: raise ValueError("A single time snapshot must be selected for this plot") # Gather the points x_min, x_max = axes[0]["MIN"], axes[0]["MAX"] y_min, y_max = axes[1]["MIN"], axes[1]["MAX"] z = np.transpose(np.asarray(list(gen)[0])) norm = _autonorm(norm, z) plot_function = ax.contourf if contour_plot else ax.pcolormesh if rasterized: # Rasterizing in contourf is a bit tricky # Cf. http://stackoverflow.com/questions/33250005/size-of-matplotlib-contourf-image-files plot = plot_function(axes[0]["LIST"], axes[1]["LIST"], z, norm=norm, cmap=cmap, zorder=-9) ax.set_rasterization_zorder(-1) else: plot = plot_function(axes[0]["LIST"], axes[1]["LIST"], z, norm=norm, cmap=cmap) ax.set_xlim(x_min, x_max) ax.set_ylim(y_min, y_max) ax.set_title(_create_label(title_name, title_units, latex_label)) _fix_colorbar(fig.colorbar(plot)) if output_path: # "" or None plt.savefig(output_path, dpi=dpi) if show: plt.show() else: plt.close() return fig, ax
def run_config_grid(config, run_dir, prefix=None, run_name="osiris_run", remote_dir=None, clean_dir=True, prolog="", epilog="", create_only=False): """ Queue a OSIRIS run in a compatible grid (e.g., Sun Grid Engine). Args: config (`ConfigFile`): The instance describing the configuration file. run_dir (str): Folder where the run will be carried. prefix (str): A prefix to run the command. If None, "mpirun -np X " will be used when a config with multiple nodes is provided. run_name (str): Name of the job in the engine. remote_dir (str): If provided, a remote directory where the run will be carried, which might be only available in the node selected by the engine. Note that if this option is used, the returned Run instance will not access the remote_dir, but the run_dir. If the remote node is unable to access the path (trying to create it if needed), OSIRIS will be started in the run dir and errors will be logged by the queue system. clean_dir (bool): Whether to remove the files in the directory before execution. prolog (str): Shell code to run before calling OSIRIS (but once in the remote_dir if asked for). epilog (str): Shell code to run after calling OSIRIS. create_only (bool): Whether just to create the files, but not queueing the run. Returns: Run: A Run instance describing the execution. """ # Clean if needed if clean_dir: for root, dirs, files in walk(run_dir): for f in files: remove(path.join(root, f)) for root, dirs, files in walk(run_dir): for f in files: logger.warning("Could not remove file %s" % f) # Find the needed amount of nodes n = config.get_nodes() if prefix is None: prefix = "mpirun -np %d " % n if n > 1 else "" elif prefix[-1] != " ": prefix += " " # copy the input file ensure_dir_exists(run_dir) config.write(path.join(run_dir, "os-stdin")) # Copy the osiris executable osiris_path = path.abspath(path.join(run_dir, "osiris")) osiris = ifd(config.get_d(), osiris_1d, osiris_2d, osiris_3d) copyfile(osiris, osiris_path) ensure_executable(osiris_path) # Create a start.sh file with the launch script s = "".join([ "#!/bin/bash\n#\n#$ -cwd\n#$ -S /bin/bash\n#$ -N %s\n" % run_name, "#$ -pe smp %d\n" % n if n > 1 else "", "#\n", "NEW_DIR=%s\nmkdir -p $NEW_DIR\ncp -r . $NEW_DIR\ncd $NEW_DIR\n" % remote_dir if remote_dir else "", prolog + "\n", "\n%s./osiris > out.txt 2> err.txt\n" % prefix, epilog + "\n" ]) with open(path.join(run_dir, "start.sh"), 'w') as f: f.write(s) ensure_executable(path.join(run_dir, "start.sh")) if not create_only: subprocess.Popen("qsub " + path.abspath(path.join(run_dir, "start.sh")), shell=True, cwd=path.abspath(run_dir)) return Run(run_dir)
def time_1d_animation(self, output_path=None, dataset_selector=None, axes_selector=None, time_selector=None, dpi=200, fps=1, scale_mode="expand", latex_label=True, interval=200): """ Generate a plot of 1d data animated in time. If an output path with a suitable extension is supplied, the method will export it. Available formats are mp4 and gif. The returned objects allow for minimal customization and representation. For example in Jupyter you might use `IPython.display.HTML(animation.to_html5_video())`, where `animation` is the returned `FuncAnimation` instance. Note: Exporting a high resolution animated gif with many frames might eat your RAM. Args: output_path (str): The place where the plot is saved. If "" or None, the plot is shown in matplotlib. dataset_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method. axes_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method. time_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method. interval (float): Delay between frames in ms. If exporting to mp4, the fps is used instead to generate the file, although the returned objects do use this value. dpi (int): The resolution of the frames in dots per inch (only if exporting). fps (int): The frames per seconds (only if exporting to mp4). scale_mode (str): How the scale is changed through time. Available methods are: * "expand": The y limits increase when needed, but they don't decrease. * "adjust_always": Always change the y limits to those of the data. * "max": Use the maximum range from the beginning. latex_label (bool): Whether for use LaTeX code for the plot. Returns: (`matplotlib.figure.Figure`, `matplotlib.axes.Axes`, `matplotlib.animation.FuncAnimation`): Objects representing the generated plot and its animation. Raises: FileNotFoundError: If tried to export to mp4 but ffmpeg is not found in the system. """ if output_path: ensure_dir_exists(os.path.dirname(output_path)) axes = self.get_axes(dataset_selector=dataset_selector, axes_selector=axes_selector) if len(axes) != 1: raise ValueError("Expected 1 axis plot, but %d were provided" % len(axes)) axis = axes[0] gen = self.get_generator(dataset_selector=dataset_selector, axes_selector=axes_selector, time_selector=time_selector) # Set plot labels fig, ax = plt.subplots() fig.set_tight_layout(True) x_name = axis["LONG_NAME"] x_units = axis["UNITS"] y_name = self.data_name y_units = self.units ax.set_xlabel(_create_label(x_name, x_units, latex_label)) ax.set_ylabel(_create_label(y_name, y_units, latex_label)) # Plot the points x_min, x_max = axis["MIN"], axis["MAX"] plot_data, = ax.plot(axis["LIST"], next(gen)) ax.set_xlim(x_min, x_max) if scale_mode == "max": # Get a list (generator) with the mins and maxs in each time step min_max_list = map(lambda l: [min(l), max(l)], self.get_generator(dataset_selector=dataset_selector, axes_selector=axes_selector, time_selector=time_selector)) f = lambda mins, maxs: (min(mins), max(maxs)) y_min, y_max = f(*zip(*min_max_list)) ax.set_ylim(y_min, y_max) time_list = self.get_time_list(time_selector) # Prepare a function for the updates def update(i): """Update the plot, returning the artists which must be redrawn.""" try: new_dataset = next(gen) except StopIteration: logger.warning("Tried to add a frame to the animation, but all data was used.") return label = 't = {0}'.format(time_list[i]) plot_data.set_ydata(new_dataset[:]) ax.set_title(label) if not scale_mode or scale_mode == "max": pass elif scale_mode == "expand": prev = ax.get_ylim() data_limit = [min(new_dataset), max(new_dataset)] ax.set_ylim(min(prev[0], data_limit[0]), max(prev[1], data_limit[1])) elif scale_mode == "adjust_always": ax.set_ylim(min(new_dataset), max(new_dataset)) return plot_data, ax anim = FuncAnimation(fig, update, frames=range(1, len(time_list) - 2), interval=interval) if not output_path: # "" or None pass else: filename = os.path.basename(output_path) if "." in filename: extension = output_path.split(".")[-1].lower() else: extension = None if extension == "gif": anim.save(output_path, dpi=dpi, writer='imagemagick') elif extension == "mp4": metadata = dict(title=os.path.split(self.data_path)[-1], artist='duat', comment=self.data_path) writer = FFMpegWriter(fps=fps, metadata=metadata) with writer.saving(fig, output_path, dpi): # Iterate over frames for i in range(1, len(time_list) - 1): update(i) writer.grab_frame() # Keep showing the last frame for the fixed time writer.grab_frame() else: logger.warning("Unknown extension in path %s. No output produced." % output_path) plt.close() return fig, ax, anim
def run_config(config, run_dir, prefix=None, clean_dir=True, blocking=None, force=None, mpcaller=None, create_only=False): """ Initiate a OSIRIS run from a config instance. Args: config (`ConfigFile`): The instance describing the configuration file. run_dir (str): Folder where the run is carried. prefix (str): A prefix to run the command. If None, "mpirun -np X " will be used when a config with multiple nodes is provided. clean_dir (bool): Whether to remove the files in the directory before execution. blocking (bool): Whether to wait for the run to finish. force (str): Set what to do if a running executable is found in the directory. Set to "ignore" to launch anyway, possibly resulting in multiple instances running simultaneously; set to "kill" to terminate the existing processes. mpcaller (MPCaller): An instance controlling multithreaded calls. If supplied, all calls will be handled by this instance and the blocking parameter will be ignored. create_only (bool): Whether just to create the files, but not starting the run. Returns: tuple: A Run instance describing the execution. """ # Find the needed amount of nodes n = config.get_nodes() if prefix is None: prefix = "mpirun -np %d " % n if n > 1 else "" elif prefix[-1] != " ": prefix += " " # Search for possibly running processes candidates = _find_running_exe(path.join(run_dir, "osiris")) if candidates: if force == "ignore": logger.warning("Ignored %d running exe found in %s" % (len(candidates), run_dir)) elif force == "kill": logger.warning("Killing %d running exe found in %s" % (len(candidates), run_dir)) for c in candidates: try: psutil.Process(c).terminate() except psutil.NoSuchProcess: pass # If just ended else: logger.warning("Running exe found in %s. Aborting launch." % run_dir) return # Clean if needed if clean_dir: for root, dirs, files in walk(run_dir): for f in files: remove(path.join(root, f)) for root, dirs, files in walk(run_dir): for f in files: logger.warning("Could not remove file %s" % f) # If the run is restartable, make the if_restart variable explicit if "restart" in config and "ndump_fac" in config["restart"] and config[ "restart"]["ndump_fac"]: if "if_restart" not in config["restart"]: config["restart"][ "if_restart"] = False # This is the default value # copy the input file ensure_dir_exists(run_dir) config.write(path.join(run_dir, "os-stdin")) # Copy the osiris executable osiris_path = path.abspath(path.join(run_dir, "osiris")) osiris = ifd(config.get_d(), osiris_1d, osiris_2d, osiris_3d) copyfile(osiris, osiris_path) ensure_executable(osiris_path) # Create a start.sh file to ease manual launch with open(path.join(run_dir, "start.sh"), 'w') as f: f.write("#!/bin/bash\n%s./osiris > out.txt 2> err.txt" % prefix) ensure_executable(path.join(run_dir, "start.sh")) # Create a continue.sh file to ease manual relaunch of aborted executions with open(path.join(run_dir, "continue.sh"), 'w') as f: f.write( "#!/bin/bash" "\nsed -i -e \"s/if_restart = .false./if_restart = .true./g\" os-stdin" "\n./%s osiris >> out.txt 2>> err.txt" % prefix) ensure_executable(path.join(run_dir, "continue.sh")) if create_only: return Run(run_dir) if mpcaller is not None: run = Run(run_dir) # Set the run instance to update the process info when the call is made. mpcaller.add_call(Call(_execute_run, prefix, osiris_path, run_dir)) return run else: proc = subprocess.Popen(prefix + osiris_path + " > out.txt 2> err.txt", shell=True, cwd=path.abspath(run_dir)) if blocking: proc.wait() else: # Sleep a little to check for quickly appearing errors and to allow the shell to start osiris sleep(0.2) # BEWARE: Perhaps under extreme circumstances, OSIRIS might have not started despite sleeping. # This could be solved calling the update method of the Run instance. # Consider this a feature instead of a bug :P run = Run(run_dir) # Try to detect errors checking the output if run._has_error(): logger.warning( "Error detected while launching %s.\nCheck out.txt and err.txt for more information or re-run in console." % run_dir) return run