def time_2d_animation(self, output_path=None, dataset_selector=None, axes_selector=None, time_selector=None,
dpi=200, fps=1, cmap=None, norm=None, rasterized=True, z_min=None,
z_max=None, latex_label=True, interval=200):
"""
Generate a plot of 2d data as a color map which animated in time.
If an output path with a suitable extension is supplied, the method will export it. Available formats are mp4
and gif. The returned objects allow for minimal customization and representation. For example in Jupyter you
might use `IPython.display.HTML(animation.to_html5_video())`, where `animation` is the returned `FuncAnimation`
instance.
Note:
Exporting a high resolution animated gif with many frames might eat your RAM.
Args:
output_path (str): The place where the plot is saved. If "" or None, the plot is shown in matplotlib.
dataset_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method.
axes_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method.
time_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method.
interval (float): Delay between frames in ms. If exporting to mp4, the fps is used instead to generate the
file, although the returned objects do use this value.
dpi (int): The resolution of the frames in dots per inch (only if exporting).
fps (int): The frames per seconds (only if exporting to mp4).
latex_label (bool): Whether for use LaTeX code for the plot.
cmap (str or `matplotlib.colors.Colormap`): The Colormap to use in the plot.
norm (str or `matplotlib.colors.Normalize`): How to scale the colormap. For advanced manipulation, use some
Normalize subclass, e.g., colors.SymLogNorm(0.01). Automatic scales can be selected with
the following strings:
* "lin": Linear scale from minimum to maximum.
* "log": Logarithmic scale from minimum to maximum up to vmax/vmin>1E9, otherwise increasing vmin.
rasterized (bool): Whether the map is rasterized. This does not apply to axes, title... Note non-rasterized
images with large amount of data exported to PDF might challenging to handle.
Returns:
(`matplotlib.figure.Figure`, `matplotlib.axes.Axes`, `matplotlib.animation.FuncAnimation`):
Objects representing the generated plot and its animation.
Raises:
FileNotFoundError: If tried to export to mp4 but ffmpeg is not found in the system.
"""
if output_path:
ensure_dir_exists(os.path.dirname(output_path))
axes = self.get_axes(dataset_selector=dataset_selector, axes_selector=axes_selector)
if len(axes) != 2:
raise ValueError("Expected 2 axes plot, but %d were provided" % len(axes))
gen = self.get_generator(dataset_selector=dataset_selector, axes_selector=axes_selector,
time_selector=time_selector)
# Set plot labels
fig, ax = plt.subplots()
fig.set_tight_layout(True)
x_name = axes[0]["LONG_NAME"]
x_units = axes[0]["UNITS"]
y_name = axes[1]["LONG_NAME"]
y_units = axes[1]["UNITS"]
title_name = self.data_name
title_units = self.units
ax.set_xlabel(_create_label(x_name, x_units, latex_label))
ax.set_ylabel(_create_label(y_name, y_units, latex_label))
# Gather the points
x_min, x_max = axes[0]["MIN"], axes[0]["MAX"]
y_min, y_max = axes[1]["MIN"], axes[1]["MAX"]
z = np.transpose(np.asarray(next(gen)))
time_list = self.get_time_list(time_selector)
if len(time_list) < 2:
raise ValueError("At least two time snapshots are needed to make an animation")
norm = _autonorm(norm, z)
plot_function = ax.pcolormesh
if rasterized:
# Rasterizing in contourf is a bit tricky
# Cf. http://stackoverflow.com/questions/33250005/size-of-matplotlib-contourf-image-files
plot = plot_function(axes[0]["LIST"], axes[1]["LIST"], z, norm=norm, cmap=cmap, zorder=-9)
ax.set_rasterization_zorder(-1)
else:
plot = plot_function(axes[0]["LIST"], axes[1]["LIST"], z, norm=norm, cmap=cmap)
ax.set_xlim(x_min, x_max)
ax.set_ylim(y_min, y_max)
ax.set_title(_create_label(title_name, title_units, latex_label))
_fix_colorbar(fig.colorbar(plot))
# Prepare a function for the updates
def update(i):
"""Update the plot, returning the artists which must be redrawn."""
try:
new_dataset = np.transpose(np.asarray(next(gen)))
except StopIteration:
logger.warning("Tried to add a frame to the animation, but all data was used.")
return
label = 't = {0}'.format(time_list[i])
# BEWARE: The set_array syntax is rather problematic. Depending on the shading used in pcolormesh, the
# following might not work.
plot.set_array(new_dataset[:-1, :-1].ravel())
# For more details, check lumbric's answer to
# https://stackoverflow.com/questions/18797175/animation-with-pcolormesh-routine-in-matplotlib-how-do-i-initialize-the-data
ax.set_title(label)
return plot, ax
anim = FuncAnimation(fig, update, frames=range(1, len(time_list) - 2), interval=interval)
if not output_path: # "" or None
pass
else:
filename = os.path.basename(output_path)
if "." in filename:
extension = output_path.split(".")[-1].lower()
else:
extension = None
if extension == "gif":
anim.save(output_path, dpi=dpi, writer='imagemagick')
elif extension == "mp4":
metadata = dict(title=os.path.split(self.data_path)[-1], artist='duat', comment=self.data_path)
writer = FFMpegWriter(fps=fps, metadata=metadata)
with writer.saving(fig, output_path, dpi):
# Iterate over frames
for i in range(1, len(time_list) - 1):
update(i)
writer.grab_frame()
# Keep showing the last frame for the fixed time
writer.grab_frame()
else:
logger.warning("Unknown extension in path %s. No output produced." % output_path)
plt.close()
return fig, ax, anim
def axes_2d_colormap(self, output_path=None, dataset_selector=None, axes_selector=None, time_selector=None,
dpi=200, latex_label=True, cmap=None, norm=None, show=True, rasterized=True,
contour_plot=False):
"""
Generate a colormap in two axes.
A single time snapshot must be selected with the time_selector parameter. For an animated version in time see
the :func:`~duat.osiris.plot.Diagnostic.time_2d_animation` method.
Note:
For simple manipulation like labels or title you can make use of the returned tuple or a
`matplotlib.pyplot.style.context`. More advanced manipulation can be done extracting the data with the
:func:`~duat.osiris.plot.Diagnostic.get_generator` method instead.
Args:
output_path (str): The place where the plot is saved. If "" or None, the figure is not saved.
dataset_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method.
axes_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method.
time_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method.
dpi (int): The resolution of the file in dots per inch.
latex_label (bool): Whether for use LaTeX code for the plot.
cmap (str or `matplotlib.colors.Colormap`): The Colormap to use in the plot.
norm (str or `matplotlib.colors.Normalize`): How to scale the colormap. For advanced manipulation, use some
Normalize subclass, e.g., colors.SymLogNorm(0.01). Automatic scales can be selected with
the following strings:
* "lin": Linear scale from minimum to maximum.
* "log": Logarithmic scale from minimum to maximum up to vmax/vmin>1E9, otherwise increasing vmin.
show (bool): Whether to show the plot. This is blocking if matplotlib is in non-interactive mode.
rasterized (bool): Whether the map is rasterized. This does not apply to axes, title... Note non-rasterized
images with large amount of data exported to PDF might challenging to handle.
contour_plot (bool): Whether contour lines are plot instead of the density map.
Returns:
(`matplotlib.figure.Figure`, `matplotlib.axes.Axes`): Objects representing the generated plot.
"""
if output_path:
ensure_dir_exists(os.path.dirname(output_path))
axes = self.get_axes(dataset_selector=dataset_selector, axes_selector=axes_selector)
if len(axes) != 2:
raise ValueError("Expected 2 axes plot, but %d were provided" % len(axes))
gen = self.get_generator(dataset_selector=dataset_selector, axes_selector=axes_selector,
time_selector=time_selector)
# Set plot labels
fig, ax = plt.subplots()
x_name = axes[0]["LONG_NAME"]
x_units = axes[0]["UNITS"]
y_name = axes[1]["LONG_NAME"]
y_units = axes[1]["UNITS"]
title_name = self.data_name
title_units = self.units
ax.set_xlabel(_create_label(x_name, x_units, latex_label))
ax.set_ylabel(_create_label(y_name, y_units, latex_label))
time_list = self.get_time_list(time_selector)
if len(time_list) == 0:
raise ValueError("No time snapshot selected")
if len(time_list) != 1:
raise ValueError("A single time snapshot must be selected for this plot")
# Gather the points
x_min, x_max = axes[0]["MIN"], axes[0]["MAX"]
y_min, y_max = axes[1]["MIN"], axes[1]["MAX"]
z = np.transpose(np.asarray(list(gen)[0]))
norm = _autonorm(norm, z)
plot_function = ax.contourf if contour_plot else ax.pcolormesh
if rasterized:
# Rasterizing in contourf is a bit tricky
# Cf. http://stackoverflow.com/questions/33250005/size-of-matplotlib-contourf-image-files
plot = plot_function(axes[0]["LIST"], axes[1]["LIST"], z, norm=norm, cmap=cmap, zorder=-9)
ax.set_rasterization_zorder(-1)
else:
plot = plot_function(axes[0]["LIST"], axes[1]["LIST"], z, norm=norm, cmap=cmap)
ax.set_xlim(x_min, x_max)
ax.set_ylim(y_min, y_max)
ax.set_title(_create_label(title_name, title_units, latex_label))
_fix_colorbar(fig.colorbar(plot))
if output_path: # "" or None
plt.savefig(output_path, dpi=dpi)
if show:
plt.show()
else:
plt.close()
return fig, ax
def run_config_grid(config,
run_dir,
prefix=None,
run_name="osiris_run",
remote_dir=None,
clean_dir=True,
prolog="",
epilog="",
create_only=False):
"""
Queue a OSIRIS run in a compatible grid (e.g., Sun Grid Engine).
Args:
config (`ConfigFile`): The instance describing the configuration file.
run_dir (str): Folder where the run will be carried.
prefix (str): A prefix to run the command. If None, "mpirun -np X " will be used when a config with multiple
nodes is provided.
run_name (str): Name of the job in the engine.
remote_dir (str): If provided, a remote directory where the run will be carried, which might be only available
in the node selected by the engine. Note that if this option is used, the returned Run
instance will not access the remote_dir, but the run_dir. If the remote node is unable to
access the path (trying to create it if needed), OSIRIS will be started in the run dir and
errors will be logged by the queue system.
clean_dir (bool): Whether to remove the files in the directory before execution.
prolog (str): Shell code to run before calling OSIRIS (but once in the remote_dir if asked for).
epilog (str): Shell code to run after calling OSIRIS.
create_only (bool): Whether just to create the files, but not queueing the run.
Returns:
Run: A Run instance describing the execution.
"""
# Clean if needed
if clean_dir:
for root, dirs, files in walk(run_dir):
for f in files:
remove(path.join(root, f))
for root, dirs, files in walk(run_dir):
for f in files:
logger.warning("Could not remove file %s" % f)
# Find the needed amount of nodes
n = config.get_nodes()
if prefix is None:
prefix = "mpirun -np %d " % n if n > 1 else ""
elif prefix[-1] != " ":
prefix += " "
# copy the input file
ensure_dir_exists(run_dir)
config.write(path.join(run_dir, "os-stdin"))
# Copy the osiris executable
osiris_path = path.abspath(path.join(run_dir, "osiris"))
osiris = ifd(config.get_d(), osiris_1d, osiris_2d, osiris_3d)
copyfile(osiris, osiris_path)
ensure_executable(osiris_path)
# Create a start.sh file with the launch script
s = "".join([
"#!/bin/bash\n#\n#$ -cwd\n#$ -S /bin/bash\n#$ -N %s\n" % run_name,
"#$ -pe smp %d\n" % n if n > 1 else "", "#\n",
"NEW_DIR=%s\nmkdir -p $NEW_DIR\ncp -r . $NEW_DIR\ncd $NEW_DIR\n" %
remote_dir if remote_dir else "", prolog + "\n",
"\n%s./osiris > out.txt 2> err.txt\n" % prefix, epilog + "\n"
])
with open(path.join(run_dir, "start.sh"), 'w') as f:
f.write(s)
ensure_executable(path.join(run_dir, "start.sh"))
if not create_only:
subprocess.Popen("qsub " +
path.abspath(path.join(run_dir, "start.sh")),
shell=True,
cwd=path.abspath(run_dir))
return Run(run_dir)
def time_1d_animation(self, output_path=None, dataset_selector=None, axes_selector=None, time_selector=None,
dpi=200, fps=1, scale_mode="expand",
latex_label=True, interval=200):
"""
Generate a plot of 1d data animated in time.
If an output path with a suitable extension is supplied, the method will export it. Available formats are mp4
and gif. The returned objects allow for minimal customization and representation. For example in Jupyter you
might use `IPython.display.HTML(animation.to_html5_video())`, where `animation` is the returned `FuncAnimation`
instance.
Note:
Exporting a high resolution animated gif with many frames might eat your RAM.
Args:
output_path (str): The place where the plot is saved. If "" or None, the plot is shown in matplotlib.
dataset_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method.
axes_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method.
time_selector: See :func:`~duat.osiris.plot.Diagnostic.get_generator` method.
interval (float): Delay between frames in ms. If exporting to mp4, the fps is used instead to generate the
file, although the returned objects do use this value.
dpi (int): The resolution of the frames in dots per inch (only if exporting).
fps (int): The frames per seconds (only if exporting to mp4).
scale_mode (str): How the scale is changed through time. Available methods are:
* "expand": The y limits increase when needed, but they don't decrease.
* "adjust_always": Always change the y limits to those of the data.
* "max": Use the maximum range from the beginning.
latex_label (bool): Whether for use LaTeX code for the plot.
Returns:
(`matplotlib.figure.Figure`, `matplotlib.axes.Axes`, `matplotlib.animation.FuncAnimation`):
Objects representing the generated plot and its animation.
Raises:
FileNotFoundError: If tried to export to mp4 but ffmpeg is not found in the system.
"""
if output_path:
ensure_dir_exists(os.path.dirname(output_path))
axes = self.get_axes(dataset_selector=dataset_selector, axes_selector=axes_selector)
if len(axes) != 1:
raise ValueError("Expected 1 axis plot, but %d were provided" % len(axes))
axis = axes[0]
gen = self.get_generator(dataset_selector=dataset_selector, axes_selector=axes_selector,
time_selector=time_selector)
# Set plot labels
fig, ax = plt.subplots()
fig.set_tight_layout(True)
x_name = axis["LONG_NAME"]
x_units = axis["UNITS"]
y_name = self.data_name
y_units = self.units
ax.set_xlabel(_create_label(x_name, x_units, latex_label))
ax.set_ylabel(_create_label(y_name, y_units, latex_label))
# Plot the points
x_min, x_max = axis["MIN"], axis["MAX"]
plot_data, = ax.plot(axis["LIST"], next(gen))
ax.set_xlim(x_min, x_max)
if scale_mode == "max":
# Get a list (generator) with the mins and maxs in each time step
min_max_list = map(lambda l: [min(l), max(l)],
self.get_generator(dataset_selector=dataset_selector, axes_selector=axes_selector,
time_selector=time_selector))
f = lambda mins, maxs: (min(mins), max(maxs))
y_min, y_max = f(*zip(*min_max_list))
ax.set_ylim(y_min, y_max)
time_list = self.get_time_list(time_selector)
# Prepare a function for the updates
def update(i):
"""Update the plot, returning the artists which must be redrawn."""
try:
new_dataset = next(gen)
except StopIteration:
logger.warning("Tried to add a frame to the animation, but all data was used.")
return
label = 't = {0}'.format(time_list[i])
plot_data.set_ydata(new_dataset[:])
ax.set_title(label)
if not scale_mode or scale_mode == "max":
pass
elif scale_mode == "expand":
prev = ax.get_ylim()
data_limit = [min(new_dataset), max(new_dataset)]
ax.set_ylim(min(prev[0], data_limit[0]), max(prev[1], data_limit[1]))
elif scale_mode == "adjust_always":
ax.set_ylim(min(new_dataset), max(new_dataset))
return plot_data, ax
anim = FuncAnimation(fig, update, frames=range(1, len(time_list) - 2), interval=interval)
if not output_path: # "" or None
pass
else:
filename = os.path.basename(output_path)
if "." in filename:
extension = output_path.split(".")[-1].lower()
else:
extension = None
if extension == "gif":
anim.save(output_path, dpi=dpi, writer='imagemagick')
elif extension == "mp4":
metadata = dict(title=os.path.split(self.data_path)[-1], artist='duat', comment=self.data_path)
writer = FFMpegWriter(fps=fps, metadata=metadata)
with writer.saving(fig, output_path, dpi):
# Iterate over frames
for i in range(1, len(time_list) - 1):
update(i)
writer.grab_frame()
# Keep showing the last frame for the fixed time
writer.grab_frame()
else:
logger.warning("Unknown extension in path %s. No output produced." % output_path)
plt.close()
return fig, ax, anim
def run_config(config,
run_dir,
prefix=None,
clean_dir=True,
blocking=None,
force=None,
mpcaller=None,
create_only=False):
"""
Initiate a OSIRIS run from a config instance.
Args:
config (`ConfigFile`): The instance describing the configuration file.
run_dir (str): Folder where the run is carried.
prefix (str): A prefix to run the command. If None, "mpirun -np X " will be used when a config with multiple
nodes is provided.
clean_dir (bool): Whether to remove the files in the directory before execution.
blocking (bool): Whether to wait for the run to finish.
force (str): Set what to do if a running executable is found in the directory. Set to "ignore" to launch anyway,
possibly resulting in multiple instances running simultaneously; set to "kill" to terminate the
existing processes.
mpcaller (MPCaller): An instance controlling multithreaded calls. If supplied, all calls will be handled by this
instance and the blocking parameter will be ignored.
create_only (bool): Whether just to create the files, but not starting the run.
Returns:
tuple: A Run instance describing the execution.
"""
# Find the needed amount of nodes
n = config.get_nodes()
if prefix is None:
prefix = "mpirun -np %d " % n if n > 1 else ""
elif prefix[-1] != " ":
prefix += " "
# Search for possibly running processes
candidates = _find_running_exe(path.join(run_dir, "osiris"))
if candidates:
if force == "ignore":
logger.warning("Ignored %d running exe found in %s" %
(len(candidates), run_dir))
elif force == "kill":
logger.warning("Killing %d running exe found in %s" %
(len(candidates), run_dir))
for c in candidates:
try:
psutil.Process(c).terminate()
except psutil.NoSuchProcess:
pass # If just ended
else:
logger.warning("Running exe found in %s. Aborting launch." %
run_dir)
return
# Clean if needed
if clean_dir:
for root, dirs, files in walk(run_dir):
for f in files:
remove(path.join(root, f))
for root, dirs, files in walk(run_dir):
for f in files:
logger.warning("Could not remove file %s" % f)
# If the run is restartable, make the if_restart variable explicit
if "restart" in config and "ndump_fac" in config["restart"] and config[
"restart"]["ndump_fac"]:
if "if_restart" not in config["restart"]:
config["restart"][
"if_restart"] = False # This is the default value
# copy the input file
ensure_dir_exists(run_dir)
config.write(path.join(run_dir, "os-stdin"))
# Copy the osiris executable
osiris_path = path.abspath(path.join(run_dir, "osiris"))
osiris = ifd(config.get_d(), osiris_1d, osiris_2d, osiris_3d)
copyfile(osiris, osiris_path)
ensure_executable(osiris_path)
# Create a start.sh file to ease manual launch
with open(path.join(run_dir, "start.sh"), 'w') as f:
f.write("#!/bin/bash\n%s./osiris > out.txt 2> err.txt" % prefix)
ensure_executable(path.join(run_dir, "start.sh"))
# Create a continue.sh file to ease manual relaunch of aborted executions
with open(path.join(run_dir, "continue.sh"), 'w') as f:
f.write(
"#!/bin/bash"
"\nsed -i -e \"s/if_restart = .false./if_restart = .true./g\" os-stdin"
"\n./%s osiris >> out.txt 2>> err.txt" % prefix)
ensure_executable(path.join(run_dir, "continue.sh"))
if create_only:
return Run(run_dir)
if mpcaller is not None:
run = Run(run_dir)
# Set the run instance to update the process info when the call is made.
mpcaller.add_call(Call(_execute_run, prefix, osiris_path, run_dir))
return run
else:
proc = subprocess.Popen(prefix + osiris_path + " > out.txt 2> err.txt",
shell=True,
cwd=path.abspath(run_dir))
if blocking:
proc.wait()
else: # Sleep a little to check for quickly appearing errors and to allow the shell to start osiris
sleep(0.2)
# BEWARE: Perhaps under extreme circumstances, OSIRIS might have not started despite sleeping.
# This could be solved calling the update method of the Run instance.
# Consider this a feature instead of a bug :P
run = Run(run_dir)
# Try to detect errors checking the output
if run._has_error():
logger.warning(
"Error detected while launching %s.\nCheck out.txt and err.txt for more information or re-run in console."
% run_dir)
return run