def create_and_persist_plots(self, dataset: xr.Dataset): ds = dataset filename = DSUtil.get_plot_filename(dataset, "Three Phase Voltage", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: # Calculations for contour plots date = pd.to_datetime(ds.time.data[0]).strftime('%d-%b-%Y') #hi = np.ceil(ds.wind_speed.max().data + 1) #lo = np.floor(ds.wind_speed.min().data) #levels = np.arange(lo, hi, 1) # Colormaps to use #wind_cmap = cmocean.cm.deep_r #avail_cmap = cmocean.cm.amp_r # Create figure and axes objects fig, ax = plt.subplots(figsize=(16, 8), constrained_layout=True) fig.suptitle( f"Three Phase Voltage from {ds.attrs['title']} on {date}") ds.MODAQ_Va[:100].plot(ax=ax) ds.MODAQ_Vb[:100].plot(ax=ax) ds.MODAQ_Vc[:100].plot(ax=ax) # Save the figure fig.savefig(tmp_path, dpi=100) self.storage.save(tmp_path) plt.close() return
def test_plotting_utilities(dataset): expected_filename = "test.SortedDataset.a1.20211001.000000.height.png" filename = DSUtil.get_plot_filename(dataset, "height", "png") filepath = os.path.join(STORAGE_PATH, "test.SortedDataset.a1", filename) assert filename == expected_filename assert DSUtil.get_date_from_filename(filepath) == "20211001.000000" DSUtil.plot_qc(dataset, "height_out", filepath) assert DSUtil.is_image(filepath) assert not DSUtil.is_image(PROCESSED_NC)
def hook_generate_and_persist_plots(self, dataset: xr.Dataset) -> None: """------------------------------------------------------------------- Hook to allow users to create plots from the xarray dataset after processing and QC have been applied and just before the dataset is saved to disk. To save on filesystem space (which is limited when running on the cloud via a lambda function), this method should only write one plot to local storage at a time. An example of how this could be done is below: ``` filename = DSUtil.get_plot_filename(dataset, "sea_level", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: fig, ax = plt.subplots(figsize=(10,5)) ax.plot(dataset["time"].data, dataset["sea_level"].data) fig.save(tmp_path) storage.save(tmp_path) filename = DSUtil.get_plot_filename(dataset, "qc_sea_level", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: fig, ax = plt.subplots(figsize=(10,5)) DSUtil.plot_qc(dataset, "sea_level", tmp_path) storage.save(tmp_path) ``` Args: dataset (xr.Dataset): The xarray dataset with customizations and QC applied. -------------------------------------------------------------------""" def format_time_xticks(ax, start=4, stop=21, step=4, date_format="%H-%M"): ax.xaxis.set_major_locator( mpl.dates.HourLocator(byhour=range(start, stop, step))) ax.xaxis.set_major_formatter(mpl.dates.DateFormatter(date_format)) plt.setp(ax.xaxis.get_majorticklabels(), rotation=0, ha="center") def double_plot(ax, twin, data, colors, var_labels, ax_labels, **kwargs): def _add_lineplot(_ax, _data, _c, _label, _ax_label, _spine): _data.plot(ax=_ax, c=_c, label=_label, linewidth=2, **kwargs) _ax.tick_params(axis="y", which="both", colors=_c) _ax.set_ylabel(_ax_label, color=_c) _ax.spines[_spine].set_color(_c) _add_lineplot(ax, data[0], colors[0], var_labels[0], ax_labels[0], "left") _add_lineplot(twin, data[1], colors[1], var_labels[1], ax_labels[1], "right") twin.spines["left"].set_color( colors[0]) # twin overwrites ax, so set color here def add_colorbar(ax, plot, label): cb = plt.colorbar(plot, ax=ax, pad=0.01) cb.ax.set_ylabel(label, fontsize=12) cb.outline.set_linewidth(1) cb.ax.tick_params(size=0) cb.ax.minorticks_off() return cb # Useful variables ds = dataset date = pd.to_datetime(ds.time.data[0]).strftime("%d-%b-%Y") cmap = sns.color_palette("viridis", as_cmap=True) colors = [cmap(0.00), cmap(0.60)] # Create the first plot -- Surface Met Parameters filename = DSUtil.get_plot_filename(dataset, "surface_met_parameters", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: # Define data and metadata data = [ [ds.wind_speed, ds.wind_direction], [ds.pressure, ds.rh], [ds.air_temperature, ds.CTD_SST], ] var_labels = [ [ r"$\overline{\mathrm{U}}$ Cup", r"$\overline{\mathrm{\theta}}$ Cup" ], ["Pressure", "Relative Humidity"], ["Air Temperature", "Sea Surface Temperature"], ] ax_labels = [ [ r"$\overline{\mathrm{U}}$ (ms$^{-1}$)", r"$\bar{\mathrm{\theta}}$ (degrees)", ], [ r"$\overline{\mathrm{P}}$ (bar)", r"$\overline{\mathrm{RH}}$ (%)" ], [ r"$\overline{\mathrm{T}}_{air}$ ($\degree$C)", r"$\overline{\mathrm{SST}}$ ($\degree$C)", ], ] # Create figure and axes objects fig, axs = plt.subplots(nrows=3, figsize=(14, 8), constrained_layout=True) twins = [ax.twinx() for ax in axs] fig.suptitle( f"Surface Met Parameters at {ds.attrs['location_meaning']} on {date}" ) # Create the plots gill_data = [ds.gill_wind_speed, ds.gill_wind_direction] gill_labels = [ r"$\overline{\mathrm{U}}$ Gill", r"$\overline{\mathrm{\theta}}$ Gill", ] double_plot( axs[0], twins[0], data=gill_data, colors=colors, var_labels=gill_labels, linestyle="--", ax_labels=["", ""], ) for i in range(3): double_plot( axs[i], twins[i], data=data[i], colors=colors, var_labels=var_labels[i], ax_labels=ax_labels[i], ) axs[i].grid(which="both", color="lightgray", linewidth=0.5) lines = axs[i].lines + twins[i].lines labels = [line.get_label() for line in lines] axs[i].legend(lines, labels, ncol=len(labels), bbox_to_anchor=(1, -0.15)) format_time_xticks(axs[i]) axs[i].set_xlabel("Time (UTC)") twins[0].set_ylim(0, 360) # Save and close the figure fig.savefig(tmp_path, dpi=100) self.storage.save(tmp_path) plt.close() # Create the second plot -- Conductivity and Sea Surface Temperature filename = DSUtil.get_plot_filename(dataset, "conductivity", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: # Define data and metadata data = [ds.conductivity, ds.CTD_SST] var_labels = [ r"Conductivity (S m$^{-1}$)", r"$\overline{\mathrm{SST}}$ ($\degree$C)", ] ax_labels = [ r"Conductivity (S m$^{-1}$)", r"$\overline{\mathrm{SST}}$ ($\degree$C)", ] # Create the figure and axes objects fig, ax = plt.subplots(figsize=(14, 8), constrained_layout=True) fig.suptitle( f"Conductivity and Sea Surface Temperature at {ds.attrs['location_meaning']} on {date}" ) twin = ax.twinx() # Make the plot double_plot( ax, twin, data=data, colors=colors, var_labels=var_labels, ax_labels=ax_labels, ) # Set the labels and ticks ax.grid(which="both", color="lightgray", linewidth=0.5) lines = ax.lines + twin.lines labels = [line.get_label() for line in lines] ax.legend(lines, labels, ncol=len(labels), bbox_to_anchor=(1, -0.03)) format_time_xticks(ax) ax.set_xlabel("Time (UTC)") # Save and close the figure fig.savefig(tmp_path, dpi=100) self.storage.save(tmp_path) plt.close() # Create the third plot - current speed and direction filename = DSUtil.get_plot_filename(dataset, "current_velocity", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: # Reduce dimensionality of dataset for plotting ds_1H: xr.Dataset = ds.reindex({"depth": ds.depth.data[::2]}) ds_1H: xr.Dataset = ds_1H.resample(time="1H").nearest() # Calculations for contour plots levels = 30 # Calculations for quiver plot qv_slice = slice( 1, None) # Skip first to prevent weird overlap with axes borders qv_degrees = ds_1H.current_direction.data[qv_slice, qv_slice].transpose() qv_theta = (qv_degrees + 90) * (np.pi / 180) X, Y = ds_1H.time.data[qv_slice], ds_1H.depth.data[qv_slice] U, V = np.cos(-qv_theta), np.sin(-qv_theta) # Create figure and axes objects fig, ax = plt.subplots(figsize=(14, 8), constrained_layout=True) fig.suptitle( f"Current Speed and Direction at {ds.attrs['location_meaning']} on {date}" ) # Make the plots csf = ds.current_speed.plot.contourf( ax=ax, x="time", yincrease=False, levels=levels, cmap=cmocean.cm.deep_r, add_colorbar=False, ) # ds.current_speed.plot.contour(ax=ax, x="time", yincrease=False, levels=levels, colors="lightgray", linewidths=0.5) ax.quiver( X, Y, U, V, width=0.002, scale=60, color="white", pivot="middle", zorder=10, ) add_colorbar(ax, csf, r"Current Speed (mm s$^{-1}$)") # Set the labels and ticks format_time_xticks(ax) ax.set_xlabel("Time (UTC)") ax.set_ylabel("Depth (m)") # Save the figure fig.savefig(tmp_path, dpi=100) self.storage.save(tmp_path) plt.close() return
def hook_generate_and_persist_plots(self, dataset: xr.Dataset): def format_time_xticks(ax, start=4, stop=21, step=4, date_format="%H-%M"): ax.xaxis.set_major_locator( mpl.dates.HourLocator(byhour=range(start, stop, step))) ax.xaxis.set_major_formatter(mpl.dates.DateFormatter(date_format)) plt.setp(ax.xaxis.get_majorticklabels(), rotation=0, ha="center") def add_colorbar(ax, plot, label): cb = plt.colorbar(plot, ax=ax, pad=0.01) cb.ax.set_ylabel(label, fontsize=12) cb.outline.set_linewidth(1) cb.ax.tick_params(size=0) cb.ax.minorticks_off() return cb ds = dataset date = pd.to_datetime(ds.time.data[0]).strftime("%d-%b-%Y") # Colormaps to use wind_cmap = cmocean.cm.deep_r avail_cmap = cmocean.cm.amp_r # Create the first plot - Lidar Wind Speeds at several elevations filename = DSUtil.get_plot_filename(dataset, "wind_speeds", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: # Create the figure and axes objects fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(14, 8), constrained_layout=True) fig.suptitle( f"Wind Speed Time Series at {ds.attrs['location_meaning']} on {date}" ) # Select heights to plot heights = [40, 90, 140, 200] # Plot the data for i, height in enumerate(heights): velocity = ds.wind_speed.sel(height=height) velocity.plot( ax=ax, linewidth=2, c=wind_cmap(i / len(heights)), label=f"{height} m", ) # Set the labels and ticks format_time_xticks(ax) ax.legend(facecolor="white", ncol=len(heights), bbox_to_anchor=(1, -0.05)) ax.set_title("") # Remove bogus title created by xarray ax.set_xlabel("Time (UTC)") ax.set_ylabel(r"Wind Speed (ms$^{-1}$)") # Save the figure fig.savefig(tmp_path, dpi=100) self.storage.save(tmp_path) plt.close() filename = DSUtil.get_plot_filename(dataset, "wind_speed_and_direction", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: # Reduce dimensionality of dataset for plotting ds_1H: xr.Dataset = ds.resample(time="1H").nearest() # Calculations for contour plots levels = 30 # Calculations for quiver plot qv_slice = slice( 1, None) # Skip first to prevent weird overlap with axes borders qv_degrees = ds_1H.wind_direction.data[qv_slice].transpose() qv_theta = (qv_degrees + 90) * (np.pi / 180) X, Y = ds_1H.time.data[qv_slice], ds_1H.height.data U, V = np.cos(-qv_theta), np.sin(-qv_theta) # Create figure and axes objects fig, axs = plt.subplots(nrows=2, figsize=(14, 8), constrained_layout=True) fig.suptitle( f"Wind Speed and Direction at {ds.attrs['location_meaning']} on {date}" ) # Make top subplot -- contours and quiver plots for wind speed and direction csf = ds.wind_speed.plot.contourf(ax=axs[0], x="time", levels=levels, cmap=wind_cmap, add_colorbar=False) # ds.wind_speed.plot.contour(ax=axs[0], x="time", levels=levels, colors="lightgray", linewidths=0.5) axs[0].quiver( X, Y, U, V, width=0.002, scale=60, color="white", pivot="middle", zorder=10, ) add_colorbar(axs[0], csf, r"Wind Speed (ms$^{-1}$)") # Make bottom subplot -- heatmap for data availability da = ds.data_availability.plot( ax=axs[1], x="time", cmap=avail_cmap, add_colorbar=False, vmin=0, vmax=100, ) add_colorbar(axs[1], da, "Availability (%)") # Set the labels and ticks for i in range(2): format_time_xticks(axs[i]) axs[i].set_xlabel("Time (UTC)") axs[i].set_ylabel("Height ASL (m)") # Save the figure fig.savefig(tmp_path, dpi=100) self.storage.save(tmp_path) plt.close() return
def hook_generate_and_persist_plots(self, dataset: xr.Dataset) -> None: """------------------------------------------------------------------- Hook to allow users to create plots from the xarray dataset after processing and QC have been applied and just before the dataset is saved to disk. To save on filesystem space (which is limited when running on the cloud via a lambda function), this method should only write one plot to local storage at a time. An example of how this could be done is below: ``` filename = DSUtil.get_plot_filename(dataset, "sea_level", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: fig, ax = plt.subplots(figsize=(10,5)) ax.plot(dataset["time"].data, dataset["sea_level"].data) fig.savefig(tmp_path) self.storage.save(tmp_path) plt.close() filename = DSUtil.get_plot_filename(dataset, "qc_sea_level", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: fig, ax = plt.subplots(figsize=(10,5)) DSUtil.plot_qc(dataset, "sea_level", tmp_path) storage.save(tmp_path) ``` Args: dataset (xr.Dataset): The xarray dataset with customizations and QC applied. -------------------------------------------------------------------""" ds = dataset # Useful values location = ds.attrs["location_meaning"] date1, date2 = pd.to_datetime(ds.time.data[0]), pd.to_datetime( ds.time.data[-1]) hhmm1, hhmm2 = date1.strftime("%H:%M"), date2.strftime("%H:%M") date = date1.strftime("%d-%b-%Y") filename = DSUtil.get_plot_filename(dataset, "buoy_motion_histogram", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: fig, ax = plt.subplots(figsize=(14, 8), constrained_layout=True) # Create plot labels including mean roll/pitch mean_roll, mean_pitch = ds["roll"].mean().data, ds["pitch"].mean( ).data roll_label = (r"$\.{\theta}_{roll}$ [$\overline{\theta}_r$ =" + f"{mean_roll:.3f} deg]") pitch_label = (r"$\.{\theta}_{pitch}$ [$\overline{\theta}_p$ =" + f"{mean_pitch:.3f} deg]") # Plot the stepped ds["roll"].plot.hist(ax=ax, linewidth=2, edgecolor="black", histtype="step", label=roll_label) ds["pitch"].plot.hist(ax=ax, linewidth=2, edgecolor="red", histtype="step", label=pitch_label) # Set axes and figure labels fig.suptitle( f"Buoy Motion Histogram at {location} on {date} from {hhmm1} to {hhmm2}" ) ax.set_xlabel("Buoy Motion (deg)") ax.set_ylabel("Frequency") ax.set_title("") ax.legend(ncol=2, bbox_to_anchor=(1, -0.04)) # Save the figure fig.savefig(tmp_path, dpi=100) self.storage.save(tmp_path) plt.close() return
def hook_generate_and_persist_plots(self, dataset: xr.Dataset) -> None: """------------------------------------------------------------------- Hook to allow users to create plots from the xarray dataset after processing and QC have been applied and just before the dataset is saved to disk. To save on filesystem space (which is limited when running on the cloud via a lambda function), this method should only write one plot to local storage at a time. An example of how this could be done is below: ``` filename = DSUtil.get_plot_filename(dataset, "sea_level", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: fig, ax = plt.subplots(figsize=(10,5)) ax.plot(dataset["time"].data, dataset["sea_level"].data) fig.savefig(tmp_path) self.storage.save(tmp_path) plt.close() filename = DSUtil.get_plot_filename(dataset, "qc_sea_level", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: fig, ax = plt.subplots(figsize=(10,5)) DSUtil.plot_qc(dataset, "sea_level", tmp_path) storage.save(tmp_path) ``` Args: dataset (xr.Dataset): The xarray dataset with customizations and QC applied. -------------------------------------------------------------------""" def format_time_xticks(ax, start=4, stop=21, step=4, date_format="%H-%M"): ax.xaxis.set_major_locator( mpl.dates.HourLocator(byhour=range(start, stop, step))) ax.xaxis.set_major_formatter(mpl.dates.DateFormatter(date_format)) plt.setp(ax.xaxis.get_majorticklabels(), rotation=0, ha='center') # Useful variables ds = dataset date = pd.to_datetime(ds.time.data[0]).strftime('%d-%b-%Y') # Create wave statistics plot filename = DSUtil.get_plot_filename(dataset, "wave_statistics", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: # Create figure and axes objects fig, axs = plt.subplots(nrows=3, figsize=(14, 8), constrained_layout=True) fig.suptitle( f"Wave Statistics at {ds.attrs['location_meaning']} on {date}") # Plot wave heights cmap = cmocean.cm.amp_r ds.average_wave_height.plot(ax=axs[0], c=cmap(0.10), linewidth=2, label=r"H$_{avg}$") ds.significant_wave_height.plot(ax=axs[0], c=cmap(0.5), linewidth=2, label=r"H$_{sig}$") ds.max_wave_height.plot(ax=axs[0], c=cmap(0.85), linewidth=2, label=r"H$_{max}$") axs[0].set_ylabel("Wave Height (m)") axs[0].legend(bbox_to_anchor=(1, -0.10), ncol=3) # Plot wave periods cmap = cmocean.cm.dense ds.average_wave_period.plot(ax=axs[1], c=cmap(0.15), linewidth=2, label=r"T$_{avg}$") ds.significant_wave_period.plot(ax=axs[1], c=cmap(0.5), linewidth=2, label=r"T$_{sig}$") ds.mean_wave_period.plot(ax=axs[1], c=cmap(0.8), linewidth=2, label=r"$\overline{T}_{mean}$") axs[1].set_ylabel("Wave Period (s)") axs[1].legend(bbox_to_anchor=(1, -0.10), ncol=3) # Plot mean direction cmap = cmocean.cm.haline ds.mean_wave_direction.plot(ax=axs[2], c=cmap(0.4), linewidth=2, label=r"$\overline{\phi}_{mean}$") axs[2].set_ylabel(r"Wave $\overline{\phi}$ (deg)") axs[2].legend(bbox_to_anchor=(1, -0.10)) # Set xlabels and ticks for i in range(3): axs[i].set_xlabel("Time (UTC)") format_time_xticks(axs[i]) # Save figure fig.savefig(tmp_path, dpi=100) self.storage.save(tmp_path) plt.close() return
def add_colorbar(ax, plot, label): cb = plt.colorbar(plot, ax=ax, pad=0.01) cb.ax.set_ylabel(label, fontsize=12) cb.outline.set_linewidth(1) cb.ax.tick_params(size=0) cb.ax.minorticks_off() return cb # Useful variables ds = dataset date = pd.to_datetime(ds.time.data[0]).strftime('%d-%b-%Y') cmap = sns.color_palette("viridis", as_cmap=True) colors = [cmap(0.00), cmap(0.60)] # Create the first plot -- Surface Met Parameters filename = DSUtil.get_plot_filename(dataset, "surface_met_parameters", "png") with self.storage._tmp.get_temp_filepath(filename) as tmp_path: # Define data and metadata data = [[ds.wind_speed, ds.wind_direction], [ds.pressure, ds.rh], [ds.air_temperature, ds.CTD_SST]] var_labels = [[ r"$\overline{\mathrm{U}}$ Cup", r"$\overline{\mathrm{\theta}}$ Cup" ], ["Pressure", "Relative Humidity"], ["Air Temperature", "Sea Surface Temperature"]] ax_labels = [[ r"$\overline{\mathrm{U}}$ (ms$^{-1}$)", r"$\bar{\mathrm{\theta}}$ (degrees)" ], [